From e355bef79bdde4a7adc17489205c9e5f338f954c Mon Sep 17 00:00:00 2001 From: Bakke Date: Thu, 26 Sep 2024 07:22:02 +0000 Subject: [PATCH] fix: vsphere image repositories, tags and docs (#11564) The old repository for these has been deleted, leaving the previous configuration not possible to deploy, and even currently running clusters fail after a restart as the DeameonSet has ImagePullPolicy: Always. More details can be found here: kubernetes-sigs/vsphere-csi-driver#3053 As of writing, only CSI driver versions 3.1.2 to 3.3.1 is available in this registry. This "officially" supports Kubernetes 1.26 to 1.30. Since older drivers are not available, I have removed some feature-gating for those unavailable versions while I was at it. For the cloud provider, the `latest` image is now missing, and only 1.28.0 to 1.31.0 are available. I've set the latest of these as the new default. I also updated the documented default versions, as they were all out of date and not aligned with actual code defaults. --- docs/CSI/vsphere-csi.md | 22 +++++++++---------- inventory/sample/group_vars/all/vsphere.yml | 12 +++++----- .../csi_driver/vsphere/defaults/main.yml | 6 ++--- .../vsphere-csi-controller-config.yml.j2 | 2 -- .../vsphere-csi-controller-deployment.yml.j2 | 4 ++-- .../vsphere-csi-controller-rbac.yml.j2 | 2 -- .../vsphere/templates/vsphere-csi-node.yml.j2 | 2 +- .../vsphere/defaults/main.yml | 2 +- ...vsphere-cloud-controller-manager-ds.yml.j2 | 2 +- 9 files changed, 25 insertions(+), 29 deletions(-) diff --git a/docs/CSI/vsphere-csi.md b/docs/CSI/vsphere-csi.md index af58440ca..01fa9b2dc 100644 --- a/docs/CSI/vsphere-csi.md +++ b/docs/CSI/vsphere-csi.md @@ -23,20 +23,20 @@ You need to source the vSphere credentials you use to deploy your machines that | external_vsphere_password | TRUE | string | | | Password for vCenter (Can also be specified with the `VSPHERE_PASSWORD` environment variable) | | external_vsphere_datacenter | TRUE | string | | | Datacenter name to use | | external_vsphere_kubernetes_cluster_id | TRUE | string | | "kubernetes-cluster-id" | Kubernetes cluster ID to use | -| external_vsphere_version | TRUE | string | | "6.7u3" | Vmware Vsphere version where located all VMs | -| external_vsphere_cloud_controller_image_tag | TRUE | string | | "latest" | Kubernetes cluster ID to use | -| vsphere_syncer_image_tag | TRUE | string | | "v2.2.1" | Syncer image tag to use | -| vsphere_csi_attacher_image_tag | TRUE | string | | "v3.1.0" | CSI attacher image tag to use | -| vsphere_csi_controller | TRUE | string | | "v2.2.1" | CSI controller image tag to use | +| external_vsphere_version | TRUE | string | | "7.0u1" | Vmware Vsphere version where located all VMs | +| external_vsphere_cloud_controller_image_tag | TRUE | string | | "v1.31.0" | CPI manager image tag to use | +| vsphere_syncer_image_tag | TRUE | string | | "v3.3.1" | Syncer image tag to use | +| vsphere_csi_attacher_image_tag | TRUE | string | | "v4.3.0" | CSI attacher image tag to use | +| vsphere_csi_controller | TRUE | string | | "v3.3.1" | CSI controller image tag to use | | vsphere_csi_controller_replicas | TRUE | integer | | 1 | Number of pods Kubernetes should deploy for the CSI controller | -| vsphere_csi_liveness_probe_image_tag | TRUE | string | | "v2.2.0" | CSI liveness probe image tag to use | +| vsphere_csi_liveness_probe_image_tag | TRUE | string | | "v2.10.0" | CSI liveness probe image tag to use | | vsphere_csi_provisioner_image_tag | TRUE | string | | "v2.1.0" | CSI provisioner image tag to use | -| vsphere_csi_node_driver_registrar_image_tag | TRUE | string | | "v1.1.0" | CSI node driver registrar image tag to use | -| vsphere_csi_driver_image_tag | TRUE | string | | "v1.0.2" | CSI driver image tag to use | -| vsphere_csi_resizer_tag | TRUE | string | | "v1.1.0" | CSI resizer image tag to use | +| vsphere_csi_node_driver_registrar_image_tag | TRUE | string | | "v3.5.0" | CSI node driver registrar image tag to use | +| vsphere_csi_driver_image_tag | TRUE | string | | "v3.3.1" | CSI driver image tag to use | +| vsphere_csi_resizer_tag | TRUE | string | | "v1.8.0" | CSI resizer image tag to use | | vsphere_csi_aggressive_node_drain | FALSE | boolean | | false | Enable aggressive node drain strategy | -| vsphere_csi_aggressive_node_unreachable_timeout | FALSE | int | 300 | | Timeout till node will be drained when it in an unreachable state | -| vsphere_csi_aggressive_node_not_ready_timeout | FALSE | int | 300 | | Timeout till node will be drained when it in not-ready state | +| vsphere_csi_aggressive_node_unreachable_timeout | FALSE | int | | 300 | Timeout till node will be drained when it in an unreachable state | +| vsphere_csi_aggressive_node_not_ready_timeout | FALSE | int | | 300 | Timeout till node will be drained when it in not-ready state | | vsphere_csi_namespace | TRUE | string | | "kube-system" | vSphere CSI namespace to use; kube-system for backward compatibility, should be change to vmware-system-csi on the long run | ## Usage example diff --git a/inventory/sample/group_vars/all/vsphere.yml b/inventory/sample/group_vars/all/vsphere.yml index af3cfbe28..08a33f42c 100644 --- a/inventory/sample/group_vars/all/vsphere.yml +++ b/inventory/sample/group_vars/all/vsphere.yml @@ -11,14 +11,14 @@ # external_vsphere_version: "6.7u3" ## Tags for the external vSphere Cloud Provider images -## gcr.io/cloud-provider-vsphere/cpi/release/manager -# external_vsphere_cloud_controller_image_tag: "latest" -## gcr.io/cloud-provider-vsphere/csi/release/syncer -# vsphere_syncer_image_tag: "v2.5.1" +## registry.k8s.io/cloud-pv-vsphere/cloud-provider-vsphere +# external_vsphere_cloud_controller_image_tag: "v1.31.0" +## registry.k8s.io/csi-vsphere/syncer +# vsphere_syncer_image_tag: "v3.3.1" ## registry.k8s.io/sig-storage/csi-attacher # vsphere_csi_attacher_image_tag: "v3.4.0" -## gcr.io/cloud-provider-vsphere/csi/release/driver -# vsphere_csi_controller: "v2.5.1" +## registry.k8s.io/csi-vsphere/driver +# vsphere_csi_controller: "v3.3.1" ## registry.k8s.io/sig-storage/livenessprobe # vsphere_csi_liveness_probe_image_tag: "v2.6.0" ## registry.k8s.io/sig-storage/csi-provisioner diff --git a/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml b/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml index 325e3cb7d..684f6c47a 100644 --- a/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml +++ b/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml @@ -4,14 +4,14 @@ external_vsphere_insecure: "true" external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id" external_vsphere_version: "7.0u1" -vsphere_syncer_image_tag: "v3.1.0" +vsphere_syncer_image_tag: "v3.3.1" vsphere_csi_attacher_image_tag: "v4.3.0" -vsphere_csi_controller: "v3.1.0" +vsphere_csi_controller: "v3.3.1" vsphere_csi_liveness_probe_image_tag: "v2.10.0" vsphere_csi_provisioner_image_tag: "v3.5.0" vsphere_csi_snapshotter_image_tag: "v6.2.2" vsphere_csi_node_driver_registrar_image_tag: "v2.8.0" -vsphere_csi_driver_image_tag: "v3.1.0" +vsphere_csi_driver_image_tag: "v3.3.1" vsphere_csi_resizer_tag: "v1.8.0" # Set to kube-system for backward compatibility, should be change to vmware-system-csi on the long run diff --git a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2 b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2 index 274889604..caad7a334 100644 --- a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2 @@ -19,12 +19,10 @@ data: "multi-vcenter-csi-topology": "true" "csi-internal-generated-cluster-id": "true" "listview-tasks": "true" -{% if vsphere_csi_controller is version('v2.7.0', '>=') %} "improved-csi-idempotency": "true" "improved-volume-topology": "true" "use-csinode-id": "true" "list-volumes": "false" -{% endif %} kind: ConfigMap metadata: name: internal-feature-states.csi.vsphere.vmware.com diff --git a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2 b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2 index 00d6e6a92..6a1f31963 100644 --- a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2 @@ -99,7 +99,7 @@ spec: name: socket-dir {% endif %} - name: vsphere-csi-controller - image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_controller }} + image: {{ kube_image_repo }}/csi-vsphere/driver:{{ vsphere_csi_controller }} args: - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" - "--fss-namespace={{ vsphere_csi_namespace }}" @@ -168,7 +168,7 @@ spec: - name: socket-dir mountPath: {{ csi_endpoint }} - name: vsphere-syncer - image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/syncer:{{ vsphere_syncer_image_tag }} + image: {{ kube_image_repo }}/csi-vsphere/syncer:{{ vsphere_syncer_image_tag }} args: - "--leader-election" - "--leader-election-lease-duration=30s" diff --git a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2 b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2 index 013d3dc3f..447d57d0a 100644 --- a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2 @@ -33,11 +33,9 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch", "create", "update", "patch"] -{% if vsphere_csi_controller is version('v2.0.0', '>=') %} - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] -{% endif %} - apiGroups: ["storage.k8s.io"] resources: ["storageclasses","csinodes"] verbs: ["get", "list", "watch"] diff --git a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2 b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2 index e110ee300..ebf85a444 100644 --- a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2 @@ -62,7 +62,7 @@ spec: - --mode=kubelet-registration-probe initialDelaySeconds: 3 - name: vsphere-csi-node - image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_driver_image_tag }} + image: {{ kube_image_repo }}/csi-vsphere/driver:{{ vsphere_csi_driver_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" diff --git a/roles/kubernetes-apps/external_cloud_controller/vsphere/defaults/main.yml b/roles/kubernetes-apps/external_cloud_controller/vsphere/defaults/main.yml index b6fb797a8..3bbc32866 100644 --- a/roles/kubernetes-apps/external_cloud_controller/vsphere/defaults/main.yml +++ b/roles/kubernetes-apps/external_cloud_controller/vsphere/defaults/main.yml @@ -8,7 +8,7 @@ external_vsphere_insecure: "true" ## arg1: "value1" ## arg2: "value2" external_vsphere_cloud_controller_extra_args: {} -external_vsphere_cloud_controller_image_tag: "latest" +external_vsphere_cloud_controller_image_tag: "v1.31.0" external_vsphere_user: "{{ lookup('env', 'VSPHERE_USER') }}" external_vsphere_password: "{{ lookup('env', 'VSPHERE_PASSWORD') }}" diff --git a/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2 index 05e4ac300..943409964 100644 --- a/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2 +++ b/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2 @@ -36,7 +36,7 @@ spec: serviceAccountName: cloud-controller-manager containers: - name: vsphere-cloud-controller-manager - image: {{ gcr_image_repo }}/cloud-provider-vsphere/cpi/release/manager:{{ external_vsphere_cloud_controller_image_tag }} + image: {{ kube_image_repo }}/cloud-pv-vsphere/cloud-provider-vsphere:{{ external_vsphere_cloud_controller_image_tag }} args: - --v=2 - --cloud-provider=vsphere