From b396801e28b428f824743828f3328a00d5f90c13 Mon Sep 17 00:00:00 2001 From: Steven Reitsma Date: Fri, 10 Dec 2021 19:49:11 +0100 Subject: [PATCH] Update Cinder CSI to v1.22 (#8296) --- roles/download/defaults/main.yml | 26 +++--- .../aws-ebs-csi-controllerservice.yml.j2 | 5 +- .../cinder-csi-controllerplugin-rbac.yml.j2 | 85 ++++++------------- .../cinder-csi-controllerplugin.yml.j2 | 49 +++++++---- .../templates/cinder-csi-nodeplugin.yml.j2 | 30 ++++--- 5 files changed, 95 insertions(+), 100 deletions(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 8a915b7b4..4e10e9cbf 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -686,18 +686,18 @@ addon_resizer_version: "1.8.11" addon_resizer_image_repo: "{{ kube_image_repo }}/addon-resizer" addon_resizer_image_tag: "{{ addon_resizer_version }}" -csi_attacher_image_repo: "{{ quay_image_repo }}/k8scsi/csi-attacher" -csi_attacher_image_tag: "v2.2.0" -csi_provisioner_image_repo: "{{ quay_image_repo }}/k8scsi/csi-provisioner" -csi_provisioner_image_tag: "v1.6.0" -csi_snapshotter_image_repo: "{{ quay_image_repo }}/k8scsi/csi-snapshotter" -csi_snapshotter_image_tag: "v2.1.1" -csi_resizer_image_repo: "{{ quay_image_repo }}/k8scsi/csi-resizer" -csi_resizer_image_tag: "v0.5.0" -csi_node_driver_registrar_image_repo: "{{ quay_image_repo }}/k8scsi/csi-node-driver-registrar" -csi_node_driver_registrar_image_tag: "v1.3.0" -csi_livenessprobe_image_repo: "{{ quay_image_repo }}/k8scsi/livenessprobe" -csi_livenessprobe_image_tag: "v2.0.0" +csi_attacher_image_repo: "{{ kube_image_repo }}/sig-storage/csi-attacher" +csi_attacher_image_tag: "v3.3.0" +csi_provisioner_image_repo: "{{ kube_image_repo }}/sig-storage/csi-provisioner" +csi_provisioner_image_tag: "v3.0.0" +csi_snapshotter_image_repo: "{{ kube_image_repo }}/sig-storage/csi-snapshotter" +csi_snapshotter_image_tag: "v4.2.1" +csi_resizer_image_repo: "{{ kube_image_repo }}/sig-storage/csi-resizer" +csi_resizer_image_tag: "v1.3.0" +csi_node_driver_registrar_image_repo: "{{ kube_image_repo }}/sig-storage/csi-node-driver-registrar" +csi_node_driver_registrar_image_tag: "v2.4.0" +csi_livenessprobe_image_repo: "{{ kube_image_repo }}/sig-storage/livenessprobe" +csi_livenessprobe_image_tag: "v2.5.0" snapshot_controller_supported_versions: v1.22: "v4.2.1" @@ -707,7 +707,7 @@ snapshot_controller_image_repo: "{{ kube_image_repo }}/sig-storage/snapshot-cont snapshot_controller_image_tag: "{{ snapshot_controller_supported_versions[kube_major_version] }}" cinder_csi_plugin_image_repo: "{{ docker_image_repo }}/k8scloudprovider/cinder-csi-plugin" -cinder_csi_plugin_image_tag: "v1.20.0" +cinder_csi_plugin_image_tag: "v1.22.0" aws_ebs_csi_plugin_image_repo: "{{ docker_image_repo }}/amazon/aws-ebs-csi-driver" aws_ebs_csi_plugin_image_tag: "v0.5.0" diff --git a/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2 b/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2 index b08196efb..e0796765f 100644 --- a/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2 @@ -18,7 +18,7 @@ spec: spec: nodeSelector: kubernetes.io/os: linux - serviceAccount: ebs-csi-controller-sa + serviceAccountName: ebs-csi-controller-sa priorityClassName: system-cluster-critical containers: - name: ebs-plugin @@ -68,8 +68,7 @@ spec: {% if aws_ebs_csi_enable_volume_scheduling %} - --feature-gates=Topology=true {% endif %} - - --enable-leader-election - - --leader-election-type=leases + - --leader-election=true env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 index 353e68558..d40053ad3 100644 --- a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 @@ -16,17 +16,19 @@ metadata: rules: - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] + verbs: ["get", "list", "watch", "patch"] - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: ClusterRoleBinding @@ -73,7 +75,12 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["get", "list"] - + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -95,15 +102,6 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-snapshotter-role rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] @@ -116,19 +114,12 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents/status"] verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete"] - + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -157,20 +148,22 @@ rules: # verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] + verbs: ["get", "list", "watch", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch"] - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] + resources: ["pods"] verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -184,29 +177,3 @@ roleRef: kind: ClusterRole name: csi-resizer-role apiGroup: rbac.authorization.k8s.io - ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: kube-system - name: external-resizer-cfg -rules: -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-resizer-role-cfg - namespace: kube-system -subjects: - - kind: ServiceAccount - name: csi-cinder-controller-sa - namespace: kube-system -roleRef: - kind: Role - name: external-resizer-cfg - apiGroup: rbac.authorization.k8s.io diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 index fd76073dd..6bd671ade 100644 --- a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 @@ -17,7 +17,7 @@ spec: labels: app: csi-cinder-controllerplugin spec: - serviceAccount: csi-cinder-controller-sa + serviceAccountName: csi-cinder-controller-sa containers: - name: csi-attacher image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }} @@ -26,8 +26,7 @@ spec: - "--csi-address=$(ADDRESS)" - "--timeout=3m" {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} - - --leader-election - - --leader-election-namespace=kube-system + - --leader-election=true {% endif %} env: - name: ADDRESS @@ -41,13 +40,13 @@ spec: args: - "--csi-address=$(ADDRESS)" - "--timeout=3m" + - "--default-fstype=ext4" + - "--extra-create-metadata" {% if cinder_topology is defined and cinder_topology %} - --feature-gates=Topology=true {% endif %} {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} - - --enable-leader-election - - --leader-election-type=leases - - --leader-election-namespace=kube-system + - "--leader-election=true" {% endif %} env: - name: ADDRESS @@ -60,9 +59,10 @@ spec: imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--csi-address=$(ADDRESS)" + - "--timeout=3m" + - "--extra-create-metadata" {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} - - --leader-election - - --leader-election-namespace=kube-system + - --leader-election=true {% endif %} env: - name: ADDRESS @@ -75,9 +75,10 @@ spec: imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--csi-address=$(ADDRESS)" + - "--timeout=3m" + - "--handle-volume-inuse-error=false" {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} - - --leader-election - - --leader-election-namespace=kube-system + - --leader-election=true {% endif %} env: - name: ADDRESS @@ -85,26 +86,44 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: liveness-probe + image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir - name: cinder-csi-plugin image: {{ cinder_csi_plugin_image_repo }}:{{ cinder_csi_plugin_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - /bin/cinder-csi-plugin - - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--cloud-config=$(CLOUD_CONFIG)" - "--cluster=$(CLUSTER_NAME)" env: - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix://csi/csi.sock - name: CLOUD_CONFIG value: /etc/config/cloud.conf - name: CLUSTER_NAME value: kubernetes + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 10 + periodSeconds: 60 volumeMounts: - name: socket-dir mountPath: /csi diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 index 3d901ec98..d0a86bd9f 100644 --- a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 @@ -15,7 +15,7 @@ spec: labels: app: csi-cinder-nodeplugin spec: - serviceAccount: csi-cinder-node-sa + serviceAccountName: csi-cinder-node-sa hostNetwork: true containers: - name: node-driver-registrar @@ -24,10 +24,6 @@ spec: args: - "--csi-address=$(ADDRESS)" - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "rm -rf /registration/cinder.csi.openstack.org /registration/cinder.csi.openstack.org-reg.sock"] env: - name: ADDRESS value: /csi/csi.sock @@ -42,6 +38,13 @@ spec: mountPath: /csi - name: registration-dir mountPath: /registration + - name: liveness-probe + image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }} + args: + - "--csi-address=/csi/csi.sock" + volumeMounts: + - name: socket-dir + mountPath: /csi - name: cinder-csi-plugin securityContext: privileged: true @@ -52,18 +55,25 @@ spec: imagePullPolicy: {{ k8s_image_pull_policy }} args: - /bin/cinder-csi-plugin - - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--cloud-config=$(CLOUD_CONFIG)" env: - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix://csi/csi.sock - name: CLOUD_CONFIG value: /etc/config/cloud.conf + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 volumeMounts: - name: socket-dir mountPath: /csi