diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 81048af2d..981cf0ff0 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -673,15 +673,14 @@ aws_ebs_csi_plugin_image_repo: "{{ docker_image_repo }}/amazon/aws-ebs-csi-drive aws_ebs_csi_plugin_image_tag: "v0.5.0" azure_csi_image_repo: "mcr.microsoft.com/oss/kubernetes-csi" -azure_csi_provisioner_image_tag: "v1.5.0" -azure_csi_attacher_image_tag: "v1.2.0" -azure_csi_cluster_registrar_image_tag: "v1.0.1" -azure_csi_node_registrar_image_tag: "v1.1.0" -azure_csi_snapshotter_image_tag: "v2.0.0" -azure_csi_resizer_image_tag: "v0.3.0" -azure_csi_livenessprobe_image_tag: "v1.1.0" +azure_csi_provisioner_image_tag: "v2.2.2" +azure_csi_attacher_image_tag: "v3.3.0" +azure_csi_resizer_image_tag: "v1.3.0" +azure_csi_livenessprobe_image_tag: "v2.5.0" +azure_csi_node_registrar_image_tag: "v2.4.0" +azure_csi_snapshotter_image_tag: "v3.0.3" azure_csi_plugin_image_repo: "mcr.microsoft.com/k8s/csi" -azure_csi_plugin_image_tag: "v0.7.0" +azure_csi_plugin_image_tag: "v1.10.0" gcp_pd_csi_image_repo: "gke.gcr.io" gcp_pd_csi_driver_image_tag: "v0.7.0-gke.0" diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml b/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml index c1eec6401..341cc97d8 100644 --- a/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml +++ b/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml @@ -1,4 +1,6 @@ --- azure_csi_use_instance_metadata: true -azure_csi_controller_replicas: 1 +azure_csi_controller_replicas: 2 azure_csi_plugin_image_tag: latest +azure_csi_controller_affinity: {} +azure_csi_node_affinity: {} diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 index ad974d38c..16f4c989c 100644 --- a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 @@ -71,7 +71,10 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch", "create", "update", "patch"] @@ -128,15 +131,6 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-external-snapshotter-role rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] @@ -150,14 +144,11 @@ rules: resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete"] - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["create", "list", "watch", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] @@ -197,6 +188,9 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -210,3 +204,27 @@ roleRef: kind: ClusterRole name: azuredisk-external-resizer-role apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-controller-secret-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-controller-secret-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azuredisk-controller-secret-role + apiGroup: rbac.authorization.k8s.io diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 index 659c9b965..693619073 100644 --- a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 @@ -24,19 +24,23 @@ spec: effect: "NoSchedule" - key: "node-role.kubernetes.io/control-plane" effect: "NoSchedule" +{% if azure_csi_controller_affinity %} + affinity: + {{ azure_csi_controller_affinity | to_nice_yaml | indent(width=8) }} +{% endif %} containers: - name: csi-provisioner image: {{ azure_csi_image_repo }}/csi-provisioner:{{ azure_csi_provisioner_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - - "--provisioner=disk.csi.azure.com" - "--feature-gates=Topology=true" - "--csi-address=$(ADDRESS)" - - "--connection-timeout=15s" - - "--v=5" - - "--timeout=120s" - - "--enable-leader-election" - - "--leader-election-type=leases" + - "--v=2" + - "--timeout=15s" + - "--leader-election" + - "--worker-threads=40" + - "--extra-create-metadata=true" + - "--strict-topology=true" env: - name: ADDRESS value: /csi/csi.sock @@ -45,8 +49,7 @@ spec: name: socket-dir resources: limits: - cpu: 200m - memory: 200Mi + memory: 500Mi requests: cpu: 10m memory: 20Mi @@ -54,40 +57,20 @@ spec: image: {{ azure_csi_image_repo }}/csi-attacher:{{ azure_csi_attacher_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - - "-v=5" + - "-v=2" - "-csi-address=$(ADDRESS)" - - "-timeout=120s" + - "-timeout=600s" - "-leader-election" - - "-leader-election-type=leases" - env: - - name: ADDRESS - value: /csi/csi.sock - volumeMounts: - - mountPath: /csi - name: socket-dir - resources: - limits: - cpu: 200m - memory: 200Mi - requests: - cpu: 10m - memory: 20Mi - - name: cluster-driver-registrar - image: {{ azure_csi_image_repo }}/csi-cluster-driver-registrar:{{ azure_csi_cluster_registrar_image_tag }} - args: - - --csi-address=$(ADDRESS) - - --driver-requires-attachment=true - - --v=5 + - "-worker-threads=500" env: - name: ADDRESS value: /csi/csi.sock volumeMounts: - - name: socket-dir - mountPath: /csi + - mountPath: /csi + name: socket-dir resources: limits: - cpu: 200m - memory: 200Mi + memory: 500Mi requests: cpu: 10m memory: 20Mi @@ -96,7 +79,7 @@ spec: args: - "-csi-address=$(ADDRESS)" - "-leader-election" - - "--v=5" + - "-v=2" env: - name: ADDRESS value: /csi/csi.sock @@ -105,8 +88,7 @@ spec: mountPath: /csi resources: limits: - cpu: 200m - memory: 200Mi + memory: 100Mi requests: cpu: 10m memory: 20Mi @@ -114,8 +96,10 @@ spec: image: {{ azure_csi_image_repo }}/csi-resizer:{{ azure_csi_resizer_image_tag }} args: - "-csi-address=$(ADDRESS)" - - "-v=5" + - "-v=2" - "-leader-election" + - '-handle-volume-inuse-error=false' + - "-timeout=60s" env: - name: ADDRESS value: /csi/csi.sock @@ -124,8 +108,7 @@ spec: mountPath: /csi resources: limits: - cpu: 200m - memory: 200Mi + memory: 500Mi requests: cpu: 10m memory: 20Mi @@ -133,16 +116,15 @@ spec: image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }} args: - --csi-address=/csi/csi.sock - - --connection-timeout=3s + - --probe-timeout=3s - --health-port=29602 - - --v=5 + - --v=2 volumeMounts: - name: socket-dir mountPath: /csi resources: limits: - cpu: 200m - memory: 200Mi + memory: 100Mi requests: cpu: 10m memory: 20Mi @@ -152,7 +134,11 @@ spec: args: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - - "--nodeid=$(KUBE_NODE_NAME)" + - "--metrics-address=0.0.0.0:29604" + - "--disable-avset-nodes=true" + - "--drivername=disk.csi.azure.com" + - "--cloud-config-secret-name=cloud-config" + - "--cloud-config-secret-namespace=kube-system" ports: - containerPort: 29602 name: healthz @@ -184,8 +170,7 @@ spec: name: msi resources: limits: - cpu: 200m - memory: 200Mi + memory: 500Mi requests: cpu: 10m memory: 20Mi @@ -195,6 +180,3 @@ spec: - name: azure-cred secret: secretName: cloud-config - - name: msi - hostPath: - path: /var/lib/waagent/ManagedIdentity-Settings diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node-rbac.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node-rbac.yml.j2 new file mode 100644 index 000000000..d55ea0d07 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node-rbac.yml.j2 @@ -0,0 +1,30 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-azuredisk-node-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-node-secret-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-node-secret-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azuredisk-node-secret-role + apiGroup: rbac.authorization.k8s.io diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 index 45e46718f..76e797371 100644 --- a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 @@ -5,6 +5,10 @@ metadata: name: csi-azuredisk-node namespace: kube-system spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate selector: matchLabels: app: csi-azuredisk-node @@ -14,9 +18,17 @@ spec: app: csi-azuredisk-node spec: hostNetwork: true + dnsPolicy: Default + serviceAccountName: csi-azuredisk-node-sa nodeSelector: kubernetes.io/os: linux +{% if azure_csi_node_affinity %} + affinity: + {{ azure_csi_node_affinity | to_nice_yaml | indent(width=8) }} +{% endif %} priorityClassName: system-node-critical + tolerations: + - operator: Exists containers: - name: liveness-probe volumeMounts: @@ -26,13 +38,12 @@ spec: imagePullPolicy: {{ k8s_image_pull_policy }} args: - --csi-address=/csi/csi.sock - - --connection-timeout=3s + - --probe-timeout=3s - --health-port=29603 - - --v=5 + - --v=2 resources: limits: - cpu: 200m - memory: 200Mi + memory: 100Mi requests: cpu: 10m memory: 20Mi @@ -41,11 +52,15 @@ spec: args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v=5 - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "rm -rf /registration/disk.csi.azure.com-reg.sock /csi/csi.sock"] + - --v=2 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 env: - name: ADDRESS value: /csi/csi.sock @@ -58,8 +73,7 @@ spec: mountPath: /registration resources: limits: - cpu: 200m - memory: 200Mi + memory: 100Mi requests: cpu: 10m memory: 20Mi @@ -71,6 +85,11 @@ spec: - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" - "--metrics-address=0.0.0.0:29605" + - "--enable-perf-optimization=true" + - "--drivername=disk.csi.azure.com" + - "--volume-attach-limit=-1" + - "--cloud-config-secret-name=cloud-config" + - "--cloud-config-secret-namespace=kube-system" ports: - containerPort: 29603 name: healthz @@ -106,10 +125,6 @@ spec: name: mountpoint-dir - mountPath: /etc/kubernetes/ name: azure-cred - readOnly: true - - mountPath: /var/lib/waagent/ManagedIdentity-Settings - readOnly: true - name: msi - mountPath: /dev name: device-dir - mountPath: /sys/bus/scsi/devices @@ -118,7 +133,6 @@ spec: name: scsi-host-dir resources: limits: - cpu: 200m memory: 200Mi requests: cpu: 10m @@ -136,12 +150,10 @@ spec: path: /var/lib/kubelet/plugins_registry/ type: DirectoryOrCreate name: registration-dir - - name: azure-cred - secret: - secretName: cloud-config - hostPath: - path: /var/lib/waagent/ManagedIdentity-Settings - name: msi + path: /etc/kubernetes/ + type: DirectoryOrCreate + name: azure-cred - hostPath: path: /dev type: Directory diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2 deleted file mode 100644 index 7e8454df0..000000000 --- a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2 +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - creationTimestamp: null - name: csinodeinfos.csi.storage.k8s.io -spec: - group: csi.storage.k8s.io - scope: Cluster - names: - kind: CSINodeInfo - plural: csinodeinfos - versions: - - name: v1alpha1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - csiDrivers: - description: List of CSI drivers running on the node and their properties. - items: - properties: - driver: - description: The CSI driver that this object refers to. - type: string - nodeID: - description: The node from the driver point of view. - type: string - topologyKeys: - description: List of keys supported by the driver. - items: - type: string - type: array - type: array -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: []