From e573a2f6d40784ae4549bd007c37ef27f4eb0887 Mon Sep 17 00:00:00 2001 From: Daniel Strufe <2900921+dabeck@users.noreply.github.com> Date: Fri, 25 Aug 2023 03:55:17 +0200 Subject: [PATCH] Add huawei cloud controller (#10198) * Add huaweicloud as external cloud controller * Add huaweicloud example config * Rename AK,SK to ACCESS_KEY and SECRET_KEY * Add reference to huaweicloud * Fix variable naming * Fix env var name * Update example * Fix variable naming * Fix cloud_config path * Add namespace for leader election * Revert reviewers * Delete OWNERS Delete owners who are not responsible here. * Fix build validation --- inventory/sample/group_vars/all/all.yml | 2 +- .../sample/group_vars/all/huaweicloud.yml | 17 +++ .../huaweicloud/defaults/main.yml | 19 +++ .../tasks/huaweicloud-credential-check.yml | 33 +++++ .../huaweicloud/tasks/main.yml | 49 ++++++++ ...external-huawei-cloud-config-secret.yml.j2 | 10 ++ .../templates/external-huawei-cloud-config.j2 | 23 ++++ ...-huawei-cloud-controller-manager-ds.yml.j2 | 93 ++++++++++++++ ...ud-controller-manager-role-bindings.yml.j2 | 16 +++ ...awei-cloud-controller-manager-roles.yml.j2 | 117 ++++++++++++++++++ .../external_cloud_controller/meta/main.yml | 10 ++ 11 files changed, 388 insertions(+), 1 deletion(-) create mode 100644 inventory/sample/group_vars/all/huaweicloud.yml create mode 100644 roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml create mode 100644 roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/huaweicloud-credential-check.yml create mode 100644 roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/main.yml create mode 100644 roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config-secret.yml.j2 create mode 100644 roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config.j2 create mode 100644 roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-ds.yml.j2 create mode 100644 roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2 create mode 100644 roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2 diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml index 3c2d5a8de..b93f1a3ac 100644 --- a/inventory/sample/group_vars/all/all.yml +++ b/inventory/sample/group_vars/all/all.yml @@ -48,7 +48,7 @@ loadbalancer_apiserver_healthcheck_port: 8081 # cloud_provider: ## When cloud_provider is set to 'external', you can set the cloud controller to deploy -## Supported cloud controllers are: 'openstack', 'vsphere' and 'hcloud' +## Supported cloud controllers are: 'openstack', 'vsphere', 'huaweicloud' and 'hcloud' ## When openstack or vsphere are used make sure to source in the required fields # external_cloud_provider: diff --git a/inventory/sample/group_vars/all/huaweicloud.yml b/inventory/sample/group_vars/all/huaweicloud.yml new file mode 100644 index 000000000..20c720227 --- /dev/null +++ b/inventory/sample/group_vars/all/huaweicloud.yml @@ -0,0 +1,17 @@ +## Values for the external Huawei Cloud Controller +# external_huaweicloud_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP" +# external_huaweicloud_lbaas_network_id: "Neutron network ID to create LBaaS VIP" + +## Credentials to authenticate against Keystone API +## All of them are required Per default these values will be +## read from the environment. +# external_huaweicloud_auth_url: "{{ lookup('env','OS_AUTH_URL') }}" +# external_huaweicloud_access_key: "{{ lookup('env','OS_ACCESS_KEY') }}" +# external_huaweicloud_secret_key: "{{ lookup('env','OS_SECRET_KEY') }}" +# external_huaweicloud_region: "{{ lookup('env','OS_REGION_NAME') }}" +# external_huaweicloud_project_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID'),true) }}" +# external_huaweicloud_cloud: "{{ lookup('env','OS_CLOUD') }}" + +## The repo and tag of the external Huawei Cloud Controller image +# external_huawei_cloud_controller_image_repo: "swr.ap-southeast-1.myhuaweicloud.com" +# external_huawei_cloud_controller_image_tag: "v0.26.3" diff --git a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml new file mode 100644 index 000000000..6d89c5726 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml @@ -0,0 +1,19 @@ +--- +# The external cloud controller will need credentials to access +# openstack apis. Per default these values will be +# read from the environment. +external_huaweicloud_auth_url: "{{ lookup('env','OS_AUTH_URL') }}" +external_huaweicloud_access_key: "{{ lookup('env','OS_ACCESS_KEY') }}" +external_huaweicloud_secret_key: "{{ lookup('env','OS_SECRET_KEY') }}" +external_huaweicloud_region: "{{ lookup('env','OS_REGION_NAME') }}" +external_huaweicloud_project_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID'),true) }}" +external_huaweicloud_cloud: "{{ lookup('env','OS_CLOUD') }}" + +## A dictionary of extra arguments to add to the huawei cloud controller manager deployment +## Format: +## external_huawei_cloud_controller_extra_args: +## arg1: "value1" +## arg2: "value2" +external_huawei_cloud_controller_extra_args: {} +external_huawei_cloud_controller_image_repo: "swr.ap-southeast-1.myhuaweicloud.com" +external_huawei_cloud_controller_image_tag: "v0.26.3" diff --git a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/huaweicloud-credential-check.yml b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/huaweicloud-credential-check.yml new file mode 100644 index 000000000..79172ff97 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/huaweicloud-credential-check.yml @@ -0,0 +1,33 @@ +--- +- name: External Huawei Cloud Controller | check external_huaweicloud_auth_url value + fail: + msg: "external_huaweicloud_auth_url is missing" + when: external_huaweicloud_auth_url is not defined or not external_huaweicloud_auth_url + + +- name: External Huawei Cloud Controller | check external_huaweicloud_access_key value + fail: + msg: "you must set external_huaweicloud_access_key" + when: + - external_huaweicloud_access_key is not defined or not external_huaweicloud_access_key + +- name: External Huawei Cloud Controller | check external_huaweicloud_secret_key value + fail: + msg: "external_huaweicloud_secret_key is missing" + when: + - external_huaweicloud_access_key is defined + - external_huaweicloud_access_key|length > 0 + - external_huaweicloud_secret_key is not defined or not external_huaweicloud_secret_key + + +- name: External Huawei Cloud Controller | check external_huaweicloud_region value + fail: + msg: "external_huaweicloud_region is missing" + when: external_huaweicloud_region is not defined or not external_huaweicloud_region + + +- name: External Huawei Cloud Controller | check external_huaweicloud_project_id value + fail: + msg: "one of external_huaweicloud_project_id must be specified" + when: + - external_huaweicloud_project_id is not defined or not external_huaweicloud_project_id diff --git a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/main.yml b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/main.yml new file mode 100644 index 000000000..880be0dfc --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/main.yml @@ -0,0 +1,49 @@ +--- +- name: External Huawei Cloud Controller | Check Huawei credentials + include_tasks: huaweicloud-credential-check.yml + tags: external-huaweicloud + +- name: External huaweicloud Cloud Controller | Get base64 cacert + slurp: + src: "{{ external_huaweicloud_cacert }}" + register: external_huaweicloud_cacert_b64 + when: + - inventory_hostname == groups['kube_control_plane'][0] + - external_huaweicloud_cacert is defined + - external_huaweicloud_cacert | length > 0 + tags: external-huaweicloud + +- name: External huaweicloud Cloud Controller | Get base64 cloud-config + set_fact: + external_huawei_cloud_config_secret: "{{ lookup('template', 'external-huawei-cloud-config.j2') | b64encode }}" + when: inventory_hostname == groups['kube_control_plane'][0] + tags: external-huaweicloud + +- name: External Huawei Cloud Controller | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + group: "{{ kube_cert_group }}" + mode: 0640 + with_items: + - {name: external-huawei-cloud-config-secret, file: external-huawei-cloud-config-secret.yml} + - {name: external-huawei-cloud-controller-manager-roles, file: external-huawei-cloud-controller-manager-roles.yml} + - {name: external-huawei-cloud-controller-manager-role-bindings, file: external-huawei-cloud-controller-manager-role-bindings.yml} + - {name: external-huawei-cloud-controller-manager-ds, file: external-huawei-cloud-controller-manager-ds.yml} + register: external_huaweicloud_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + tags: external-huaweicloud + +- name: External Huawei Cloud Controller | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ external_huaweicloud_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + tags: external-huaweicloud diff --git a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config-secret.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config-secret.yml.j2 new file mode 100644 index 000000000..1f0bbf3d5 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config-secret.yml.j2 @@ -0,0 +1,10 @@ +# This YAML file contains secret objects, +# which are necessary to run external huaweicloud cloud controller. + +kind: Secret +apiVersion: v1 +metadata: + name: external-huawei-cloud-config + namespace: kube-system +data: + cloud-config: {{ external_huawei_cloud_config_secret }} diff --git a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config.j2 b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config.j2 new file mode 100644 index 000000000..07f1771d6 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config.j2 @@ -0,0 +1,23 @@ +[Global] +auth-url="{{ external_huaweicloud_auth_url }}" +{% if external_huaweicloud_access_key is defined and external_huaweicloud_access_key != "" %} +access-key={{ external_huaweicloud_access_key }} +{% endif %} +{% if external_huaweicloud_secret_key is defined and external_huaweicloud_secret_key != "" %} +secret-key={{ external_huaweicloud_secret_key }} +{% endif %} +region="{{ external_huaweicloud_region }}" +{% if external_huaweicloud_project_id is defined and external_huaweicloud_project_id != "" %} +project-id="{{ external_huaweicloud_project_id }}" +{% endif %} +{% if external_huaweicloud_cloud is defined and external_huaweicloud_cloud != "" %} +cloud="{{ external_huaweicloud_cloud }}" +{% endif %} + +[VPC] +{% if external_huaweicloud_lbaas_subnet_id is defined %} +subnet-id={{ external_huaweicloud_lbaas_subnet_id }} +{% endif %} +{% if external_huaweicloud_lbaas_network_id is defined %} +id={{ external_huaweicloud_lbaas_network_id }} +{% endif %} diff --git a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-ds.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-ds.yml.j2 new file mode 100644 index 000000000..5e4b424f5 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-ds.yml.j2 @@ -0,0 +1,93 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: huawei-cloud-provider +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: huawei-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: huawei-cloud-controller-manager +spec: + selector: + matchLabels: + k8s-app: huawei-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: huawei-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + securityContext: + runAsUser: 1001 + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + serviceAccountName: cloud-controller-manager + containers: + - name: huawei-cloud-controller-manager + image: {{ external_huawei_cloud_controller_image_repo }}/k8s-cloudprovider/huawei-cloud-controller-manager:{{ external_huawei_cloud_controller_image_tag }} + args: + - /bin/huawei-cloud-controller-manager + - --v=1 + - --cloud-config=$(CLOUD_CONFIG) + - --cloud-provider=huaweicloud + - --use-service-account-credentials=true +{% for key, value in external_huawei_cloud_controller_extra_args.items() %} + - "{{ '--' + key + '=' + value }}" +{% endfor %} + volumeMounts: + - mountPath: /etc/kubernetes + name: k8s-certs + readOnly: true + - mountPath: /etc/ssl/certs + name: ca-certs + readOnly: true + - mountPath: /etc/config + name: cloud-config-volume + readOnly: true +{% if kubelet_flexvolumes_plugins_dir is defined %} + - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + name: flexvolume-dir +{% endif %} + resources: + requests: + cpu: 200m + env: + - name: CLOUD_CONFIG + value: /etc/config/cloud-config + hostNetwork: true + volumes: +{% if kubelet_flexvolumes_plugins_dir is defined %} + - name: flexvolume-dir + hostPath: + path: "{{ kubelet_flexvolumes_plugins_dir }}" + type: DirectoryOrCreate +{% endif %} + - name: k8s-certs + hostPath: + path: /etc/kubernetes + type: DirectoryOrCreate + - name: ca-certs + hostPath: + path: /etc/ssl/certs + type: DirectoryOrCreate + - name: cloud-config-volume + secret: + secretName: external-huawei-cloud-config diff --git a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2 new file mode 100644 index 000000000..bbdf3364a --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2 @@ -0,0 +1,16 @@ +apiVersion: v1 +items: +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: List +metadata: {} diff --git a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2 new file mode 100644 index 000000000..2e2d8b64e --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2 @@ -0,0 +1,117 @@ +apiVersion: v1 +items: +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - resources: + - tokenreviews + verbs: + - get + - list + - watch + - create + - update + - patch + apiGroups: + - authentication.k8s.io + - resources: + - configmaps + - endpoints + - pods + - services + - secrets + - serviceaccounts + - serviceaccounts/token + verbs: + - get + - list + - watch + - create + - update + - patch + apiGroups: + - '' + - resources: + - nodes + verbs: + - get + - list + - watch + - delete + - patch + - update + apiGroups: + - '' + - resources: + - services/status + - pods/status + verbs: + - update + - patch + apiGroups: + - '' + - resources: + - nodes/status + verbs: + - patch + - update + apiGroups: + - '' + - resources: + - events + - endpoints + verbs: + - create + - patch + - update + apiGroups: + - '' + - resources: + - leases + verbs: + - get + - update + - create + - delete + apiGroups: + - coordination.k8s.io + - resources: + - customresourcedefinitions + verbs: + - get + - update + - create + - delete + apiGroups: + - apiextensions.k8s.io + - resources: + - ingresses + verbs: + - get + - list + - watch + - update + - create + - patch + - delete + apiGroups: + - networking.k8s.io + - resources: + - ingresses/status + verbs: + - update + - patch + apiGroups: + - networking.k8s.io + - resources: + - endpointslices + verbs: + - get + - list + - watch + apiGroups: + - discovery.k8s.io +kind: List +metadata: {} diff --git a/roles/kubernetes-apps/external_cloud_controller/meta/main.yml b/roles/kubernetes-apps/external_cloud_controller/meta/main.yml index 6e8c235dd..b1fc4ad69 100644 --- a/roles/kubernetes-apps/external_cloud_controller/meta/main.yml +++ b/roles/kubernetes-apps/external_cloud_controller/meta/main.yml @@ -30,3 +30,13 @@ dependencies: tags: - external-cloud-controller - external-hcloud + - role: kubernetes-apps/external_cloud_controller/huaweicloud + when: + - cloud_provider is defined + - cloud_provider == "external" + - external_cloud_provider is defined + - external_cloud_provider == "huaweicloud" + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - external-cloud-controller + - external-huaweicloud