Feat: Add external OCI cloud controller manager (#11378)

* Feat: add external OCI cloud controller manager template & variable Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> * Feat: add external OCI cloud controller manager workflow Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> * Feat: migrate external OCI CCM config check from OCI cloud provider Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> * cloud_controller: oracle: simpler asserts Make the asserts check for Oracle Cloud Infrastructure external cloud controller more compact, and hence readable. Allows to put them back in the main tasks for less back and forth when reading the code. --------- Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> Co-authored-by: Max Gautier <mg@max.gautier.name>
7 months ago · 20df44521d
8 changed files with 395 additions and 0 deletions
--- a/inventory/sample/group_vars/all/oci.yml
+++ b/inventory/sample/group_vars/all/oci.yml
@ -1,3 +1,30 @@
+## When External Oracle Cloud Infrastructure is used, set these variables
+## External OCI Cloud Controller Manager
+## https://github.com/oracle/oci-cloud-controller-manager/blob/v1.29.0/manifests/provider-config-example.yaml
+# external_oracle_auth_region: ""
+# external_oracle_auth_tenancy: ""
+# external_oracle_auth_user: ""
+# external_oracle_auth_key: ""
+# external_oracle_auth_passphrase: ""
+# external_oracle_auth_fingerprint: ""
+# external_oracle_auth_use_instance_principals: false
+
+# external_oracle_compartment: ""
+# external_oracle_vcn: ""
+# external_oracle_load_balancer_subnet1: ""
+# external_oracle_load_balancer_subnet2: ""
+# external_oracle_load_balancer_security_list_management_mode: All
+# external_oracle_load_balancer_security_lists: {}
+
+# external_oracle_ratelimiter_qps_read: 20.0
+# external_oracle_ratelimiter_bucket_read: 5
+# external_oracle_ratelimiter_qps_write: 20.0
+# external_oracle_ratelimiter_bucket_write: 5
+
+# external_oracle_cloud_controller_image_repo: ghcr.io/oracle/cloud-provider-oci
+# external_oracle_cloud_controller_image_tag: "v1.29.0"
+
+
 ## When Oracle Cloud Infrastructure is used, set these variables
 # oci_private_key:
 # oci_region_id:
--- a/roles/kubernetes-apps/external_cloud_controller/meta/main.yml
+++ b/roles/kubernetes-apps/external_cloud_controller/meta/main.yml
@ -40,3 +40,13 @@ dependencies:
    tags:
      - external-cloud-controller
      - external-huaweicloud
+  - role: kubernetes-apps/external_cloud_controller/oci
+    when:
+      - cloud_provider is defined
+      - cloud_provider == "external"
+      - external_cloud_provider is defined
+      - external_cloud_provider == "oci"
+      - inventory_hostname == groups['kube_control_plane'][0]
+    tags:
+      - external-cloud-controller
+      - external-oci
--- a/roles/kubernetes-apps/external_cloud_controller/oci/defaults/main.yml
+++ b/roles/kubernetes-apps/external_cloud_controller/oci/defaults/main.yml
@ -0,0 +1,25 @@
+---
+## External Oracle Cloud Controller Manager
+## https://github.com/oracle/oci-cloud-controller-manager/blob/v1.29.0/manifests/provider-config-example.yaml
+external_oracle_auth_region: ""
+external_oracle_auth_tenancy: ""
+external_oracle_auth_user: ""
+external_oracle_auth_key: ""
+external_oracle_auth_passphrase: ""
+external_oracle_auth_fingerprint: ""
+external_oracle_auth_use_instance_principals: false
+
+external_oracle_compartment: ""
+external_oracle_vcn: ""
+external_oracle_load_balancer_subnet1: ""
+external_oracle_load_balancer_subnet2: ""
+external_oracle_load_balancer_security_list_management_mode: All
+external_oracle_load_balancer_security_lists: {}
+
+external_oracle_ratelimiter_qps_read: 20.0
+external_oracle_ratelimiter_bucket_read: 5
+external_oracle_ratelimiter_qps_write: 20.0
+external_oracle_ratelimiter_bucket_write: 5
+
+external_oracle_cloud_controller_image_repo: ghcr.io/oracle/cloud-provider-oci
+external_oracle_cloud_controller_image_tag: "v1.29.0"
--- a/roles/kubernetes-apps/external_cloud_controller/oci/tasks/main.yml
+++ b/roles/kubernetes-apps/external_cloud_controller/oci/tasks/main.yml
@ -0,0 +1,53 @@
+---
+- name: "External OCI Cloud Controller Manager | Check credentials"
+  ansible.builtin.assert:
+    that:
+      - external_oracle_auth_key | length > 0
+      - external_oracle_auth_region | length > 0
+      - external_oracle_auth_tenancy | length > 0
+      - external_oracle_auth_user | length > 0
+      - external_oracle_auth_fingerprint | length > 0
+  when: not external_oracle_auth_use_instance_principals
+
+- name: "External OCI Cloud Controller Manager | Check settings"
+  ansible.builtin.assert:
+    that:
+      - external_oracle_compartment | length > 0
+      - external_oracle_vcn | length > 0
+      - external_oracle_load_balancer_subnet1 | length > 0
+      - external_oracle_load_balancer_subnet2 | length > 0
+      - external_oracle_load_balancer_security_list_management_mode in ["All", "Frontend", "None"]
+
+- name: External OCI Cloud Controller Manager | Get base64 cloud-config
+  set_fact:
+    external_oracle_cloud_config_secret: "{{ lookup('template', 'external-oci-cloud-config.j2') | b64encode }}"
+  when: inventory_hostname == groups['kube_control_plane'][0]
+  tags: external-oci
+
+- name: External OCI Cloud Controller Manager | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    group: "{{ kube_cert_group }}"
+    mode: "0640"
+  with_items:
+    - {name: external-oci-cloud-config-secret, file: external-oci-cloud-config-secret.yml}
+    - {name: external-oci-cloud-controller-manager-rbac, file: external-oci-cloud-controller-manager-rbac.yml}
+    - {name: external-oci-cloud-controller-manager, file: external-oci-cloud-controller-manager.yml}
+  register: external_oracle_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+  tags: external-oci
+
+- name: External OCI Cloud Controller Manager | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ external_oracle_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+  tags: external-oci
--- a/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config-secret.yml.j2
+++ b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config-secret.yml.j2
@ -0,0 +1,10 @@
+# This YAML file contains secret objects,
+# which are necessary to run external oci cloud controller.
+
+kind: Secret
+apiVersion: v1
+metadata:
+  name: oci-cloud-controller-manager
+  namespace: kube-system
+data:
+  cloud-provider.yaml: {{ external_oracle_cloud_config_secret }}
--- a/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config.yml.j2
+++ b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config.yml.j2
@ -0,0 +1,45 @@
+{% if external_oracle_auth_use_instance_principals %}
+useInstancePrincipals: true
+{% endif %}
+
+auth:
+{% if external_oracle_auth_use_instance_principals %}
+  useInstancePrincipals: true
+{% else %}
+  useInstancePrincipals: false
+  region: {{ external_oracle_auth_region }}
+  tenancy: {{ external_oracle_auth_tenancy }}
+  user: {{ external_oracle_auth_user }}
+  key: |
+    {{ external_oracle_auth_key }}
+  {% if external_oracle_auth_passphrase is defined %}
+  # Omit if there is not a password for the key
+  passphrase: {{ external_oracle_auth_passphrase }}
+  {% endif %}
+  fingerprint: {{ external_oracle_auth_fingerprint }}
+{% endif %}
+
+compartment: {{ external_oracle_compartment }}
+
+vcn: {{ external_oracle_vcn }}
+
+loadBalancer:
+  subnet1: {{ external_oracle_load_balancer_subnet1 }}
+  subnet2: {{ external_oracle_load_balancer_subnet2 }}
+
+  securityListManagementMode: {{ external_oracle_load_balancer_security_list_management_mode }}
+
+{% if external_oracle_security_lists is defined and external_oracle_security_lists | length > 0 %}
+  # Optional specification of which security lists to modify per subnet. This does not apply if security list management is off.
+  securityLists:
+  {% for subnet_ocid, list_ocid in external_oracle_load_balancer_security_lists.items() %}
+    {{ subnet_ocid }}: {{ list_ocid }}
+  {% endfor %}
+{% endif %}
+
+# Optional rate limit controls for accessing OCI API
+rateLimiter:
+  rateLimitQPSRead: {{ external_oracle_ratelimiter_qps_read }}
+  rateLimitBucketRead: {{ external_oracle_ratelimiter_bucket_read }}
+  rateLimitQPSWrite: {{ external_oracle_ratelimiter_qps_write }}
+  rateLimitBucketWrite: {{ external_oracle_ratelimiter_bucket_write }}
--- a/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager-rbac.yml.j2
+++ b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager-rbac.yml.j2
@ -0,0 +1,166 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:cloud-controller-manager
+  labels:
+    kubernetes.io/cluster-service: "true"
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - watch
+  - patch
+  - get
+
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - patch
+  - get
+  - update
+
+- apiGroups:
+    - ""
+  resources:
+    - configmaps
+  resourceNames:
+    - "extension-apiserver-authentication"
+  verbs:
+    - get
+
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - patch
+  - update
+
+# For leader election
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  resourceNames:
+  - "cloud-controller-manager"
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+
+- apiGroups:
+    - "coordination.k8s.io"
+  resources:
+    - leases
+  verbs:
+    - get
+    - create
+    - update
+    - delete
+    - patch
+    - watch
+
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  resourceNames:
+  - "cloud-controller-manager"
+  verbs:
+  - get
+  - update
+
+- apiGroups:
+    - ""
+  resources:
+    - configmaps
+  resourceNames:
+    - "extension-apiserver-authentication"
+  verbs:
+    - get
+    - list
+    - watch
+
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - list
+  - get
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+
+# For the PVL
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - list
+  - watch
+  - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: oci-cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system
--- a/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager.yml.j2
+++ b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager.yml.j2
@ -0,0 +1,59 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: oci-cloud-controller-manager
+  namespace: kube-system
+  labels:
+    k8s-app: oci-cloud-controller-manager
+spec:
+  selector:
+    matchLabels:
+      component: oci-cloud-controller-manager
+      tier: control-plane
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        component: oci-cloud-controller-manager
+        tier: control-plane
+    spec:
+      serviceAccountName: cloud-controller-manager
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      tolerations:
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      volumes:
+        - name: cfg
+          secret:
+            secretName: oci-cloud-controller-manager
+        - name: kubernetes
+          hostPath:
+            path: /etc/kubernetes
+      containers:
+        - name: oci-cloud-controller-manager
+          image: {{ external_oracle_cloud_controller_image_repo }}:{{ external_oracle_cloud_controller_image_tag }}
+          command: ["/usr/local/bin/oci-cloud-controller-manager"]
+          args:
+            - --cloud-config=/etc/oci/cloud-provider.yaml
+            - --cloud-provider=oci
+            - --leader-elect-resource-lock=leases
+            - --concurrent-service-syncs=3
+            - --v=2
+          volumeMounts:
+            - name: cfg
+              mountPath: /etc/oci
+              readOnly: true
+            - name: kubernetes
+              mountPath: /etc/kubernetes
+              readOnly: true