Change MetalLB to one of addons (#6238)

This changes MetalLB contrib to one of addons for deploying MetalLB with
Kubernetes cluster deployment. By the default, Kubespray doesn't deploy
MetalLB addon.

contrib/metallb/library

@@ -1 +0,0 @@
-../../library

contrib/metallb/metallb.yml

@@ -1,12 +0,0 @@
----
-- hosts: bastion[0]
-  gather_facts: False
-  roles:
-    - { role: kubespray-defaults}
-    - { role: bastion-ssh-config, tags: ["localhost", "bastion"]}
-- hosts: kube-master[0]
-  tags:
-    - "provision"
-  roles:
-    - { role: kubespray-defaults}
-    - { role: provision }

contrib/metallb/roles/provision/defaults/main.yml

@@ -1,16 +0,0 @@
----
-metallb:
-  ip_range:
-    - "10.5.0.50-10.5.0.99"
-  protocol: "layer2"
-  # additional_address_pools:
-  #   kube_service_pool:
-  #     ip_range:
-  #       - 10.5.1.50-10.5.1.99"
-  #     protocol: "layer2"
-  #     auto_assign: false
-  limits:
-    cpu: "100m"
-    memory: "100Mi"
-  port: "7472"
-  version: v0.9.3

contrib/metallb/roles/provision/templates/metallb-config.yml.j2

@@ -1,25 +0,0 @@
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  namespace: metallb-system
-  name: config
-data:
-  config: |
-    address-pools:
-    - name: loadbalanced
-      protocol: {{ metallb.protocol }}
-      addresses:
-{% for ip_range in metallb.ip_range %}
-      - {{ ip_range }}
-{% endfor %}
-{% if metallb.additional_address_pools is defined %}{% for pool in metallb.additional_address_pools %}
-    - name: {{ pool }}
-      protocol: {{ metallb.additional_address_pools[pool].protocol }}
-      addresses:
-{% for ip_range in metallb.additional_address_pools[pool].ip_range %}
-      - {{ ip_range }}
-{% endfor %}
-      auto-assign: {{ metallb.additional_address_pools[pool].auto_assign }}
-{% endfor %}
-{% endif %}

contrib/metallb/roles/provision/templates/metallb.yml.j2

@@ -1,266 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: metallb-system
-  labels:
-    app: metallb
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  namespace: metallb-system
-  name: controller
-  labels:
-    app: metallb
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  namespace: metallb-system
-  name: speaker
-  labels:
-    app: metallb
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: metallb-system:controller
-  labels:
-    app: metallb
-rules:
-- apiGroups: [""]
-  resources: ["services"]
-  verbs: ["get", "list", "watch", "update"]
-- apiGroups: [""]
-  resources: ["services/status"]
-  verbs: ["update"]
-- apiGroups: [""]
-  resources: ["events"]
-  verbs: ["create", "patch"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: metallb-system:speaker
-  labels:
-    app: metallb
-rules:
-- apiGroups: [""]
-  resources: ["services", "endpoints", "nodes"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: [""]
-  resources: ["events"]
-  verbs: ["create"]
-{% if podsecuritypolicy_enabled %}
-- apiGroups: ["policy"]
-  resourceNames: ["metallb"]
-  resources: ["podsecuritypolicies"]
-  verbs: ["use"]
----
-apiVersion: policy/v1beta1
-kind: PodSecurityPolicy
-metadata:
-  name: metallb
-  annotations:
-    seccomp.security.alpha.kubernetes.io/defaultProfileName:  'runtime/default'
-    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default'
-{% if apparmor_enabled %}
-    apparmor.security.beta.kubernetes.io/defaultProfileName:  'runtime/default'
-    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
-{% endif %}
-  labels:
-    app: metallb
-spec:
-  privileged: true
-  allowPrivilegeEscalation: false
-  allowedCapabilities:
-  - net_raw
-  volumes:
-  - secret
-  hostNetwork: true
-  hostPorts:
-  - min: {{ metallb.port }}
-    max: {{ metallb.port }}
-  hostIPC: false
-  hostPID: false
-  runAsUser:
-    rule: 'RunAsAny'
-  seLinux:
-    rule: 'RunAsAny'
-  supplementalGroups:
-    rule: 'RunAsAny'
-  fsGroup:
-    rule: 'RunAsAny'
-  readOnlyRootFilesystem: true
-{% endif %}
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  namespace: metallb-system
-  name: config-watcher
-  labels:
-    app: metallb
-rules:
-- apiGroups: [""]
-  resources: ["configmaps"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: [""]
-  resources: ["events"]
-  verbs: ["create"]
----
-
-## Role bindings
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: metallb-system:controller
-  labels:
-    app: metallb
-subjects:
-- kind: ServiceAccount
-  name: controller
-  namespace: metallb-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: metallb-system:controller
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: metallb-system:speaker
-  labels:
-    app: metallb
-subjects:
-- kind: ServiceAccount
-  name: speaker
-  namespace: metallb-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: metallb-system:speaker
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  namespace: metallb-system
-  name: config-watcher
-  labels:
-    app: metallb
-subjects:
-- kind: ServiceAccount
-  name: controller
-- kind: ServiceAccount
-  name: speaker
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: config-watcher
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  namespace: metallb-system
-  name: speaker
-  labels:
-    app: metallb
-    component: speaker
-spec:
-  selector:
-    matchLabels:
-      app: metallb
-      component: speaker
-  template:
-    metadata:
-      labels:
-        app: metallb
-        component: speaker
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "{{ metallb.port }}"
-    spec:
-      serviceAccountName: speaker
-      terminationGracePeriodSeconds: 0
-      hostNetwork: true
-      containers:
-      - name: speaker
-        image: metallb/speaker:{{ metallb.version }}
-        imagePullPolicy: IfNotPresent
-        args:
-        - --port={{ metallb.port }}
-        - --config=config
-        env:
-        - name: METALLB_NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        ports:
-        - name: monitoring
-          containerPort: {{ metallb.port }}
-        resources:
-          limits:
-            cpu: {{ metallb.limits.cpu }}
-            memory: {{ metallb.limits.memory }}
-        securityContext:
-          allowPrivilegeEscalation: false
-          readOnlyRootFilesystem: true
-          capabilities:
-            drop:
-            - all
-            add:
-            - net_raw
-
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  namespace: metallb-system
-  name: controller
-  labels:
-    app: metallb
-    component: controller
-spec:
-  revisionHistoryLimit: 3
-  selector:
-    matchLabels:
-      app: metallb
-      component: controller
-  template:
-    metadata:
-      labels:
-        app: metallb
-        component: controller
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "{{ metallb.port }}"
-    spec:
-      serviceAccountName: controller
-      terminationGracePeriodSeconds: 0
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534 # nobody
-      containers:
-      - name: controller
-        image: metallb/controller:{{ metallb.version }}
-        imagePullPolicy: IfNotPresent
-        args:
-        - --port={{ metallb.port }}
-        - --config=config
-        ports:
-        - name: monitoring
-          containerPort: {{ metallb.port }}
-        resources:
-          limits:
-            cpu: {{ metallb.limits.cpu }}
-            memory: {{ metallb.limits.memory }}
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - all
-          readOnlyRootFilesystem: true
-
----

inventory/sample/group_vars/k8s-cluster/addons.yml

@@ -119,3 +119,19 @@ ingress_alb_enabled: false
 # Cert manager deployment
 cert_manager_enabled: false
 # cert_manager_namespace: "cert-manager"
+
+# MetalLB deployment
+metallb_enabled: false
+# metallb_ip_range:
+#   - "10.5.0.50-10.5.0.99"
+# metallb_version: v0.9.3
+# metallb_protocol: "layer2"
+# metallb_port: "7472"
+# metallb_limits_cpu: "100m"
+# metallb_limits_mem: "100Mi"
+# metallb_additional_address_pools:
+#   kube_service_pool:
+#     ip_range:
+#       - "10.5.1.50-10.5.1.99"
+#     protocol: "layer2"
+#     auto_assign: false

roles/kubernetes-apps/meta/main.yml

@@ -97,3 +97,10 @@ dependencies:
       - inventory_hostname == groups['kube-master'][0]
     tags:
       - oci
+
+  - role: kubernetes-apps/metallb
+    when:
+      - metallb_enabled
+      - inventory_hostname == groups['kube-master'][0]
+    tags:
+      - metallb

roles/kubernetes-apps/metallb/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+metallb_enabled: false
+metallb_version: v0.9.3
+metallb_protocol: "layer2"
+metallb_port: "7472"
+metallb_limits_cpu: "100m"
+metallb_limits_mem: "100Mi"

contrib/metallb/roles/provision/tasks/main.yml → roles/kubernetes-apps/metallb/tasks/main.yml

@@ -5,6 +5,12 @@
   when:
     - "kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp"
 
+- name: Kubernetes Apps | Check cluster settings for MetalLB
+  fail:
+    msg: "metallb_ip_range is mandatory to be specified for MetalLB"
+  when:
+    - metallb_ip_range is not defined or not metallb_ip_range
+
 - name: Kubernetes Apps | Check AppArmor status
   command: which apparmor_parser
   register: apparmor_status
@@ -38,3 +44,25 @@
   with_items: "{{ rendering.results }}"
   when:
     - "inventory_hostname == groups['kube-master'][0]"
+
+- name: Kubernetes Apps | Check existing secret of MetalLB
+  command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf -n metallb-system get secret memberlist"
+  register: metallb_secret
+  become: true
+  ignore_errors: yes
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: Kubernetes Apps | Create random bytes for MetalLB
+  command: "openssl rand -base64 32"
+  register: metallb_rand
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - metallb_secret.rc != 0
+
+- name: Kubernetes Apps | Install secret of MetalLB if not existing
+  command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf -n metallb-system create secret generic memberlist --from-literal=secretkey={{ metallb_rand.stdout }}"
+  become: true
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - metallb_secret.rc != 0

roles/kubernetes-apps/metallb/templates/metallb-config.yml.j2

@@ -0,0 +1,25 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: loadbalanced
+      protocol: {{ metallb_protocol }}
+      addresses:
+{% for ip_range in metallb_ip_range %}
+      - {{ ip_range }}
+{% endfor %}
+{% if metallb_additional_address_pools is defined %}{% for pool in metallb_additional_address_pools %}
+    - name: {{ pool }}
+      protocol: {{ metallb_additional_address_pools[pool].protocol }}
+      addresses:
+{% for ip_range in metallb_additional_address_pools[pool].ip_range %}
+      - {{ ip_range }}
+{% endfor %}
+      auto-assign: {{ metallb_additional_address_pools[pool].auto_assign }}
+{% endfor %}
+{% endif %}

roles/kubernetes-apps/metallb/templates/metallb.yml.j2

@@ -0,0 +1,398 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: metallb-system
+  labels:
+    app: metallb
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  labels:
+    app: metallb
+  name: controller
+  namespace: metallb-system
+spec:
+  allowPrivilegeEscalation: false
+  allowedCapabilities: []
+  allowedHostPaths: []
+  defaultAddCapabilities: []
+  defaultAllowPrivilegeEscalation: false
+  fsGroup:
+    ranges:
+    - max: 65535
+      min: 1
+    rule: MustRunAs
+  hostIPC: false
+  hostNetwork: false
+  hostPID: false
+  privileged: false
+  readOnlyRootFilesystem: true
+  requiredDropCapabilities:
+  - ALL
+  runAsUser:
+    ranges:
+    - max: 65535
+      min: 1
+    rule: MustRunAs
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    ranges:
+    - max: 65535
+      min: 1
+    rule: MustRunAs
+  volumes:
+  - configMap
+  - secret
+  - emptyDir
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  labels:
+    app: metallb
+  name: speaker
+  namespace: metallb-system
+spec:
+  allowPrivilegeEscalation: false
+  allowedCapabilities:
+  - NET_ADMIN
+  - NET_RAW
+  - SYS_ADMIN
+  allowedHostPaths: []
+  defaultAddCapabilities: []
+  defaultAllowPrivilegeEscalation: false
+  fsGroup:
+    rule: RunAsAny
+  hostIPC: false
+  hostNetwork: true
+  hostPID: false
+  hostPorts:
+  - max: {{ metallb_port }}
+    min: {{ metallb_port }}
+  privileged: true
+  readOnlyRootFilesystem: true
+  requiredDropCapabilities:
+  - ALL
+  runAsUser:
+    rule: RunAsAny
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  volumes:
+  - configMap
+  - secret
+  - emptyDir
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: metallb
+  name: controller
+  namespace: metallb-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: metallb
+  name: speaker
+  namespace: metallb-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:controller
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ''
+  resources:
+  - services/status
+  verbs:
+  - update
+- apiGroups:
+  - ''
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - policy
+  resourceNames:
+  - controller
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:speaker
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - services
+  - endpoints
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - policy
+  resourceNames:
+  - speaker
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: metallb
+  name: config-watcher
+  namespace: metallb-system
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: metallb
+  name: pod-lister
+  namespace: metallb-system
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - pods
+  verbs:
+  - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:controller
+subjects:
+- kind: ServiceAccount
+  name: controller
+  namespace: metallb-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:speaker
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:speaker
+subjects:
+- kind: ServiceAccount
+  name: speaker
+  namespace: metallb-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: config-watcher
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: config-watcher
+subjects:
+- kind: ServiceAccount
+  name: controller
+- kind: ServiceAccount
+  name: speaker
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: pod-lister
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pod-lister
+subjects:
+- kind: ServiceAccount
+  name: speaker
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: metallb
+    component: speaker
+  name: speaker
+  namespace: metallb-system
+spec:
+  selector:
+    matchLabels:
+      app: metallb
+      component: speaker
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: '{{ metallb_port }}'
+        prometheus.io/scrape: 'true'
+      labels:
+        app: metallb
+        component: speaker
+    spec:
+      containers:
+      - args:
+        - --port={{ metallb_port }}
+        - --config=config
+        env:
+        - name: METALLB_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: METALLB_HOST
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: METALLB_ML_BIND_ADDR
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: METALLB_ML_LABELS
+          value: "app=metallb,component=speaker"
+        - name: METALLB_ML_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: METALLB_ML_SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: memberlist
+              key: secretkey
+        image: metallb/speaker:{{ metallb_version }}
+        imagePullPolicy: Always
+        name: speaker
+        ports:
+        - containerPort: {{ metallb_port }}
+          name: monitoring
+        resources:
+          limits:
+            cpu: {{ metallb_limits_cpu }}
+            memory: {{ metallb_limits_mem }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+            - SYS_ADMIN
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      serviceAccountName: speaker
+      terminationGracePeriodSeconds: 2
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: metallb
+    component: controller
+  name: controller
+  namespace: metallb-system
+spec:
+  revisionHistoryLimit: 3
+  selector:
+    matchLabels:
+      app: metallb
+      component: controller
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: '{{ metallb_port }}'
+        prometheus.io/scrape: 'true'
+      labels:
+        app: metallb
+        component: controller
+    spec:
+      containers:
+      - args:
+        - --port={{ metallb_port }}
+        - --config=config
+        image: metallb/controller:{{ metallb_version }}
+        imagePullPolicy: Always
+        name: controller
+        ports:
+        - containerPort: {{ metallb_port }}
+          name: monitoring
+        resources:
+          limits:
+            cpu: {{ metallb_limits_cpu }}
+            memory: {{ metallb_limits_mem }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+      serviceAccountName: controller
+      terminationGracePeriodSeconds: 0

roles/kubespray-defaults/defaults/main.yaml

@@ -324,6 +324,7 @@ ingress_ambassador_enabled: false
 ingress_alb_enabled: false
 cert_manager_enabled: false
 expand_persistent_volumes: false
+metallb_enabled: false
 
 ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461)
 # openstack_blockstorage_version: "v1/v2/auto (default)"