Update MetalLB deployment, wait for resource. (#9995)

* Update MetalLB deployment, wait for resource. Signed-off-by: Jeroen Rijken <jeroen.rijken@xs4all.nl> * yml to yaml, add basic test for metallb Signed-off-by: Jeroen Rijken <jeroen.rijken@xs4all.nl> --------- Signed-off-by: Jeroen Rijken <jeroen.rijken@xs4all.nl>
1 year ago · ea7dcd46d7
5 changed files with 181 additions and 84 deletions
--- a/docs/metallb.md
+++ b/docs/metallb.md
@ -26,15 +26,39 @@ By default only the MetalLB BGP speaker is allowed to run on control plane nodes
 ```yaml
 metallb_config:
  controller:
+    nodeselector:
+      kubernetes.io/os: linux
    tolerations:
-      - key: "node-role.kubernetes.io/master"
-        operator: "Equal"
-        value: ""
-        effect: "NoSchedule"
-      - key: "node-role.kubernetes.io/control-plane"
-        operator: "Equal"
-        value: ""
-        effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/master"
+      operator: "Equal"
+      value: ""
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/control-plane"
+      operator: "Equal"
+      value: ""
+      effect: "NoSchedule"
+```
+
+If you'd like to set additional nodeSelector and tolerations values, you can do so in the following fasion:
+
+```yaml
+metallb_config:
+  controller:
+    nodeselector:
+      kubernetes.io/os: linux
+    tolerations:
+    - key: "node-role.kubernetes.io/control-plane"
+      operator: "Equal"
+      value: ""
+      effect: "NoSchedule"
+  speaker:
+    nodeselector:
+      kubernetes.io/os: linux
+    tolerations:
+    - key: "node-role.kubernetes.io/control-plane"
+      operator: "Equal"
+      value: ""
+      effect: "NoSchedule"
 ```

 ## Pools
@ -137,7 +161,6 @@ In this scenario you should disable the MetalLB speaker and configure the `calic

 ```yaml
 metallb_speaker_enabled: false
-metallb_avoid_buggy_ips: true
 metallb_config:
  address_pools:
    primary:
@ -177,22 +200,22 @@ metallb_config:
      vpn-only: "1234:1"
      NO_ADVERTISE: "65535:65282"
    metallb_peers:
-        peer1:
-          peer_address: 10.6.0.1
-          peer_asn: 64512
-          my_asn: 4200000000
-          communities:
-            - vpn-only
-          address_pool:
-            - pool1
-        peer2:
-          peer_address: 10.10.0.1
-          peer_asn: 64513
-          my_asn: 4200000000
-          communities:
-            - NO_ADVERTISE
-          address_pool:
-            - pool2
+      peer1:
+        peer_address: 10.6.0.1
+        peer_asn: 64512
+        my_asn: 4200000000
+        communities:
+          - vpn-only
+        address_pool:
+          - pool1
+      peer2:
+        peer_address: 10.10.0.1
+        peer_asn: 64513
+        my_asn: 4200000000
+        communities:
+          - NO_ADVERTISE
+        address_pool:
+          - pool2
 calico_advertise_service_loadbalancer_ips:
  - 10.5.0.0/16
  - 10.6.0.0/16
--- a/roles/kubernetes-apps/metallb/defaults/main.yml
+++ b/roles/kubernetes-apps/metallb/defaults/main.yml
@ -1,10 +1,8 @@
 ---
 metallb_enabled: false
 metallb_log_level: info
-metallb_protocol: "layer2"
 metallb_port: "7472"
 metallb_memberlist_port: "7946"
-metallb_peers: []
 metallb_speaker_enabled: "{{ metallb_enabled }}"
 metallb_speaker_nodeselector:
  kubernetes.io/os: "linux"
@ -18,6 +16,3 @@ metallb_speaker_tolerations:
    key: node-role.kubernetes.io/control-plane
    operator: Exists
 metallb_controller_tolerations: []
-metallb_pool_name: "loadbalanced"
-metallb_auto_assign: true
-metallb_avoid_buggy_ips: false
--- a/roles/kubernetes-apps/metallb/tasks/main.yml
+++ b/roles/kubernetes-apps/metallb/tasks/main.yml
@ -5,13 +5,6 @@
  when:
    - "kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp"

- name: Kubernetes Apps | Check BGP peers for MetalLB
-  fail:
-    msg: "metallb_peers is mandatory when metallb_protocol is bgp and metallb_speaker_enabled"
-  when:
-    - metallb_config.layer3 is defined and metallb_speaker_enabled
-    - metallb_config.metallb_peers is not defined or not metallb_config.metallb_peers
-
 - name: Kubernetes Apps | Check that the deprecated 'matallb_auto_assign' variable is not used anymore
  fail:
    msg: "'matallb_auto_assign' configuration variable is deprecated, please use 'metallb_auto_assign' instead"
@ -36,46 +29,95 @@
 - name: Kubernetes Apps | Lay Down MetalLB
  become: true
  template:
-    src: "{{ item }}.j2"
-    dest: "{{ kube_config_dir }}/{{ item }}"
+    src: "metallb.yaml.j2"
+    dest: "{{ kube_config_dir }}/metallb.yaml"
    mode: 0644
-  with_items: ["metallb.yml", "pools.yaml", "layer2.yaml", "layer3.yaml"]
-  register: "rendering"
+  register: metallb_rendering
  when:
-    - "inventory_hostname == groups['kube_control_plane'][0]"
-
- name: Kubernetes Apps | Create MetalLB resources and replace existing
-  k8s:
-    definition: "{{ lookup('template', 'metallb.yaml') }}"
-
- name: Kubernetes Apps | Wait for MetalLB controller to be running
-  k8s_info:
-    kind: Deployment
-    namespace: metallb-system
-    name: controller
-    wait: True
-    wait_sleep: 10
-    wait_timeout: 360
-    wait_condition:
-      status: "True"
-      type: Available
-  register: result
-  until: result is not failed
+    - inventory_hostname == groups['kube_control_plane'][0]

 - name: Kubernetes Apps | Install and configure MetalLB
  kube:
    name: "MetalLB"
    kubectl: "{{ bin_dir }}/kubectl"
-    filename: "{{ kube_config_dir }}/{{ item.item }}"
-    state: "{{ item.changed | ternary('latest','present') }}"
+    filename: "{{ kube_config_dir }}/metallb.yaml"
+    state: "{{ metallb_rendering.changed | ternary('latest','present') }}"
+    wait: true
  become: true
-  with_items: "{{ rendering.results }}"
  when:
-    - "inventory_hostname == groups['kube_control_plane'][0]"
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kubernetes Apps | Wait for MetalLB controller to be running
+  command: "{{ bin_dir }}/kubectl -n metallb-system wait --for=condition=ready pod -l app=metallb,component=controller"
+  become: true
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: MetalLB | Address pools
+  block:
+    - name: MetalLB | Layout address pools template
+      ansible.builtin.template:
+        src: pools.yaml.j2
+        dest: "{{ kube_config_dir }}/pools.yaml"
+        mode: 0644
+      register: pools_rendering
+
+    - name: MetalLB | Create address pools configuration
+      kube:
+        name: "MetalLB"
+        kubectl: "{{ bin_dir }}/kubectl"
+        filename: "{{ kube_config_dir }}/pools.yaml"
+        state: "{{ pools_rendering.changed | ternary('latest','present') }}"
+      become: true
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - metallb_config.address_pools is defined
+
+- name: MetalLB | Layer2
+  block:
+    - name: MetalLB | Layout layer2 template
+      ansible.builtin.template:
+        src: layer2.yaml.j2
+        dest: "{{ kube_config_dir }}/layer2.yaml"
+        mode: 0644
+      register: layer2_rendering
+
+    - name: MetalLB | Create layer2 configuration
+      kube:
+        name: "MetalLB"
+        kubectl: "{{ bin_dir }}/kubectl"
+        filename: "{{ kube_config_dir }}/layer2.yaml"
+        state: "{{ layer2_rendering.changed | ternary('latest','present') }}"
+      become: true
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - metallb_config.layer2 is defined
+
+- name: MetalLB | Layer3
+  block:
+    - name: MetalLB | Layout layer3 template
+      ansible.builtin.template:
+        src: layer3.yaml.j2
+        dest: "{{ kube_config_dir }}/layer3.yaml"
+        mode: 0644
+      register: layer3_rendering
+
+    - name: MetalLB | Create layer3 configuration
+      kube:
+        name: "MetalLB"
+        kubectl: "{{ bin_dir }}/kubectl"
+        filename: "{{ kube_config_dir }}/layer3.yaml"
+        state: "{{ layer3_rendering.changed | ternary('latest','present') }}"
+      become: true
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - metallb_config.layer3 is defined
+

 - name: Kubernetes Apps | Delete MetalLB ConfigMap
-  k8s:
+  kube:
    name: config
-    kind: ConfigMap
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: ConfigMap
    namespace: metallb-system
    state: absent
--- a/roles/kubernetes-apps/metallb/templates/metallb.yaml.j2
+++ b/roles/kubernetes-apps/metallb/templates/metallb.yaml.j2
@ -1,3 +1,13 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/warn: privileged
+  name: metallb-system
+
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@ -1703,8 +1713,8 @@ spec:
  template:
    metadata:
      annotations:
-        prometheus.io/port: "{{ metallb_port }}"
-        prometheus.io/scrape: "true"
+        prometheus.io/port: '{{ metallb_port }}'
+        prometheus.io/scrape: 'true'
      labels:
        app: metallb
        component: controller
@ -1719,7 +1729,7 @@ spec:
          value: memberlist
        - name: METALLB_DEPLOYMENT
          value: controller
-        image: {{ metallb_controller_image_repo }}:{{ metallb_version }}
+        image: "{{ metallb_controller_image_repo }}:{{ metallb_version }}"
        livenessProbe:
          failureThreshold: 3
          httpGet:
@ -1755,14 +1765,15 @@ spec:
        - mountPath: /tmp/k8s-webhook-server/serving-certs
          name: cert
          readOnly: true
-{% if metallb_config.controller.tolerations %}
+{% if metallb_config.controller is defined and metallb_config.controller.tolerations is defined %}
      tolerations:
        {{ metallb_config.controller.tolerations | to_nice_yaml(indent=2) | indent(width=8) }}
-{% endif %}
-{% if metallb_controller_nodeselector %}
-      nodeSelector:
-        {{ metallb_controller_nodeselector | to_nice_yaml | indent(width=8) }}
 {%- endif %}
+      nodeSelector:
+        {{ metallb_controller_nodeselector | to_nice_yaml | indent(width=8) -}}
+        {% if metallb_config.controller is defined and metallb_config.controller.nodeselector is defined %}
+        {{ metallb_config.controller.nodeselector | to_nice_yaml | indent(width=8) -}}
+        {%- endif %}
      securityContext:
        fsGroup: 65534
        runAsNonRoot: true
@ -1793,8 +1804,8 @@ spec:
  template:
    metadata:
      annotations:
-        prometheus.io/port: "{{ metallb_port }}"
-        prometheus.io/scrape: "true"
+        prometheus.io/port: '{{ metallb_port }}'
+        prometheus.io/scrape: 'true'
      labels:
        app: metallb
        component: speaker
@ -1823,7 +1834,7 @@ spec:
            secretKeyRef:
              key: secretkey
              name: memberlist
-        image: {{ metallb_speaker_image_repo }}:{{ metallb_version }}
+        image: "{{ metallb_speaker_image_repo }}:{{ metallb_version }}"
        livenessProbe:
          failureThreshold: 3
          httpGet:
@ -1860,16 +1871,19 @@ spec:
            - ALL
          readOnlyRootFilesystem: true
      hostNetwork: true
-{% if metallb_speaker_nodeselector %}
      nodeSelector:
-        {{ metallb_speaker_nodeselector | to_nice_yaml | indent(width=8) }}
-{%- endif %}
+        {{ metallb_speaker_nodeselector | to_nice_yaml | indent(width=8) -}}
+        {% if metallb_config.speaker is defined and metallb_config.speaker.nodeselector is defined %}
+        {{ metallb_config.speaker.nodeselector | to_nice_yaml | indent(width=8) -}}
+        {%- endif %}
+
      serviceAccountName: speaker
      terminationGracePeriodSeconds: 2
-{% if metallb_speaker_tolerations %}
      tolerations:
-        {{ metallb_speaker_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}
-{% endif %}
+        {{ metallb_speaker_tolerations | to_nice_yaml(indent=2) | indent(width=8) -}}
+        {% if metallb_config.speaker is defined and metallb_config.speaker.tolerations is defined %}
+        {{ metallb_config.speaker.tolerations | to_nice_yaml(indent=2) | indent(width=8) -}}
+        {% endif %}
 {% endif %}

 ---
@ -2004,7 +2018,7 @@ webhooks:
  clientConfig:
    service:
      name: webhook-service
-  namespace: metallb-system
+      namespace: metallb-system
      path: /validate-metallb-io-v1beta1-l2advertisement
  failurePolicy: Fail
  name: l2advertisementvalidationwebhook.metallb.io
--- a/tests/files/packet_centos7-flannel-addons-ha.yml
+++ b/tests/files/packet_centos7-flannel-addons-ha.yml
@ -49,3 +49,26 @@ kube_vip_enabled: true
 kube_vip_arp_enabled: true
 kube_vip_controlplane_enabled: true
 kube_vip_address: 192.168.1.100
+
+# MetalLB
+metallb_enabled: true
+metallb_speaker_enabled: true
+metallb_config:
+  address_pools:
+    primary:
+      ip_range:
+        - 192.0.1.0-192.0.1.254
+      auto_assign: true
+    pool1:
+      ip_range:
+        - 192.0.2.1-192.0.2.1
+      auto_assign: false
+    pool2:
+      ip_range:
+        - 192.0.2.2-192.0.2.2
+      auto_assign: false
+
+  layer2:
+    - primary
+    - pool1
+    - pool2