Update multus to v4.1.0 and clarify cilium compatibility (#11434)

* Update multus to v4.1.0 and clarify cilium compatibility

* Fix: bug introduced by #10934 where the template would break if multus was defined

* Set priorityClassName to system-node-critical for multus pods

docs/CNI/multus.md

@@ -17,6 +17,16 @@ kube_network_plugin_multus: true
 
 will install Multus and Calico and configure Multus to use Calico as the primary network plugin.
 
+### Cilium compatibility
+
+If you are using `cilium` as the primary CNI you'll have to set `cilium_cni_exclusive` to `false` to avoid cillium reverting multus config.
+
+```yml
+kube_network_plugin: cilium
+kube_network_plugin_multus: true
+cilium_cni_exclusive: false
+```
+
 ## Using Multus
 
 Once Multus is installed, you can create CNI configurations (as a CRD objects) for additional networks, in this case a macvlan CNI configuration is defined. You may replace the config field with any valid CNI configuration where the CNI binary is available on the nodes.

roles/kubernetes-apps/network_plugin/multus/tasks/main.yml

@@ -9,7 +9,7 @@
     state: "latest"
   delegate_to: "{{ groups['kube_control_plane'][0] }}"
   run_once: true
-  with_items: "{{ (multus_manifest_1.results | default([])) + (multus_nodes_list | map('extract', hostvars, 'multus_manifest_2.results') | default([]) | list) }}"
+  with_items: "{{ (multus_manifest_1.results | default([])) + (multus_nodes_list | map('extract', hostvars, 'multus_manifest_2') | map('default', []) | list | json_query('[].results')) }}"
   loop_control:
     label: "{{ item.item.name if item != None else 'skipped' }}"
   vars:

roles/kubespray-defaults/defaults/main/download.yml

@@ -122,7 +122,7 @@ cilium_enable_hubble: false
 kube_ovn_version: "v1.12.21"
 kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}"
 kube_router_version: "v2.0.0"
-multus_version: "v3.8"
+multus_version: "v4.1.0"
 helm_version: "v3.15.4"
 nerdctl_version: "1.7.6"
 krew_version: "v0.4.4"

roles/network_plugin/multus/defaults/main.yml

@@ -6,5 +6,4 @@ multus_cni_run_dir_host: "/run"
 multus_cni_conf_dir: "{{ ('/host', multus_cni_conf_dir_host) | join }}"
 multus_cni_bin_dir: "{{ ('/host', multus_cni_bin_dir_host) | join }}"
 multus_cni_run_dir: "{{ ('/host', multus_cni_run_dir_host) | join }}"
-multus_cni_version: "0.4.0"
 multus_kubeconfig_file_host: "{{ (multus_cni_conf_dir_host, '/multus.d/multus.kubeconfig') | join }}"

roles/network_plugin/multus/templates/multus-daemonset.yml.j2

@@ -24,6 +24,7 @@ spec:
     spec:
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
+      priorityClassName: system-node-critical
       nodeSelector:
         kubernetes.io/arch: {{ image_arch }}
 {% if container_manager_types | length >= 2 %}
@@ -32,16 +33,34 @@ spec:
       tolerations:
       - operator: Exists
       serviceAccountName: multus
+      initContainers:
+      - name: install-multus-binary
+        image: {{ multus_image_repo }}:{{ multus_image_tag }}
+        command: ["/install_multus"]
+        args:
+        - "--type"
+        - "thin"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - name: cnibin
+          mountPath: {{ multus_cni_bin_dir }}
+          mountPropagation: Bidirectional
       containers:
       - name: kube-multus
         image: {{ multus_image_repo }}:{{ multus_image_tag }}
-        command: ["/entrypoint.sh"]
+        command: ["/thin_entrypoint"]
         args:
         - "--cni-conf-dir={{ multus_cni_conf_dir }}"
+        - "--multus-autoconfig-dir={{ multus_cni_conf_dir }}"
         - "--cni-bin-dir={{ multus_cni_bin_dir }}"
         - "--multus-conf-file={{ multus_conf_file }}"
         - "--multus-kubeconfig-file-host={{ multus_kubeconfig_file_host }}"
-        - "--cni-version={{ multus_cni_version }}"
         resources:
           requests:
             cpu: "100m"
@@ -55,6 +74,7 @@ spec:
           capabilities:
             add: ["SYS_ADMIN"]
 {% endif %}
+        terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
 {% if container_manager == 'crio' %}
         - name: run