From 86437730de3dcd044a672babb6ad3f9d74a02ec0 Mon Sep 17 00:00:00 2001 From: ChengHao Yang <17496418+tico88612@users.noreply.github.com> Date: Tue, 1 Apr 2025 12:36:36 +0100 Subject: [PATCH] Use cilium-cli install Cilium Signed-off-by: ChengHao Yang <17496418+tico88612@users.noreply.github.com> --- roles/network_plugin/cilium/defaults/main.yml | 55 ++++----- roles/network_plugin/cilium/tasks/apply.yml | 3 + .../cilium/templates/values.yaml.j2 | 112 ++++++++++++++++++ 3 files changed, 138 insertions(+), 32 deletions(-) create mode 100644 roles/network_plugin/cilium/templates/values.yaml.j2 diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml index 6dfb49094..99a4d3cbe 100644 --- a/roles/network_plugin/cilium/defaults/main.yml +++ b/roles/network_plugin/cilium/defaults/main.yml @@ -3,7 +3,7 @@ cilium_min_version_required: "1.15" # Log-level cilium_debug: false -cilium_mtu: "" +cilium_mtu: "0" cilium_enable_ipv4: "{{ ipv4_stack }}" cilium_enable_ipv6: "{{ ipv6_stack }}" @@ -26,7 +26,7 @@ cilium_agent_health_port: "9879" # - --synchronize-k8s-nodes # - --identity-allocation-mode=kvstore # - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations -cilium_identity_allocation_mode: kvstore +cilium_identity_allocation_mode: crd # Etcd SSL dirs cilium_cert_dir: /etc/cilium/certs @@ -55,8 +55,8 @@ cilium_enable_prometheus: false cilium_enable_portmap: false # Monitor aggregation level (none/low/medium/maximum) cilium_monitor_aggregation: medium -# Kube Proxy Replacement mode (strict/partial) -cilium_kube_proxy_replacement: partial +# Kube Proxy Replacement mode (true/false) +cilium_kube_proxy_replacement: false # If upgrading from Cilium < 1.5, you may want to override some of these options # to prevent service disruptions. See also: @@ -94,8 +94,8 @@ cilium_encryption_enabled: false cilium_encryption_type: "ipsec" # Enable encryption for pure node to node traffic. -# This option is only effective when `cilium_encryption_type` is set to `ipsec`. -cilium_ipsec_node_encryption: false +# This option is only effective when `cilium_encryption_type` is set to `wireguard`. +cilium_encryption_node_encryption: false # If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation. # When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard, @@ -109,6 +109,7 @@ cilium_wireguard_userspace_fallback: false # In case they select the Pod at egress, then the bandwidth enforcement will be disabled for those Pods. # Bandwidth Manager requires a v5.1.x or more recent Linux kernel. cilium_enable_bandwidth_manager: false +cilium_enable_bandwidth_manager_bbr: false # IP Masquerade Agent # https://docs.cilium.io/en/stable/concepts/networking/masquerading/ @@ -131,6 +132,7 @@ cilium_non_masquerade_cidrs: ### Indicates whether to masquerade traffic to the link local prefix. ### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list. cilium_masq_link_local: false +cilium_masq_link_local_ipv6: false ### A time interval at which the agent attempts to reload config from disk cilium_ip_masq_resync_interval: 60s @@ -139,10 +141,10 @@ cilium_ip_masq_resync_interval: 60s cilium_enable_hubble: false ### Enable Hubble-ui cilium_enable_hubble_ui: "{{ cilium_enable_hubble }}" -### Enable Hubble Metrics +### Enable Hubble Metrics (deprecated) cilium_enable_hubble_metrics: false ### if cilium_enable_hubble_metrics: true -cilium_hubble_metrics: {} +cilium_hubble_metrics: [] # - dns # - drop # - tcp @@ -184,7 +186,8 @@ cilium_ipam_mode: cluster-pool # Extra arguments for the Cilium agent -cilium_agent_custom_args: [] +cilium_agent_custom_args: [] # deprecated +cilium_agent_extra_args: [] # For adding and mounting extra volumes to the cilium agent cilium_agent_extra_volumes: [] @@ -208,13 +211,19 @@ cilium_operator_extra_volumes: [] cilium_operator_extra_volume_mounts: [] # Extra arguments for the Cilium Operator -cilium_operator_custom_args: [] +cilium_operator_custom_args: [] # deprecated +cilium_operator_extra_args: [] # Tolerations of the cilium operator cilium_operator_tolerations: - operator: "Exists" +# Unique ID of the cluster. Must be unique across all connected +# clusters and in the range of 1 to 255. Only required for Cluster Mesh, +# may be 0 if Cluster Mesh is not used. +cilium_cluster_id: 0 # Name of the cluster. Only relevant when building a mesh of clusters. +# The "default" name cannot be used if the Cluster ID is different from 0. cilium_cluster_name: default # Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. @@ -257,7 +266,7 @@ cilium_enable_bpf_masquerade: false # host stack (true) or directly and more efficiently out of BPF (false) if # the kernel supports it. The latter has the implication that it will also # bypass netfilter in the host namespace. -cilium_enable_host_legacy_routing: true +cilium_enable_host_legacy_routing: false # -- Enable use of the remote node identity. # ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity @@ -301,9 +310,9 @@ cilium_rolling_restart_wait_retries_count: 30 cilium_rolling_restart_wait_retries_delay_seconds: 10 # Cilium changed the default metrics exporter ports in 1.12 -cilium_agent_scrape_port: "{{ cilium_version is version('1.12', '>=') | ternary('9962', '9090') }}" -cilium_operator_scrape_port: "{{ cilium_version is version('1.12', '>=') | ternary('9963', '6942') }}" -cilium_hubble_scrape_port: "{{ cilium_version is version('1.12', '>=') | ternary('9965', '9091') }}" +cilium_agent_scrape_port: "9962" +cilium_operator_scrape_port: "9963" +cilium_hubble_scrape_port: "9965" # Cilium certgen args for generate certificate for hubble mTLS cilium_certgen_args: @@ -322,24 +331,6 @@ cilium_certgen_args: hubble-relay-client-cert-secret-name: hubble-relay-client-certs hubble-relay-server-cert-generate: false -# A list of extra rules variables to add to clusterrole for cilium operator, formatted like: -# cilium_clusterrole_rules_operator_extra_vars: -# - apiGroups: -# - '""' -# resources: -# - pods -# verbs: -# - delete -# - apiGroups: -# - '""' -# resources: -# - nodes -# verbs: -# - list -# - watch -# resourceNames: -# - toto -cilium_clusterrole_rules_operator_extra_vars: [] cilium_enable_host_firewall: false cilium_policy_audit_mode: false diff --git a/roles/network_plugin/cilium/tasks/apply.yml b/roles/network_plugin/cilium/tasks/apply.yml index cb891875e..04a761c47 100644 --- a/roles/network_plugin/cilium/tasks/apply.yml +++ b/roles/network_plugin/cilium/tasks/apply.yml @@ -1,4 +1,7 @@ --- +- name: Cilium | Install + command: "{{ bin_dir }}/cilium install --version {{ cilium_version }} -f {{ kube_config_dir }}/cilium-values.yaml" + when: inventory_hostname == groups['kube_control_plane'][0] - name: Cilium | Wait for pods to run command: "{{ kubectl }} -n kube-system get pods -l k8s-app=cilium -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'" # noqa literal-compare diff --git a/roles/network_plugin/cilium/templates/values.yaml.j2 b/roles/network_plugin/cilium/templates/values.yaml.j2 new file mode 100644 index 000000000..636dd06ff --- /dev/null +++ b/roles/network_plugin/cilium/templates/values.yaml.j2 @@ -0,0 +1,112 @@ +MTU: {{ cilium_mtu }} +debug: + enabled: {{ cilium_debug }} +ipv4: + enabled: {{ cilium_enable_ipv4 }} +ipv6: + enabled: {{ cilium_enable_ipv6 }} + +l2announcements: + enabled: {{ cilium_l2announcements }} + +healthPort: {{ cilium_agent_health_port }} + +identityAllocationMode: {{ cilium_identity_allocation_mode }} + +tunnelProtocol: {{ cilium_tunnel_mode }} + +loadbalancer: + mode: {{ cilium_loadbalancer_mode }} + +kubeProxyReplacement: {{ cilium_kube_proxy_replacement }} + +extraVolumes: + {{ cilium_agent_extra_volumes | to_nice_yaml(indent=2) | indent(2) }} + +extraVolumeMounts: + {{ cilium_agent_extra_volume_mounts | to_nice_yaml(indent=2) | indent(2) }} + +extraArgs: + {{ cilium_agent_extra_args | to_nice_yaml(indent=2) | indent(2) }} + +bpf: + masquerade: {{ cilium_enable_bpf_masquerade }} + hostLegacyRouting: {{ cilium_enable_host_legacy_routing }} + monitorAggregation: {{ cilium_monitor_aggregation }} + preallocateMaps: {{ cilium_preallocate_bpf_maps }} + mapDynamicSizeRatio: {{ cilium_bpf_map_dynamic_size_ratio }} + +cni: + exclusive: {{ cilium_cni_exclusive }} + logFile: {{ cilium_cni_log_file }} + +autoDirectNodeRoutes: {{ cilium_auto_direct_node_routes }} + +ipv4NativeRoutingCIDR: {{ cilium_native_routing_cidr }} +ipv6NativeRoutingCIDR: {{ cilium_native_routing_cidr_ipv6 }} + +encryption: + enabled: {{ cilium_encryption_enabled }} + {% if cilium_encryption_enabled %} + type: {{ cilium_encryption_type }} + {% if cilium_encryption_type == 'wireguard' %} + nodeEncryption: {{ cilium_encryption_node_encryption }} + {% endif %} + + {% endif %} + +bandwidthManager: + enabled: {{ cilium_enable_bandwidth_manager }} + bbr: {{ cilium_enable_bandwidth_manager_bbr }} + +ipMasqAgent: + enabled: {{ cilium_ip_masq_agent_enable }} + {% if cilium_ip_masq_agent_enable %} + config: + nonMasqueradeCIDRs: {{ cilium_non_masquerade_cidrs }} + masqLinkLocal: {{ cilium_masq_link_local }} + masqLinkLocalIPv6: {{ cilium_masq_link_local_ipv6 }} + # cilium_ip_masq_resync_interval + {% endif %} + +hubble: + enabled: {{ cilium_enable_hubble }} + ui: + enabled: {{ cilium_enable_hubble_ui }} + metrics: + enabled: {{ cilium_hubble_metrics }} + +ipam: + mode: {{ cilium_ipam_mode }} + operator: + clusterPoolIPv4PodCIDRList: + - {{ cilium_pool_cidr | default(kube_pods_subnet) }} + clusterPoolIPv4MaskSize: {{ cilium_pool_mask_size | default(kube_network_node_prefix) }} + + clusterPoolIPv6PodCIDRList: + - {{ cilium_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }} + clusterPoolIPv6MaskSize: {{ cilium_pool_mask_size_ipv6 | default(kube_network_node_prefix_ipv6) }} + +cgroup: + autoMount: + enabled: {{ cilium_cgroup_auto_mount }} + hostRoot: {{ cilium_cgroup_host_root }} + +operator: + replicas: {{ cilium_operator_replicas }} + extraArgs: + {{ cilium_operator_extra_args | to_nice_yaml(indent=2) | indent(4) }} + extraVolumes: + {{ cilium_operator_extra_volumes | to_nice_yaml(indent=2) | indent(4) }} + extraVolumeMounts: + {{ cilium_operator_extra_volume_mounts | to_nice_yaml(indent=2) | indent(4) }} + +cluster: + id: {{ cilium_cluster_id }} + name: {{ cilium_cluster_name }} + +enableIPv4Masquerade: {{ cilium_enable_ipv4_masquerade }} +enableIPv6Masquerade: {{ cilium_enable_ipv6_masquerade }} + +hostFirewall: + enabled: {{ cilium_enable_host_firewall }}