From 1533d4041126625d69f2d35f10ce28083941686f Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Wed, 4 Sep 2024 14:02:53 +0200 Subject: [PATCH 1/4] Fix kube_reserved_cgroups_for_service_slice The default value is used across kubespray but only defined in kubernetes/node. Move it to kubespray-defaults --- roles/kubernetes/node/defaults/main.yml | 1 - roles/kubespray-defaults/defaults/main/main.yml | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 4d4b011a7..bb374f04a 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -36,7 +36,6 @@ kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnet }} {{ kube_n # Reserve this space for kube resources # Whether to run kubelet and container-engine daemons in a dedicated cgroup. (Not required for resource reservations). kube_reserved: false -kube_reserved_cgroups_for_service_slice: kube.slice kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}" kube_memory_reserved: 256Mi kube_cpu_reserved: 100m diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index e2d0ec22e..70d6b1647 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -29,6 +29,9 @@ kube_proxy_mode: ipvs ## The timeout for init first control-plane kubeadm_init_timeout: 300s +# TODO: remove this +kube_reserved_cgroups_for_service_slice: kube.slice + ## List of kubeadm init phases that should be skipped during control plane setup ## By default 'addon/coredns' is skipped ## 'addon/kube-proxy' gets skipped for some network plugins From 872d7171055b93751d9f4b6af8af5d11ea5061af Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Wed, 4 Sep 2024 13:44:11 +0200 Subject: [PATCH 2/4] Add kube|system_reserved CI testing --- tests/files/packet_ubuntu24-calico-all-in-one.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/files/packet_ubuntu24-calico-all-in-one.yml b/tests/files/packet_ubuntu24-calico-all-in-one.yml index 5d7f55878..310bf349f 100644 --- a/tests/files/packet_ubuntu24-calico-all-in-one.yml +++ b/tests/files/packet_ubuntu24-calico-all-in-one.yml @@ -22,3 +22,6 @@ containerd_registries_mirrors: - host: http://172.19.16.11:5000 capabilities: ["pull", "resolve", "push"] skip_verify: true + +kube_reserved: true +system_reserved: true From 1bc61c9f35d17aade956482f52abe337be792097 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Thu, 23 Nov 2023 17:18:47 +0100 Subject: [PATCH 3/4] Simplify kubelet-config template Remove system|kube_master__reserved variables. Those variables are unnecessary because users can simply use the variables in group_vars if they which to differentiate control plane nodes from other nodes. Set conservative defaults for ephemeral-storage and pids for both kube and system reserved resources. --- roles/kubernetes/node/defaults/main.yml | 26 +++------ .../templates/kubelet-config.v1beta1.yaml.j2 | 58 +++---------------- 2 files changed, 17 insertions(+), 67 deletions(-) diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index bb374f04a..2e0a143de 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -37,29 +37,19 @@ kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnet }} {{ kube_n # Whether to run kubelet and container-engine daemons in a dedicated cgroup. (Not required for resource reservations). kube_reserved: false kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}" -kube_memory_reserved: 256Mi -kube_cpu_reserved: 100m -# kube_ephemeral_storage_reserved: 2Gi -# kube_pid_reserved: "1000" -# Reservation for control plane hosts -kube_master_memory_reserved: 512Mi -kube_master_cpu_reserved: 200m -# kube_master_ephemeral_storage_reserved: 2Gi -# kube_master_pid_reserved: "1000" +kube_memory_reserved: "256Mi" +kube_cpu_reserved: "100m" +kube_ephemeral_storage_reserved: "500Mi" +kube_pid_reserved: "1000" # Set to true to reserve resources for system daemons system_reserved: false system_reserved_cgroups_for_service_slice: system.slice system_reserved_cgroups: "/{{ system_reserved_cgroups_for_service_slice }}" -system_memory_reserved: 512Mi -system_cpu_reserved: 500m -# system_ephemeral_storage_reserved: 2Gi -# system_pid_reserved: "1000" -# Reservation for control plane hosts -system_master_memory_reserved: 256Mi -system_master_cpu_reserved: 250m -# system_master_ephemeral_storage_reserved: 2Gi -# system_master_pid_reserved: "1000" +system_memory_reserved: "512Mi" +system_cpu_reserved: "500m" +system_ephemeral_storage_reserved: "500Mi" +system_pid_reserved: 1000 ## Eviction Thresholds to avoid system OOMs # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds diff --git a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 index 870383c04..3357aef48 100644 --- a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 @@ -60,56 +60,16 @@ clusterDNS: - {{ dns_address }} {% endfor %} {# Node reserved CPU/memory #} -{% if kube_reserved | bool %} -kubeReservedCgroup: {{ kube_reserved_cgroups }} +{% for scope in "kube", "system" %} +{% if lookup('ansible.builtin.vars', scope + "_reserved") | bool %} +{{ scope }}ReservedCgroup: {{ lookup('ansible.builtin.vars', scope + '_reserved_cgroups') }} {% endif %} -kubeReserved: -{% if 'kube_control_plane' in group_names %} - cpu: "{{ kube_master_cpu_reserved }}" - memory: {{ kube_master_memory_reserved }} -{% if kube_master_ephemeral_storage_reserved is defined %} - ephemeral-storage: {{ kube_master_ephemeral_storage_reserved }} -{% endif %} -{% if kube_master_pid_reserved is defined %} - pid: "{{ kube_master_pid_reserved }}" -{% endif %} -{% else %} - cpu: "{{ kube_cpu_reserved }}" - memory: {{ kube_memory_reserved }} -{% if kube_ephemeral_storage_reserved is defined %} - ephemeral-storage: {{ kube_ephemeral_storage_reserved }} -{% endif %} -{% if kube_pid_reserved is defined %} - pid: "{{ kube_pid_reserved }}" -{% endif %} -{% endif %} -{% if system_reserved | bool %} -systemReservedCgroup: {{ system_reserved_cgroups }} -systemReserved: -{% if 'kube_control_plane' in group_names %} - cpu: "{{ system_master_cpu_reserved }}" - memory: {{ system_master_memory_reserved }} -{% if system_master_ephemeral_storage_reserved is defined %} - ephemeral-storage: {{ system_master_ephemeral_storage_reserved }} -{% endif %} -{% if system_master_pid_reserved is defined %} - pid: "{{ system_master_pid_reserved }}" -{% endif %} -{% else %} - cpu: "{{ system_cpu_reserved }}" - memory: {{ system_memory_reserved }} -{% if system_ephemeral_storage_reserved is defined %} - ephemeral-storage: {{ system_ephemeral_storage_reserved }} -{% endif %} -{% if system_pid_reserved is defined %} - pid: "{{ system_pid_reserved }}" -{% endif %} -{% endif %} -{% endif %} -{% if ('kube_control_plane' in group_names) and (eviction_hard_control_plane is defined) and eviction_hard_control_plane %} -evictionHard: - {{ eviction_hard_control_plane | to_nice_yaml(indent=2) | indent(2) }} -{% elif ('kube_control_plane' not in group_names) and (eviction_hard is defined) and eviction_hard %} +{{ scope }}Reserved: +{% for resource in "cpu", "memory", "ephemeral-storage", "pid" %} + {{ resource }}: "{{ lookup('ansible.builtin.vars', scope + '_' ~ (resource | replace('-', '_')) + '_reserved') }}" +{% endfor %} +{% endfor %} +{% if eviction_hard is defined and eviction_hard %} evictionHard: {{ eviction_hard | to_nice_yaml(indent=2) | indent(2) }} {% endif %} From fe60832a023c39be42e76c0c72fd34f253c8cfa6 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Thu, 23 Nov 2023 21:13:55 +0100 Subject: [PATCH 4/4] Remove kubelet_node_{custom_flags,config_extra_args} There is no need to have an extra variables for this, just use different values per host (using Ansible group_vars, for example) --- docs/ansible/vars.md | 8 ++------ roles/kubernetes/node/defaults/main.yml | 6 ------ .../node/templates/kubelet-config.v1beta1.yaml.j2 | 3 --- roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 | 2 +- 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/docs/ansible/vars.md b/docs/ansible/vars.md index b172f4ada..4015fa878 100644 --- a/docs/ansible/vars.md +++ b/docs/ansible/vars.md @@ -296,8 +296,8 @@ node_taints: For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. -Extra flags for the kubelet can be specified using these variables, -in the form of dicts of key-value pairs of configuration parameters that will be inserted into the kubelet YAML config file. The `kubelet_node_config_extra_args` apply kubelet settings only to nodes and not control planes. Example: +Extra flags for the kubelet can be specified using these variables, in the form of dicts of key-value pairs of +configuration parameters that will be inserted into the kubelet YAML config file. Example: ```yml kubelet_config_extra_args: @@ -312,14 +312,10 @@ kubelet_config_extra_args: The possible vars are: * *kubelet_config_extra_args* -* *kubelet_node_config_extra_args* Previously, the same parameters could be passed as flags to kubelet binary with the following vars: * *kubelet_custom_flags* -* *kubelet_node_custom_flags* - -The `kubelet_node_custom_flags` apply kubelet settings only to nodes and not control planes. Example: ```yml kubelet_custom_flags: diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 2e0a143de..1df9d6418 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -125,9 +125,6 @@ kubelet_config_extra_args_cgroupfs: systemCgroups: /system.slice cgroupRoot: / -## Support parameters to be passed to kubelet via kubelet-config.yaml only on nodes, not control plane nodes -kubelet_node_config_extra_args: {} - # Maximum number of container log files that can be present for a container. kubelet_logfiles_max_nr: 5 @@ -137,9 +134,6 @@ kubelet_logfiles_max_size: 10Mi ## Support custom flags to be passed to kubelet kubelet_custom_flags: [] -## Support custom flags to be passed to kubelet only on nodes, not control plane nodes -kubelet_node_custom_flags: [] - # If non-empty, will use this string as identification instead of the actual hostname kube_override_hostname: >- {%- if cloud_provider is defined and cloud_provider in ['aws'] -%} diff --git a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 index 3357aef48..998009fda 100644 --- a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 @@ -77,9 +77,6 @@ resolvConf: "{{ kube_resolv_conf }}" {% if kubelet_config_extra_args %} {{ kubelet_config_extra_args | to_nice_yaml(indent=2) }} {% endif %} -{% if inventory_hostname in groups['kube_node'] and kubelet_node_config_extra_args %} -{{ kubelet_node_config_extra_args | to_nice_yaml(indent=2) }} -{% endif %} {% if kubelet_feature_gates or kube_feature_gates %} featureGates: {% for feature in (kubelet_feature_gates | default(kube_feature_gates, true)) %} diff --git a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 index a5aa369df..7a77101c9 100644 --- a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 +++ b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 @@ -15,7 +15,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" --runtime-cgroups={{ kubelet_runtime_cgroups }} \ {% endset %} -KUBELET_ARGS="{{ kubelet_args_base }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube_node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" +KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_custom_flags | join(' ') }}" {% if kubelet_flexvolumes_plugins_dir is defined %} KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}" {% endif %}