From 6e5b9e0ebfa5d0c037356da8825be883e24069fc Mon Sep 17 00:00:00 2001 From: Pasquale Toscano Date: Fri, 5 Nov 2021 15:59:54 +0100 Subject: [PATCH] Fix Kubelet and Containerd when using cgroupfs as cgroup driver (#8123) --- docs/vars.md | 2 +- .../containerd/defaults/main.yml | 16 +++++++++------- .../container-engine/containerd/tasks/facts.yml | 6 ++++++ .../container-engine/containerd/tasks/main.yml | 4 ++++ .../containerd/templates/config.toml.j2 | 10 +++++++++- roles/kubernetes/node/defaults/main.yml | 12 ++++++++++-- roles/kubernetes/node/tasks/facts.yml | 17 +++++++++++++++++ .../templates/kubelet-config.v1beta1.yaml.j2 | 2 +- 8 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 roles/container-engine/containerd/tasks/facts.yml diff --git a/docs/vars.md b/docs/vars.md index 51129cfc2..be812042a 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -119,7 +119,7 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m ``--insecure-registry=myregistry.mydomain:5000`` * *docker_plugins* - This list can be used to define [Docker plugins](https://docs.docker.com/engine/extend/) to install. * *containerd_default_runtime* - Sets the default Containerd runtime used by the Kubernetes CRI plugin. -* *containerd_runtimes* - Sets the Containerd runtime attributes used by the Kubernetes CRI plugin. +* *containerd_additional_runtimes* - Sets the additional Containerd runtimes used by the Kubernetes CRI plugin. [Default config](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/container-engine/containerd/defaults/main.yml) can be overriden in inventory vars. * *http_proxy/https_proxy/no_proxy/no_proxy_exclude_workers/additional_no_proxy* - Proxy variables for deploying behind a proxy. Note that no_proxy defaults to all internal cluster IPs and hostnames diff --git a/roles/container-engine/containerd/defaults/main.yml b/roles/container-engine/containerd/defaults/main.yml index bfab4aaa8..f01f85f38 100644 --- a/roles/container-engine/containerd/defaults/main.yml +++ b/roles/container-engine/containerd/defaults/main.yml @@ -7,13 +7,15 @@ containerd_oom_score: 0 containerd_default_runtime: "runc" # containerd_snapshotter: "native" -containerd_runtimes: - - name: runc - type: "io.containerd.runc.v2" - engine: "" - root: "" - options: - systemdCgroup: "true" +containerd_runc_runtime: + name: runc + type: "io.containerd.runc.v2" + engine: "" + root: "" + options: + systemCgroup: "true" + +containerd_additional_runtimes: [] # Example for Kata Containers as additional runtime: # - name: kata # type: "io.containerd.kata.v2" diff --git a/roles/container-engine/containerd/tasks/facts.yml b/roles/container-engine/containerd/tasks/facts.yml new file mode 100644 index 000000000..987b78458 --- /dev/null +++ b/roles/container-engine/containerd/tasks/facts.yml @@ -0,0 +1,6 @@ +--- + +- name: set kubelet_config_extra_args options when cgroupfs is used + set_fact: + containerd_runc_runtime: "{{ containerd_runc_runtime | combine({'options':{'systemCgroup':'false'}}) }}" + when: not containerd_use_systemd_cgroup diff --git a/roles/container-engine/containerd/tasks/main.yml b/roles/container-engine/containerd/tasks/main.yml index 7088f340d..39779e78c 100644 --- a/roles/container-engine/containerd/tasks/main.yml +++ b/roles/container-engine/containerd/tasks/main.yml @@ -1,4 +1,8 @@ --- +- import_tasks: facts.yml + tags: + - facts + - name: Fail containerd setup if distribution is not supported fail: msg: "{{ ansible_distribution }} is not supported by containerd." diff --git a/roles/container-engine/containerd/templates/config.toml.j2 b/roles/container-engine/containerd/templates/config.toml.j2 index 35c4f933a..48f3628e0 100644 --- a/roles/container-engine/containerd/templates/config.toml.j2 +++ b/roles/container-engine/containerd/templates/config.toml.j2 @@ -22,7 +22,15 @@ oom_score = {{ containerd_oom_score }} default_runtime_name = "{{ containerd_default_runtime | default('runc') }}" snapshotter = "{{ containerd_snapshotter | default('overlayfs') }}" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] -{% for runtime in containerd_runtimes %} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ containerd_runc_runtime.name }}] + runtime_type = "{{ containerd_runc_runtime.type }}" + runtime_engine = "{{ containerd_runc_runtime.engine}}" + runtime_root = "{{ containerd_runc_runtime.root }}" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ containerd_runc_runtime.name }}.options] +{% for key, value in containerd_runc_runtime.options.items() %} + {{ key }} = {{ value }} +{% endfor %} +{% for runtime in containerd_additional_runtimes %} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}] runtime_type = "{{ runtime.type }}" runtime_engine = "{{ runtime.engine }}" diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 686e2e609..d42fa555a 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -14,11 +14,14 @@ kube_resolv_conf: "/etc/resolv.conf" # Set to empty to avoid cgroup creation kubelet_enforce_node_allocatable: "\"\"" -# Set runtime cgroups +# Set runtime and kubelet cgroups when using systemd as cgroup driver (default) kubelet_runtime_cgroups: "/systemd/system.slice" -# Set kubelet cgroups kubelet_kubelet_cgroups: "/systemd/system.slice" +# Set runtime and kubelet cgroups when using cgroupfs as cgroup driver +kubelet_runtime_cgroups_cgroupfs: "/system.slice/containerd.service" +kubelet_kubelet_cgroups_cgroupfs: "/system.slice/kubelet.slice" + ### fail with swap on (default true) kubelet_fail_swap_on: true @@ -66,6 +69,11 @@ kubelet_max_pods: 110 ## Support parameters to be passed to kubelet via kubelet-config.yaml kubelet_config_extra_args: {} +## Parameters to be passed to kubelet via kubelet-config.yaml when cgroupfs is used as cgroup driver +kubelet_config_extra_args_cgroupfs: + systemCgroups: /system.slice + cgroupRoot: / + ## Support parameters to be passed to kubelet via kubelet-config.yaml only on nodes, not masters kubelet_node_config_extra_args: {} diff --git a/roles/kubernetes/node/tasks/facts.yml b/roles/kubernetes/node/tasks/facts.yml index b7b3ad011..d4bd428f5 100644 --- a/roles/kubernetes/node/tasks/facts.yml +++ b/roles/kubernetes/node/tasks/facts.yml @@ -27,6 +27,23 @@ {%- if containerd_use_systemd_cgroup -%}systemd{%- else -%}cgroupfs{%- endif -%} when: container_manager == 'containerd' +- name: set kubelet_cgroup_driver + set_fact: + kubelet_cgroup_driver: "{{ kubelet_cgroup_driver_detected }}" + when: kubelet_cgroup_driver is undefined + +- name: set kubelet_cgroups options when cgroupfs is used + set_fact: + kubelet_runtime_cgroups: "{{ kubelet_runtime_cgroups_cgroupfs }}" + kubelet_kubelet_cgroups: "{{ kubelet_kubelet_cgroups_cgroupfs }}" + when: kubelet_cgroup_driver == 'cgroupfs' + +- name: set kubelet_config_extra_args options when cgroupfs is used + vars: + set_fact: + kubelet_config_extra_args: "{{ kubelet_config_extra_args | combine(kubelet_config_extra_args_cgroupfs) }}" + when: kubelet_cgroup_driver == 'cgroupfs' + - name: os specific vars include_vars: "{{ item }}" with_first_found: diff --git a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 index 673c61788..13ed5f4c4 100644 --- a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 @@ -23,7 +23,7 @@ enforceNodeAllocatable: {% endfor %} {% endif %} staticPodPath: {{ kube_manifest_dir }} -cgroupDriver: {{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} +cgroupDriver: {{ kubelet_cgroup_driver }} containerLogMaxFiles: {{ kubelet_logfiles_max_nr }} containerLogMaxSize: {{ kubelet_logfiles_max_size }} maxPods: {{ kubelet_max_pods }}