From 9fda84b1c95f9663965d7c485ccc46484a34b693 Mon Sep 17 00:00:00 2001 From: Sergey Date: Mon, 9 Dec 2019 12:43:09 +0300 Subject: [PATCH] set node label via kubectl label command (#5257) * set varios node label via kubectl label command, not kubelet options * remove node_labels from KUBELET_ARGS --- cluster.yml | 1 + roles/kubernetes/node-label/tasks/main.yml | 47 +++++++++++++++++++ .../node/templates/kubelet.env.v1beta1.j2 | 18 +------ scale.yml | 1 + upgrade-cluster.yml | 2 + 5 files changed, 52 insertions(+), 17 deletions(-) create mode 100644 roles/kubernetes/node-label/tasks/main.yml diff --git a/cluster.yml b/cluster.yml index 3e8b78115..1d2f15bbf 100644 --- a/cluster.yml +++ b/cluster.yml @@ -78,6 +78,7 @@ - { role: kubespray-defaults} - { role: kubernetes/kubeadm, tags: kubeadm} - { role: network_plugin, tags: network } + - { role: kubernetes/node-label } - hosts: calico-rr any_errors_fatal: "{{ any_errors_fatal | default(true) }}" diff --git a/roles/kubernetes/node-label/tasks/main.yml b/roles/kubernetes/node-label/tasks/main.yml new file mode 100644 index 000000000..646fd0981 --- /dev/null +++ b/roles/kubernetes/node-label/tasks/main.yml @@ -0,0 +1,47 @@ +--- +- name: Kubernetes Apps | Wait for kube-apiserver + uri: + url: "{{ kube_apiserver_endpoint }}/healthz" + validate_certs: no + client_cert: "{{ kube_apiserver_client_cert }}" + client_key: "{{ kube_apiserver_client_key }}" + register: result + until: result.status == 200 + retries: 10 + delay: 6 + when: inventory_hostname == groups['kube-master'][0] + +- name: Set role node label to empty list + set_fact: + role_node_labels: [] + +- name: Node label for nvidia GPU nodes + set_fact: + role_node_labels: "{{ role_node_labels + [ 'nvidia.com/gpu=true' ] }}" + when: + - nvidia_gpu_nodes is defined + - nvidia_accelerator_enabled|bool + - inventory_hostname in nvidia_gpu_nodes + +- name: Set inventory node label to empty list + set_fact: + inventory_node_labels: [] + +- name: Populate inventory node label + set_fact: + inventory_node_labels: "{{ inventory_node_labels + [ '%s=%s'|format(item.key, item.value) ] }}" + loop: "{{ node_labels|d({})|dict2items }}" + when: + - node_labels is defined + - node_labels is mapping + +- debug: var=role_node_labels +- debug: var=inventory_node_labels + +- name: Set label to node + command: >- + {{ bin_dir }}/kubectl label node {{ inventory_hostname }} {{ item }} --overwrite=true + loop: "{{ role_node_labels + inventory_node_labels }}" + delegate_to: "{{ groups['kube-master'][0] }}" + changed_when: false +... diff --git a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 index 0a37fe3f5..20aed6d9a 100644 --- a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 +++ b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 @@ -23,22 +23,6 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" --runtime-cgroups={{ kubelet_runtime_cgroups }} \ {% endset %} -{# Kubelet node labels #} -{% set role_node_labels = [] %} -{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %} -{% if inventory_hostname in nvidia_gpu_nodes %} -{% set dummy = role_node_labels.append('nvidia.com/gpu=true') %} -{% endif %} -{% endif %} - -{% set inventory_node_labels = [] %} -{% if node_labels is defined and node_labels is mapping %} -{% for labelname, labelvalue in node_labels.items() %} -{% set dummy = inventory_node_labels.append('%s=%s'|format(labelname, labelvalue)) %} -{% endfor %} -{% endif %} -{% set all_node_labels = role_node_labels + inventory_node_labels %} - {# Kubelet node taints for gpu #} {% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %} {% if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %} @@ -49,7 +33,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} {% endif %} -KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" +KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" {% if kubelet_flexvolumes_plugins_dir is defined %} KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}" {% endif %} diff --git a/scale.yml b/scale.yml index 4f53a1201..8b53b04f1 100644 --- a/scale.yml +++ b/scale.yml @@ -46,4 +46,5 @@ - { role: kubernetes/node, tags: node } - { role: kubernetes/kubeadm, tags: kubeadm } - { role: network_plugin, tags: network } + - { role: kubernetes/node-label } environment: "{{ proxy_env }}" diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml index a5b2bf781..b3d17f530 100644 --- a/upgrade-cluster.yml +++ b/upgrade-cluster.yml @@ -72,6 +72,7 @@ - { role: kubernetes/node, tags: node } - { role: kubernetes/master, tags: master, upgrade_cluster_setup: true } - { role: kubernetes/client, tags: client } + - { role: kubernetes/node-label } - { role: kubernetes-apps/cluster_roles, tags: cluster-roles } - { role: upgrade/post-upgrade, tags: post-upgrade } environment: "{{ proxy_env }}" @@ -95,6 +96,7 @@ - { role: upgrade/pre-upgrade, tags: pre-upgrade } - { role: kubernetes/node, tags: node } - { role: kubernetes/kubeadm, tags: kubeadm } + - { role: kubernetes/node-label } - { role: upgrade/post-upgrade, tags: post-upgrade } environment: "{{ proxy_env }}"