Merge pull request #3941 from riverzhang/gpu

Fix GPU node Scheduling

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -88,6 +88,12 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% else %}
 {%   set dummy = role_node_labels.append("node-role.kubernetes.io/node=''") %}
 {% endif %}
+{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
+{%   if inventory_hostname in nvidia_gpu_nodes %}
+{%     set dummy = role_node_labels.append('nvidia.com/gpu=true')  %}
+{%   endif %}
+{% endif %}
+
 {% set inventory_node_labels = [] %}
 {% if node_labels is defined and node_labels is mapping %}
 {%   for labelname, labelvalue in node_labels.items() %}
@@ -96,6 +102,16 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% endif %}
 {% set all_node_labels = role_node_labels + inventory_node_labels %}
 
+{# Kubelet node taints for gpu #}
+{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
+{%   if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %}
+{%       set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
+{%   elif inventory_hostname in nvidia_gpu_nodes and node_taints is not defined %}
+{%       set node_taints = [] %}
+{%       set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
+{%   endif %}
+{% endif %}
+
 KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
 {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium", "kube-router"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"