kubernetes/preinstall: dns setting cleanup(dhclient, resolvconf)

We use a lot of facts where variables are enough, and format too early,
which prevent reusing the variables in different contexts.

- Moves set_fact variables to the vars directory, remove unnecessary
 intermediate variables, and render them at usage sites to only do logic
 on native Ansible/Jinja lists.
- Use defaults/ rather than default filters for several variables.

roles/container-engine/docker/tasks/set_facts_dns.yml

@@ -10,12 +10,12 @@
 
 - name: Add upstream dns servers
   set_fact:
-    docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers | default([]) }}"
+    docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers }}"
   when: dns_mode in ['coredns', 'coredns_dual']
 
 - name: Add global searchdomains
   set_fact:
-    docker_dns_search_domains: "{{ docker_dns_search_domains + searchdomains | default([]) }}"
+    docker_dns_search_domains: "{{ docker_dns_search_domains + searchdomains }}"
 
 - name: Check system nameservers
   shell: set -o pipefail && grep "^nameserver" /etc/resolv.conf | sed -r 's/^nameserver\s*([^#\s]+)\s*(#.*)?/\1/'

roles/kubernetes-apps/ansible/tasks/main.yml

@@ -66,7 +66,7 @@
       {{ primaryClusterIP }}
       {%- endif -%}
     upstreamForwardTarget: >-
-      {%- if upstream_dns_servers is defined and upstream_dns_servers | length > 0 -%}
+      {%- if upstream_dns_servers | length > 0 -%}
       {{ upstream_dns_servers | join(' ') }}
       {%- else -%}
       /etc/resolv.conf

roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2

@@ -68,7 +68,7 @@ data:
 {% endif %}
         }
         prometheus :9153
-        forward . {{ upstream_dns_servers | join(' ') if upstream_dns_servers is defined and upstream_dns_servers | length > 0 else '/etc/resolv.conf' }} {
+        forward . {{ upstream_dns_servers | join(' ') if upstream_dns_servers | length > 0 else '/etc/resolv.conf' }} {
           prefer_udp
           max_concurrent 1000
 {% if dns_upstream_forward_extra_opts is defined %}

roles/kubernetes/preinstall/defaults/main.yml

@@ -2,6 +2,9 @@
 # Set to true to allow pre-checks to fail and continue deployment
 ignore_assert_errors: false
 
+nameservers: []
+cloud_resolver: []
+disable_host_nameservers: false
 epel_enabled: false
 # Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf
 dns_late: false

roles/kubernetes/preinstall/tasks/0020-set_facts.yml

@@ -77,7 +77,7 @@
 - name: Set default dns if remove_default_searchdomains is false
   set_fact:
     default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"]
-  when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | default([]) | length==0)
+  when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | length == 0)
 
 - name: Set dns facts
   set_fact:
@@ -151,43 +151,6 @@
     dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate
   when: ansible_os_family == "Debian"
 
-- name: Generate search domains to resolvconf
-  set_fact:
-    searchentries:
-      search {{ (default_searchdomains | default([]) + searchdomains | default([])) | join(' ') }}
-    domainentry:
-      domain {{ dns_domain }}
-    supersede_search:
-      supersede domain-search "{{ (default_searchdomains | default([]) + searchdomains | default([])) | join('", "') }}";
-    supersede_domain:
-      supersede domain-name "{{ dns_domain }}";
-
-# This task should only run after cluster/nodelocal DNS is up, otherwise all DNS lookups will timeout
-- name: Generate nameservers for resolvconf, including cluster DNS
-  set_fact:
-    nameserverentries: |-
-      {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique | join(',') }}
-    dhclient_supersede_nameserver_entries_list: |-
-      {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique }}
-  when: not dns_early or dns_late
-
-# This task should run instead of the above task when cluster/nodelocal DNS hasn't
-# been deployed yet (like scale.yml/cluster.yml) or when it's down (reset.yml)
-- name: Generate nameservers for resolvconf, not including cluster DNS
-  set_fact:
-    nameserverentries: |-
-      {{ (nameservers | d([]) + cloud_resolver | d([]) + configured_nameservers | d([])) | unique | join(',') }}
-    dhclient_supersede_nameserver_entries_list: |-
-      {{ (nameservers | d([]) + cloud_resolver | d([])) | unique }}
-  when: dns_early and not dns_late
-
-- name: Generate supersede_nameserver from dhclient_supersede_nameserver_entries_list
-  set_fact:
-    supersede_nameserver: |-
-      {%- if dhclient_supersede_nameserver_entries_list | length > 0 -%}
-      supersede domain-name-servers {{ dhclient_supersede_nameserver_entries_list | join(', ') }};
-      {%- endif -%}
-
 - name: Set etcd vars if using kubeadm mode
   set_fact:
     etcd_cert_dir: "{{ kube_cert_dir }}"

roles/kubernetes/preinstall/tasks/0060-resolvconf.yml

@@ -7,10 +7,9 @@
   blockinfile:
     path: "{{ resolvconffile }}"
     block: |-
-      {% for item in [domainentry] + [searchentries] -%}
-      {{ item }}
-      {% endfor %}
-      {% for item in nameserverentries.split(',') %}
+      domain {{ dns_domain }}
+      search {{ (default_searchdomains + searchdomains) | join(' ') }}
+      {% for item in nameserverentries %}
       nameserver {{ item }}
       {% endfor %}
       options ndots:{{ ndots }} timeout:{{ dns_timeout | default('2') }} attempts:{{ dns_attempts | default('2') }}

roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml

@@ -4,11 +4,12 @@
     path: /etc/NetworkManager/conf.d/dns.conf
     section: global-dns-domain-*
     option: servers
-    value: "{{ nameserverentries }}"
+    value: "{{ nameserverentries | join(',') }}"
     mode: '0600'
     backup: true
   when:
-    - nameserverentries != "127.0.0.53" or systemd_resolved_enabled.rc != 0
+    - ('127.0.0.53' not in nameserverentries
+       or systemd_resolved_enabled.rc != 0)
   notify: Preinstall | update resolvconf for networkmanager
 
 - name: Set default dns if remove_default_searchdomains is false
@@ -21,7 +22,7 @@
     path: /etc/NetworkManager/conf.d/dns.conf
     section: global-dns
     option: searches
-    value: "{{ (default_searchdomains | default([]) + searchdomains | default([])) | join(',') }}"
+    value: "{{ (default_searchdomains | default([]) + searchdomains) | join(',') }}"
     mode: '0600'
     backup: true
   notify: Preinstall | update resolvconf for networkmanager

roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml

@@ -1,9 +1,10 @@
 ---
 - name: Configure dhclient to supersede search/domain/nameservers
   blockinfile:
+    # 1 is the 2nd item of a tuple in items()
     block: |-
-      {% for item in [supersede_domain, supersede_search, supersede_nameserver] | reject('equalto', '') -%}
-      {{ item }}
+      {% for key, val in dhclient_supersede.items() | rejectattr(1, '==', []) -%}
+      supersede {{ key }} {{ val | join(',') }};
       {% endfor %}
     path: "{{ dhclientconffile }}"
     create: true

roles/kubernetes/preinstall/templates/resolved.conf.j2

@@ -1,12 +1,12 @@
 [Resolve]
 {% if not dns_early and dns_late %}
-DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server )| list | join(' ') }}
+DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server) | list | join(' ') }}
 {% endif %}
-FallbackDNS={{ ( upstream_dns_servers|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(' ') }}
-{% if remove_default_searchdomains and searchdomains|default([])|length != 0 %}
-Domains={{ searchdomains|default([]) | join(' ') }}
+FallbackDNS={{ ( upstream_dns_servers + nameservers + cloud_resolver) | unique | join(' ') }}
+{% if remove_default_searchdomains and searchdomains | length != 0 %}
+Domains={{ searchdomains | join(' ') }}
 {% else %}
-Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}
+Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains) | join(' ') }}
 {% endif %}
 DNSSEC=no
 Cache=no-negative

roles/kubernetes/preinstall/vars/main.yml

@@ -76,4 +76,17 @@ coredns_server_by_mode:
   coredns_dual: "{{ [skydns_server, skydns_server_secondary] }}"
   manual: "{{ manual_dns_server.split(',') }}"
   none: []
-coredns_server: "{{ upstream_dns_server if dns_early else coredns_server_by_mode[dns_mode] }}"
+coredns_server: "{{ upstream_dns_servers if dns_early else coredns_server_by_mode[dns_mode] }}"
+
+_nameserverentries:
+  late:
+    - "{{ nodelocaldns_ip if enable_nodelocaldns else coredns_server }}"
+  early:
+    - "{{ nameservers }}"
+    - "{{ cloud_resolver }}"
+    - "{{ configured_nameservers if not disable_host_nameservers else [] }}"
+nameserverentries: "{{ ((_nameserverentries['late'] if not dns_early else []) + _nameserverentries['early']) | flatten | unique }}"
+dhclient_supersede:
+  domain-name-servers: "{{ ([nameservers, cloud_resolver] | flatten | unique) if dns_early else nameserverentries }}"
+  domain-name: "{{ [dns_domain] }}"
+  domain-search: "{{ default_searchdomains + searchdomains }}"

roles/kubespray-defaults/defaults/main/main.yml

@@ -114,6 +114,10 @@ dns_mode: coredns
 # Enable dns autoscaler
 enable_dns_autoscaler: true
 
+# DNS servers added after the cluster DNS
+# These will also be used as upstream by Coredns for out-cluster queries
+upstream_dns_servers: []
+
 # Enable nodelocal dns cache
 enable_nodelocaldns: true
 enable_nodelocaldns_secondary: false
@@ -137,6 +141,7 @@ dns_domain: "{{ cluster_name }}"
 docker_dns_search_domains:
 - 'default.svc.{{ dns_domain }}'
 - 'svc.{{ dns_domain }}'
+searchdomains: []
 
 kube_dns_servers:
   coredns: ["{{ skydns_server }}"]