diff --git a/roles/container-engine/docker/tasks/set_facts_dns.yml b/roles/container-engine/docker/tasks/set_facts_dns.yml index d0ccd745a..42a2c64bc 100644 --- a/roles/container-engine/docker/tasks/set_facts_dns.yml +++ b/roles/container-engine/docker/tasks/set_facts_dns.yml @@ -10,12 +10,12 @@ - name: Add upstream dns servers set_fact: - docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers | default([]) }}" + docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers }}" when: dns_mode in ['coredns', 'coredns_dual'] - name: Add global searchdomains set_fact: - docker_dns_search_domains: "{{ docker_dns_search_domains + searchdomains | default([]) }}" + docker_dns_search_domains: "{{ docker_dns_search_domains + searchdomains }}" - name: Check system nameservers shell: set -o pipefail && grep "^nameserver" /etc/resolv.conf | sed -r 's/^nameserver\s*([^#\s]+)\s*(#.*)?/\1/' diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index 5622f3b89..3f998f58d 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -66,7 +66,7 @@ {{ primaryClusterIP }} {%- endif -%} upstreamForwardTarget: >- - {%- if upstream_dns_servers is defined and upstream_dns_servers | length > 0 -%} + {%- if upstream_dns_servers | length > 0 -%} {{ upstream_dns_servers | join(' ') }} {%- else -%} /etc/resolv.conf diff --git a/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 index 841dc6501..7671c0c95 100644 --- a/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 @@ -68,7 +68,7 @@ data: {% endif %} } prometheus :9153 - forward . {{ upstream_dns_servers | join(' ') if upstream_dns_servers is defined and upstream_dns_servers | length > 0 else '/etc/resolv.conf' }} { + forward . {{ upstream_dns_servers | join(' ') if upstream_dns_servers | length > 0 else '/etc/resolv.conf' }} { prefer_udp max_concurrent 1000 {% if dns_upstream_forward_extra_opts is defined %} diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index ec0309d30..74b962c3e 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -2,6 +2,9 @@ # Set to true to allow pre-checks to fail and continue deployment ignore_assert_errors: false +nameservers: [] +cloud_resolver: [] +disable_host_nameservers: false epel_enabled: false # Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf dns_late: false diff --git a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml index 3a0f8d8a3..c8303e61d 100644 --- a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml @@ -77,7 +77,7 @@ - name: Set default dns if remove_default_searchdomains is false set_fact: default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"] - when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | default([]) | length==0) + when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | length == 0) - name: Set dns facts set_fact: @@ -151,43 +151,6 @@ dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate when: ansible_os_family == "Debian" -- name: Generate search domains to resolvconf - set_fact: - searchentries: - search {{ (default_searchdomains | default([]) + searchdomains | default([])) | join(' ') }} - domainentry: - domain {{ dns_domain }} - supersede_search: - supersede domain-search "{{ (default_searchdomains | default([]) + searchdomains | default([])) | join('", "') }}"; - supersede_domain: - supersede domain-name "{{ dns_domain }}"; - -# This task should only run after cluster/nodelocal DNS is up, otherwise all DNS lookups will timeout -- name: Generate nameservers for resolvconf, including cluster DNS - set_fact: - nameserverentries: |- - {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique | join(',') }} - dhclient_supersede_nameserver_entries_list: |- - {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique }} - when: not dns_early or dns_late - -# This task should run instead of the above task when cluster/nodelocal DNS hasn't -# been deployed yet (like scale.yml/cluster.yml) or when it's down (reset.yml) -- name: Generate nameservers for resolvconf, not including cluster DNS - set_fact: - nameserverentries: |- - {{ (nameservers | d([]) + cloud_resolver | d([]) + configured_nameservers | d([])) | unique | join(',') }} - dhclient_supersede_nameserver_entries_list: |- - {{ (nameservers | d([]) + cloud_resolver | d([])) | unique }} - when: dns_early and not dns_late - -- name: Generate supersede_nameserver from dhclient_supersede_nameserver_entries_list - set_fact: - supersede_nameserver: |- - {%- if dhclient_supersede_nameserver_entries_list | length > 0 -%} - supersede domain-name-servers {{ dhclient_supersede_nameserver_entries_list | join(', ') }}; - {%- endif -%} - - name: Set etcd vars if using kubeadm mode set_fact: etcd_cert_dir: "{{ kube_cert_dir }}" diff --git a/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml b/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml index 9aad0dba8..b7e9a1ec9 100644 --- a/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml +++ b/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml @@ -7,10 +7,9 @@ blockinfile: path: "{{ resolvconffile }}" block: |- - {% for item in [domainentry] + [searchentries] -%} - {{ item }} - {% endfor %} - {% for item in nameserverentries.split(',') %} + domain {{ dns_domain }} + search {{ (default_searchdomains + searchdomains) | join(' ') }} + {% for item in nameserverentries %} nameserver {{ item }} {% endfor %} options ndots:{{ ndots }} timeout:{{ dns_timeout | default('2') }} attempts:{{ dns_attempts | default('2') }} diff --git a/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml b/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml index 6dfa72426..d223a9468 100644 --- a/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml +++ b/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml @@ -4,11 +4,12 @@ path: /etc/NetworkManager/conf.d/dns.conf section: global-dns-domain-* option: servers - value: "{{ nameserverentries }}" + value: "{{ nameserverentries | join(',') }}" mode: '0600' backup: true when: - - nameserverentries != "127.0.0.53" or systemd_resolved_enabled.rc != 0 + - ('127.0.0.53' not in nameserverentries + or systemd_resolved_enabled.rc != 0) notify: Preinstall | update resolvconf for networkmanager - name: Set default dns if remove_default_searchdomains is false @@ -21,7 +22,7 @@ path: /etc/NetworkManager/conf.d/dns.conf section: global-dns option: searches - value: "{{ (default_searchdomains | default([]) + searchdomains | default([])) | join(',') }}" + value: "{{ (default_searchdomains | default([]) + searchdomains) | join(',') }}" mode: '0600' backup: true notify: Preinstall | update resolvconf for networkmanager diff --git a/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml b/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml index 9745ab261..24ae1bbc9 100644 --- a/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml +++ b/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml @@ -1,9 +1,10 @@ --- - name: Configure dhclient to supersede search/domain/nameservers blockinfile: + # 1 is the 2nd item of a tuple in items() block: |- - {% for item in [supersede_domain, supersede_search, supersede_nameserver] | reject('equalto', '') -%} - {{ item }} + {% for key, val in dhclient_supersede.items() | rejectattr(1, '==', []) -%} + supersede {{ key }} {{ val | join(',') }}; {% endfor %} path: "{{ dhclientconffile }}" create: true diff --git a/roles/kubernetes/preinstall/templates/resolved.conf.j2 b/roles/kubernetes/preinstall/templates/resolved.conf.j2 index edafbf9b9..85f02468d 100644 --- a/roles/kubernetes/preinstall/templates/resolved.conf.j2 +++ b/roles/kubernetes/preinstall/templates/resolved.conf.j2 @@ -1,12 +1,12 @@ [Resolve] {% if not dns_early and dns_late %} -DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server )| list | join(' ') }} +DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server) | list | join(' ') }} {% endif %} -FallbackDNS={{ ( upstream_dns_servers|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(' ') }} -{% if remove_default_searchdomains and searchdomains|default([])|length != 0 %} -Domains={{ searchdomains|default([]) | join(' ') }} +FallbackDNS={{ ( upstream_dns_servers + nameservers + cloud_resolver) | unique | join(' ') }} +{% if remove_default_searchdomains and searchdomains | length != 0 %} +Domains={{ searchdomains | join(' ') }} {% else %} -Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }} +Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains) | join(' ') }} {% endif %} DNSSEC=no Cache=no-negative diff --git a/roles/kubernetes/preinstall/vars/main.yml b/roles/kubernetes/preinstall/vars/main.yml index 12458b557..4052cb260 100644 --- a/roles/kubernetes/preinstall/vars/main.yml +++ b/roles/kubernetes/preinstall/vars/main.yml @@ -76,4 +76,17 @@ coredns_server_by_mode: coredns_dual: "{{ [skydns_server, skydns_server_secondary] }}" manual: "{{ manual_dns_server.split(',') }}" none: [] -coredns_server: "{{ upstream_dns_server if dns_early else coredns_server_by_mode[dns_mode] }}" +coredns_server: "{{ upstream_dns_servers if dns_early else coredns_server_by_mode[dns_mode] }}" + +_nameserverentries: + late: + - "{{ nodelocaldns_ip if enable_nodelocaldns else coredns_server }}" + early: + - "{{ nameservers }}" + - "{{ cloud_resolver }}" + - "{{ configured_nameservers if not disable_host_nameservers else [] }}" +nameserverentries: "{{ ((_nameserverentries['late'] if not dns_early else []) + _nameserverentries['early']) | flatten | unique }}" +dhclient_supersede: + domain-name-servers: "{{ ([nameservers, cloud_resolver] | flatten | unique) if dns_early else nameserverentries }}" + domain-name: "{{ [dns_domain] }}" + domain-search: "{{ default_searchdomains + searchdomains }}" diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index 32a78545a..93e957ccf 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -114,6 +114,10 @@ dns_mode: coredns # Enable dns autoscaler enable_dns_autoscaler: true +# DNS servers added after the cluster DNS +# These will also be used as upstream by Coredns for out-cluster queries +upstream_dns_servers: [] + # Enable nodelocal dns cache enable_nodelocaldns: true enable_nodelocaldns_secondary: false @@ -137,6 +141,7 @@ dns_domain: "{{ cluster_name }}" docker_dns_search_domains: - 'default.svc.{{ dns_domain }}' - 'svc.{{ dns_domain }}' +searchdomains: [] kube_dns_servers: coredns: ["{{ skydns_server }}"]