diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 45912786b..7dae4774e 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -27,6 +27,28 @@ cluster_name: cluster.local # set this variable to calico if needed. keep it empty if flannel is used kube_network_plugin: calico +# For some environments, each node has a pubilcally accessible +# address and an address it should bind services to. These are +# really inventory level variables, but described here for consistency. +# +# When advertising access, the access_ip will be used, but will defer to +# ip and then the default ansible ip when unspecified. +# +# When binding to restrict access, the ip variable will be used, but will +# defer to the default ansible ip when unspecified. +# +# The ip variable is used for specific address binding, e.g. listen address +# for etcd. This is use to help with environments like Vagrant or multi-nic +# systems where one address should be preferred over another. +# ip: 10.2.2.2 +# +# The access_ip variable is used to define how other nodes should access +# the node. This is used in flannel to allow other flannel nodes to see +# this node for example. The access_ip is really useful AWS and Google +# environments where the nodes are accessed remotely by the "public" ip, +# but don't know about that address themselves. +# access_ip: 1.1.1.1 + # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 diff --git a/roles/etcd/templates/etcd.j2 b/roles/etcd/templates/etcd.j2 index 2b9ab0820..91daf38ca 100644 --- a/roles/etcd/templates/etcd.j2 +++ b/roles/etcd/templates/etcd.j2 @@ -6,12 +6,12 @@ ETCD_DATA_DIR="/var/lib/etcd" {% set _dummy = etcd.update({'name':"etcd"+loop.index|string}) %} {% endif %} {% endfor %} -ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379" -ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380" +ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)) }}:2379" +ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)) }}:2380" ETCD_INITIAL_CLUSTER_STATE="new" ETCD_INITIAL_CLUSTER_TOKEN="k8s_etcd" ETCD_LISTEN_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380" ETCD_NAME="{{ etcd.name }}" {% endif %} -ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}=http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2380{% if not loop.last %},{% endif %}{% endfor %}" +ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}=http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2380{% if not loop.last %},{% endif %}{% endfor %}" ETCD_LISTEN_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379,http://127.0.0.1:2379" diff --git a/roles/kubernetes/master/templates/kube-apiserver.j2 b/roles/kubernetes/master/templates/kube-apiserver.j2 index 3dabca551..2f900fdc8 100644 --- a/roles/kubernetes/master/templates/kube-apiserver.j2 +++ b/roles/kubernetes/master/templates/kube-apiserver.j2 @@ -24,7 +24,7 @@ KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{ KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}" # Location of the etcd cluster -KUBE_ETCD_SERVERS="--etcd_servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2379{% if not loop.last %},{% endif %}{% endfor %}" +KUBE_ETCD_SERVERS="--etcd_servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %}" # default admission control policies KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota" diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 0d8cfb026..1a0b6a2cb 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -10,7 +10,7 @@ spec: command: - /hyperkube - apiserver - - --etcd-servers={% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %} + - --etcd-servers={% for srv in groups['etcd'] %}http://{{ hostvars[srv]['access_ip'] | default(hostvars[srv]['ip']|default(hostvars[srv]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %} - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota - --service-cluster-ip-range={{ kube_service_addresses }} diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index 04a0a6c31..fd59bc10f 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -7,7 +7,7 @@ KUBE_LOGGING="--logtostderr=true" {% endif %} KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}" KUBE_ALLOW_PRIV="--allow_privileged=true" -KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}" +KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}" # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces) KUBELET_ADDRESS="--address=0.0.0.0" # The port for the info server to serve on diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index 942baac44..f3083f639 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -18,7 +18,7 @@ spec: {% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} - --master=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }} {% else %} - - --master=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{ kube_apiserver_port }} + - --master=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }} {% endif%} - --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml {% endif %} diff --git a/roles/kubernetes/node/templates/openssl.conf.j2 b/roles/kubernetes/node/templates/openssl.conf.j2 index c594e3337..c0f253ebe 100644 --- a/roles/kubernetes/node/templates/openssl.conf.j2 +++ b/roles/kubernetes/node/templates/openssl.conf.j2 @@ -14,7 +14,8 @@ DNS.3 = kubernetes.default.svc.{{ dns_domain }} DNS.4 = {{ apiserver_loadbalancer_domain_name }} {% endif %} {% for host in groups['kube-master'] %} -IP.{{ loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} +IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} +IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} {% endfor %} -{% set idx = groups['kube-master'] | length | int + 1 %} +{% set idx = groups['kube-master'] | length | int * 2 + 1 %} IP.{{ idx | string }} = {{ kube_apiserver_ip }} diff --git a/roles/kubernetes/preinstall/tasks/etchosts.yml b/roles/kubernetes/preinstall/tasks/etchosts.yml index 20d1e8150..4706de03a 100644 --- a/roles/kubernetes/preinstall/tasks/etchosts.yml +++ b/roles/kubernetes/preinstall/tasks/etchosts.yml @@ -2,8 +2,8 @@ - name: Hosts | populate inventory into hosts file lineinfile: dest: /etc/hosts - regexp: "^{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address) }} {{ item }}$" - line: "{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address) }} {{ item }}" + regexp: "^{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }}$" + line: "{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }}" state: present backup: yes when: hostvars[item].ansible_default_ipv4.address is defined diff --git a/roles/network_plugin/defaults/main.yml b/roles/network_plugin/defaults/main.yml index 04fece73f..83de96452 100644 --- a/roles/network_plugin/defaults/main.yml +++ b/roles/network_plugin/defaults/main.yml @@ -1,6 +1,9 @@ --- -## defines the IP used to talk to the node -# flannel_public_ip: + +# Flannel public IP +# The address that flannel should advertise as how to access the system +flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}" ## interface that should be used for flannel operations +## This is actually an inventory node-level item # flannel_interface: diff --git a/roles/network_plugin/templates/calico/calico.conf.j2 b/roles/network_plugin/templates/calico/calico.conf.j2 index 685c68c18..7c816bdae 100644 --- a/roles/network_plugin/templates/calico/calico.conf.j2 +++ b/roles/network_plugin/templates/calico/calico.conf.j2 @@ -9,7 +9,7 @@ ETCD_AUTHORITY=127.0.0.1:2379 {% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} KUBE_API_ROOT=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}/api/v1/ {% else %} -KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{kube_apiserver_port}}/api/v1/ +KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{kube_apiserver_port}}/api/v1/ {% endif %} # Kubernetes authentication token {% if calico_token is defined | default('') %}