Add variables and defaults for multiple types of ip addresses.

Each node can have 3 IPs.
1. ansible_default_ip4 - whatever ansible things is the first IPv4 address
   usually with the default gw.
2. ip - An address to use on the local node to bind listeners and do local
   communication.  For example, Vagrant boxes have a first address that is the
   NAT bridge and is common for all nodes.  The second address/interface should
   be used.
3. access_ip - An address to use for node-to-node access.  This is assumed to
   be used by other nodes to access the node and may not be actually assigned
   on the node.  For example, AWS public ip that is not assigned to node.

This updates the places addresses are used to use either ip or access_ip and walk
up the list to find an address.

inventory/group_vars/all.yml

@@ -27,6 +27,28 @@ cluster_name: cluster.local
 # set this variable to calico if needed. keep it empty if flannel is used
 kube_network_plugin: calico
 
+# For some environments, each node has a pubilcally accessible
+# address and an address it should bind services to.  These are
+# really inventory level variables, but described here for consistency.
+#
+# When advertising access, the access_ip will be used, but will defer to
+# ip and then the default ansible ip when unspecified.
+#
+# When binding to restrict access, the ip variable will be used, but will
+# defer to the default ansible ip when unspecified.
+#
+# The ip variable is used for specific address binding, e.g. listen address
+# for etcd.  This is use to help with environments like Vagrant or multi-nic
+# systems where one address should be preferred over another.
+# ip: 10.2.2.2
+#
+# The access_ip variable is used to define how other nodes should access
+# the node.  This is used in flannel to allow other flannel nodes to see
+# this node for example.  The access_ip is really useful AWS and Google
+# environments where the nodes are accessed remotely by the "public" ip,
+# but don't know about that address themselves.
+# access_ip: 1.1.1.1
+
 # Kubernetes internal network for services, unused block of space.
 kube_service_addresses: 10.233.0.0/18
 

roles/etcd/templates/etcd.j2

@@ -6,12 +6,12 @@ ETCD_DATA_DIR="/var/lib/etcd"
 {%             set _dummy = etcd.update({'name':"etcd"+loop.index|string}) %}
 {%         endif %}
 {%     endfor %}
-ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379"
-ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2380"
+ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)) }}:2379"
+ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address))  }}:2380"
 ETCD_INITIAL_CLUSTER_STATE="new"
 ETCD_INITIAL_CLUSTER_TOKEN="k8s_etcd"
 ETCD_LISTEN_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2380"
 ETCD_NAME="{{ etcd.name }}"
 {% endif %}
-ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}=http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
+ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}=http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
 ETCD_LISTEN_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2379,http://127.0.0.1:2379"

roles/kubernetes/master/templates/kube-apiserver.j2

@@ -24,7 +24,7 @@ KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{
 KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}"
 
 # Location of the etcd cluster
-KUBE_ETCD_SERVERS="--etcd_servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2379{% if not loop.last %},{% endif %}{% endfor %}"
+KUBE_ETCD_SERVERS="--etcd_servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %}"
 
 # default admission control policies
 KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -10,7 +10,7 @@ spec:
     command:
     - /hyperkube
     - apiserver
-    - --etcd-servers={% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %}
+    - --etcd-servers={% for srv in groups['etcd'] %}http://{{ hostvars[srv]['access_ip'] | default(hostvars[srv]['ip']|default(hostvars[srv]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %}
 
     - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
     - --service-cluster-ip-range={{ kube_service_addresses }}

roles/kubernetes/node/templates/kubelet.j2

@@ -7,7 +7,7 @@ KUBE_LOGGING="--logtostderr=true"
 {% endif %}
 KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}"
 KUBE_ALLOW_PRIV="--allow_privileged=true"
-KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}"
+KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
 KUBELET_ADDRESS="--address=0.0.0.0"
 # The port for the info server to serve on

roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2

@@ -18,7 +18,7 @@ spec:
 {%   if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
     - --master=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}
 {%   else %}
-    - --master=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{ kube_apiserver_port }}
+    - --master=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}
 {%   endif%}
     - --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml
 {% endif %}

roles/kubernetes/node/templates/openssl.conf.j2

@@ -14,7 +14,8 @@ DNS.3 = kubernetes.default.svc.{{ dns_domain }}
 DNS.4 = {{ apiserver_loadbalancer_domain_name }}
 {% endif %}
 {% for host in groups['kube-master'] %}
-IP.{{ loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
 {% endfor %}
-{% set idx =  groups['kube-master'] | length | int + 1 %}
+{% set idx =  groups['kube-master'] | length | int * 2 + 1 %}
 IP.{{ idx | string }} = {{ kube_apiserver_ip }}

roles/kubernetes/preinstall/tasks/etchosts.yml

@@ -2,8 +2,8 @@
 - name: Hosts | populate inventory into hosts file
   lineinfile:
     dest: /etc/hosts
-    regexp: "^{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address) }} {{ item }}$"
-    line: "{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address) }} {{ item }}"
+    regexp: "^{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }}$"
+    line: "{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }}"
     state: present
     backup: yes
   when: hostvars[item].ansible_default_ipv4.address is defined

roles/network_plugin/defaults/main.yml

@@ -1,6 +1,9 @@
 ---
-## defines the IP used to talk to the node
-# flannel_public_ip:
+
+# Flannel public IP
+# The address that flannel should advertise as how to access the system
+flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}"
 
 ## interface that should be used for flannel operations
+## This is actually an inventory node-level item
 # flannel_interface:

roles/network_plugin/templates/calico/calico.conf.j2

@@ -9,7 +9,7 @@ ETCD_AUTHORITY=127.0.0.1:2379
 {% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
 KUBE_API_ROOT=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}/api/v1/
 {% else %}
-KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{kube_apiserver_port}}/api/v1/
+KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{kube_apiserver_port}}/api/v1/
 {% endif %}
 # Kubernetes authentication token
 {% if calico_token is defined | default('') %}