From bedcca922c5feb91ae379404923ec131f2626108 Mon Sep 17 00:00:00 2001 From: Greg Althaus Date: Tue, 26 Jan 2016 14:41:42 -0600 Subject: [PATCH] Add variables and defaults for multiple types of ip addresses. Each node can have 3 IPs. 1. ansible_default_ip4 - whatever ansible things is the first IPv4 address usually with the default gw. 2. ip - An address to use on the local node to bind listeners and do local communication. For example, Vagrant boxes have a first address that is the NAT bridge and is common for all nodes. The second address/interface should be used. 3. access_ip - An address to use for node-to-node access. This is assumed to be used by other nodes to access the node and may not be actually assigned on the node. For example, AWS public ip that is not assigned to node. This updates the places addresses are used to use either ip or access_ip and walk up the list to find an address. --- inventory/group_vars/all.yml | 22 +++++++++++++++++++ roles/etcd/templates/etcd.j2 | 6 ++--- .../master/templates/kube-apiserver.j2 | 2 +- .../manifests/kube-apiserver.manifest.j2 | 2 +- roles/kubernetes/node/templates/kubelet.j2 | 2 +- .../manifests/kube-proxy.manifest.j2 | 2 +- .../kubernetes/node/templates/openssl.conf.j2 | 5 +++-- .../kubernetes/preinstall/tasks/etchosts.yml | 4 ++-- roles/network_plugin/defaults/main.yml | 7 ++++-- .../templates/calico/calico.conf.j2 | 2 +- 10 files changed, 40 insertions(+), 14 deletions(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 45912786b..7dae4774e 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -27,6 +27,28 @@ cluster_name: cluster.local # set this variable to calico if needed. keep it empty if flannel is used kube_network_plugin: calico +# For some environments, each node has a pubilcally accessible +# address and an address it should bind services to. These are +# really inventory level variables, but described here for consistency. +# +# When advertising access, the access_ip will be used, but will defer to +# ip and then the default ansible ip when unspecified. +# +# When binding to restrict access, the ip variable will be used, but will +# defer to the default ansible ip when unspecified. +# +# The ip variable is used for specific address binding, e.g. listen address +# for etcd. This is use to help with environments like Vagrant or multi-nic +# systems where one address should be preferred over another. +# ip: 10.2.2.2 +# +# The access_ip variable is used to define how other nodes should access +# the node. This is used in flannel to allow other flannel nodes to see +# this node for example. The access_ip is really useful AWS and Google +# environments where the nodes are accessed remotely by the "public" ip, +# but don't know about that address themselves. +# access_ip: 1.1.1.1 + # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 diff --git a/roles/etcd/templates/etcd.j2 b/roles/etcd/templates/etcd.j2 index 2b9ab0820..91daf38ca 100644 --- a/roles/etcd/templates/etcd.j2 +++ b/roles/etcd/templates/etcd.j2 @@ -6,12 +6,12 @@ ETCD_DATA_DIR="/var/lib/etcd" {% set _dummy = etcd.update({'name':"etcd"+loop.index|string}) %} {% endif %} {% endfor %} -ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379" -ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380" +ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)) }}:2379" +ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)) }}:2380" ETCD_INITIAL_CLUSTER_STATE="new" ETCD_INITIAL_CLUSTER_TOKEN="k8s_etcd" ETCD_LISTEN_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380" ETCD_NAME="{{ etcd.name }}" {% endif %} -ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}=http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2380{% if not loop.last %},{% endif %}{% endfor %}" +ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}=http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2380{% if not loop.last %},{% endif %}{% endfor %}" ETCD_LISTEN_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379,http://127.0.0.1:2379" diff --git a/roles/kubernetes/master/templates/kube-apiserver.j2 b/roles/kubernetes/master/templates/kube-apiserver.j2 index 3dabca551..2f900fdc8 100644 --- a/roles/kubernetes/master/templates/kube-apiserver.j2 +++ b/roles/kubernetes/master/templates/kube-apiserver.j2 @@ -24,7 +24,7 @@ KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{ KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}" # Location of the etcd cluster -KUBE_ETCD_SERVERS="--etcd_servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2379{% if not loop.last %},{% endif %}{% endfor %}" +KUBE_ETCD_SERVERS="--etcd_servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %}" # default admission control policies KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota" diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 0d8cfb026..1a0b6a2cb 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -10,7 +10,7 @@ spec: command: - /hyperkube - apiserver - - --etcd-servers={% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %} + - --etcd-servers={% for srv in groups['etcd'] %}http://{{ hostvars[srv]['access_ip'] | default(hostvars[srv]['ip']|default(hostvars[srv]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %} - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota - --service-cluster-ip-range={{ kube_service_addresses }} diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index 04a0a6c31..fd59bc10f 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -7,7 +7,7 @@ KUBE_LOGGING="--logtostderr=true" {% endif %} KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}" KUBE_ALLOW_PRIV="--allow_privileged=true" -KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}" +KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}" # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces) KUBELET_ADDRESS="--address=0.0.0.0" # The port for the info server to serve on diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index 942baac44..f3083f639 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -18,7 +18,7 @@ spec: {% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} - --master=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }} {% else %} - - --master=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{ kube_apiserver_port }} + - --master=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }} {% endif%} - --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml {% endif %} diff --git a/roles/kubernetes/node/templates/openssl.conf.j2 b/roles/kubernetes/node/templates/openssl.conf.j2 index c594e3337..c0f253ebe 100644 --- a/roles/kubernetes/node/templates/openssl.conf.j2 +++ b/roles/kubernetes/node/templates/openssl.conf.j2 @@ -14,7 +14,8 @@ DNS.3 = kubernetes.default.svc.{{ dns_domain }} DNS.4 = {{ apiserver_loadbalancer_domain_name }} {% endif %} {% for host in groups['kube-master'] %} -IP.{{ loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} +IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} +IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} {% endfor %} -{% set idx = groups['kube-master'] | length | int + 1 %} +{% set idx = groups['kube-master'] | length | int * 2 + 1 %} IP.{{ idx | string }} = {{ kube_apiserver_ip }} diff --git a/roles/kubernetes/preinstall/tasks/etchosts.yml b/roles/kubernetes/preinstall/tasks/etchosts.yml index 20d1e8150..4706de03a 100644 --- a/roles/kubernetes/preinstall/tasks/etchosts.yml +++ b/roles/kubernetes/preinstall/tasks/etchosts.yml @@ -2,8 +2,8 @@ - name: Hosts | populate inventory into hosts file lineinfile: dest: /etc/hosts - regexp: "^{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address) }} {{ item }}$" - line: "{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address) }} {{ item }}" + regexp: "^{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }}$" + line: "{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }}" state: present backup: yes when: hostvars[item].ansible_default_ipv4.address is defined diff --git a/roles/network_plugin/defaults/main.yml b/roles/network_plugin/defaults/main.yml index 04fece73f..83de96452 100644 --- a/roles/network_plugin/defaults/main.yml +++ b/roles/network_plugin/defaults/main.yml @@ -1,6 +1,9 @@ --- -## defines the IP used to talk to the node -# flannel_public_ip: + +# Flannel public IP +# The address that flannel should advertise as how to access the system +flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}" ## interface that should be used for flannel operations +## This is actually an inventory node-level item # flannel_interface: diff --git a/roles/network_plugin/templates/calico/calico.conf.j2 b/roles/network_plugin/templates/calico/calico.conf.j2 index 685c68c18..7c816bdae 100644 --- a/roles/network_plugin/templates/calico/calico.conf.j2 +++ b/roles/network_plugin/templates/calico/calico.conf.j2 @@ -9,7 +9,7 @@ ETCD_AUTHORITY=127.0.0.1:2379 {% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} KUBE_API_ROOT=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}/api/v1/ {% else %} -KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{kube_apiserver_port}}/api/v1/ +KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{kube_apiserver_port}}/api/v1/ {% endif %} # Kubernetes authentication token {% if calico_token is defined | default('') %}