From 4f92417a5dba3c9e955f1dc9fae1c780e839e2fe Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Sat, 30 Jan 2016 16:04:47 +0100 Subject: [PATCH] split network plugins into distinct roles --- README.md | 13 +++--- apps.yml | 4 -- cluster.yml | 4 +- inventory/group_vars/all.yml | 6 +-- roles/docker/tasks/main.yml | 2 +- roles/docker/vars/debian.yml | 3 +- roles/docker/vars/ubuntu.yml | 21 +++++++++ roles/download/defaults/main.yml | 25 +++++++---- .../{node => master}/files/kube-gen-token.sh | 0 .../master/tasks/gen_kube_tokens.yml | 7 +++ roles/kubernetes/master/tasks/main.yml | 5 --- roles/kubernetes/node/handlers/main.yml | 4 -- .../node/tasks/gen_calico_tokens.yml | 27 ------------ roles/kubernetes/node/tasks/main.yml | 30 +++---------- roles/kubernetes/node/tasks/secrets.yml | 2 - .../node/templates/cni-calico.conf.j2 | 9 ++++ roles/kubernetes/node/templates/kubelet.j2 | 2 +- roles/kubernetes/preinstall/tasks/main.yml | 35 +++++++++++++++ roles/network_plugin/calico/defaults/main.yml | 2 + roles/network_plugin/calico/handlers/main.yml | 15 +++++++ .../calico.yml => calico/tasks/main.yml} | 44 +++++++++++++++---- .../templates}/calico-node.service.j2 | 4 +- .../templates}/deb-calico.initd.j2 | 0 .../{ => calico}/templates/docker | 0 .../calico/templates/network-environment.j2 | 9 ++++ .../templates}/rh-calico.initd.j2 | 0 .../templates/systemd-docker.service | 0 .../{ => flannel}/defaults/main.yml | 3 -- .../{ => flannel}/handlers/main.yml | 11 ----- .../flannel.yml => flannel/tasks/main.yml} | 27 ++++++++++-- roles/network_plugin/flannel/templates/docker | 6 +++ .../templates}/flannel-pod.yml | 0 .../templates}/network.json | 0 .../flannel/templates/systemd-docker.service | 28 ++++++++++++ roles/network_plugin/meta/main.yml | 6 +++ roles/network_plugin/tasks/main.yml | 30 ------------- .../templates/calico/calico.conf.j2 | 17 ------- .../templates/calico/network-environment.j2 | 2 - 38 files changed, 235 insertions(+), 168 deletions(-) create mode 100644 roles/docker/vars/ubuntu.yml rename roles/kubernetes/{node => master}/files/kube-gen-token.sh (100%) delete mode 100644 roles/kubernetes/node/tasks/gen_calico_tokens.yml create mode 100644 roles/kubernetes/node/templates/cni-calico.conf.j2 create mode 100644 roles/network_plugin/calico/defaults/main.yml create mode 100644 roles/network_plugin/calico/handlers/main.yml rename roles/network_plugin/{tasks/calico.yml => calico/tasks/main.yml} (73%) rename roles/network_plugin/{templates/calico => calico/templates}/calico-node.service.j2 (59%) rename roles/network_plugin/{templates/calico => calico/templates}/deb-calico.initd.j2 (100%) rename roles/network_plugin/{ => calico}/templates/docker (100%) create mode 100644 roles/network_plugin/calico/templates/network-environment.j2 rename roles/network_plugin/{templates/calico => calico/templates}/rh-calico.initd.j2 (100%) rename roles/network_plugin/{ => calico}/templates/systemd-docker.service (100%) rename roles/network_plugin/{ => flannel}/defaults/main.yml (93%) rename roles/network_plugin/{ => flannel}/handlers/main.yml (71%) rename roles/network_plugin/{tasks/flannel.yml => flannel/tasks/main.yml} (58%) create mode 100644 roles/network_plugin/flannel/templates/docker rename roles/network_plugin/{templates/flannel => flannel/templates}/flannel-pod.yml (100%) rename roles/network_plugin/{templates/flannel => flannel/templates}/network.json (100%) create mode 100644 roles/network_plugin/flannel/templates/systemd-docker.service create mode 100644 roles/network_plugin/meta/main.yml delete mode 100644 roles/network_plugin/tasks/main.yml delete mode 100644 roles/network_plugin/templates/calico/calico.conf.j2 delete mode 100644 roles/network_plugin/templates/calico/network-environment.j2 diff --git a/README.md b/README.md index 950ce225f..dc35e7ff6 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ in order to avoid any issue during deployment you should disable your firewall ### Components * [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.1.4 * [etcd](https://github.com/coreos/etcd/releases) v2.2.4 -* [calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.14.0 +* [calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.16.0 * [flanneld](https://github.com/coreos/flannel/releases) v0.5.5 * [docker](https://www.docker.com/) v1.9.1 @@ -107,21 +107,20 @@ kube-master ### Playbook ``` --- - - hosts: k8s-cluster roles: + - { role: adduser, tags: adduser } - { role: download, tags: download } - { role: kubernetes/preinstall, tags: preinstall } + - { role: etcd, tags: etcd } - { role: docker, tags: docker } - { role: kubernetes/node, tags: node } - - { role: etcd, tags: etcd } + - { role: network_plugin, tags: network } - { role: dnsmasq, tags: dnsmasq } - - { role: network_plugin, tags: ['calico', 'flannel', 'network'] } - hosts: kube-master roles: - { role: kubernetes/master, tags: master } - ``` ### Run @@ -143,14 +142,14 @@ the server address has to be present on both groups 'kube-master' and 'kube-node In order to do so, some variables have to be used '**loadbalancer_apiserver**' and '**apiserver_loadbalancer_domain_name**' -### Network Overlay +### Network Plugin You can choose between 2 network plugins. Only one must be chosen. * **flannel**: gre/vxlan (layer 2) networking. ([official docs](https://github.com/coreos/flannel)) * **calico**: bgp (layer 3) networking. ([official docs](http://docs.projectcalico.org/en/0.13/)) -The choice is defined with the variable '**kube_network_plugin**' +The choice is defined with the variable **kube_network_plugin** ### Check cluster status diff --git a/apps.yml b/apps.yml index a7f0078e8..ee9d9f36b 100644 --- a/apps.yml +++ b/apps.yml @@ -9,7 +9,6 @@ - { role: apps/k8s-elasticsearch, tags: 'elasticsearch' } - { role: apps/k8s-memcached, tags: 'memcached' } - { role: apps/k8s-redis, tags: 'redis' } - - { role: apps/k8s-mongodb-simple, tags: 'mongodb-simple' } # Msg Broker - { role: apps/k8s-rabbitmq, tags: 'rabbitmq' } @@ -28,6 +27,3 @@ # ETCD - { role: apps/k8s-etcd, tags: 'etcd'} - - # Chat Apps - - { role: apps/k8s-rocketchat, tags: 'rocketchat'} \ No newline at end of file diff --git a/cluster.yml b/cluster.yml index c0e23169a..56d945302 100644 --- a/cluster.yml +++ b/cluster.yml @@ -4,11 +4,11 @@ - { role: adduser, tags: adduser } - { role: download, tags: download } - { role: kubernetes/preinstall, tags: preinstall } + - { role: etcd, tags: etcd } - { role: docker, tags: docker } - { role: kubernetes/node, tags: node } - - { role: etcd, tags: etcd } + - { role: network_plugin, tags: network } - { role: dnsmasq, tags: dnsmasq } - - { role: network_plugin, tags: ['calico', 'flannel', 'network'] } - hosts: kube-master roles: diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 7dae4774e..2cc35d707 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -24,9 +24,6 @@ kube_users: # Kubernetes cluster name, also will be used as DNS domain cluster_name: cluster.local -# set this variable to calico if needed. keep it empty if flannel is used -kube_network_plugin: calico - # For some environments, each node has a pubilcally accessible # address and an address it should bind services to. These are # really inventory level variables, but described here for consistency. @@ -49,6 +46,9 @@ kube_network_plugin: calico # but don't know about that address themselves. # access_ip: 1.1.1.1 +# Choose network plugin (calico or flannel) +kube_network_plugin: calico + # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index 9ffff7205..f39e0612f 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -41,7 +41,7 @@ action: "{{ docker_package_info.pkg_mgr }}" args: pkg: "{{item}}" - state: latest + state: present with_items: docker_package_info.pkgs when: docker_package_info.pkgs|length > 0 diff --git a/roles/docker/vars/debian.yml b/roles/docker/vars/debian.yml index f97dd9116..b4d8209f0 100644 --- a/roles/docker/vars/debian.yml +++ b/roles/docker/vars/debian.yml @@ -1,9 +1,10 @@ docker_kernel_min_version: '3.2' +docker_version: 1.9.1-0~{{ ansible_distribution_release|lower }} docker_package_info: pkg_mgr: apt pkgs: - - docker-engine + - docker-engine={{ docker_version }} docker_repo_key_info: pkg_key: apt_key diff --git a/roles/docker/vars/ubuntu.yml b/roles/docker/vars/ubuntu.yml new file mode 100644 index 000000000..311d73a54 --- /dev/null +++ b/roles/docker/vars/ubuntu.yml @@ -0,0 +1,21 @@ +docker_kernel_min_version: '3.2' +docker_version: 1.9.0-0~{{ ansible_distribution_release }} + +docker_package_info: + pkg_mgr: apt + pkgs: + - docker-engine={{ docker_version }} + +docker_repo_key_info: + pkg_key: apt_key + keyserver: hkp://p80.pool.sks-keyservers.net:80 + repo_keys: + - 58118E89F3A912897C070ADBF76221572C52609D + +docker_repo_info: + pkg_repo: apt_repository + repos: + - > + deb https://apt.dockerproject.org/repo + {{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }} + main diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index e5e81e188..d4f5af875 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -4,18 +4,20 @@ local_release_dir: /tmp # Versions kube_version: v1.1.4 etcd_version: v2.2.4 -calico_version: v0.14.0 -calico_plugin_version: v0.7.0 +calico_version: v0.16.0 +calico_cni_version: v1.0.0 # Download URL's kube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64" etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz" calico_download_url: "https://github.com/Metaswitch/calico-docker/releases/download/{{calico_version}}/calicoctl" -calico_plugin_download_url: "https://github.com/projectcalico/calico-kubernetes/releases/download/{{calico_plugin_version}}/calico_kubernetes" +calico_cni_download_url: "https://github.com/projectcalico/calico-cni/releases/download/{{calico_cni_version}}/calico" +calico_cni_ipam_download_url: "https://github.com/projectcalico/calico-cni/releases/download/{{calico_cni_version}}/calico-ipam" # Checksums -calico_checksum: "f251d7a8583233906aa6d059447c1e4fb32bf1369a51fdf96a68d50466d6a69c" -calico_plugin_checksum: "032f582f5eeec6fb26191d2fbcbf8bca4da3b14abb579db7baa7b3504d4dffec" +calico_checksum: "cfbbcad4b3b7d79be9a25bcdc153ec1d139eecd54840914a363b0710eebc5c51" +calico_cni_checksum: "cfbb95d4416cb65845a188f3bd991fff232bd5ce3463b2919d586ab77967aecd" +calico_cni_ipam_checksum: "93ebf8756b26314e1e3f612f1e824418cbb0a8df2942664422e697bcb109fbb2" etcd_checksum: "6c4e5cdeaaac1a70b8f06b5dd6b82c37ff19993c9bca81248975610e555c4b9b" kubectl_checksum: "873ba19926d17a3287dc8639ea1434fe3cd0cb4e61d82101ba754922cfc7a633" kubelet_checksum: "f2d1eae3fa6e304f6cbc9b2621e4b86fc3bcb4e74a15d35f58bf00e45c706e0a" @@ -29,10 +31,17 @@ downloads: owner: "root" mode: "0755" - - name: calico-plugin + - name: calico-cni-plugin dest: calico/bin/calico - sha256: "{{ calico_plugin_checksum }}" - url: "{{ calico_plugin_download_url }}" + sha256: "{{ calico_cni_checksum }}" + url: "{{ calico_cni_download_url }}" + owner: "root" + mode: "0755" + + - name: calico-cni-plugin-ipam + dest: calico/bin/calico-ipam + sha256: "{{ calico_cni_ipam_checksum }}" + url: "{{ calico_cni_ipam_download_url }}" owner: "root" mode: "0755" diff --git a/roles/kubernetes/node/files/kube-gen-token.sh b/roles/kubernetes/master/files/kube-gen-token.sh similarity index 100% rename from roles/kubernetes/node/files/kube-gen-token.sh rename to roles/kubernetes/master/files/kube-gen-token.sh diff --git a/roles/kubernetes/master/tasks/gen_kube_tokens.yml b/roles/kubernetes/master/tasks/gen_kube_tokens.yml index 43726a46d..62b26e2fe 100644 --- a/roles/kubernetes/master/tasks/gen_kube_tokens.yml +++ b/roles/kubernetes/master/tasks/gen_kube_tokens.yml @@ -1,4 +1,11 @@ --- +- name: tokens | copy the token gen script + copy: + src=kube-gen-token.sh + dest={{ kube_script_dir }} + mode=u+x + when: inventory_hostname == groups['kube-master'][0] + - name: tokens | generate tokens for master components command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}" environment: diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index 26e5896cb..5eb0de96f 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -69,11 +69,6 @@ shell: setcap cap_net_bind_service+ep {{ bin_dir }}/kube-apiserver changed_when: false -- name: Restart apiserver - command: "/bin/true" - notify: restart kube-apiserver - when: is_gentoken_calico|default(false) - - meta: flush_handlers - include: start.yml diff --git a/roles/kubernetes/node/handlers/main.yml b/roles/kubernetes/node/handlers/main.yml index 81d7ca2c7..7b2c3df71 100644 --- a/roles/kubernetes/node/handlers/main.yml +++ b/roles/kubernetes/node/handlers/main.yml @@ -9,10 +9,6 @@ - reload systemd - reload kubelet -- name: set is_gentoken_calico fact - set_fact: - is_gentoken_calico: true - - name: reload kubelet service: name: kubelet diff --git a/roles/kubernetes/node/tasks/gen_calico_tokens.yml b/roles/kubernetes/node/tasks/gen_calico_tokens.yml deleted file mode 100644 index c9a4f2955..000000000 --- a/roles/kubernetes/node/tasks/gen_calico_tokens.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- name: tokens | copy the token gen script - copy: - src=kube-gen-token.sh - dest={{ kube_script_dir }} - mode=u+x - when: inventory_hostname == groups['kube-master'][0] - -- name: tokens | generate tokens for calico - command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}" - environment: - TOKEN_DIR: "{{ kube_token_dir }}" - with_nested: - - [ "system:calico" ] - - "{{ groups['k8s-cluster'] }}" - register: gentoken_calico - changed_when: "'Added' in gentoken_calico.stdout" - when: kube_network_plugin == "calico" - delegate_to: "{{ groups['kube-master'][0] }}" - notify: set is_gentoken_calico fact - -- name: tokens | get the calico token values - slurp: - src: "{{ kube_token_dir }}/system:calico-{{ inventory_hostname }}.token" - register: calico_token - when: kube_network_plugin == "calico" - delegate_to: "{{ groups['kube-master'][0] }}" diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 18d293615..3af211902 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -1,32 +1,12 @@ --- -- name: Create kubernetes config directory - file: - path: "{{ kube_config_dir }}" - state: directory - owner: kube - -- name: Create kubernetes script directory - file: - path: "{{ kube_script_dir }}" - state: directory - owner: kube - -- name: Create kubernetes manifests directory - file: - path: "{{ kube_manifest_dir }}" - state: directory - owner: kube - -- name: Create kubernetes logs directory - file: - path: "{{ kube_log_dir }}" - state: directory +- name: Write Calico cni config + template: + src: "cni-calico.conf.j2" + dest: "/etc/cni/net.d/10-calico.conf" owner: kube - when: init_system == "sysvinit" + when: kube_network_plugin == "calico" - include: secrets.yml - tags: - - secrets - include: install.yml diff --git a/roles/kubernetes/node/tasks/secrets.yml b/roles/kubernetes/node/tasks/secrets.yml index 8a2997ca5..49b7f154f 100644 --- a/roles/kubernetes/node/tasks/secrets.yml +++ b/roles/kubernetes/node/tasks/secrets.yml @@ -16,8 +16,6 @@ - include: gen_certs.yml when: inventory_hostname == groups['kube-master'][0] -- include: gen_calico_tokens.yml - # Sync certs between nodes - name: Secrets | create user user: diff --git a/roles/kubernetes/node/templates/cni-calico.conf.j2 b/roles/kubernetes/node/templates/cni-calico.conf.j2 new file mode 100644 index 000000000..c11067965 --- /dev/null +++ b/roles/kubernetes/node/templates/cni-calico.conf.j2 @@ -0,0 +1,9 @@ +{ + "name": "calico-k8s-network", + "type": "calico", + "etcd_authority": "127.0.0.1:2379", + "log_level": "info", + "ipam": { + "type": "calico-ipam" + } +} diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index fd59bc10f..55842874e 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -24,7 +24,7 @@ KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} - KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}" {% endif %} {% if kube_network_plugin is defined and kube_network_plugin == "calico" %} -KUBELET_NETWORK_PLUGIN="--network_plugin={{ kube_network_plugin }}" +KUBELET_NETWORK_PLUGIN="--network_plugin=cni --network-plugin-dir=/etc/cni/net.d" {% endif %} # Should this cluster be allowed to run privileged docker containers KUBE_ALLOW_PRIV="--allow_privileged=true" diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 365720ba9..7ceda9b1f 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -33,6 +33,41 @@ always_run: True tags: always +- name: Create kubernetes config directory + file: + path: "{{ kube_config_dir }}" + state: directory + owner: kube + +- name: Create kubernetes script directory + file: + path: "{{ kube_script_dir }}" + state: directory + owner: kube + +- name: Create kubernetes manifests directory + file: + path: "{{ kube_manifest_dir }}" + state: directory + owner: kube + +- name: Create kubernetes logs directory + file: + path: "{{ kube_log_dir }}" + state: directory + owner: kube + when: init_system == "sysvinit" + +- name: Create cni directories + file: + path: "{{ item }}" + state: directory + owner: kube + with_items: + - "/etc/cni/net.d" + - "/opt/cni/bin" + when: kube_network_plugin == "calico" + - name: Update package management cache (APT) apt: update_cache=yes when: ansible_pkg_mgr == 'apt' diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml new file mode 100644 index 000000000..9ee31209a --- /dev/null +++ b/roles/network_plugin/calico/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# cloud_provider: no diff --git a/roles/network_plugin/calico/handlers/main.yml b/roles/network_plugin/calico/handlers/main.yml new file mode 100644 index 000000000..35b1ae759 --- /dev/null +++ b/roles/network_plugin/calico/handlers/main.yml @@ -0,0 +1,15 @@ +--- +- name: restart calico-node + command: /bin/true + notify: + - reload systemd + - reload calico-node + +- name : reload systemd + shell: systemctl daemon-reload + when: init_system == "systemd" + +- name: reload calico-node + service: + name: calico-node + state: restarted diff --git a/roles/network_plugin/tasks/calico.yml b/roles/network_plugin/calico/tasks/main.yml similarity index 73% rename from roles/network_plugin/tasks/calico.yml rename to roles/network_plugin/calico/tasks/main.yml index 748d044a2..d8f51fa1a 100644 --- a/roles/network_plugin/tasks/calico.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -1,9 +1,36 @@ --- +- name: Calico | Set docker daemon options + template: + src: docker + dest: "/etc/default/docker" + owner: root + group: root + mode: 0644 + notify: + - restart docker + +- name: Calico | Write docker.service systemd file + template: + src: systemd-docker.service + dest: /lib/systemd/system/docker.service + notify: restart docker + when: init_system == "systemd" + +- meta: flush_handlers + - name: Calico | Install calicoctl bin command: rsync -piu "{{ local_release_dir }}/calico/bin/calicoctl" "{{ bin_dir }}/calicoctl" register: calico_copy changed_when: false +- name: Calico | Install calico cni bin + command: rsync -piu "{{ local_release_dir }}/calico/bin/calico" "/opt/cni/bin/calico" + changed_when: false + +- name: Calico | Install calico-ipam cni bin + command: rsync -piu "{{ local_release_dir }}/calico/bin/calico" "/opt/cni/bin/calico-ipam" + changed_when: false + - name: Calico | install calicoctl file: path={{ bin_dir }}/calicoctl mode=0755 state=file @@ -51,33 +78,32 @@ ( not calico_pools.json['node']['nodes'][0]['key'] | search(".*{{ kube_pods_subnet | ipaddr('network') }}.*") ) run_once: true -- name: Calico | Write calico-node configuration - template: src=calico/calico.conf.j2 dest=/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/calico_kubernetes.ini - notify: restart calico-node - - name: Calico | Write /etc/network-environment - template: src=calico/network-environment.j2 dest=/etc/network-environment + template: src=network-environment.j2 dest=/etc/network-environment when: init_system == "sysvinit" - name: Calico | Write calico-node systemd init file - template: src=calico/calico-node.service.j2 dest=/etc/systemd/system/calico-node.service + template: src=calico-node.service.j2 dest=/etc/systemd/system/calico-node.service when: init_system == "systemd" notify: restart calico-node - name: Calico | Write calico-node initd script - template: src=calico/deb-calico.initd.j2 dest=/etc/init.d/calico-node owner=root mode=0755 + template: src=deb-calico.initd.j2 dest=/etc/init.d/calico-node owner=root mode=0755 when: init_system == "sysvinit" and ansible_os_family == "Debian" notify: restart calico-node - name: Calico | Write calico-node initd script - template: src=calico/rh-calico.initd.j2 dest=/etc/init.d/calico-node owner=root mode=0755 + template: src=rh-calico.initd.j2 dest=/etc/init.d/calico-node owner=root mode=0755 when: init_system == "sysvinit" and ansible_os_family == "RedHat" notify: restart calico-node - meta: flush_handlers - name: Calico | Enable calico-node - service: name=calico-node enabled=yes state=started + service: + name: calico-node + state: started + enabled: yes - name: Calico | Restart calico if binary changed service: diff --git a/roles/network_plugin/templates/calico/calico-node.service.j2 b/roles/network_plugin/calico/templates/calico-node.service.j2 similarity index 59% rename from roles/network_plugin/templates/calico/calico-node.service.j2 rename to roles/network_plugin/calico/templates/calico-node.service.j2 index 5ee52305e..8c8af6971 100644 --- a/roles/network_plugin/templates/calico/calico-node.service.j2 +++ b/roles/network_plugin/calico/templates/calico-node.service.j2 @@ -8,9 +8,9 @@ After=docker.service etcd.service User=root PermissionsStartOnly=true {% if inventory_hostname in groups['kube-node'] and peer_with_router|default(false)%} -ExecStart={{ bin_dir }}/calicoctl node --kubernetes --ip={{ip | default(ansible_default_ipv4.address) }} --as={{ local_as }} --detach=false +ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --as={{ local_as }} --detach=false {% else %} -ExecStart={{ bin_dir }}/calicoctl node --kubernetes --ip={{ip | default(ansible_default_ipv4.address) }} --detach=false +ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --detach=false {% endif %} Restart=always Restart=10 diff --git a/roles/network_plugin/templates/calico/deb-calico.initd.j2 b/roles/network_plugin/calico/templates/deb-calico.initd.j2 similarity index 100% rename from roles/network_plugin/templates/calico/deb-calico.initd.j2 rename to roles/network_plugin/calico/templates/deb-calico.initd.j2 diff --git a/roles/network_plugin/templates/docker b/roles/network_plugin/calico/templates/docker similarity index 100% rename from roles/network_plugin/templates/docker rename to roles/network_plugin/calico/templates/docker diff --git a/roles/network_plugin/calico/templates/network-environment.j2 b/roles/network_plugin/calico/templates/network-environment.j2 new file mode 100644 index 000000000..9a588cfc4 --- /dev/null +++ b/roles/network_plugin/calico/templates/network-environment.j2 @@ -0,0 +1,9 @@ +# This host's IPv4 address (the source IP address used to reach other nodes +# in the Kubernetes cluster). +DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }} + +# The Kubernetes master IP +KUBERNETES_MASTER={{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }} + +# IP and port of etcd instance used by Calico +ETCD_AUTHORITY=127.0.0.1:2379 diff --git a/roles/network_plugin/templates/calico/rh-calico.initd.j2 b/roles/network_plugin/calico/templates/rh-calico.initd.j2 similarity index 100% rename from roles/network_plugin/templates/calico/rh-calico.initd.j2 rename to roles/network_plugin/calico/templates/rh-calico.initd.j2 diff --git a/roles/network_plugin/templates/systemd-docker.service b/roles/network_plugin/calico/templates/systemd-docker.service similarity index 100% rename from roles/network_plugin/templates/systemd-docker.service rename to roles/network_plugin/calico/templates/systemd-docker.service diff --git a/roles/network_plugin/defaults/main.yml b/roles/network_plugin/flannel/defaults/main.yml similarity index 93% rename from roles/network_plugin/defaults/main.yml rename to roles/network_plugin/flannel/defaults/main.yml index e648133bc..42ce76535 100644 --- a/roles/network_plugin/defaults/main.yml +++ b/roles/network_plugin/flannel/defaults/main.yml @@ -1,5 +1,4 @@ --- - # Flannel public IP # The address that flannel should advertise as how to access the system flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}" @@ -7,5 +6,3 @@ flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address ## interface that should be used for flannel operations ## This is actually an inventory node-level item # flannel_interface: - -# cloud_provider: no \ No newline at end of file diff --git a/roles/network_plugin/handlers/main.yml b/roles/network_plugin/flannel/handlers/main.yml similarity index 71% rename from roles/network_plugin/handlers/main.yml rename to roles/network_plugin/flannel/handlers/main.yml index 14b526b67..1683b8c35 100644 --- a/roles/network_plugin/handlers/main.yml +++ b/roles/network_plugin/flannel/handlers/main.yml @@ -1,10 +1,4 @@ --- -- name: restart calico-node - command: /bin/true - notify: - - reload systemd - - reload calico-node - - name: restart docker command: /bin/true notify: @@ -21,11 +15,6 @@ shell: systemctl daemon-reload when: init_system == "systemd" -- name: reload calico-node - service: - name: calico-node - state: restarted - - name: reload docker service: name: docker diff --git a/roles/network_plugin/tasks/flannel.yml b/roles/network_plugin/flannel/tasks/main.yml similarity index 58% rename from roles/network_plugin/tasks/flannel.yml rename to roles/network_plugin/flannel/tasks/main.yml index fc5dcaa2f..5cdbf24ac 100644 --- a/roles/network_plugin/tasks/flannel.yml +++ b/roles/network_plugin/flannel/tasks/main.yml @@ -1,13 +1,13 @@ --- - name: Flannel | Write flannel configuration template: - src: flannel/network.json + src: network.json dest: /etc/flannel-network.json backup: yes - name: Flannel | Create flannel pod manifest template: - src: flannel/flannel-pod.yml + src: flannel-pod.yml dest: /etc/kubernetes/manifests/flannel-pod.manifest notify: delete default docker bridge @@ -16,7 +16,7 @@ path: /run/flannel/subnet.env delay: 5 -- name: Get flannel_subnet from subnet.env +- name: Flannel | Get flannel_subnet from subnet.env shell: cat /run/flannel/subnet.env | awk -F'=' '$1 == "FLANNEL_SUBNET" {print $2}' register: flannel_subnet_output changed_when: false @@ -24,10 +24,29 @@ - set_fact: flannel_subnet: "{{ flannel_subnet_output.stdout }}" -- name: Get flannel_mtu from subnet.env +- name: Flannel | Get flannel_mtu from subnet.env shell: cat /run/flannel/subnet.env | awk -F'=' '$1 == "FLANNEL_MTU" {print $2}' register: flannel_mtu_output changed_when: false - set_fact: flannel_mtu: "{{ flannel_mtu_output.stdout }}" + +- name: Flannel | Set docker daemon options + template: + src: docker + dest: "/etc/default/docker" + owner: root + group: root + mode: 0644 + notify: + - restart docker + +- name: Flannel | Write docker.service systemd file + template: + src: systemd-docker.service + dest: /lib/systemd/system/docker.service + notify: restart docker + when: init_system == "systemd" + +- meta: flush_handlers diff --git a/roles/network_plugin/flannel/templates/docker b/roles/network_plugin/flannel/templates/docker new file mode 100644 index 000000000..eefd150e1 --- /dev/null +++ b/roles/network_plugin/flannel/templates/docker @@ -0,0 +1,6 @@ +# Deployed by Ansible +{% if init_system == "sysvinit" and kube_network_plugin == "flannel" and ansible_os_family == "Debian" %} +DOCKER_OPTS="--bip={{ flannel_subnet }} --mtu={{ flannel_mtu }}" +{% elif kube_network_plugin == "flannel" %} +OPTIONS="--bip={{ flannel_subnet }} --mtu={{ flannel_mtu }}" +{% endif %} diff --git a/roles/network_plugin/templates/flannel/flannel-pod.yml b/roles/network_plugin/flannel/templates/flannel-pod.yml similarity index 100% rename from roles/network_plugin/templates/flannel/flannel-pod.yml rename to roles/network_plugin/flannel/templates/flannel-pod.yml diff --git a/roles/network_plugin/templates/flannel/network.json b/roles/network_plugin/flannel/templates/network.json similarity index 100% rename from roles/network_plugin/templates/flannel/network.json rename to roles/network_plugin/flannel/templates/network.json diff --git a/roles/network_plugin/flannel/templates/systemd-docker.service b/roles/network_plugin/flannel/templates/systemd-docker.service new file mode 100644 index 000000000..3275c6e24 --- /dev/null +++ b/roles/network_plugin/flannel/templates/systemd-docker.service @@ -0,0 +1,28 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=http://docs.docker.com +{% if ansible_os_family == "RedHat" %} +After=network.target +Wants=docker-storage-setup.service +{% elif ansible_os_family == "Debian" %} +After=network.target docker.socket +Requires=docker.socket +{% endif %} + +[Service] +Type=notify +EnvironmentFile=-/etc/default/docker +Environment=GOTRACEBACK=crash +ExecStart=/usr/bin/docker daemon \ + $OPTIONS \ + $DOCKER_STORAGE_OPTIONS \ + $DOCKER_NETWORK_OPTIONS \ + $INSECURE_REGISTRY +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +MountFlags=slave +TimeoutStartSec=1min + +[Install] +WantedBy=multi-user.target diff --git a/roles/network_plugin/meta/main.yml b/roles/network_plugin/meta/main.yml new file mode 100644 index 000000000..6d9e8fbd7 --- /dev/null +++ b/roles/network_plugin/meta/main.yml @@ -0,0 +1,6 @@ +--- +dependencies: + - role: network_plugin/calico + when: kube_network_plugin == 'calico' + - role: network_plugin/flannel + when: kube_network_plugin == 'flannel' diff --git a/roles/network_plugin/tasks/main.yml b/roles/network_plugin/tasks/main.yml deleted file mode 100644 index 2d098d865..000000000 --- a/roles/network_plugin/tasks/main.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -- name: "Test if network plugin is defined" - fail: msg="ERROR, One network_plugin variable must be defined (Flannel or Calico)" - when: ( kube_network_plugin is defined and kube_network_plugin == "calico" and kube_network_plugin == "flannel" ) or - kube_network_plugin is not defined - -- include: flannel.yml - when: kube_network_plugin == "flannel" - -- name: Set docker daemon options - template: - src: docker - dest: "{{ '/etc/sysconfig/docker-network' if ansible_os_family == 'RedHat' else '/etc/default/docker' }}" - owner: root - group: root - mode: 0644 - notify: - - restart docker - -- name: Write docker.service systemd file - template: - src: systemd-docker.service - dest: /lib/systemd/system/docker.service - notify: restart docker - when: init_system == "systemd" - -- meta: flush_handlers - -- include: calico.yml - when: kube_network_plugin == "calico" diff --git a/roles/network_plugin/templates/calico/calico.conf.j2 b/roles/network_plugin/templates/calico/calico.conf.j2 deleted file mode 100644 index 7c816bdae..000000000 --- a/roles/network_plugin/templates/calico/calico.conf.j2 +++ /dev/null @@ -1,17 +0,0 @@ -[config] -CALICO_IPAM=true - -# Location of etcd cluster used by Calico. By default, this uses the etcd -# instance running on the Kubernetes Master -ETCD_AUTHORITY=127.0.0.1:2379 - -# The kubernetes-apiserver location - used by the calico plugin -{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} -KUBE_API_ROOT=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}/api/v1/ -{% else %} -KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{kube_apiserver_port}}/api/v1/ -{% endif %} -# Kubernetes authentication token -{% if calico_token is defined | default('') %} -KUBE_AUTH_TOKEN={{ calico_token.content|b64decode }} -{% endif %} diff --git a/roles/network_plugin/templates/calico/network-environment.j2 b/roles/network_plugin/templates/calico/network-environment.j2 deleted file mode 100644 index 3d3f87b65..000000000 --- a/roles/network_plugin/templates/calico/network-environment.j2 +++ /dev/null @@ -1,2 +0,0 @@ -DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }} -ETCD_AUTHORITY=127.0.0.1:2379