From 23fd3461bcdfd0eafb385bfa0320b682cb8035e0 Mon Sep 17 00:00:00 2001 From: Aivars Sterns Date: Thu, 23 Aug 2018 17:17:18 +0300 Subject: [PATCH] calico upgrade to v3 (#3086) * calico upgrade to v3 * update calico_rr version * add missing file * change contents of main.yml as it was left old version * enable network policy by default * remove unneeded task * Fix kubelet calico settings * fix when statement * switch back to node-kubeconfig.yaml --- README.md | 2 +- docs/calico.md | 9 +- inventory/sample/group_vars/k8s-cluster.yml | 3 - roles/download/defaults/main.yml | 10 +- .../network_plugin/calico/tasks/main.yml | 12 +- .../policy_controller/calico/tasks/main.yml | 23 +- .../templates/calico-kube-controllers.yml.j2 | 15 +- .../calico/templates/calico-kube-cr.yml.j2 | 8 + roles/kubernetes/kubeadm/tasks/main.yml | 9 + .../node/templates/kubelet-container.j2 | 1 + .../node/templates/kubelet.rkt.service.j2 | 7 + roles/kubernetes/preinstall/tasks/main.yml | 1 + .../preinstall/tasks/set_resolv_facts.yml | 2 +- roles/kubespray-defaults/defaults/main.yaml | 2 +- roles/network_plugin/calico/defaults/main.yml | 11 +- roles/network_plugin/calico/tasks/check.yml | 37 ++++ roles/network_plugin/calico/tasks/install.yml | 189 +++++++++++++++++ roles/network_plugin/calico/tasks/main.yml | 197 +----------------- roles/network_plugin/calico/tasks/pre.yml | 16 ++ roles/network_plugin/calico/tasks/upgrade.yml | 26 +++ .../calico/templates/calico-node.yml.j2 | 14 +- .../calico/templates/cni-calico.conflist.j2 | 10 +- .../calico/templates/etcdv2-store.yml.j2 | 9 + .../calico/templates/etcdv3-store.yml.j2 | 9 + .../canal/templates/canal-node.yaml.j2 | 6 + roles/reset/tasks/main.yml | 1 + upgrade-cluster.yml | 22 +- 27 files changed, 399 insertions(+), 252 deletions(-) create mode 100644 roles/network_plugin/calico/tasks/check.yml create mode 100644 roles/network_plugin/calico/tasks/install.yml create mode 100644 roles/network_plugin/calico/tasks/pre.yml create mode 100644 roles/network_plugin/calico/tasks/upgrade.yml create mode 100644 roles/network_plugin/calico/templates/etcdv2-store.yml.j2 create mode 100644 roles/network_plugin/calico/templates/etcdv3-store.yml.j2 diff --git a/README.md b/README.md index 021a5a078..e8475d76b 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Supported Components - [docker](https://www.docker.com/) v17.03 (see note) - [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2) - Network Plugin - - [calico](https://github.com/projectcalico/calico) v2.6.8 + - [calico](https://github.com/projectcalico/calico) v3.1.3 - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) - [cilium](https://github.com/cilium/cilium) v1.2.0 - [contiv](https://github.com/contiv/install) v1.1.7 diff --git a/docs/calico.md b/docs/calico.md index b8cdc90cb..f6adaa662 100644 --- a/docs/calico.md +++ b/docs/calico.md @@ -1,6 +1,13 @@ Calico =========== +--- + **N.B. Version 2.6.5 upgrade to 3.1.1 is upgrading etcd store to etcdv3** + If you create automated backups of etcdv2 please switch for creating etcdv3 backups, as kubernetes and calico now uses etcdv3 + After migration you can check `/tmp/calico_upgrade/` directory for converted items to etcdv3. + **PLEASE TEST upgrade before upgrading production cluster.** + --- + Check if the calico-node container is running ``` @@ -86,7 +93,7 @@ To do so you can deploy BGP route reflectors and peer `calico-node` with them as recommended here: * https://hub.docker.com/r/calico/routereflector/ -* http://docs.projectcalico.org/v2.0/reference/private-cloud/l3-interconnect-fabric +* https://docs.projectcalico.org/v3.1/reference/private-cloud/l3-interconnect-fabric You need to edit your inventory and add: diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index ded955066..119d5af09 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -83,9 +83,6 @@ kube_network_plugin: calico # weave_no_masq_local: true # weave_extra_args: ~ -# Enable kubernetes network policies -enable_network_policy: false - # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index c726b40bd..97c8d8562 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -32,11 +32,11 @@ kubeadm_version: "{{ kube_version }}" etcd_version: v3.2.18 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download -calico_version: "v2.6.8" -calico_ctl_version: "v1.6.3" -calico_cni_version: "v1.11.4" -calico_policy_version: "v1.0.3" -calico_rr_version: "v0.4.2" +calico_version: "v3.1.3" +calico_ctl_version: "v3.1.3" +calico_cni_version: "v3.1.3" +calico_policy_version: "v3.1.3" +calico_rr_version: "v0.6.1" flannel_version: "v0.10.0" flannel_cni_version: "v0.3.0" vault_version: 0.10.1 diff --git a/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml b/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml index 4c8295c1e..013d3b056 100644 --- a/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml +++ b/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml @@ -7,5 +7,13 @@ resource: "{{item.item.type}}" filename: "{{kube_config_dir}}/{{item.item.file}}" state: "latest" - with_items: "{{ calico_node_manifests.results }}" - when: inventory_hostname == groups['kube-master'][0] and not item|skipped + with_items: + - "{{ calico_node_manifests.results }}" + when: + - inventory_hostname == groups['kube-master'][0] and not item|skipped + +- name: "calico upgrade complete" + shell: "{{ bin_dir }}/calico-upgrade complete --no-prompts --apiconfigv1 /etc/calico/etcdv2.yml --apiconfigv3 /etc/calico/etcdv3.yml" + when: + - inventory_hostname == groups['kube-master'][0] + - calico_upgrade_needed|default(False) diff --git a/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml b/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml index bc2fdf7e7..7e9377da4 100644 --- a/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml +++ b/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml @@ -2,20 +2,12 @@ - name: Set cert dir set_fact: calico_cert_dir: "{{ canal_cert_dir }}" - when: kube_network_plugin == 'canal' + when: + - kube_network_plugin == 'canal' tags: - facts - canal -- name: Delete the old calico-policy-controller if it exist - kube: - name: calico-policy-controller - kubectl: "{{bin_dir}}/kubectl" - resource: rs - namespace: "kube-system" - state: absent - run_once: true - - name: Create calico-kube-controllers manifests template: src: "{{item.file}}.j2" @@ -26,7 +18,9 @@ - {name: calico-kube-controllers, file: calico-kube-cr.yml, type: clusterrole} - {name: calico-kube-controllers, file: calico-kube-crb.yml, type: clusterrolebinding} register: calico_kube_manifests - when: inventory_hostname == groups['kube-master'][0] and not item|skipped + when: + - inventory_hostname == groups['kube-master'][0] + - rbac_enabled or item.type not in rbac_resources - name: Start of Calico kube controllers kube: @@ -36,5 +30,8 @@ resource: "{{item.item.type}}" filename: "{{kube_config_dir}}/{{item.item.file}}" state: "latest" - with_items: "{{ calico_kube_manifests.results }}" - when: inventory_hostname == groups['kube-master'][0] and not item|skipped + with_items: + - "{{ calico_kube_manifests.results }}" + when: + - inventory_hostname == groups['kube-master'][0] + - not item|skipped diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 index 5dcafbbc5..d99466248 100644 --- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 +++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 @@ -6,8 +6,12 @@ metadata: labels: k8s-app: calico-kube-controllers kubernetes.io/cluster-service: "true" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' spec: replicas: 1 + strategy: + type: Recreate selector: matchLabels: kubernetes.io/cluster-service: "true" @@ -45,17 +49,6 @@ spec: value: "{{ calico_cert_dir }}/cert.crt" - name: ETCD_KEY_FILE value: "{{ calico_cert_dir }}/key.pem" - # Location of the Kubernetes API - this shouldn't need to be - # changed so long as it is used in conjunction with - # CONFIGURE_ETC_HOSTS="true". - - name: K8S_API - value: "https://kubernetes.default" - # Configure /etc/hosts within the container to resolve - # the kubernetes.default Service to the correct clusterIP - # using the environment provided by the kubelet. - # This removes the need for KubeDNS to resolve the Service. - - name: CONFIGURE_ETC_HOSTS - value: "true" volumeMounts: - mountPath: {{ calico_cert_dir }} name: etcd-certs diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 index d05e986a4..bf582c769 100644 --- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 +++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 @@ -12,6 +12,14 @@ rules: - pods - namespaces - networkpolicies + - nodes verbs: - watch - list + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - watch + - list \ No newline at end of file diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index c0d436b66..5697076e8 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -88,6 +88,15 @@ - is_kube_master - kubeadm_discovery_address != kube_apiserver_endpoint +# FIXME(mattymo): Reconcile kubelet kubeconfig filename for both deploy modes +- name: Symlink kubelet kubeconfig for calico/canal + file: + src: "{{ kube_config_dir }}/kubelet.conf" + dest: "{{ kube_config_dir }}/node-kubeconfig.yaml" + state: link + force: yes + when: kube_network_plugin in ['calico','canal'] + - name: Restart all kube-proxy pods to ensure that they load the new configmap shell: "{{ bin_dir }}/kubectl delete pod -n kube-system -l k8s-app=kube-proxy" delegate_to: "{{groups['kube-master']|first}}" diff --git a/roles/kubernetes/node/templates/kubelet-container.j2 b/roles/kubernetes/node/templates/kubelet-container.j2 index dcf86c327..58e54560d 100644 --- a/roles/kubernetes/node/templates/kubelet-container.j2 +++ b/roles/kubernetes/node/templates/kubelet-container.j2 @@ -22,6 +22,7 @@ -v {{ docker_daemon_graph }}:{{ docker_daemon_graph }}:rw \ -v /var/log:/var/log:rw \ -v /var/lib/kubelet:/var/lib/kubelet:shared \ + -v /var/lib/calico:/var/lib/calico:shared \ -v /var/lib/cni:/var/lib/cni:shared \ -v /var/run:/var/run:rw \ {# we can run into issues with double mounting /var/lib/kubelet #} diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 index d75f4c662..46557d273 100644 --- a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 @@ -37,6 +37,10 @@ ExecStart=/usr/bin/rkt run \ --volume etc-cni,kind=host,source=/etc/cni,readOnly=true \ --volume opt-cni,kind=host,source=/opt/cni,readOnly=true \ --volume var-lib-cni,kind=host,source=/var/lib/cni,readOnly=false \ +{% endif %} +{% if kube_network_plugin in ["calico", "canal"] %} + --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=false \ +{% endif %} {# we can run into issues with double mounting /var/lib/kubelet #} {# surely there's a better way to do this #} {% if '/var/lib/kubelet' not in kubelet_flexvolumes_plugins_dir %} @@ -55,6 +59,9 @@ ExecStart=/usr/bin/rkt run \ --mount volume=etc-cni,target=/etc/cni \ --mount volume=opt-cni,target=/opt/cni \ --mount volume=var-lib-cni,target=/var/lib/cni \ +{% if kube_network_plugin in ["calico", "canal"] %} + --mount volume=var-lib-calico,target=/var/lib/calico \ +{% endif %} {% endif %} --mount volume=os-release,target=/etc/os-release \ --mount volume=dns,target=/etc/resolv.conf \ diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 4db366ced..83347b59f 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -89,6 +89,7 @@ with_items: - "/etc/cni/net.d" - "/opt/cni/bin" + - "/var/lib/calico" when: - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium"] - inventory_hostname in groups['k8s-cluster'] diff --git a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml index eb8f3f43f..4e535fb0f 100644 --- a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml @@ -98,7 +98,7 @@ {%- elif dns_mode == 'coredns_dual' and not dns_early|bool -%} {{ [ skydns_server ] + [ skydns_server_secondary ] + upstream_dns_servers|default([]) }} {%- elif dns_mode == 'manual' and not dns_early|bool -%} - {{ [ manual_dns_server ] + upstream_dns_servers|default([]) }} + {{ ( manual_dns_server.split(',') | list) + upstream_dns_servers|default([]) }} {%- elif dns_early|bool -%} {{ upstream_dns_servers|default([]) }} {%- else -%} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 888cbc3c8..4808daf16 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -187,7 +187,7 @@ dashboard_enabled: true efk_enabled: false helm_enabled: false registry_enabled: false -enable_network_policy: false +enable_network_policy: true local_volume_provisioner_enabled: "{{ local_volumes_enabled | default('false') }}" persistent_volumes_enabled: false cephfs_provisioner_enabled: false diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml index 800bbb6fc..e65d569ed 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico/defaults/main.yml @@ -2,9 +2,12 @@ # Enables Internet connectivity from containers nat_outgoing: true +#add default ippool name +calico_pool_name: "default-pool" + # Use IP-over-IP encapsulation across hosts ipip: true -ipip_mode: always # change to "cross-subnet" if you only want ipip encapsulation on traffic going across subnets +ipip_mode: Always # change to "CrossSubnet" if you only want ipip encapsulation on traffic going across subnets # Set to true if you want your calico cni binaries to overwrite the # ones from hyperkube while leaving other cni plugins intact. @@ -37,6 +40,9 @@ calico_felix_prometheusmetricsport: 9091 calico_felix_prometheusgometricsenabled: "true" calico_felix_prometheusprocessmetricsenabled: "true" +### check latest version https://github.com/projectcalico/calico-upgrade/releases +calico_upgrade_version: v1.0.5 + # Should calico ignore kernel's RPF check setting, # see https://github.com/projectcalico/felix/blob/ab8799eaea66627e5db7717e62fca61fd9c08646/python/calico/felix/config.py#L198 calico_node_ignorelooserpf: false @@ -48,3 +54,6 @@ calico_node_ignorelooserpf: false # calico_ip_auto_method: "interface=eth.*" calico_baremetal_nodename: "{{ inventory_hostname }}" + +### do not enable this, this is detected in scope of tasks, this is just a default value +calico_upgrade_needed: false \ No newline at end of file diff --git a/roles/network_plugin/calico/tasks/check.yml b/roles/network_plugin/calico/tasks/check.yml new file mode 100644 index 000000000..4eca9cc65 --- /dev/null +++ b/roles/network_plugin/calico/tasks/check.yml @@ -0,0 +1,37 @@ +--- +- name: "Check vars defined correctly" + assert: + that: + - "calico_pool_name is defined" + - "calico_pool_name | match('^[a-zA-Z0-9-_\\\\.]{2,63}$')" + - "ipip_mode is defined" + - "ipip_mode in ['Always', 'CrossSubnet', 'Never']" + msg: "Check variable definitions seems something is wrong" + run_once: yes + +- name: "Get current version of calico cluster version" + shell: "{{ bin_dir }}/calicoctl version | grep 'Cluster Version' | awk '{ print $3}'" + register: calico_version_on_server + run_once: yes + delegate_to: "{{ groups['kube-master'][0] }}" + +- name: "Determine if calico upgrade is needed" + block: + - name: "Check that calico version is enought for upgrade" + assert: + that: + - calico_version_on_server.stdout|version_compare('v2.6.5', '>=') + msg: "Your version of calico is not fresh enough for upgrade" + + - name: "Set upgrade flag when version needs to be updated" + set_fact: + calico_upgrade_needed: True + when: + - calico_version_on_server.stdout|version_compare('v2.6.5', '>=') + - calico_version_on_server.stdout|version_compare('v3.0.0', '<') + + when: + - 'calico_version_on_server.stdout is defined' + - 'calico_version_on_server.stdout != ""' + - inventory_hostname == groups['kube-master'][0] + run_once: yes \ No newline at end of file diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml new file mode 100644 index 000000000..d658d575e --- /dev/null +++ b/roles/network_plugin/calico/tasks/install.yml @@ -0,0 +1,189 @@ +--- + +- name: Calico | Write Calico cni config + template: + src: "cni-calico.conflist.j2" + dest: "/etc/cni/net.d/10-calico.conflist" + owner: kube + +- name: Calico | Create calico certs directory + file: + dest: "{{ calico_cert_dir }}" + state: directory + mode: 0750 + owner: root + group: root + +- name: Calico | Link etcd certificates for calico-node + file: + src: "{{ etcd_cert_dir }}/{{ item.s }}" + dest: "{{ calico_cert_dir }}/{{ item.d }}" + state: hard + force: yes + with_items: + - {s: "ca.pem", d: "ca_cert.crt"} + - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"} + - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"} + +- name: Calico | Install calicoctl container script + template: + src: calicoctl-container.j2 + dest: "{{ bin_dir }}/calicoctl" + mode: 0755 + owner: root + group: root + changed_when: false + +- name: Calico | Copy cni plugins from hyperkube + command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/" + register: cni_task_result + until: cni_task_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + tags: + - hyperkube + - upgrade + +- name: Calico | Copy cni plugins from calico/cni container + command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp /opt/cni/bin/* /cnibindir/'" + register: cni_task_result + until: cni_task_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + when: + - "overwrite_hyperkube_cni|bool" + tags: + - hyperkube + - upgrade + +- name: Calico | Set cni directory permissions + file: + path: /opt/cni/bin + state: directory + owner: kube + recurse: true + mode: 0755 + +- name: Calico | wait for etcd + uri: + url: "{{ etcd_access_addresses.split(',') | first }}/health" + validate_certs: no + client_cert: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" + client_key: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + register: result + until: result.status == 200 or result.status == 401 + retries: 10 + delay: 5 + run_once: true + +- name: Calico | Check if calico network pool has already been configured + shell: > + {{ bin_dir }}/calicoctl get ippool | grep -w "{{ kube_pods_subnet }}" | wc -l + register: calico_conf + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + delegate_to: "{{ groups['kube-master'][0] }}" + run_once: true + +- name: Calico | Configure calico network pool + shell: > + echo " + { "kind": "IPPool", + "apiVersion": "projectcalico.org/v3", + "metadata": { + "name": "{{ calico_pool_name }}", + }, + "spec": { + "cidr": "{{ kube_pods_subnet }}", + "ipipMode": "{{ ipip_mode|capitalize }}", + "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl create -f - + run_once: true + delegate_to: "{{ groups['kube-master'][0] }}" + when: + - 'calico_conf.stdout == "0"' + +- name: "Determine nodeToNodeMesh needed state" + set_fact: + nodeToNodeMeshEnabled: "false" + when: + - peer_with_router|default(false) or peer_with_calico_rr|default(false) + - inventory_hostname in groups['k8s-cluster'] + run_once: yes + + +- name: Calico | Set global as_num + shell: > + echo ' + { "kind": "BGPConfiguration", + "apiVersion": "projectcalico.org/v3", + "metadata": { + "name": "default", + }, + "spec": { + "logSeverityScreen": "Info", + "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled|default('true') }} , + "asNumber": {{ global_as_num }} }} ' | {{ bin_dir }}/calicoctl --skip-exists create -f - + run_once: true + delegate_to: "{{ groups['kube-master'][0] }}" + +- name: Calico | Configure peering with router(s) + shell: > + echo '{ + "apiVersion": "projectcalico.org/v3", + "kind": "bgpPeer", + "metadata": { + "name": "{{ inventory_hostname }}-bgp" + }, + "spec": { + "asNumber": "{{ item.as }}", + "node": "{{ inventory_hostname }}", + "scope": "node", + "peerIP": "{{ item.router_id }}" + }}' | {{ bin_dir }}/calicoctl create --skip-exists -f - + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ peers|default([]) }}" + when: + - peer_with_router|default(false) + - inventory_hostname in groups['k8s-cluster'] + +- name: Calico | Configure peering with route reflectors + shell: > + echo '{ + "apiVersion": "projectcalico.org/v3", + "kind": "bgpPeer", + "metadata": { + "name": "{{ inventory_hostname }}" + }, + "spec": { + "asNumber": "{{ local_as | default(global_as_num)}}", + "scope": "node", + "node": "{{ inventory_hostname }}", + "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}" + }}' | {{ bin_dir }}/calicoctl create --skip-exists -f - + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ groups['calico-rr'] | default([]) }}" + when: + - (peer_with_calico_rr|default(false) + - inventory_hostname in groups['k8s-cluster'] + - hostvars[item]['cluster_id'] == cluster_id) + +- name: Calico | Create calico manifests + template: + src: "{{item.file}}.j2" + dest: "{{kube_config_dir}}/{{item.file}}" + with_items: + - {name: calico-config, file: calico-config.yml, type: cm} + - {name: calico-node, file: calico-node.yml, type: ds} + - {name: calico, file: calico-node-sa.yml, type: sa} + - {name: calico, file: calico-cr.yml, type: clusterrole} + - {name: calico, file: calico-crb.yml, type: clusterrolebinding} + register: calico_node_manifests + when: + - inventory_hostname in groups['kube-master'] + - rbac_enabled or item.type not in rbac_resources \ No newline at end of file diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index 7ce84cfdc..72d8e3e11 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -1,193 +1,12 @@ --- -- name: Calico | Disable calico-node service if it exists - service: - name: calico-node - state: stopped - enabled: yes - failed_when: false +- include_tasks: check.yml -- name: Calico | Get kubelet hostname - shell: >- - {{ bin_dir }}/kubectl get node -o custom-columns='NAME:.metadata.name,INTERNAL-IP:.status.addresses[?(@.type=="InternalIP")].address' - | egrep "{{ ansible_all_ipv4_addresses | join('$|') }}$" | cut -d" " -f1 - register: calico_kubelet_name - delegate_to: "{{ groups['kube-master'][0] }}" - when: cloud_provider is defined - -- name: Calico | Write Calico cni config - template: - src: "cni-calico.conflist.j2" - dest: "/etc/cni/net.d/10-calico.conflist" - owner: kube - -- name: Calico | Create calico certs directory - file: - dest: "{{ calico_cert_dir }}" - state: directory - mode: 0750 - owner: root - group: root - -- name: Calico | Link etcd certificates for calico-node - file: - src: "{{ etcd_cert_dir }}/{{ item.s }}" - dest: "{{ calico_cert_dir }}/{{ item.d }}" - state: hard - force: yes - with_items: - - {s: "ca.pem", d: "ca_cert.crt"} - - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"} - - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"} - -- name: Calico | Install calicoctl container script - template: - src: calicoctl-container.j2 - dest: "{{ bin_dir }}/calicoctl" - mode: 0755 - owner: root - group: root - changed_when: false - -- name: Calico | Copy cni plugins from hyperkube - command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -rf /opt/cni/bin/. /cnibindir/" - register: cni_task_result - until: cni_task_result.rc == 0 - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - changed_when: false - tags: - - hyperkube - - upgrade - -- name: Calico | Copy cni plugins from calico/cni container - command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp /opt/cni/bin/* /cnibindir/'" - register: cni_task_result - until: cni_task_result.rc == 0 - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - changed_when: false - when: overwrite_hyperkube_cni|bool - tags: - - hyperkube - - upgrade - -- name: Calico | Set cni directory permissions - file: - path: /opt/cni/bin - state: directory - owner: kube - recurse: true - mode: 0755 - -- name: Calico | wait for etcd - uri: - url: "{{ etcd_access_addresses.split(',') | first }}/health" - validate_certs: no - client_cert: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - client_key: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" - register: result - until: result.status == 200 or result.status == 401 - retries: 10 - delay: 5 - run_once: true - -- name: Calico | Check if calico network pool has already been configured - command: |- - curl \ - --cacert {{ etcd_cert_dir }}/ca.pem \ - --cert {{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem \ - --key {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem \ - {{ etcd_access_addresses.split(',') | first }}/v2/keys/calico/v1/ipam/v4/pool - register: calico_conf - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - run_once: true - changed_when: false +- include_tasks: pre.yml -- name: Calico | Configure calico network pool - shell: > - echo '{ - "kind": "ipPool", - "spec": {"disabled": false, "ipip": {"enabled": {{ ipip }}, "mode": "{{ ipip_mode }}"}, - "nat-outgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }}}, - "apiVersion": "v1", - "metadata": {"cidr": "{{ kube_pods_subnet }}"} - }' - | {{ bin_dir }}/calicoctl apply -f - - environment: - NO_DEFAULT_POOLS: true - run_once: true - when: ("Key not found" in calico_conf.stdout or "nodes" not in calico_conf.stdout) - -- name: Calico | Get calico configuration from etcd - command: |- - curl \ - --cacert {{ etcd_cert_dir }}/ca.pem \ - --cert {{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem \ - --key {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem \ - {{ etcd_access_addresses.split(',') | first }}/v2/keys/calico/v1/ipam/v4/pool - register: calico_pools_raw - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - run_once: true - -- set_fact: - calico_pools: "{{ calico_pools_raw.stdout | from_json }}" - run_once: true - -- name: Calico | Set global as_num - command: "{{ bin_dir}}/calicoctl config set asNumber {{ global_as_num }}" - run_once: true - -- name: Calico | Disable node mesh - shell: "{{ bin_dir }}/calicoctl config set nodeToNodeMesh off" - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - when: ((peer_with_router|default(false) or peer_with_calico_rr|default(false)) - and inventory_hostname in groups['k8s-cluster']) - run_once: true - -- name: Calico | Configure peering with router(s) - shell: > - echo '{ - "kind": "bgpPeer", - "spec": {"asNumber": "{{ item.as }}"}, - "apiVersion": "v1", - "metadata": {"node": "{{ inventory_hostname }}", "scope": "node", "peerIP": "{{ item.router_id }}"} - }' - | {{ bin_dir }}/calicoctl create --skip-exists -f - - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - with_items: "{{ peers|default([]) }}" - when: peer_with_router|default(false) and inventory_hostname in groups['k8s-cluster'] - -- name: Calico | Configure peering with route reflectors - shell: > - echo '{ - "kind": "bgpPeer", - "spec": {"asNumber": "{{ local_as | default(global_as_num)}}"}, - "apiVersion": "v1", - "metadata": {"node": "{{ inventory_hostname }}", - "scope": "node", - "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}"} - }' - | {{ bin_dir }}/calicoctl create --skip-exists -f - - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - with_items: "{{ groups['calico-rr'] | default([]) }}" - when: (peer_with_calico_rr|default(false) and inventory_hostname in groups['k8s-cluster'] - and hostvars[item]['cluster_id'] == cluster_id) - -- name: Calico | Create calico manifests - template: - src: "{{item.file}}.j2" - dest: "{{kube_config_dir}}/{{item.file}}" - with_items: - - {name: calico-config, file: calico-config.yml, type: cm} - - {name: calico-node, file: calico-node.yml, type: ds} - - {name: calico, file: calico-node-sa.yml, type: sa} - - {name: calico, file: calico-cr.yml, type: clusterrole} - - {name: calico, file: calico-crb.yml, type: clusterrolebinding} - register: calico_node_manifests +- include_tasks: upgrade.yml when: - - inventory_hostname in groups['kube-master'] + - calico_upgrade_needed + run_once: yes + delegate_to: "{{ groups['kube-master'][0] }}" + +- include_tasks: install.yml diff --git a/roles/network_plugin/calico/tasks/pre.yml b/roles/network_plugin/calico/tasks/pre.yml new file mode 100644 index 000000000..4781541bd --- /dev/null +++ b/roles/network_plugin/calico/tasks/pre.yml @@ -0,0 +1,16 @@ +--- +- name: Calico | Disable calico-node service if it exists + service: + name: calico-node + state: stopped + enabled: no + failed_when: false + +- name: Calico | Get kubelet hostname + shell: >- + {{ bin_dir }}/kubectl get node -o custom-columns='NAME:.metadata.name,INTERNAL-IP:.status.addresses[?(@.type=="InternalIP")].address' + | egrep "{{ ansible_all_ipv4_addresses | join('$|') }}$" | cut -d" " -f1 + register: calico_kubelet_name + delegate_to: "{{ groups['kube-master'][0] }}" + when: + - "cloud_provider is defined" \ No newline at end of file diff --git a/roles/network_plugin/calico/tasks/upgrade.yml b/roles/network_plugin/calico/tasks/upgrade.yml new file mode 100644 index 000000000..443362c98 --- /dev/null +++ b/roles/network_plugin/calico/tasks/upgrade.yml @@ -0,0 +1,26 @@ +--- +- name: "Download calico-upgrade tool (force version)" + get_url: + url: "https://github.com/projectcalico/calico-upgrade/releases/download/{{ calico_upgrade_version }}/calico-upgrade" + dest: "{{ bin_dir }}/calico-upgrade" + mode: 0755 + owner: root + group: root + force: yes + +- name: "Create etcdv2 and etcdv3 calicoApiConfig" + template: + src: "{{ item }}-store.yml.j2" + dest: "/etc/calico/{{ item }}.yml" + with_items: + - "etcdv2" + - "etcdv3" + +- name: "Tests data migration (dry-run)" + shell: "{{ bin_dir }}/calico-upgrade dry-run --output-dir=/tmp --apiconfigv1 /etc/calico/etcdv2.yml --apiconfigv3 /etc/calico/etcdv3.yml" + register: calico_upgrade_test_data + failed_when: '"Successfully" not in calico_upgrade_test_data.stdout' + +- name: "If test migration is success continue with calico data real migration" + shell: "{{ bin_dir }}/calico-upgrade start --no-prompts --apiconfigv1 /etc/calico/etcdv2.yml --apiconfigv3 /etc/calico/etcdv3.yml --output-dir=/tmp/calico_upgrade" + register: calico_upgrade_migration_data \ No newline at end of file diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2 index 54dacba8f..38469624e 100644 --- a/roles/network_plugin/calico/templates/calico-node.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-node.yml.j2 @@ -77,12 +77,6 @@ spec: # Set Felix logging to "info" - name: FELIX_LOGSEVERITYSCREEN value: "info" - # Disable autocreation of pools - - name: CALICO_NO_DEFAULT_POOLS - value: "true" - # Enable libnetwork - - name: CALICO_LIBNETWORK_ENABLED - value: "true" # Set MTU for tunnel device used if ipip is enabled {% if calico_mtu is defined %} - name: FELIX_IPINIPMTU @@ -158,6 +152,8 @@ spec: readOnly: true - mountPath: /var/run/calico name: var-run-calico + - mountPath: /var/lib/calico + name: var-lib-calico readOnly: false - mountPath: /calico-secrets name: etcd-certs @@ -169,6 +165,9 @@ spec: - name: var-run-calico hostPath: path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico # Used to install CNI. - name: cni-bin-dir hostPath: @@ -183,5 +182,4 @@ spec: updateStrategy: rollingUpdate: maxUnavailable: {{ serial | default('20%') }} - type: RollingUpdate - + type: RollingUpdate \ No newline at end of file diff --git a/roles/network_plugin/calico/templates/cni-calico.conflist.j2 b/roles/network_plugin/calico/templates/cni-calico.conflist.j2 index 443e3b43b..e84ab7d61 100644 --- a/roles/network_plugin/calico/templates/cni-calico.conflist.j2 +++ b/roles/network_plugin/calico/templates/cni-calico.conflist.j2 @@ -3,7 +3,7 @@ "cniVersion":"0.3.1", "plugins":[ { - {% if cloud_provider is defined %} +{% if cloud_provider is defined %} "nodename": "{{ calico_kubelet_name.stdout }}", {% else %} "nodename": "{{ calico_baremetal_nodename }}", @@ -19,14 +19,14 @@ "assign_ipv4": "true", "ipv4_pools": ["{{ kube_pods_subnet }}"] }, - {% if enable_network_policy %} +{% if enable_network_policy %} "policy": { "type": "k8s" }, - {%- endif %} - {% if calico_mtu is defined and calico_mtu is number %} +{%- endif %} +{% if calico_mtu is defined and calico_mtu is number %} "mtu": {{ calico_mtu }}, - {%- endif %} +{%- endif %} "kubernetes": { "kubeconfig": "{{ kube_config_dir }}/node-kubeconfig.yaml" } diff --git a/roles/network_plugin/calico/templates/etcdv2-store.yml.j2 b/roles/network_plugin/calico/templates/etcdv2-store.yml.j2 new file mode 100644 index 000000000..c65728838 --- /dev/null +++ b/roles/network_plugin/calico/templates/etcdv2-store.yml.j2 @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: calicoApiConfig +metadata: +spec: + datastoreType: "etcdv2" + etcdEndpoints: "{{ etcd_access_addresses }}" + etcdKeyFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + etcdCertFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" + etcdCACertFile: "{{ etcd_cert_dir }}/ca.pem" diff --git a/roles/network_plugin/calico/templates/etcdv3-store.yml.j2 b/roles/network_plugin/calico/templates/etcdv3-store.yml.j2 new file mode 100644 index 000000000..3dc566d0f --- /dev/null +++ b/roles/network_plugin/calico/templates/etcdv3-store.yml.j2 @@ -0,0 +1,9 @@ +apiVersion: projectcalico.org/v3 +kind: CalicoAPIConfig +metadata: +spec: + datastoreType: "etcdv3" + etcdEndpoints: "{{ etcd_access_addresses }}" + etcdKeyFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + etcdCertFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" + etcdCACertFile: "{{ etcd_cert_dir }}/ca.pem" \ No newline at end of file diff --git a/roles/network_plugin/canal/templates/canal-node.yaml.j2 b/roles/network_plugin/canal/templates/canal-node.yaml.j2 index 7e3ecf8d5..6ef251af8 100644 --- a/roles/network_plugin/canal/templates/canal-node.yaml.j2 +++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2 @@ -28,6 +28,9 @@ spec: - name: lib-modules hostPath: path: /lib/modules + - name: var-lib-calico + hostPath: + path: /var/lib/calico - name: var-run-calico hostPath: path: /var/run/calico @@ -202,6 +205,9 @@ spec: - mountPath: /var/run/calico name: var-run-calico readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false - name: "canal-certs" mountPath: "{{ canal_cert_dir }}" readOnly: true diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index fe72c2bec..47b51546f 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -136,6 +136,7 @@ - "{{ bin_dir }}/kubeadm" - "{{ bin_dir }}/helm" - "{{ bin_dir }}/calicoctl" + - "{{ bin_dir }}/calico-upgrade" - "{{ bin_dir }}/weave" - /var/lib/rkt - /etc/vault diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml index 3d75883dd..abc89e18f 100644 --- a/upgrade-cluster.yml +++ b/upgrade-cluster.yml @@ -78,10 +78,19 @@ - { role: kubernetes/master, tags: master } - { role: kubernetes/client, tags: client } - { role: kubernetes-apps/cluster_roles, tags: cluster-roles } - - { role: network_plugin, tags: network } - { role: upgrade/post-upgrade, tags: post-upgrade } environment: "{{proxy_env}}" +#Upgrade calico on all masters and nodes +- hosts: kube-master:kube-node + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + serial: "{{ serial | default('20%') }}" + roles: + - { role: kubespray-defaults} + - { role: network_plugin, tags: network } + - { role: kubernetes-apps/network_plugin, tags: network } + - { role: kubernetes-apps/policy_controller, tags: policy-controller } + #Finally handle worker upgrades, based on given batch size - hosts: kube-node:!kube-master any_errors_fatal: "{{ any_errors_fatal | default(true) }}" @@ -90,7 +99,6 @@ - { role: kubespray-defaults} - { role: upgrade/pre-upgrade, tags: pre-upgrade } - { role: kubernetes/node, tags: node } - - { role: network_plugin, tags: network } - { role: upgrade/post-upgrade, tags: post-upgrade } - { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" } - { role: kubespray-defaults} @@ -102,14 +110,6 @@ - { role: kubespray-defaults} - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" } -- hosts: kube-master - any_errors_fatal: true - roles: - - { role: kubespray-defaults} - - { role: kubernetes-apps/network_plugin, tags: network } - - { role: kubernetes-apps/policy_controller, tags: policy-controller } - - { role: kubernetes/client, tags: client } - - hosts: calico-rr any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: @@ -127,4 +127,4 @@ any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: - { role: kubespray-defaults} - - { role: kubernetes-apps, tags: apps } + - { role: kubernetes-apps, tags: apps } \ No newline at end of file