From b294db5aed93e5bddabc87dda60799311f0576ab Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 15 Sep 2017 13:19:37 +0100 Subject: [PATCH] fix apply for netchecker upgrade (#1659) * fix apply for netchecker upgrade and graceful upgrade * Speed up daemonset upgrades. Make check wait for ds upgrades. --- .../kubernetes-apps/ansible/tasks/netchecker.yml | 1 + .../ansible/templates/netchecker-agent-ds.yml.j2 | 3 +-- .../templates/netchecker-agent-hostnet-ds.yml.j2 | 2 +- roles/kubernetes/master/tasks/pre-upgrade.yml | 10 +++------- .../calico/templates/calico-node.yml.j2 | 2 +- .../canal/templates/canal-node.yaml.j2 | 2 +- .../flannel/templates/cni-flannel.yml.j2 | 4 ++-- .../weave/templates/weave-net.yml.j2 | 2 +- tests/testcases/030_check-network.yml | 16 ++++++++++------ 9 files changed, 21 insertions(+), 21 deletions(-) diff --git a/roles/kubernetes-apps/ansible/tasks/netchecker.yml b/roles/kubernetes-apps/ansible/tasks/netchecker.yml index 4e91da224..3b9168c03 100644 --- a/roles/kubernetes-apps/ansible/tasks/netchecker.yml +++ b/roles/kubernetes-apps/ansible/tasks/netchecker.yml @@ -10,6 +10,7 @@ kube: name: "netchecker-server" namespace: "{{ netcheck_namespace }}" + filename: "{{ netchecker_server_manifest.stat.path }}" kubectl: "{{bin_dir}}/kubectl" resource: "deploy" state: latest diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 index d73004242..4f32214eb 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 @@ -42,6 +42,5 @@ spec: memory: {{ netchecker_agent_memory_requests }} updateStrategy: rollingUpdate: - maxUnavailable: 1 + maxUnavailable: 100% type: RollingUpdate - diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 index 70194c900..76fca4812 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 @@ -46,5 +46,5 @@ spec: memory: {{ netchecker_agent_memory_requests }} updateStrategy: rollingUpdate: - maxUnavailable: 1 + maxUnavailable: 100% type: RollingUpdate diff --git a/roles/kubernetes/master/tasks/pre-upgrade.yml b/roles/kubernetes/master/tasks/pre-upgrade.yml index 7cd650cbd..2e1aa269c 100644 --- a/roles/kubernetes/master/tasks/pre-upgrade.yml +++ b/roles/kubernetes/master/tasks/pre-upgrade.yml @@ -13,22 +13,18 @@ kube_apiserver_storage_backend: "etcd2" when: old_data_exists.rc == 0 and not force_etcd3|bool -- name: "Pre-upgrade | Delete master manifests on all kube-masters" +- name: "Pre-upgrade | Delete master manifests" file: - path: "/etc/kubernetes/manifests/{{item[1]}}.manifest" + path: "/etc/kubernetes/manifests/{{item}}.manifest" state: absent - delegate_to: "{{item[0]}}" with_nested: - - "{{groups['kube-master']}}" - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"] register: kube_apiserver_manifest_replaced when: (secret_changed|default(false) or etcd_secret_changed|default(false)) -- name: "Pre-upgrade | Delete master containers forcefully on all kube-masters" +- name: "Pre-upgrade | Delete master containers forcefully" shell: "docker ps -f name=k8s-{{item}}* -q | xargs --no-run-if-empty docker rm -f" - delegate_to: "{{item[0]}}" with_nested: - - "{{groups['kube-master']}}" - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"] when: kube_apiserver_manifest_replaced.changed run_once: true diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2 index 8acb28327..4c0538c8c 100644 --- a/roles/network_plugin/calico/templates/calico-node.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-node.yml.j2 @@ -161,6 +161,6 @@ spec: path: "{{ calico_cert_dir }}" updateStrategy: rollingUpdate: - maxUnavailable: 1 + maxUnavailable: {{ serial | default('20%') }} type: RollingUpdate diff --git a/roles/network_plugin/canal/templates/canal-node.yaml.j2 b/roles/network_plugin/canal/templates/canal-node.yaml.j2 index 972b02d5f..07754c089 100644 --- a/roles/network_plugin/canal/templates/canal-node.yaml.j2 +++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2 @@ -190,5 +190,5 @@ spec: readOnly: true updateStrategy: rollingUpdate: - maxUnavailable: 1 + maxUnavailable: {{ serial | default('20%') }} type: RollingUpdate diff --git a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 index 0012228d7..165395c24 100644 --- a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 +++ b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 @@ -121,5 +121,5 @@ spec: path: /opt/cni/bin updateStrategy: rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate \ No newline at end of file + maxUnavailable: {{ serial | default('20%') }} + type: RollingUpdate diff --git a/roles/network_plugin/weave/templates/weave-net.yml.j2 b/roles/network_plugin/weave/templates/weave-net.yml.j2 index c61f2e7e4..67c04d9be 100644 --- a/roles/network_plugin/weave/templates/weave-net.yml.j2 +++ b/roles/network_plugin/weave/templates/weave-net.yml.j2 @@ -156,6 +156,6 @@ items: path: /lib/modules updateStrategy: rollingUpdate: - maxUnavailable: 1 + maxUnavailable: {{ serial | default('20%') }} type: RollingUpdate diff --git a/tests/testcases/030_check-network.yml b/tests/testcases/030_check-network.yml index 7c934c592..7269dab35 100644 --- a/tests/testcases/030_check-network.yml +++ b/tests/testcases/030_check-network.yml @@ -12,16 +12,11 @@ bin_dir: "/usr/local/bin" when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] - - name: Check kubectl output - shell: "{{bin_dir}}/kubectl get pods --all-namespaces -owide" - register: get_pods - - - debug: msg="{{get_pods.stdout.split('\n')}}" - name: Get pod names shell: "{{bin_dir}}/kubectl get pods -o json" register: pods - until: '"ContainerCreating" not in pods.stdout' + until: '"ContainerCreating" not in pods.stdout and "Terminating" not in pods.stdout' retries: 60 delay: 2 no_log: true @@ -30,11 +25,20 @@ command: "{{bin_dir}}/kubectl get pods -o jsonpath='{range .items[?(.spec.hostNetwork)]}{.metadata.name} {.status.podIP} {.status.containerStatuses} {end}'" register: hostnet_pods + no_log: true - name: Get running pods command: "{{bin_dir}}/kubectl get pods -o jsonpath='{range .items[?(.status.phase==\"Running\")]}{.metadata.name} {.status.podIP} {.status.containerStatuses} {end}'" register: running_pods + no_log: true + + - name: Check kubectl output + shell: "{{bin_dir}}/kubectl get pods --all-namespaces -owide" + register: get_pods + no_log: true + + - debug: msg="{{get_pods.stdout.split('\n')}}" - set_fact: kube_pods_subnet: 10.233.64.0/18