From e9f795c5cee624551ceef019f2466e29845bf2c4 Mon Sep 17 00:00:00 2001
From: Chad Swenson <chadswen@gmail.com>
Date: Wed, 8 Nov 2017 01:40:33 -0600
Subject: [PATCH] Master component and kubelet container upgrade fixes

* Fixes an issue where apiserver and friends (controller manager, scheduler) were prevented from restarting after manifests/secrets are changed. This occurred when a replaced kubelet doesn't reconcile new master manifests, which caused old master component versions to linger during deployment. In my case this was causing upgrades from k8s 1.6/1.7 -> k8s 1.8 to fail
* Improves transitions from kubelet container to host kubelet by preventing issues where kubelet container reappeared during the deployment
---
 roles/kubernetes/master/handlers/main.yml     | 27 +++++++++++++++++++
 roles/kubernetes/master/tasks/pre-upgrade.yml |  2 +-
 .../master/tasks/static-pod-setup.yml         |  6 ++---
 roles/kubernetes/node/tasks/pre_upgrade.yml   | 19 ++++++++++---
 4 files changed, 47 insertions(+), 7 deletions(-)

diff --git a/roles/kubernetes/master/handlers/main.yml b/roles/kubernetes/master/handlers/main.yml
index 1c6dc956c..dd3b03264 100644
--- a/roles/kubernetes/master/handlers/main.yml
+++ b/roles/kubernetes/master/handlers/main.yml
@@ -13,6 +13,24 @@
     - Master | wait for kube-scheduler
     - Master | wait for kube-controller-manager
 
+- name: Master | Restart apiserver
+  command: /bin/true
+  notify:
+    - Master | Remove apiserver container
+    - Master | wait for the apiserver to be running
+
+- name: Master | Restart kube-scheduler
+  command: /bin/true
+  notify:
+    - Master | Remove scheduler container
+    - Master | wait for kube-scheduler
+
+- name: Master | Restart kube-controller-manager
+  command: /bin/true
+  notify:
+    - Master | Remove controller manager container
+    - Master | wait for kube-controller-manager
+
 - name: Master | reload systemd
   command: systemctl daemon-reload
 
@@ -21,6 +39,15 @@
     name: kubelet
     state: restarted
 
+- name: Master | Remove apiserver container
+  shell: "docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f"
+
+- name: Master | Remove scheduler container
+  shell: "docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty docker rm -f"
+
+- name: Master | Remove controller manager container
+  shell: "docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty docker rm -f"
+
 - name: Master | wait for kube-scheduler
   uri:
     url: http://localhost:10251/healthz
diff --git a/roles/kubernetes/master/tasks/pre-upgrade.yml b/roles/kubernetes/master/tasks/pre-upgrade.yml
index b546f0ca9..81da33433 100644
--- a/roles/kubernetes/master/tasks/pre-upgrade.yml
+++ b/roles/kubernetes/master/tasks/pre-upgrade.yml
@@ -24,7 +24,7 @@
   when: (secret_changed|default(false) or etcd_secret_changed|default(false))
 
 - name: "Pre-upgrade | Delete master containers forcefully"
-  shell: "docker ps -f name=k8s-{{item}}* -q | xargs --no-run-if-empty docker rm -f"
+  shell: "docker ps -af name=k8s_{{item}}* -q | xargs --no-run-if-empty docker rm -f"
   with_items:
     - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
   when: kube_apiserver_manifest_replaced.changed
diff --git a/roles/kubernetes/master/tasks/static-pod-setup.yml b/roles/kubernetes/master/tasks/static-pod-setup.yml
index 79f95d860..8110cfe8c 100644
--- a/roles/kubernetes/master/tasks/static-pod-setup.yml
+++ b/roles/kubernetes/master/tasks/static-pod-setup.yml
@@ -3,7 +3,7 @@
   template:
     src: manifests/kube-apiserver.manifest.j2
     dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
-  notify: Master | wait for the apiserver to be running
+  notify: Master | Restart apiserver
   tags:
     - kube-apiserver
 
@@ -20,7 +20,7 @@
   template:
     src: manifests/kube-scheduler.manifest.j2
     dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
-  notify: Master | wait for kube-scheduler
+  notify: Master | Restart kube-scheduler
   tags:
     - kube-scheduler
 
@@ -35,6 +35,6 @@
   template:
     src: manifests/kube-controller-manager.manifest.j2
     dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
-  notify: Master | wait for kube-controller-manager
+  notify: Master | Restart kube-controller-manager
   tags:
     - kube-controller-manager
diff --git a/roles/kubernetes/node/tasks/pre_upgrade.yml b/roles/kubernetes/node/tasks/pre_upgrade.yml
index 2701808be..8919bf0be 100644
--- a/roles/kubernetes/node/tasks/pre_upgrade.yml
+++ b/roles/kubernetes/node/tasks/pre_upgrade.yml
@@ -1,12 +1,25 @@
 ---
+- name: "Pre-upgrade | check if kubelet container exists"
+  shell: docker ps -af name=kubelet | grep kubelet
+  failed_when: false
+  changed_when: false
+  register: kubelet_container_check
+
 - name: "Pre-upgrade | copy /var/lib/cni from kubelet"
   command: docker cp kubelet:/var/lib/cni /var/lib/cni
   args:
     creates: "/var/lib/cni"
   failed_when: false
+  when: kubelet_container_check.rc == 0
+
+- name: "Pre-upgrade | ensure kubelet container service is stopped if using host deployment"
+  service:
+    name: kubelet
+    state: stopped
+  when: kubelet_deployment_type == 'host' and kubelet_container_check.rc == 0
 
-- name: "Pre-upgrade | ensure kubelet container is stopped if using host deployment"
-  command: docker stop kubelet
+- name: "Pre-upgrade | ensure kubelet container is removed if using host deployment"
+  command: docker rm -fv kubelet
   failed_when: false
   changed_when: false
-  when: kubelet_deployment_type == 'host'
+  when: kubelet_deployment_type == 'host' and kubelet_container_check.rc == 0
\ No newline at end of file