From 5cba8b16147e337dc678ed47c39e641636c978a4 Mon Sep 17 00:00:00 2001 From: Rick Haan Date: Mon, 8 Jun 2020 16:25:53 +0200 Subject: [PATCH] Wait for kube-apiserver availability before starting upgrade (#6243) * Wait for kube-apiserver availability before starting upgrade I am experiencing a timing issue when upgrading from kubespray 2.11.0(k8s 1.15.3) to kubespray 2.12.6(k8s 1.16.9). The certificates get replaced in `kubeadm-secondary-legacy.yml` and kube-apiserver notices a mismatch (for a fraction of a second) between `apiserver.crt` and `apiserver.key` which causes it to restart. And sometimes ( ~ 1 out of 5 upgrades) the kube-apiserver isn't back on time for the start of the upgrade task. It fails when kubeadm checks with the kube-apiserver to start the upgrade. The kube-apiserver returns a `connect: connection refused`. I have created this small task to check the availability of the kube-apiserver before starting the upgrade, so that the upgrade will run without an issue. Signed-off-by: Rick Haan * Fix markdownlint * Remove old CI Co-authored-by: Maxime Guyot --- .gitlab-ci/terraform.yml | 67 ------------------- docs/integration.md | 2 +- .../master/tasks/kubeadm-upgrade.yml | 11 +++ 3 files changed, 12 insertions(+), 68 deletions(-) diff --git a/.gitlab-ci/terraform.yml b/.gitlab-ci/terraform.yml index dfe15e905..358da3e07 100644 --- a/.gitlab-ci/terraform.yml +++ b/.gitlab-ci/terraform.yml @@ -92,70 +92,3 @@ tf-validate-aws: # TF_VAR_facility: ams1 # TF_VAR_public_key_path: "" # TF_VAR_operating_system: ubuntu_18_04 - -.ovh_variables: &ovh_variables - OS_AUTH_URL: https://auth.cloud.ovh.net/v3 - OS_PROJECT_ID: 8d3cd5d737d74227ace462dee0b903fe - OS_PROJECT_NAME: "9361447987648822" - OS_USER_DOMAIN_NAME: Default - OS_PROJECT_DOMAIN_ID: default - OS_USERNAME: 8XuhBMfkKVrk - OS_REGION_NAME: UK1 - OS_INTERFACE: public - OS_IDENTITY_API_VERSION: "3" - -tf-ovh_ubuntu18-calico: - extends: .terraform_apply - when: on_success - variables: - <<: *ovh_variables - TF_VERSION: 0.12.12 - PROVIDER: openstack - CLUSTER: $CI_COMMIT_REF_NAME - ANSIBLE_TIMEOUT: "60" - SSH_USER: ubuntu - TF_VAR_number_of_k8s_masters: "0" - TF_VAR_number_of_k8s_masters_no_floating_ip: "1" - TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0" - TF_VAR_number_of_etcd: "0" - TF_VAR_number_of_k8s_nodes: "0" - TF_VAR_number_of_k8s_nodes_no_floating_ip: "1" - TF_VAR_number_of_gfs_nodes_no_floating_ip: "0" - TF_VAR_number_of_bastions: "0" - TF_VAR_number_of_k8s_masters_no_etcd: "0" - TF_VAR_use_neutron: "0" - TF_VAR_floatingip_pool: "Ext-Net" - TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b" - TF_VAR_network_name: "Ext-Net" - TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229" # s1-8 - TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229" # s1-8 - TF_VAR_image: "Ubuntu 18.04" - TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]' - -tf-ovh_coreos-calico: - extends: .terraform_apply - when: on_success - variables: - <<: *ovh_variables - TF_VERSION: 0.12.12 - PROVIDER: openstack - CLUSTER: $CI_COMMIT_REF_NAME - ANSIBLE_TIMEOUT: "60" - SSH_USER: core - TF_VAR_number_of_k8s_masters: "0" - TF_VAR_number_of_k8s_masters_no_floating_ip: "1" - TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0" - TF_VAR_number_of_etcd: "0" - TF_VAR_number_of_k8s_nodes: "0" - TF_VAR_number_of_k8s_nodes_no_floating_ip: "1" - TF_VAR_number_of_gfs_nodes_no_floating_ip: "0" - TF_VAR_number_of_bastions: "0" - TF_VAR_number_of_k8s_masters_no_etcd: "0" - TF_VAR_use_neutron: "0" - TF_VAR_floatingip_pool: "Ext-Net" - TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b" - TF_VAR_network_name: "Ext-Net" - TF_VAR_flavor_k8s_master: "4d4fd037-9493-4f2b-9afe-b542b5248eac" # b2-7 - TF_VAR_flavor_k8s_node: "4d4fd037-9493-4f2b-9afe-b542b5248eac" # b2-7 - TF_VAR_image: "CoreOS Stable" - TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]' diff --git a/docs/integration.md b/docs/integration.md index 09908bf01..4d01bc49b 100644 --- a/docs/integration.md +++ b/docs/integration.md @@ -7,7 +7,7 @@ 2. Add **forked repo** as submodule to desired folder in your existent ansible repo(for example 3d/kubespray): ```git submodule add https://github.com/YOUR_GITHUB/kubespray.git kubespray``` - Git will create _.gitmodules_ file in your existent ansible repo: + Git will create `.gitmodules` file in your existent ansible repo: ```ini [submodule "3d/kubespray"] diff --git a/roles/kubernetes/master/tasks/kubeadm-upgrade.yml b/roles/kubernetes/master/tasks/kubeadm-upgrade.yml index d8a80cd02..d1473fb5e 100644 --- a/roles/kubernetes/master/tasks/kubeadm-upgrade.yml +++ b/roles/kubernetes/master/tasks/kubeadm-upgrade.yml @@ -1,4 +1,15 @@ --- +- name: "kubeadm | Wait for master kube-apiserver" + uri: + url: "https://{{ kube_apiserver_access_address }}:{{ kube_apiserver_port }}/version" + status_code: 200 + validate_certs: false + register: kube_api_server_available + until: kube_api_server_available.status == 200 + retries: 180 + delay: 1 + when: inventory_hostname == groups['kube-master'] + - name: kubeadm | Upgrade first master command: >- timeout -k 600s 600s