From ad6fecefa879515578f3f7810ce6ebac14ac1d3d Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Mon, 25 Dec 2017 08:57:45 +0000 Subject: [PATCH] Update Kubernetes to v1.9.0 (#2100) Update checksum for kubeadm Use v1.9.0 kubeadm params Include hash of ca.crt for kubeadm join Update tag for testing upgrades Add workaround for testing upgrades Remove scale CI scenarios because of slow inventory parsing in ansible 2.4.x. Change region for tests to us-central1 to improve ansible performance --- .gitlab-ci.yml | 4 +++- README.md | 2 +- inventory/group_vars/k8s-cluster.yml | 2 +- roles/download/defaults/main.yml | 20 ++++++++----------- .../templates/openstack-storage-class.yml | 1 + roles/kubernetes/kubeadm/tasks/main.yml | 12 ++++++++++- .../kubeadm/templates/kubeadm-client.conf.j2 | 2 ++ .../kubernetes/master/tasks/kubeadm-setup.yml | 8 ++++---- .../master/templates/kubeadm-config.yaml.j2 | 4 +++- roles/kubespray-defaults/defaults/main.yaml | 2 +- tests/files/centos7-calico-ha.yml | 5 +++-- tests/files/centos7-flannel-addons.yml | 2 +- tests/files/coreos-alpha-weave-ha.yml | 5 +++-- tests/files/coreos-calico-aio.yml | 2 +- tests/files/coreos-canal.yml | 2 +- tests/files/rhel7-canal-sep.yml | 2 +- tests/files/rhel7-weave.yml | 2 +- tests/files/ubuntu-canal-ha.yml | 2 +- tests/files/ubuntu-canal-kubeadm.yml | 2 +- tests/files/ubuntu-contiv-sep.yml | 2 +- tests/files/ubuntu-flannel-sep.yml | 2 +- tests/files/ubuntu-rkt-sep.yml | 2 +- tests/files/ubuntu-weave-sep.yml | 2 +- 23 files changed, 52 insertions(+), 37 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7fbcbc984..c674c2e99 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -94,9 +94,11 @@ before_script: # Check out latest tag if testing upgrade # Uncomment when gitlab kargo repo has tags #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1)) - - test "${UPGRADE_TEST}" != "false" && git checkout 72ae7638bcc94c66afa8620dfa4ad9a9249327ea + - test "${UPGRADE_TEST}" != "false" && git checkout ba0a03a8ba2d97a73d06242ec4bb3c7e2012e58c # Checkout the CI vars file so it is available - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml + # Workaround https://github.com/kubernetes-incubator/kubespray/issues/2021 + - 'sh -c "echo ignore_assert_errors: true | tee -a tests/files/${CI_JOB_NAME}.yml"' # Create cluster diff --git a/README.md b/README.md index abd1548ab..0554a5fc0 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Versions of supported components -------------------------------- -[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.8.4
+[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.9.0
[etcd](https://github.com/coreos/etcd/releases) v3.2.4
[flanneld](https://github.com/coreos/flannel/releases) v0.8.0
[calico](https://docs.projectcalico.org/v2.5/releases/) v2.5.0
diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index f8210f291..43b2d3e32 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -23,7 +23,7 @@ kube_users_dir: "{{ kube_config_dir }}/users" kube_api_anonymous_auth: false ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.8.4 +kube_version: v1.9.0 # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index b642a8801..8f5c5d3a7 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -24,7 +24,7 @@ download_always_pull: False download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}" # Versions -kube_version: v1.8.4 +kube_version: v1.9.0 kubeadm_version: "{{ kube_version }}" etcd_version: v3.2.4 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults @@ -36,27 +36,21 @@ calico_policy_version: "v1.0.0" calico_rr_version: "v0.4.0" flannel_version: "v0.9.1" flannel_cni_version: "v0.3.0" +istio_version: "0.2.6" +vault_version: 0.8.1 weave_version: 2.0.5 pod_infra_version: 3.0 contiv_version: 1.1.7 # Download URLs +istioctl_download_url: "https://storage.googleapis.com/istio-release/releases/{{ istio_version }}/istioctl/istioctl-linux" kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/amd64/kubeadm" +vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip" # Checksums -kubeadm_checksum: "08c93bb83c1af8703d49027b863fee08721cb96900f8d70d4d45b50dd1e5bc2c" - -istio_version: "0.2.6" - -istioctl_download_url: "https://storage.googleapis.com/istio-release/releases/{{ istio_version }}/istioctl/istioctl-linux" istioctl_checksum: fd703063c540b8c0ab943f478c05ab257d88ae27224c746a27d0526ddbf7c370 - -vault_version: 0.8.1 +kubeadm_checksum: 069e386f620e7274e114226ab7532c2320be7f65328c1e55b23a69b73122b828 vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188 -vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip" -vault_image_repo: "vault" -vault_image_tag: "{{ vault_version }}" - # Containers etcd_image_repo: "quay.io/coreos/etcd" @@ -127,6 +121,8 @@ helm_image_repo: "lachlanevenson/k8s-helm" helm_image_tag: "{{ helm_version }}" tiller_image_repo: "gcr.io/kubernetes-helm/tiller" tiller_image_tag: "{{ helm_version }}" +vault_image_repo: "vault" +vault_image_tag: "{{ vault_version }}" downloads: netcheck_server: diff --git a/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml b/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml index c643cfa09..02d39dd97 100644 --- a/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml +++ b/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml @@ -1,3 +1,4 @@ +--- kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 14a577106..7be760458 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -16,6 +16,13 @@ path: "{{ kube_config_dir }}/kubelet.conf" register: kubelet_conf + +- name: Calculate kubeadm CA cert hash + shell: openssl x509 -pubkey -in {{ kube_config_dir }}/ssl/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //' + register: kubeadm_ca_hash + delegate_to: "{{ groups['kube-master'][0] }}" + run_once: true + - name: Create kubeadm client config template: src: kubeadm-client.conf.j2 @@ -25,7 +32,10 @@ register: kubeadm_client_conf - name: Join to cluster if needed - command: "{{ bin_dir }}/kubeadm join --config {{ kube_config_dir}}/kubeadm-client.conf --skip-preflight-checks" + command: >- + {{ bin_dir }}/kubeadm join + --config {{ kube_config_dir}}/kubeadm-client.conf + --ignore-preflight-errors=all register: kubeadm_join when: not is_kube_master and (kubeadm_client_conf.changed or not kubelet_conf.stat.exists) diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 index 3c8ede9ad..18c6c2af6 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 @@ -4,3 +4,5 @@ caCertPath: {{ kube_config_dir }}/ssl/ca.crt token: {{ kubeadm_token }} discoveryTokenAPIServers: - {{ kubeadm_discovery_address | replace("https://", "")}} +DiscoveryTokenCACertHashes: +- sha256:{{ kubeadm_ca_hash.stdout }} diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index c901bc4fa..1405a9dd0 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -72,7 +72,7 @@ register: kubeadm_config - name: kubeadm | Initialize first master - command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks + command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all register: kubeadm_init # Retry is because upload config sometimes fails retries: 3 @@ -86,7 +86,7 @@ {{ bin_dir }}/kubeadm upgrade apply -y {{ kube_version }} --config={{ kube_config_dir }}/kubeadm-config.yaml - --skip-preflight-checks + --ignore-preflight-errors=all --allow-experimental-upgrades --allow-release-candidate-upgrades register: kubeadm_upgrade @@ -135,7 +135,7 @@ when: inventory_hostname != groups['kube-master']|first - name: kubeadm | Init other uninitialized masters - command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks + command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all register: kubeadm_init when: inventory_hostname != groups['kube-master']|first and not kubeadm_ca.stat.exists failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr @@ -147,7 +147,7 @@ {{ bin_dir }}/kubeadm upgrade apply -y {{ kube_version }} --config={{ kube_config_dir }}/kubeadm-config.yaml - --skip-preflight-checks + --ignore-preflight-errors=all --allow-experimental-upgrades --allow-release-candidate-upgrades register: kubeadm_upgrade diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index 774a7810f..26e3b46a4 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -16,7 +16,9 @@ networking: serviceSubnet: {{ kube_service_addresses }} podSubnet: {{ kube_pods_subnet }} kubernetesVersion: {{ kube_version }} -cloudProvider: {{ cloud_provider|default('') }} +{% if cloud_provider is defined and cloud_provider != "gce" %} +cloudProvider: {{ cloud_provider }} +{% endif %} authorizationModes: {% for mode in authorization_modes %} - {{ mode }} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 579eb7b83..f0febcf39 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -13,7 +13,7 @@ kube_api_anonymous_auth: false is_atomic: false ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.8.4 +kube_version: v1.9.0 # Set to true to allow pre-checks to fail and continue deployment ignore_assert_errors: false diff --git a/tests/files/centos7-calico-ha.yml b/tests/files/centos7-calico-ha.yml index a34ab2dfb..0bca5842e 100644 --- a/tests/files/centos7-calico-ha.yml +++ b/tests/files/centos7-calico-ha.yml @@ -1,7 +1,8 @@ # Instance settings cloud_image_family: centos-7 -cloud_region: europe-west1-b -mode: ha-scale +cloud_region: us-central1-c +cloud_machine_type: "n1-standard-1" +mode: ha # Deployment settings kube_network_plugin: calico diff --git a/tests/files/centos7-flannel-addons.yml b/tests/files/centos7-flannel-addons.yml index 8824df4a1..f2d77dbca 100644 --- a/tests/files/centos7-flannel-addons.yml +++ b/tests/files/centos7-flannel-addons.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: centos-7 -cloud_region: us-west1-a +cloud_region: us-central1-c cloud_machine_type: "n1-standard-1" mode: default diff --git a/tests/files/coreos-alpha-weave-ha.yml b/tests/files/coreos-alpha-weave-ha.yml index d8087c621..dd579c032 100644 --- a/tests/files/coreos-alpha-weave-ha.yml +++ b/tests/files/coreos-alpha-weave-ha.yml @@ -1,7 +1,8 @@ # Instance settings cloud_image_family: coreos-alpha -cloud_region: us-west1-a -mode: ha-scale +cloud_region: us-central1-a +cloud_machine_type: "n1-standard-1" +mode: ha startup_script: 'systemctl disable locksmithd && systemctl stop locksmithd' # Deployment settings diff --git a/tests/files/coreos-calico-aio.yml b/tests/files/coreos-calico-aio.yml index 37ff7ac8f..b1d06fc6f 100644 --- a/tests/files/coreos-calico-aio.yml +++ b/tests/files/coreos-calico-aio.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: coreos-stable -cloud_region: us-west1-b +cloud_region: us-central1-a cloud_machine_type: "n1-standard-2" mode: aio ##user-data to simply turn off coreos upgrades diff --git a/tests/files/coreos-canal.yml b/tests/files/coreos-canal.yml index afbedc30c..a3a750fd9 100644 --- a/tests/files/coreos-canal.yml +++ b/tests/files/coreos-canal.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: coreos-stable -cloud_region: us-east1-b +cloud_region: us-central1-c mode: default startup_script: 'systemctl disable locksmithd && systemctl stop locksmithd' diff --git a/tests/files/rhel7-canal-sep.yml b/tests/files/rhel7-canal-sep.yml index 2fc39cbb1..e3c679629 100644 --- a/tests/files/rhel7-canal-sep.yml +++ b/tests/files/rhel7-canal-sep.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: rhel-7 -cloud_region: us-east1-b +cloud_region: us-central1-a mode: separate # Deployment settings diff --git a/tests/files/rhel7-weave.yml b/tests/files/rhel7-weave.yml index 66804df5c..df80a556f 100644 --- a/tests/files/rhel7-weave.yml +++ b/tests/files/rhel7-weave.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: rhel-7 -cloud_region: europe-west1-b +cloud_region: us-central1-b mode: default # Deployment settings diff --git a/tests/files/ubuntu-canal-ha.yml b/tests/files/ubuntu-canal-ha.yml index 7900c055b..241c7d5a2 100644 --- a/tests/files/ubuntu-canal-ha.yml +++ b/tests/files/ubuntu-canal-ha.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: ubuntu-1604-lts -cloud_region: europe-west1-b +cloud_region: us-central1-c mode: ha # Deployment settings diff --git a/tests/files/ubuntu-canal-kubeadm.yml b/tests/files/ubuntu-canal-kubeadm.yml index 93574118f..1f8fd2d76 100644 --- a/tests/files/ubuntu-canal-kubeadm.yml +++ b/tests/files/ubuntu-canal-kubeadm.yml @@ -1,7 +1,7 @@ # Instance settings cloud_image_family: ubuntu-1604-lts cloud_machine_type: "n1-standard-1" -cloud_region: europe-west1-b +cloud_region: us-central1-c mode: ha # Deployment settings diff --git a/tests/files/ubuntu-contiv-sep.yml b/tests/files/ubuntu-contiv-sep.yml index 0489817b7..0b3b575ab 100644 --- a/tests/files/ubuntu-contiv-sep.yml +++ b/tests/files/ubuntu-contiv-sep.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: ubuntu-1604-lts -cloud_region: us-west1-a +cloud_region: us-central1-b mode: separate # Deployment settings diff --git a/tests/files/ubuntu-flannel-sep.yml b/tests/files/ubuntu-flannel-sep.yml index 6292926c8..df77a46b3 100644 --- a/tests/files/ubuntu-flannel-sep.yml +++ b/tests/files/ubuntu-flannel-sep.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: ubuntu-1604-lts -cloud_region: europe-west1-b +cloud_region: us-central1-a mode: separate # Deployment settings diff --git a/tests/files/ubuntu-rkt-sep.yml b/tests/files/ubuntu-rkt-sep.yml index 297ce5be0..b15989231 100644 --- a/tests/files/ubuntu-rkt-sep.yml +++ b/tests/files/ubuntu-rkt-sep.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: ubuntu-1604-lts -cloud_region: us-central1-b +cloud_region: us-central1-c mode: separate # Deployment settings diff --git a/tests/files/ubuntu-weave-sep.yml b/tests/files/ubuntu-weave-sep.yml index 9ab13c278..133bd907a 100644 --- a/tests/files/ubuntu-weave-sep.yml +++ b/tests/files/ubuntu-weave-sep.yml @@ -1,6 +1,6 @@ # Instance settings cloud_image_family: ubuntu-1604-lts -cloud_region: us-central1-b +cloud_region: us-central1-c mode: separate # Deployment settings