Update Kubernetes to v1.9.0 (#2100)

Update checksum for kubeadm
Use v1.9.0 kubeadm params
Include hash of ca.crt for kubeadm join
Update tag for testing upgrades
Add workaround for testing upgrades
Remove scale CI scenarios because of slow inventory parsing
in ansible 2.4.x.

Change region for tests to us-central1 to
improve ansible performance

.gitlab-ci.yml

@@ -94,9 +94,11 @@ before_script:
     # Check out latest tag if testing upgrade
     # Uncomment when gitlab kargo repo has tags
     #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
-    - test "${UPGRADE_TEST}" != "false" && git checkout 72ae7638bcc94c66afa8620dfa4ad9a9249327ea
+    - test "${UPGRADE_TEST}" != "false" && git checkout ba0a03a8ba2d97a73d06242ec4bb3c7e2012e58c
     # Checkout the CI vars file so it is available
     - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
+    # Workaround https://github.com/kubernetes-incubator/kubespray/issues/2021
+    - 'sh -c "echo ignore_assert_errors: true | tee -a tests/files/${CI_JOB_NAME}.yml"'
 
 
     # Create cluster

README.md

@@ -54,7 +54,7 @@ Versions of supported components
 --------------------------------
 
 
-[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.8.4 <br>
+[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.9.0 <br>
 [etcd](https://github.com/coreos/etcd/releases) v3.2.4 <br>
 [flanneld](https://github.com/coreos/flannel/releases) v0.8.0 <br>
 [calico](https://docs.projectcalico.org/v2.5/releases/) v2.5.0 <br>

inventory/group_vars/k8s-cluster.yml

@@ -23,7 +23,7 @@ kube_users_dir: "{{ kube_config_dir }}/users"
 kube_api_anonymous_auth: false
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.8.4
+kube_version: v1.9.0
 
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)

roles/download/defaults/main.yml

@@ -24,7 +24,7 @@ download_always_pull: False
 download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}"
 
 # Versions
-kube_version: v1.8.4
+kube_version: v1.9.0
 kubeadm_version: "{{ kube_version }}"
 etcd_version: v3.2.4
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
@@ -36,27 +36,21 @@ calico_policy_version: "v1.0.0"
 calico_rr_version: "v0.4.0"
 flannel_version: "v0.9.1"
 flannel_cni_version: "v0.3.0"
+istio_version: "0.2.6"
+vault_version: 0.8.1
 weave_version: 2.0.5
 pod_infra_version: 3.0
 contiv_version: 1.1.7
 
 # Download URLs
+istioctl_download_url: "https://storage.googleapis.com/istio-release/releases/{{ istio_version }}/istioctl/istioctl-linux"
 kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/amd64/kubeadm"
+vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip"
 
 # Checksums
-kubeadm_checksum: "08c93bb83c1af8703d49027b863fee08721cb96900f8d70d4d45b50dd1e5bc2c"
-
-istio_version: "0.2.6"
-
-istioctl_download_url: "https://storage.googleapis.com/istio-release/releases/{{ istio_version }}/istioctl/istioctl-linux"
 istioctl_checksum: fd703063c540b8c0ab943f478c05ab257d88ae27224c746a27d0526ddbf7c370
-
-vault_version: 0.8.1
+kubeadm_checksum: 069e386f620e7274e114226ab7532c2320be7f65328c1e55b23a69b73122b828
 vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188
-vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip"
-vault_image_repo: "vault"
-vault_image_tag: "{{ vault_version }}"
-
 
 # Containers
 etcd_image_repo: "quay.io/coreos/etcd"
@@ -127,6 +121,8 @@ helm_image_repo: "lachlanevenson/k8s-helm"
 helm_image_tag: "{{ helm_version }}"
 tiller_image_repo: "gcr.io/kubernetes-helm/tiller"
 tiller_image_tag: "{{ helm_version }}"
+vault_image_repo: "vault"
+vault_image_tag: "{{ vault_version }}"
 
 downloads:
   netcheck_server:

roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml

@@ -1,3 +1,4 @@
+---
 kind: StorageClass
 apiVersion: storage.k8s.io/v1
 metadata:

roles/kubernetes/kubeadm/tasks/main.yml

@@ -16,6 +16,13 @@
     path: "{{ kube_config_dir }}/kubelet.conf"
   register: kubelet_conf
 
+
+- name: Calculate kubeadm CA cert hash
+  shell: openssl x509 -pubkey -in {{ kube_config_dir }}/ssl/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
+  register: kubeadm_ca_hash
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  run_once: true
+
 - name: Create kubeadm client config
   template:
     src: kubeadm-client.conf.j2
@@ -25,7 +32,10 @@
   register: kubeadm_client_conf
 
 - name: Join to cluster if needed
-  command: "{{ bin_dir }}/kubeadm join --config {{ kube_config_dir}}/kubeadm-client.conf --skip-preflight-checks"
+  command: >-
+    {{ bin_dir }}/kubeadm join
+    --config {{ kube_config_dir}}/kubeadm-client.conf
+    --ignore-preflight-errors=all
   register: kubeadm_join
   when: not is_kube_master and (kubeadm_client_conf.changed or not kubelet_conf.stat.exists)
 

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2

@@ -4,3 +4,5 @@ caCertPath: {{ kube_config_dir }}/ssl/ca.crt
 token: {{ kubeadm_token }}
 discoveryTokenAPIServers:
 - {{ kubeadm_discovery_address | replace("https://", "")}}
+DiscoveryTokenCACertHashes:
+- sha256:{{ kubeadm_ca_hash.stdout }}

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -72,7 +72,7 @@
   register: kubeadm_config
 
 - name: kubeadm | Initialize first master
-  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
+  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all
   register: kubeadm_init
   # Retry is because upload config sometimes fails
   retries: 3
@@ -86,7 +86,7 @@
     {{ bin_dir }}/kubeadm
     upgrade apply -y {{ kube_version }}
     --config={{ kube_config_dir }}/kubeadm-config.yaml
-    --skip-preflight-checks
+    --ignore-preflight-errors=all
     --allow-experimental-upgrades
     --allow-release-candidate-upgrades
   register: kubeadm_upgrade
@@ -135,7 +135,7 @@
   when: inventory_hostname != groups['kube-master']|first
 
 - name: kubeadm | Init other uninitialized masters
-  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
+  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all
   register: kubeadm_init
   when: inventory_hostname != groups['kube-master']|first and not kubeadm_ca.stat.exists
   failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
@@ -147,7 +147,7 @@
     {{ bin_dir }}/kubeadm
     upgrade apply -y {{ kube_version }}
     --config={{ kube_config_dir }}/kubeadm-config.yaml
-    --skip-preflight-checks
+    --ignore-preflight-errors=all
     --allow-experimental-upgrades
     --allow-release-candidate-upgrades
   register: kubeadm_upgrade

roles/kubernetes/master/templates/kubeadm-config.yaml.j2

@@ -16,7 +16,9 @@ networking:
   serviceSubnet: {{ kube_service_addresses }}
   podSubnet: {{ kube_pods_subnet }}
 kubernetesVersion: {{ kube_version }}
-cloudProvider: {{ cloud_provider|default('') }}
+{% if cloud_provider is defined and cloud_provider != "gce" %}
+cloudProvider: {{ cloud_provider }}
+{% endif %}
 authorizationModes:
 {% for mode in authorization_modes %}
 - {{ mode }}

roles/kubespray-defaults/defaults/main.yaml

@@ -13,7 +13,7 @@ kube_api_anonymous_auth: false
 is_atomic: false
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.8.4
+kube_version: v1.9.0
 
 # Set to true to allow pre-checks to fail and continue deployment
 ignore_assert_errors: false

tests/files/centos7-calico-ha.yml

@@ -1,7 +1,8 @@
 # Instance settings
 cloud_image_family: centos-7
-cloud_region: europe-west1-b
-mode: ha-scale
+cloud_region: us-central1-c
+cloud_machine_type: "n1-standard-1"
+mode: ha
 
 # Deployment settings
 kube_network_plugin: calico

tests/files/centos7-flannel-addons.yml

@@ -1,6 +1,6 @@
 # Instance settings
 cloud_image_family: centos-7
-cloud_region: us-west1-a
+cloud_region: us-central1-c
 cloud_machine_type: "n1-standard-1"
 mode: default
 

tests/files/coreos-alpha-weave-ha.yml

@@ -1,7 +1,8 @@
 # Instance settings
 cloud_image_family: coreos-alpha
-cloud_region: us-west1-a
-mode: ha-scale
+cloud_region: us-central1-a
+cloud_machine_type: "n1-standard-1"
+mode: ha
 startup_script: 'systemctl disable locksmithd && systemctl stop locksmithd'
 
 # Deployment settings

tests/files/coreos-calico-aio.yml

@@ -1,6 +1,6 @@
 # Instance settings
 cloud_image_family: coreos-stable
-cloud_region: us-west1-b
+cloud_region: us-central1-a
 cloud_machine_type: "n1-standard-2"
 mode: aio
 ##user-data to simply turn off coreos upgrades

tests/files/coreos-canal.yml

@@ -1,6 +1,6 @@
 # Instance settings
 cloud_image_family: coreos-stable
-cloud_region: us-east1-b
+cloud_region: us-central1-c
 mode: default
 startup_script: 'systemctl disable locksmithd && systemctl stop locksmithd'
 

tests/files/rhel7-canal-sep.yml

@@ -1,6 +1,6 @@
 # Instance settings
 cloud_image_family: rhel-7
-cloud_region: us-east1-b
+cloud_region: us-central1-a
 mode: separate
 
 # Deployment settings

tests/files/rhel7-weave.yml

@@ -1,6 +1,6 @@
 # Instance settings
 cloud_image_family: rhel-7
-cloud_region: europe-west1-b
+cloud_region: us-central1-b
 mode: default
 
 # Deployment settings

tests/files/ubuntu-canal-ha.yml

@@ -1,6 +1,6 @@
 # Instance settings
 cloud_image_family: ubuntu-1604-lts
-cloud_region: europe-west1-b
+cloud_region: us-central1-c
 mode: ha
 
 # Deployment settings

tests/files/ubuntu-canal-kubeadm.yml

@@ -1,7 +1,7 @@
 # Instance settings
 cloud_image_family:  ubuntu-1604-lts
 cloud_machine_type: "n1-standard-1"
-cloud_region: europe-west1-b
+cloud_region: us-central1-c
 mode: ha
 
 # Deployment settings

tests/files/ubuntu-contiv-sep.yml

@@ -1,6 +1,6 @@
 # Instance settings
 cloud_image_family: ubuntu-1604-lts
-cloud_region: us-west1-a
+cloud_region: us-central1-b
 mode: separate
 
 # Deployment settings

tests/files/ubuntu-flannel-sep.yml

@@ -1,6 +1,6 @@
 # Instance settings
 cloud_image_family: ubuntu-1604-lts
-cloud_region: europe-west1-b
+cloud_region: us-central1-a
 mode: separate
 
 # Deployment settings

tests/files/ubuntu-rkt-sep.yml

@@ -1,6 +1,6 @@
 # Instance settings
 cloud_image_family: ubuntu-1604-lts
-cloud_region: us-central1-b
+cloud_region: us-central1-c
 mode: separate
 
 # Deployment settings

tests/files/ubuntu-weave-sep.yml

@@ -1,6 +1,6 @@
 # Instance settings
 cloud_image_family: ubuntu-1604-lts
-cloud_region: us-central1-b
+cloud_region: us-central1-c
 mode: separate
 
 # Deployment settings