From ec53b8b66a99b05bdfd3732310064164e7aae9c5 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Thu, 26 Oct 2017 09:10:33 +0100 Subject: [PATCH] Move cluster roles and system namespace to new role This should be done after kubeconfig is set for admin and before network plugins are up. --- .gitlab-ci.yml | 1 - cluster.yml | 3 +- extra_playbooks/upgrade-only-k8s.yml | 2 + roles/kubernetes-apps/ansible/tasks/main.yml | 19 +------ .../cluster_roles/tasks/main.yml | 56 +++++++++++++++++++ .../cluster_roles}/templates/namespace.j2 | 0 .../templates/node-crb.yml.j2 | 0 .../master/tasks/static-pod-setup.yml | 28 ---------- tests/ansible.cfg | 1 + upgrade-cluster.yml | 2 + 10 files changed, 64 insertions(+), 48 deletions(-) create mode 100644 roles/kubernetes-apps/cluster_roles/tasks/main.yml rename roles/{kubernetes/master => kubernetes-apps/cluster_roles}/templates/namespace.j2 (100%) rename roles/kubernetes-apps/{ansible => cluster_roles}/templates/node-crb.yml.j2 (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0971e6ba9..b1daab715 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,6 @@ variables: before_script: - pip install -r tests/requirements.txt - mkdir -p /.ssh - - cp tests/ansible.cfg . .job: &job tags: diff --git a/cluster.yml b/cluster.yml index 5ebed30c5..f3e42eec2 100644 --- a/cluster.yml +++ b/cluster.yml @@ -68,6 +68,8 @@ roles: - { role: kubespray-defaults} - { role: kubernetes/master, tags: master } + - { role: kubernetes/client, tags: client } + - { role: kubernetes-apps/cluster_roles, tags: cluster-roles } - hosts: k8s-cluster any_errors_fatal: "{{ any_errors_fatal | default(true) }}" @@ -83,7 +85,6 @@ - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" } - { role: kubernetes-apps/network_plugin, tags: network } - { role: kubernetes-apps/policy_controller, tags: policy-controller } - - { role: kubernetes/client, tags: client } - hosts: calico-rr any_errors_fatal: "{{ any_errors_fatal | default(true) }}" diff --git a/extra_playbooks/upgrade-only-k8s.yml b/extra_playbooks/upgrade-only-k8s.yml index 90ee84ec9..b9263cb02 100644 --- a/extra_playbooks/upgrade-only-k8s.yml +++ b/extra_playbooks/upgrade-only-k8s.yml @@ -47,6 +47,8 @@ - { role: upgrade/pre-upgrade, tags: pre-upgrade } - { role: kubernetes/node, tags: node } - { role: kubernetes/master, tags: master } + - { role: kubernetes/client, tags: client } + - { role: kubernetes-apps/cluster_roles, tags: cluster-roles } - { role: upgrade/post-upgrade, tags: post-upgrade } #Finally handle worker upgrades, based on given batch size diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index 26a0a1f99..025b4fab6 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -5,26 +5,9 @@ register: result until: result.status == 200 retries: 10 - delay: 6 + delay: 2 when: inventory_hostname == groups['kube-master'][0] -- name: Kubernetes Apps | Add ClusterRoleBinding to admit nodes - template: - src: "node-crb.yml.j2" - dest: "{{ kube_config_dir }}/node-crb.yml" - register: node_crb_manifest - when: rbac_enabled - -- name: Apply workaround to allow all nodes with cert O=system:nodes to register - kube: - name: "system:node" - kubectl: "{{bin_dir}}/kubectl" - resource: "clusterrolebinding" - filename: "{{ kube_config_dir }}/node-crb.yml" - when: - - rbac_enabled - - node_crb_manifest.changed - - name: Kubernetes Apps | Delete old kubedns resources kube: name: "kubedns" diff --git a/roles/kubernetes-apps/cluster_roles/tasks/main.yml b/roles/kubernetes-apps/cluster_roles/tasks/main.yml new file mode 100644 index 000000000..24f94aac5 --- /dev/null +++ b/roles/kubernetes-apps/cluster_roles/tasks/main.yml @@ -0,0 +1,56 @@ +--- +- name: Kubernetes Apps | Wait for kube-apiserver + uri: + url: "{{ kube_apiserver_insecure_endpoint }}/healthz" + register: result + until: result.status == 200 + retries: 10 + delay: 6 + when: inventory_hostname == groups['kube-master'][0] + +- name: Kubernetes Apps | Add ClusterRoleBinding to admit nodes + template: + src: "node-crb.yml.j2" + dest: "{{ kube_config_dir }}/node-crb.yml" + register: node_crb_manifest + when: rbac_enabled + +- name: Apply workaround to allow all nodes with cert O=system:nodes to register + kube: + name: "system:node" + kubectl: "{{bin_dir}}/kubectl" + resource: "clusterrolebinding" + filename: "{{ kube_config_dir }}/node-crb.yml" + state: latest + when: + - rbac_enabled + - node_crb_manifest.changed + +# This is not a cluster role, but should be run after kubeconfig is set on master +- name: Write kube system namespace manifest + template: + src: namespace.j2 + dest: "{{kube_config_dir}}/{{system_namespace}}-ns.yml" + when: inventory_hostname == groups['kube-master'][0] + tags: + - apps + +- name: Check if kube system namespace exists + command: "{{ bin_dir }}/kubectl get ns {{system_namespace}}" + register: 'kubesystem' + changed_when: False + failed_when: False + when: inventory_hostname == groups['kube-master'][0] + tags: + - apps + +- name: Create kube system namespace + command: "{{ bin_dir }}/kubectl create -f {{kube_config_dir}}/{{system_namespace}}-ns.yml" + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + register: create_system_ns + until: create_system_ns.rc == 0 + changed_when: False + when: inventory_hostname == groups['kube-master'][0] and kubesystem.rc != 0 + tags: + - apps diff --git a/roles/kubernetes/master/templates/namespace.j2 b/roles/kubernetes-apps/cluster_roles/templates/namespace.j2 similarity index 100% rename from roles/kubernetes/master/templates/namespace.j2 rename to roles/kubernetes-apps/cluster_roles/templates/namespace.j2 diff --git a/roles/kubernetes-apps/ansible/templates/node-crb.yml.j2 b/roles/kubernetes-apps/cluster_roles/templates/node-crb.yml.j2 similarity index 100% rename from roles/kubernetes-apps/ansible/templates/node-crb.yml.j2 rename to roles/kubernetes-apps/cluster_roles/templates/node-crb.yml.j2 diff --git a/roles/kubernetes/master/tasks/static-pod-setup.yml b/roles/kubernetes/master/tasks/static-pod-setup.yml index a68ffb137..79f95d860 100644 --- a/roles/kubernetes/master/tasks/static-pod-setup.yml +++ b/roles/kubernetes/master/tasks/static-pod-setup.yml @@ -9,34 +9,6 @@ - meta: flush_handlers -- name: Write kube system namespace manifest - template: - src: namespace.j2 - dest: "{{kube_config_dir}}/{{system_namespace}}-ns.yml" - when: inventory_hostname == groups['kube-master'][0] - tags: - - apps - -- name: Check if kube system namespace exists - command: "{{ bin_dir }}/kubectl get ns {{system_namespace}}" - register: 'kubesystem' - changed_when: False - failed_when: False - when: inventory_hostname == groups['kube-master'][0] - tags: - - apps - -- name: Create kube system namespace - command: "{{ bin_dir }}/kubectl create -f {{kube_config_dir}}/{{system_namespace}}-ns.yml" - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - register: create_system_ns - until: create_system_ns.rc == 0 - changed_when: False - when: inventory_hostname == groups['kube-master'][0] and kubesystem.rc != 0 - tags: - - apps - - name: Write kube-scheduler kubeconfig template: src: kube-scheduler-kubeconfig.yaml.j2 diff --git a/tests/ansible.cfg b/tests/ansible.cfg index 780e1524b..9e734403e 100644 --- a/tests/ansible.cfg +++ b/tests/ansible.cfg @@ -8,4 +8,5 @@ gathering = smart fact_caching = jsonfile fact_caching_connection = /tmp stdout_callback = skippy +library = ./library:../library callback_whitelist = profile_tasks diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml index 747ed6023..652ae9a08 100644 --- a/upgrade-cluster.yml +++ b/upgrade-cluster.yml @@ -67,6 +67,8 @@ - { role: upgrade/pre-upgrade, tags: pre-upgrade } - { role: kubernetes/node, tags: node } - { role: kubernetes/master, tags: master } + - { role: kubernetes/client, tags: client } + - { role: kubernetes-apps/cluster_roles, tags: cluster-roles } - { role: network_plugin, tags: network } - { role: upgrade/post-upgrade, tags: post-upgrade }