From aad78840a08a0b36a844cbd828362e7cc25d9c65 Mon Sep 17 00:00:00 2001 From: David Louks <2402775+dlouks@users.noreply.github.com> Date: Tue, 9 Feb 2021 03:53:22 -0600 Subject: [PATCH] Updated etcd cert check tasks to detect when new cert gen is required (#7219) * Added force_etcd_cert_refresh var to maintain existing functionality. Broke out etcd node cert syncing from member and admin cert sync logic. Now first etcd will sync node certs to other etcd members on every run to keep all etcds up to date after adding additional worker nodes to the cluster * Updated etcd cert check tasks to better detect when new certificates need to be generated * Move usage of force_etcd_cert_refresh var to gen_certs fact set * Force etcd cert generation per server if force_etcd_cert_refresh is set to true * Include gathering of node certs even if k8s-cluster member and in etcd group. * Removed run_once due to when statement --- roles/etcd/defaults/main.yml | 1 + roles/etcd/tasks/check_certs.yml | 108 ++++++++++++++++++++------ roles/etcd/tasks/gen_certs_script.yml | 38 +++++++-- 3 files changed, 115 insertions(+), 32 deletions(-) diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index 8da2df988..6014a9ea0 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -12,6 +12,7 @@ etcd_data_dir: "/var/lib/etcd" # Number of etcd backups to retain. Set to a value < 0 to retain all backups etcd_backup_retention_count: -1 +force_etcd_cert_refresh: true etcd_config_dir: /etc/ssl/etcd etcd_cert_dir: "{{ etcd_config_dir }}/ssl" etcd_cert_dir_mode: "0700" diff --git a/roles/etcd/tasks/check_certs.yml b/roles/etcd/tasks/check_certs.yml index d3aaa9c23..21e79015a 100644 --- a/roles/etcd/tasks/check_certs.yml +++ b/roles/etcd/tasks/check_certs.yml @@ -1,8 +1,8 @@ --- -- name: "Check_certs | check if all certs have already been generated on first master" +- name: "Check_certs | Register certs that have already been generated on first etcd node" find: paths: "{{ etcd_cert_dir }}" - patterns: "ca.pem,node*.pem" + patterns: "ca.pem,node*.pem,member*.pem,admin*.pem" get_checksum: true delegate_to: "{{ groups['etcd'][0] }}" register: etcdcert_master @@ -14,40 +14,66 @@ gen_certs: false etcd_secret_changed: false -- name: "Check certs | check if a cert already exists on node" +- name: "Check certs | Register ca and etcd admin/member certs on etcd hosts" stat: path: "{{ etcd_cert_dir }}/{{ item }}" - register: etcdcert_node + register: etcd_member_certs + when: inventory_hostname in groups['etcd'] with_items: - ca.pem + - member-{{ inventory_hostname }}.pem + - member-{{ inventory_hostname }}-key.pem + - admin-{{ inventory_hostname }}.pem + - admin-{{ inventory_hostname }}-key.pem + +- name: "Check certs | Register ca and etcd node certs on kubernetes hosts" + stat: + path: "{{ etcd_cert_dir }}/{{ item }}" + register: etcd_node_certs + when: (('calico-rr' in groups and inventory_hostname in groups['calico-rr']) or + inventory_hostname in groups['k8s-cluster']) + with_items: + - ca.pem + - node-{{ inventory_hostname }}.pem - node-{{ inventory_hostname }}-key.pem -- name: "Check_certs | Set 'gen_certs' to true" +- name: "Check_certs | Set 'gen_certs' to true if expected certificates are not on the first etcd node" set_fact: gen_certs: true - when: not item in etcdcert_master.files|map(attribute='path') | list + when: force_etcd_cert_refresh or not item in etcdcert_master.files|map(attribute='path') | list run_once: true with_items: "{{ expected_files }}" vars: expected_files: >- ['{{ etcd_cert_dir }}/ca.pem', - {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort %} - {% for host in all_etcd_hosts %} - '{{ etcd_cert_dir }}/node-{{ host }}-key.pem', + {% set etcd_members = groups['etcd'] %} + {% for host in etcd_members %} + '{{ etcd_cert_dir }}/admin-{{ host }}.pem', '{{ etcd_cert_dir }}/admin-{{ host }}-key.pem', - '{{ etcd_cert_dir }}/member-{{ host }}-key.pem' + '{{ etcd_cert_dir }}/member-{{ host }}.pem', + '{{ etcd_cert_dir }}/member-{{ host }}-key.pem', + {% endfor %} + {% set k8s_nodes = groups['k8s-cluster']|union(groups['calico-rr']|default([]))|unique|sort %} + {% for host in k8s_nodes %} + '{{ etcd_cert_dir }}/node-{{ host }}.pem', + '{{ etcd_cert_dir }}/node-{{ host }}-key.pem' {% if not loop.last %}{{','}}{% endif %} {% endfor %}] -- name: "Check_certs | Set 'gen_master_certs' to true" +- name: "Check_certs | Set 'gen_master_certs' object to track whether member and admin certs exist on first etcd node" set_fact: gen_master_certs: |- { - {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort -%} + {% set etcd_members = groups['etcd'] -%} {% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %} - {% for host in all_etcd_hosts -%} - {% set host_cert = "%s/member-%s-key.pem"|format(etcd_cert_dir, host) %} - {% if host_cert in existing_certs -%} + {% for host in etcd_members -%} + {% set member_cert = "%s/member-%s.pem"|format(etcd_cert_dir, host) %} + {% set member_key = "%s/member-%s-key.pem"|format(etcd_cert_dir, host) %} + {% set admin_cert = "%s/admin-%s.pem"|format(etcd_cert_dir, host) %} + {% set admin_key = "%s/admin-%s-key.pem"|format(etcd_cert_dir, host) %} + {% if force_etcd_cert_refresh -%} + "{{ host }}": True, + {% elif member_cert in existing_certs and member_key in existing_certs and admin_cert in existing_certs and admin_key in existing_certs -%} "{{ host }}": False, {% else -%} "{{ host }}": True, @@ -56,15 +82,18 @@ } run_once: true -- name: "Check_certs | Set 'gen_node_certs' to true" +- name: "Check_certs | Set 'gen_node_certs' object to track whether node certs exist on first etcd node" set_fact: gen_node_certs: |- { - {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort -%} + {% set k8s_nodes = groups['k8s-cluster']|union(groups['calico-rr']|default([]))|unique|sort -%} {% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %} - {% for host in all_etcd_hosts -%} - {% set host_cert = "%s/node-%s-key.pem"|format(etcd_cert_dir, host) %} - {% if host_cert in existing_certs -%} + {% for host in k8s_nodes -%} + {% set host_cert = "%s/node-%s.pem"|format(etcd_cert_dir, host) %} + {% set host_key = "%s/node-%s-key.pem"|format(etcd_cert_dir, host) %} + {% if force_etcd_cert_refresh -%} + "{{ host }}": True, + {% elif host_cert in existing_certs and host_key in existing_certs -%} "{{ host }}": False, {% else -%} "{{ host }}": True, @@ -73,12 +102,41 @@ } run_once: true +- name: "Check_certs | Set 'etcd_member_requires_sync' to true if ca or member/admin cert and key don't exist on etcd member or checksum doesn't match" + set_fact: + etcd_member_requires_sync: true + when: + - inventory_hostname in groups['etcd'] + - (not etcd_member_certs.results[0].stat.exists|default(false)) or + (not etcd_member_certs.results[1].stat.exists|default(false)) or + (not etcd_member_certs.results[2].stat.exists|default(false)) or + (not etcd_member_certs.results[3].stat.exists|default(false)) or + (not etcd_member_certs.results[4].stat.exists|default(false)) or + (etcd_member_certs.results[0].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[0].stat.path)|map(attribute="checksum")|first|default('')) or + (etcd_member_certs.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[1].stat.path)|map(attribute="checksum")|first|default('')) or + (etcd_member_certs.results[2].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[2].stat.path)|map(attribute="checksum")|first|default('')) or + (etcd_member_certs.results[3].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[3].stat.path)|map(attribute="checksum")|first|default('')) or + (etcd_member_certs.results[4].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[4].stat.path)|map(attribute="checksum")|first|default('')) + +- name: "Check_certs | Set 'kubernetes_host_requires_sync' to true if ca or node cert and key don't exist on kubernetes host or checksum doesn't match" + set_fact: + kubernetes_host_requires_sync: true + when: + - (('calico-rr' in groups and inventory_hostname in groups['calico-rr']) or + inventory_hostname in groups['k8s-cluster']) and + inventory_hostname not in groups['etcd'] + - (not etcd_node_certs.results[0].stat.exists|default(false)) or + (not etcd_node_certs.results[1].stat.exists|default(false)) or + (not etcd_node_certs.results[2].stat.exists|default(false)) or + (etcd_node_certs.results[0].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[0].stat.path)|map(attribute="checksum")|first|default('')) or + (etcd_node_certs.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[1].stat.path)|map(attribute="checksum")|first|default('')) or + (etcd_node_certs.results[2].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[2].stat.path)|map(attribute="checksum")|first|default('')) + - name: "Check_certs | Set 'sync_certs' to true" set_fact: sync_certs: true when: - - gen_node_certs[inventory_hostname] or - gen_master_certs[inventory_hostname] or - (not etcdcert_node.results[0].stat.exists|default(false)) or - (not etcdcert_node.results[1].stat.exists|default(false)) or - (etcdcert_node.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcdcert_node.results[1].stat.path)|map(attribute="checksum")|first|default('')) + - etcd_member_requires_sync|default(false) or + kubernetes_host_requires_sync|default(false) or + (inventory_hostname in gen_master_certs and gen_master_certs[inventory_hostname]) or + (inventory_hostname in gen_node_certs and gen_node_certs[inventory_hostname]) diff --git a/roles/etcd/tasks/gen_certs_script.yml b/roles/etcd/tasks/gen_certs_script.yml index 36a8e2fc6..893e61c19 100644 --- a/roles/etcd/tasks/gen_certs_script.yml +++ b/roles/etcd/tasks/gen_certs_script.yml @@ -66,11 +66,10 @@ {% endfor %}" run_once: yes delegate_to: "{{ groups['etcd'][0] }}" - when: - - gen_certs|default(false) + when: gen_certs|default(false) notify: set etcd_secret_changed -- name: Gen_certs | Gather etcd master certs +- name: Gen_certs | Gather etcd member and admin certs from first etcd node slurp: src: "{{ item }}" register: etcd_master_certs @@ -83,6 +82,33 @@ '{{ etcd_cert_dir }}/member-{{ node }}.pem', '{{ etcd_cert_dir }}/member-{{ node }}-key.pem', {% endfor %}]" + delegate_to: "{{ groups['etcd'][0] }}" + when: + - inventory_hostname in groups['etcd'] + - sync_certs|default(false) + - inventory_hostname != groups['etcd'][0] + notify: set etcd_secret_changed + +- name: Gen_certs | Write etcd member and admin certs to other etcd nodes + copy: + dest: "{{ item.item }}" + content: "{{ item.content | b64decode }}" + group: "{{ etcd_cert_group }}" + owner: kube + mode: 0640 + with_items: "{{ etcd_master_certs.results }}" + when: + - inventory_hostname in groups['etcd'] + - sync_certs|default(false) + - inventory_hostname != groups['etcd'][0] + loop_control: + label: "{{ item.item }}" + +- name: Gen_certs | Gather node certs from first etcd node + slurp: + src: "{{ item }}" + register: etcd_master_node_certs + with_items: - "[{% for node in (groups['k8s-cluster'] + groups['calico-rr']|default([]))|unique %} '{{ etcd_cert_dir }}/node-{{ node }}.pem', '{{ etcd_cert_dir }}/node-{{ node }}-key.pem', @@ -90,21 +116,19 @@ delegate_to: "{{ groups['etcd'][0] }}" when: - inventory_hostname in groups['etcd'] - - sync_certs|default(false) - inventory_hostname != groups['etcd'][0] notify: set etcd_secret_changed -- name: Gen_certs | Write etcd master certs +- name: Gen_certs | Write node certs to other etcd nodes copy: dest: "{{ item.item }}" content: "{{ item.content | b64decode }}" group: "{{ etcd_cert_group }}" owner: kube mode: 0640 - with_items: "{{ etcd_master_certs.results }}" + with_items: "{{ etcd_master_node_certs.results }}" when: - inventory_hostname in groups['etcd'] - - sync_certs|default(false) - inventory_hostname != groups['etcd'][0] loop_control: label: "{{ item.item }}"