More idempotency fixes

Fixed sync_tokens fact
Fixed sync_certs for k8s tokens fact
Disabled register docker images changability
Fixed CNI dir permission
Fix idempotency for etcd pre upgrade checks

roles/download/tasks/set_docker_image_facts.yml

@@ -13,6 +13,7 @@
   no_log: true
   register: docker_images_raw
   failed_when: false
+  changed_when: false
   check_mode: no
   when: not download_always_pull|bool
 

roles/etcd/tasks/check_certs.yml

@@ -3,6 +3,7 @@
   find:
     paths: "{{ etcd_cert_dir }}"
     patterns: "ca.pem,node*.pem"
+    get_checksum: true
   delegate_to: "{{groups['etcd'][0]}}"
   register: etcdcert_master
   run_once: true
@@ -58,7 +59,7 @@
     sync_certs: true
   when: >-
       {%- set certs = {'sync': False} -%}
-      {% if gen_node_certs[inventory_hostname] or 
+      {% if gen_node_certs[inventory_hostname] or
         (not etcdcert_node.results[0].stat.exists|default(False)) or
           (not etcdcert_node.results[1].stat.exists|default(False)) or
             (etcdcert_node.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcdcert_node.results[1].stat.path)|first|map(attribute="checksum")|default('')) -%}

roles/etcd/tasks/gen_certs_script.yml

@@ -107,38 +107,38 @@
         sync_certs|default(false) and inventory_hostname not in groups['etcd']
   notify: set etcd_secret_changed
 
-#NOTE(mattymo): Use temporary file to copy master certs because we have a ~200k 
-#char limit when using shell command                                            
-                                                                                
-#FIXME(mattymo): Use tempfile module in ansible 2.3                             
-- name: Gen_certs | Prepare tempfile for unpacking certs                        
-  shell: mktemp /tmp/certsXXXXX.tar.gz                                          
-  register: cert_tempfile                                                       
-                                                                                
-- name: Gen_certs | Write master certs to tempfile                              
-  copy:                                                                         
-    content: "{{etcd_master_cert_data.stdout}}"                                      
-    dest: "{{cert_tempfile.stdout}}"                                            
-    owner: root                                                                 
-    mode: "0600"                                                                
+#NOTE(mattymo): Use temporary file to copy master certs because we have a ~200k
+#char limit when using shell command
+
+#FIXME(mattymo): Use tempfile module in ansible 2.3
+- name: Gen_certs | Prepare tempfile for unpacking certs
+  shell: mktemp /tmp/certsXXXXX.tar.gz
+  register: cert_tempfile
+
+- name: Gen_certs | Write master certs to tempfile
+  copy:
+    content: "{{etcd_master_cert_data.stdout}}"
+    dest: "{{cert_tempfile.stdout}}"
+    owner: root
+    mode: "0600"
   when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
-        inventory_hostname != groups['etcd'][0]                          
-                                                                                
-- name: Gen_certs | Unpack certs on masters                                     
+        inventory_hostname != groups['etcd'][0]
+
+- name: Gen_certs | Unpack certs on masters
   shell: "base64 -d < {{ cert_tempfile.stdout }} | tar xz -C {{ etcd_cert_dir }}"
-  no_log: true                                                                  
-  changed_when: false                                                           
-  check_mode: no                                                                
+  no_log: true
+  changed_when: false
+  check_mode: no
   when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
-        inventory_hostname != groups['etcd'][0]                          
-  notify: set secret_changed                                                    
-                                                                                
-- name: Gen_certs | Cleanup tempfile                                            
-  file:                                                                         
-    path: "{{cert_tempfile.stdout}}"                                            
-    state: absent                                                               
+        inventory_hostname != groups['etcd'][0]
+  notify: set secret_changed
+
+- name: Gen_certs | Cleanup tempfile
+  file:
+    path: "{{cert_tempfile.stdout}}"
+    state: absent
   when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
-        inventory_hostname != groups['etcd'][0]  
+        inventory_hostname != groups['etcd'][0]
 
 - name: Gen_certs | Copy certs on nodes
   shell: "base64 -d <<< '{{etcd_node_cert_data.stdout|quote}}' | tar xz -C {{ etcd_cert_dir }}"

roles/etcd/tasks/main.yml

@@ -16,7 +16,7 @@
   tags: etcd-secrets
 
 - include: sync_etcd_node_certs.yml
-  when: cert_management == "vault" and inventory_hostname in etcd_node_cert_hosts 
+  when: cert_management == "vault" and inventory_hostname in etcd_node_cert_hosts
   tags: etcd-secrets
 
 - include: gen_certs_vault.yml

roles/etcd/tasks/pre_upgrade.yml

@@ -28,6 +28,7 @@
 - name: "Pre-upgrade | find etcd-proxy container"
   command: "{{ docker_bin_dir }}/docker ps -aq --filter 'name=etcd-proxy*'"
   register: etcd_proxy_container
+  changed_when: false
   failed_when: false
 
 - name: "Pre-upgrade | remove etcd-proxy if it exists"
@@ -47,6 +48,7 @@
   until: etcd_member_list.rc != 2
   run_once: true
   when: etcdctl_installed.stat.exists
+  changed_when: false
   failed_when: false
 
 - name: "Pre-upgrade | change peer names to SSL"

roles/kubernetes/master/tasks/main.yml

@@ -13,7 +13,6 @@
 
 - name: Install kubectl bash completion
   shell: "{{ bin_dir }}/kubectl completion bash >/etc/bash_completion.d/kubectl.sh"
-  #no_log: true
   when: ansible_os_family in ["Debian","RedHat"]
   tags: kubectl
 

roles/kubernetes/secrets/tasks/check-certs.yml

@@ -3,6 +3,7 @@
   find:
     paths: "{{ kube_cert_dir }}"
     patterns: "*.pem"
+    get_checksum: true
   delegate_to: "{{groups['kube-master'][0]}}"
   register: kubecert_master
   run_once: true
@@ -58,7 +59,7 @@
       {% if gen_node_certs[inventory_hostname] or    
         (not kubecert_node.results[0].stat.exists|default(False)) or
           (not kubecert_node.results[1].stat.exists|default(False)) or
-            (kubecert_node.results[1].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[1].stat.path)|first|map(attribute="checksum")|default('')) -%}
+            (kubecert_node.results[1].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[1].stat.path)|map(attribute="checksum")|first|default('')) -%}
               {%- set _ = certs.update({'sync': True}) -%}
       {% endif %}
       {{ certs.sync }}

roles/kubernetes/secrets/tasks/check-tokens.yml

@@ -19,7 +19,7 @@
 
 - name: "Check tokens | check if a cert already exists"
   stat:
-    path: "{{ kube_cert_dir }}/ca.pem"
+    path: "{{ kube_token_dir }}/known_tokens.csv"
   register: known_tokens
 
 - name: "Check_tokens | Set 'sync_tokens' to true"

roles/kubernetes/secrets/tasks/gen_certs_script.yml

@@ -106,6 +106,8 @@
 - name: Gen_certs | Prepare tempfile for unpacking certs
   shell: mktemp /tmp/certsXXXXX.tar.gz
   register: cert_tempfile
+  when: inventory_hostname in groups['kube-master'] and sync_certs|default(false) and
+        inventory_hostname != groups['kube-master'][0]
 
 - name: Gen_certs | Write master certs to tempfile
   copy:
@@ -149,13 +151,9 @@
     path: "{{ kube_cert_dir }}"
     group: "{{ kube_cert_group }}"
     owner: kube
+    mode: "u=rwX,g-rwx,o-rwx"
     recurse: yes
 
-- name: Gen_certs | set permissions on keys
-  shell: chmod 0600 {{ kube_cert_dir}}/*key.pem
-  when: inventory_hostname in groups['kube-master']
-  changed_when: false
-
 - name: Gen_certs | target ca-certificates path
   set_fact:
     ca_cert_path: |-

roles/kubernetes/secrets/tasks/gen_tokens.yml

@@ -39,9 +39,9 @@
 - name: Gen_tokens | Get list of tokens from first master
   shell: "(find {{ kube_token_dir }} -maxdepth 1 -type f)"
   register: tokens_list
-  changed_when: false
   check_mode: no
   delegate_to: "{{groups['kube-master'][0]}}"
+  run_once: true
   when: sync_tokens|default(false)
 
 - name: Gen_tokens | Gather tokens
@@ -54,6 +54,5 @@
 
 - name: Gen_tokens | Copy tokens on masters
   shell: "echo '{{ tokens_data.stdout|quote }}' | base64 -d | tar xz -C /"
-  changed_when: false
   when: inventory_hostname in groups['kube-master'] and sync_tokens|default(false) and
         inventory_hostname != groups['kube-master'][0]

roles/network_plugin/calico/tasks/main.yml

@@ -41,7 +41,7 @@
   notify: restart calico-node
 
 - name: Calico | Copy cni plugins from hyperkube
-  command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/"
+  command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -ac /opt/cni/bin/ /cnibindir/"
   register: cni_task_result
   until: cni_task_result.rc == 0
   retries: 4
@@ -59,6 +59,14 @@
   when: "{{ overwrite_hyperkube_cni|bool }}"
   tags: [hyperkube, upgrade]
 
+- name: Calico | Set cni directory permissions
+  file:
+    path: /opt/cni/bin
+    state: directory
+    owner: kube
+    recurse: true
+    mode: 0755
+
 - name: Calico | wait for etcd
   uri:
     url: https://localhost:2379/health
@@ -80,6 +88,7 @@
   register: calico_conf
   delegate_to: "{{groups['etcd'][0]}}"
   run_once: true
+  changed_when: false
 
 - name: Calico | Configure calico network pool
   shell: >

roles/network_plugin/canal/tasks/main.yml

@@ -44,7 +44,7 @@
   register: canal_node_manifest
 
 - name: Canal | Copy cni plugins from hyperkube
-  command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/"
+  command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -ac /opt/cni/bin/ /cnibindir/"
   register: cni_task_result
   until: cni_task_result.rc == 0
   retries: 4
@@ -61,6 +61,14 @@
   changed_when: false
   tags: [hyperkube, upgrade]
 
+- name: Canal | Set cni directory permissions
+  file:
+    path: /opt/cni/bin
+    state: directory
+    owner: kube
+    recurse: true
+    mode: 0755
+
 - name: Canal | Install calicoctl container script
   template:
     src: calicoctl-container.j2

roles/network_plugin/cloud/tasks/main.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: Cloud | Copy cni plugins from hyperkube
   command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/"
   register: cni_task_result
@@ -7,3 +6,12 @@
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   changed_when: false
+
+- name: Cloud | Set cni directory permissions
+  file:
+    path: /opt/cni/bin
+    state: directory
+    owner: kube
+    recurse: true
+    mode: "u=rwX,g-rwx,o-rwx"
+