From 39d247a2384339ed8969b9bd78634080583fded2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20Kr=C3=BCger?= <andreas@kruger.nu>
Date: Tue, 13 Mar 2018 10:31:15 +0100
Subject: [PATCH 1/7] Add support to kubeadm too

Explicitly defines the --kubelet-preferred-address-types parameter #2418

Fixes #2453
---
 roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2
index ed1cc7add..e4657a601 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2
@@ -37,6 +37,7 @@ apiServerExtraArgs:
   admission-control: {{ kube_apiserver_admission_control | join(',') }}
   apiserver-count: "{{ kube_apiserver_count }}"
   service-node-port-range: {{ kube_apiserver_node_port_range }}
+  kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
 {% if kube_basic_auth|default(true) %}
   basic-auth-file: {{ kube_users_dir }}/known_users.csv
 {% endif %}

From d1e6632e6ae02c6e9c2246cc3aff892c1c8d2b81 Mon Sep 17 00:00:00 2001
From: zhengchuan hu <zhengchuan.hu@easystack.cn>
Date: Wed, 14 Mar 2018 17:18:55 +0800
Subject: [PATCH 2/7] Fix err in kubelet.kubeadm.env.j2

1. 404 link url
2. kubelet_authentication_token_webhook is not work
3. kube_reserved variable set twice
---
 roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
index f67c72bf8..5be20d533 100644
--- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
+++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
@@ -1,4 +1,4 @@
-### Upstream source https://github.com/kubernetes/release/blob/master/debian/xenial/kubeadm/channel/stable/etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+### Upstream source https://github.com/kubernetes/release/blob/master/debian/xenial/kubeadm/channel/stable/etc/systemd/system/kubelet.service.d/
 ### All upstream values should be present in this file
 
 # logging to stderr means we get it in the systemd journal
@@ -23,13 +23,14 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% if kubelet_authentication_token_webhook %}
 --authentication-token-webhook \
 {% endif %}
+{% if kubelet_authorization_mode_webhook %}
 --authorization-mode=Webhook \
+{% endif %}
 --client-ca-file={{ kube_cert_dir }}/ca.crt \
 --pod-manifest-path={{ kube_manifest_dir }} \
 --cadvisor-port={{ kube_cadvisor_port }} \
 {# end kubeadm specific settings #}
 --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
---kube-reserved cpu={{ kube_cpu_reserved }},memory={{ kube_memory_reserved|regex_replace('Mi', 'M') }} \
 --node-status-update-frequency={{ kubelet_status_update_frequency }} \
 --cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \
 --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \

From 788e41a315b36c86d60e68fa71f3493bb286c895 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20Kr=C3=BCger?= <andreas@kruger.nu>
Date: Wed, 14 Mar 2018 19:23:43 +0100
Subject: [PATCH 3/7] Make sure output from extra args is strings

Setting the following:

```
kube_kubeadm_controller_extra_args:
  address: 0.0.0.0
  terminated-pod-gc-threshold: "100"
```

Results in `terminated-pod-gc-threshold: 100` in the kubeadm config file. But it has to be a string to work.
---
 roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2
index ed1cc7add..a8ffbbb17 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2
@@ -59,7 +59,7 @@ apiServerExtraArgs:
 {% endif %}
   allow-privileged: "true"
 {% for key in kube_kubeadm_apiserver_extra_args %}
-  {{ key }}: {{ kube_kubeadm_apiserver_extra_args[key] }}
+  {{ key }}: {{ kube_kubeadm_apiserver_extra_args[key]|string }}
 {% endfor %}
 controllerManagerExtraArgs:
   node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
@@ -69,12 +69,12 @@ controllerManagerExtraArgs:
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
 {% for key in kube_kubeadm_controller_extra_args %}
-  {{ key }}: {{ kube_kubeadm_controller_extra_args[key] }}
+  {{ key }}: {{ kube_kubeadm_controller_extra_args[key]|string }}
 {% endfor %}
 {% if kube_kubeadm_scheduler_extra_args|length > 0 %}
 schedulerExtraArgs:
 {% for key in kube_kubeadm_scheduler_extra_args %}
-  {{ key }}: {{ kube_kubeadm_scheduler_extra_args[key] }}
+  {{ key }}: {{ kube_kubeadm_scheduler_extra_args[key]|string }}
 {% endfor %}
 {% endif %}
 apiServerCertSANs:

From d843e3d562080681d4da7fe218211c8b829d37e6 Mon Sep 17 00:00:00 2001
From: Oleg Vyukov <vyukov.oleg@gmail.com>
Date: Thu, 15 Mar 2018 22:18:18 +0300
Subject: [PATCH 4/7] Fix indent  Custom ConfigMap ingress-nginx (#2447)

---
 .../ingress_nginx/templates/ingress-nginx-cm.yml.j2             | 2 +-
 .../templates/ingress-nginx-tcp-servicecs-cm.yml.j2             | 2 +-
 .../templates/ingress-nginx-udp-servicecs-cm.yml.j2             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2
index 79b9e17e7..7e47e81b1 100644
--- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2
+++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2
@@ -7,4 +7,4 @@ metadata:
   labels:
     k8s-app: ingress-nginx
 data:
-  {{ ingress_nginx_configmap | to_nice_yaml }}
+  {{ ingress_nginx_configmap | to_nice_yaml | indent(2) }}
diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2
index 5fb875940..0a87e91b7 100644
--- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2
+++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2
@@ -7,4 +7,4 @@ metadata:
   labels:
     k8s-app: ingress-nginx
 data:
-  {{ ingress_nginx_configmap_tcp_services | to_nice_yaml }}
+  {{ ingress_nginx_configmap_tcp_services | to_nice_yaml | indent(2) }}
diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2
index bcb004bc9..d943e5718 100644
--- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2
+++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2
@@ -7,4 +7,4 @@ metadata:
   labels:
     k8s-app: ingress-nginx
 data:
-  {{ ingress_nginx_configmap_udp_services | to_nice_yaml }}
+  {{ ingress_nginx_configmap_udp_services | to_nice_yaml | indent(2) }}

From 3d6fd491795adb8a38493afe6c2968a46051d5ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20Kr=C3=BCger?= <andreas@kruger.nu>
Date: Thu, 15 Mar 2018 20:20:05 +0100
Subject: [PATCH 5/7] Added option for encrypting secrets to etcd v.2 (#2428)

* Added option for encrypting secrets to etcd

* Fix keylength to 32

* Forgot the default

* Rename secrets.yaml to secrets_encryption.yaml

* Fix static path for secrets file to use ansible variable

* Rename secrets.yaml.j2 to secrets_encryption.yaml.j2

* Base64 encode the token

* Fixed merge error

* Changed path to credentials dir

* Update path to secrets file which is now readable inside the apiserver container. Set better file permissions

* Add encryption option to k8s-cluster.yml
---
 inventory/sample/group_vars/k8s-cluster.yml           |  5 ++++-
 roles/kubernetes/master/defaults/main.yml             |  5 +++++
 roles/kubernetes/master/tasks/encrypt-at-rest.yml     | 10 ++++++++++
 roles/kubernetes/master/tasks/main.yml                |  3 +++
 .../master/templates/kubeadm-config.yaml.j2           |  3 +++
 .../templates/manifests/kube-apiserver.manifest.j2    |  3 +++
 .../master/templates/secrets_encryption.yaml.j2       | 11 +++++++++++
 7 files changed, 39 insertions(+), 1 deletion(-)
 create mode 100644 roles/kubernetes/master/tasks/encrypt-at-rest.yml
 create mode 100644 roles/kubernetes/master/templates/secrets_encryption.yaml.j2

diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml
index 128e8cc99..8f69afc25 100644
--- a/inventory/sample/group_vars/k8s-cluster.yml
+++ b/inventory/sample/group_vars/k8s-cluster.yml
@@ -111,7 +111,10 @@ kube_apiserver_insecure_port: 8080 # (http)
 
 # Kube-proxy proxyMode configuration.
 # Can be ipvs, iptables
-kube_proxy_mode: iptables 
+kube_proxy_mode: iptables
+
+## Encrypting Secret Data at Rest (experimental)
+kube_encrypt_secret_data: false
 
 # DNS configuration.
 # Kubernetes cluster name, also will be used as DNS domain
diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml
index 59e528822..a1b506d4e 100644
--- a/roles/kubernetes/master/defaults/main.yml
+++ b/roles/kubernetes/master/defaults/main.yml
@@ -92,3 +92,8 @@ kube_kubeadm_scheduler_extra_args: {}
 
 ## Variable for influencing kube-scheduler behaviour
 volume_cross_zone_attachment: false
+
+## Encrypting Secret Data at Rest
+kube_encrypt_secret_data: false
+kube_encrypt_token: "{{ lookup('password', inventory_dir + '/credentials/kube_encrypt_token length=32 chars=ascii_letters,digits') }}"
+kube_encryption_algorithm: "aescbc" # Must be either: aescbc, secretbox or aesgcm
diff --git a/roles/kubernetes/master/tasks/encrypt-at-rest.yml b/roles/kubernetes/master/tasks/encrypt-at-rest.yml
new file mode 100644
index 000000000..2e569b08b
--- /dev/null
+++ b/roles/kubernetes/master/tasks/encrypt-at-rest.yml
@@ -0,0 +1,10 @@
+---
+- name: Write secrets for encrypting secret data at rest
+  template:
+    src: secrets_encryption.yaml.j2
+    dest: "{{ kube_config_dir }}/ssl/secrets_encryption.yaml"
+    owner: root
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  tags:
+    - kube-apiserver
diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml
index 04ad307fd..daa10fd79 100644
--- a/roles/kubernetes/master/tasks/main.yml
+++ b/roles/kubernetes/master/tasks/main.yml
@@ -12,6 +12,9 @@
 - import_tasks: users-file.yml
   when: kube_basic_auth|default(true)
 
+- import_tasks: encrypt-at-rest.yml
+  when: kube_encrypt_secret_data
+
 - name: Compare host kubectl with hyperkube container
   command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/cmp /hyperkube /systembindir/kubectl"
   register: kubectl_task_compare_result
diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2
index ed1cc7add..cd266ed3d 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2
@@ -52,6 +52,9 @@ apiServerExtraArgs:
 {%   if kube_oidc_groups_claim is defined %}
   oidc-groups-claim: {{ kube_oidc_groups_claim }}
 {%   endif %}
+{% endif %}
+{% if kube_encrypt_secret_data %}
+  experimental-encryption-provider-config: {{ kube_config_dir }}/ssl/secrets_encryption.yaml
 {% endif %}
   storage-backend: {{ kube_apiserver_storage_backend }}
 {% if kube_api_runtime_config is defined %}
diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
index 0dbe93cab..c1685410d 100644
--- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
@@ -103,6 +103,9 @@ spec:
 {% if authorization_modes %}
     - --authorization-mode={{ authorization_modes|join(',') }}
 {% endif %}
+{% if kube_encrypt_secret_data %}
+    - --experimental-encryption-provider-config={{ kube_config_dir }}/ssl/secrets_encryption.yaml
+{% endif %}
 {% if kube_feature_gates %}
     - --feature-gates={{ kube_feature_gates|join(',') }}
 {% endif %}
diff --git a/roles/kubernetes/master/templates/secrets_encryption.yaml.j2 b/roles/kubernetes/master/templates/secrets_encryption.yaml.j2
new file mode 100644
index 000000000..84c6a4ea8
--- /dev/null
+++ b/roles/kubernetes/master/templates/secrets_encryption.yaml.j2
@@ -0,0 +1,11 @@
+kind: EncryptionConfig
+apiVersion: v1
+resources:
+  - resources:
+    - secrets
+    providers:
+    - {{ kube_encryption_algorithm }}:
+        keys:
+        - name: key
+          secret: {{ kube_encrypt_token | b64encode }}
+    - identity: {}

From 40c0f3756bbabbbf8b9f05eaf3e86bce600a7e11 Mon Sep 17 00:00:00 2001
From: woopstar <andreas@kruger.nu>
Date: Thu, 15 Mar 2018 20:27:19 +0100
Subject: [PATCH 6/7] Encapsulate item instead of casting to string

---
 roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2
index a8ffbbb17..bbe329b5f 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2
@@ -59,7 +59,7 @@ apiServerExtraArgs:
 {% endif %}
   allow-privileged: "true"
 {% for key in kube_kubeadm_apiserver_extra_args %}
-  {{ key }}: {{ kube_kubeadm_apiserver_extra_args[key]|string }}
+  {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
 {% endfor %}
 controllerManagerExtraArgs:
   node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
@@ -69,12 +69,12 @@ controllerManagerExtraArgs:
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
 {% for key in kube_kubeadm_controller_extra_args %}
-  {{ key }}: {{ kube_kubeadm_controller_extra_args[key]|string }}
+  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
 {% endfor %}
 {% if kube_kubeadm_scheduler_extra_args|length > 0 %}
 schedulerExtraArgs:
 {% for key in kube_kubeadm_scheduler_extra_args %}
-  {{ key }}: {{ kube_kubeadm_scheduler_extra_args[key]|string }}
+  {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
 {% endfor %}
 {% endif %}
 apiServerCertSANs:

From 1a35948ff67369d856b08d8ccb747374330ab82d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20Kr=C3=BCger?= <andreas@kruger.nu>
Date: Thu, 15 Mar 2018 20:33:57 +0100
Subject: [PATCH 7/7] Enable encrypting the secrets

Enable the CI test to check the encryption of secrets
---
 tests/files/gce_centos7-flannel-addons.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/files/gce_centos7-flannel-addons.yml b/tests/files/gce_centos7-flannel-addons.yml
index 0e4346f67..8ac8a901b 100644
--- a/tests/files/gce_centos7-flannel-addons.yml
+++ b/tests/files/gce_centos7-flannel-addons.yml
@@ -15,3 +15,4 @@ etcd_deployment_type: host
 deploy_netchecker: true
 kubedns_min_replicas: 1
 cloud_provider: gce
+kube_encrypt_secret_data: true