From a8b66fd207cd4768df49d11be6be0633cf69d658 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 10 Sep 2024 13:43:02 +0200
Subject: [PATCH 1/4] Deduplicate kubeadm_patches tasks

kubernetes/control-plane and kubernetes/kubeadm roles both push kubeadm
patches in the same way.
Extract that code and make it a dependency of both.
This is safe because it's only configuration for kubeadm, which only
takes effect when kubeadm is run.
---
 roles/kubernetes/control-plane/meta/main.yml      |  1 +
 .../control-plane/tasks/kubeadm-setup.yml         | 15 ---------------
 roles/kubernetes/kubeadm/meta/main.yml            |  3 +++
 roles/kubernetes/kubeadm/tasks/main.yml           | 15 ---------------
 roles/kubernetes/kubeadm_common/tasks/main.yml    | 15 +++++++++++++++
 5 files changed, 19 insertions(+), 30 deletions(-)
 create mode 100644 roles/kubernetes/kubeadm/meta/main.yml
 create mode 100644 roles/kubernetes/kubeadm_common/tasks/main.yml

diff --git a/roles/kubernetes/control-plane/meta/main.yml b/roles/kubernetes/control-plane/meta/main.yml
index 7d793f92f..7b2cfe365 100644
--- a/roles/kubernetes/control-plane/meta/main.yml
+++ b/roles/kubernetes/control-plane/meta/main.yml
@@ -1,5 +1,6 @@
 ---
 dependencies:
+  - role: kubernetes/kubeadm_common
   - role: kubernetes/tokens
     when: kube_token_auth
     tags:
diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
index c13b6e833..d7783036b 100644
--- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
+++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
@@ -176,21 +176,6 @@
     - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed
     - not kube_external_ca_mode
 
-- name: Kubeadm | Create directory to store kubeadm patches
-  file:
-    path: "{{ kubeadm_patches.dest_dir }}"
-    state: directory
-    mode: "0640"
-  when: kubeadm_patches is defined and kubeadm_patches.enabled
-
-- name: Kubeadm | Copy kubeadm patches from inventory files
-  copy:
-    src: "{{ kubeadm_patches.source_dir }}/"
-    dest: "{{ kubeadm_patches.dest_dir }}"
-    owner: "root"
-    mode: "0644"
-  when: kubeadm_patches is defined and kubeadm_patches.enabled
-
 - name: Kubeadm | Initialize first control plane node
   command: >-
     timeout -k {{ kubeadm_init_timeout }} {{ kubeadm_init_timeout }}
diff --git a/roles/kubernetes/kubeadm/meta/main.yml b/roles/kubernetes/kubeadm/meta/main.yml
new file mode 100644
index 000000000..4e7278559
--- /dev/null
+++ b/roles/kubernetes/kubeadm/meta/main.yml
@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: kubernetes/kubeadm_common
diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml
index 9e01f5fe5..cb29e2a6a 100644
--- a/roles/kubernetes/kubeadm/tasks/main.yml
+++ b/roles/kubernetes/kubeadm/tasks/main.yml
@@ -83,21 +83,6 @@
     mode: "0640"
   when: ('kube_control_plane' not in group_names)
 
-- name: Kubeadm | Create directory to store kubeadm patches
-  file:
-    path: "{{ kubeadm_patches.dest_dir }}"
-    state: directory
-    mode: "0640"
-  when: kubeadm_patches is defined and kubeadm_patches.enabled
-
-- name: Kubeadm | Copy kubeadm patches from inventory files
-  copy:
-    src: "{{ kubeadm_patches.source_dir }}/"
-    dest: "{{ kubeadm_patches.dest_dir }}"
-    owner: "root"
-    mode: "0644"
-  when: kubeadm_patches is defined and kubeadm_patches.enabled
-
 - name: Join to cluster if needed
   environment:
     PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}:/sbin"
diff --git a/roles/kubernetes/kubeadm_common/tasks/main.yml b/roles/kubernetes/kubeadm_common/tasks/main.yml
new file mode 100644
index 000000000..b1f316e22
--- /dev/null
+++ b/roles/kubernetes/kubeadm_common/tasks/main.yml
@@ -0,0 +1,15 @@
+---
+- name: Kubeadm | Create directory to store kubeadm patches
+  file:
+    path: "{{ kubeadm_patches.dest_dir }}"
+    state: directory
+    mode: "0640"
+  when: kubeadm_patches is defined and kubeadm_patches.enabled
+
+- name: Kubeadm | Copy kubeadm patches from inventory files
+  copy:
+    src: "{{ kubeadm_patches.source_dir }}/"
+    dest: "{{ kubeadm_patches.dest_dir }}"
+    owner: "root"
+    mode: "0644"
+  when: kubeadm_patches is defined and kubeadm_patches.enabled

From 8e254ec1e82377f4401775f5111353a4e0b43abb Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 10 Sep 2024 12:00:26 +0200
Subject: [PATCH 2/4] kubeadm: allow to provide patch inline in inventories

Specifying one directory for kubeadm patches is not ideal:
1. It does not allow working with multiples inventories easily
2. No ansible templating of the patch
3. Ansible path searching can sometimes be confusing

Instead, provide the patch directly in a variable, and add some quality
of life to handle components targeting and patch ordering more
explicitly (`target` and `type` which are translated to the kubeadm
scheme which is based on the file name)
---
 .../control-plane/tasks/kubeadm-upgrade.yml        |  4 ++--
 .../templates/kubeadm-config.v1beta3.yaml.j2       |  4 ++--
 .../templates/kubeadm-controlplane.v1beta3.yaml.j2 |  4 ++--
 .../templates/kubeadm-client.conf.v1beta3.j2       |  4 ++--
 roles/kubernetes/kubeadm_common/defaults/main.yml  | 14 ++++++++++++++
 roles/kubernetes/kubeadm_common/tasks/main.yml     | 12 +++++++-----
 6 files changed, 29 insertions(+), 13 deletions(-)
 create mode 100644 roles/kubernetes/kubeadm_common/defaults/main.yml

diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml
index 343724c47..9609c2f3d 100644
--- a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml
+++ b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml
@@ -18,7 +18,7 @@
     --ignore-preflight-errors=all
     --allow-experimental-upgrades
     --etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | bool | lower }}
-    {% if kubeadm_patches is defined and kubeadm_patches.enabled %}--patches={{ kubeadm_patches.dest_dir }}{% endif %}
+    {% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %}
     --force
   register: kubeadm_upgrade
   # Retry is because upload config sometimes fails
@@ -39,7 +39,7 @@
     --ignore-preflight-errors=all
     --allow-experimental-upgrades
     --etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | bool | lower }}
-    {% if kubeadm_patches is defined and kubeadm_patches.enabled %}--patches={{ kubeadm_patches.dest_dir }}{% endif %}
+    {% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %}
     --force
   register: kubeadm_upgrade
   # Retry is because upload config sometimes fails
diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2
index ca48a3a91..9dd5e4376 100644
--- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2
+++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2
@@ -28,9 +28,9 @@ nodeRegistration:
   kubeletExtraArgs:
     cloud-provider: external
 {% endif %}
-{% if kubeadm_patches is defined and kubeadm_patches.enabled %}
+{% if kubeadm_patches | length > 0 %}
 patches:
-  directory: {{ kubeadm_patches.dest_dir }}
+  directory: {{ kubeadm_patches_dir }}
 {% endif %}
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
diff --git a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2
index cd19b5c2e..bc9f3bdf9 100644
--- a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2
+++ b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2
@@ -31,7 +31,7 @@ nodeRegistration:
 {% else %}
   taints: []
 {% endif %}
-{% if kubeadm_patches is defined and kubeadm_patches.enabled %}
+{% if kubeadm_patches | length > 0 %}
 patches:
-  directory: {{ kubeadm_patches.dest_dir }}
+  directory: {{ kubeadm_patches_dir }}
 {% endif %}
diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2
index 3b3bc57de..5016df9c3 100644
--- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2
+++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2
@@ -38,7 +38,7 @@ nodeRegistration:
   - effect: NoSchedule
     key: node-role.kubernetes.io/calico-rr
 {% endif %}
-{% if kubeadm_patches is defined and kubeadm_patches.enabled %}
+{% if kubeadm_patches | length > 0 %}
 patches:
-  directory: {{ kubeadm_patches.dest_dir }}
+  directory: {{ kubeadm_patches_dir }}
 {% endif %}
diff --git a/roles/kubernetes/kubeadm_common/defaults/main.yml b/roles/kubernetes/kubeadm_common/defaults/main.yml
new file mode 100644
index 000000000..f7d70691a
--- /dev/null
+++ b/roles/kubernetes/kubeadm_common/defaults/main.yml
@@ -0,0 +1,14 @@
+---
+kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
+kubeadm_patches: []
+# kubeadm_patches:
+# - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration
+#   type: strategic(default)|json|merge
+#   patch:
+#    metadata:
+#      annotations:
+#        example.com/test: "true"
+#      labels:
+#        example.com/prod_level: "{{ prod_level }}"
+# - ...
+# Patches are applied in the order they are specified.
diff --git a/roles/kubernetes/kubeadm_common/tasks/main.yml b/roles/kubernetes/kubeadm_common/tasks/main.yml
index b1f316e22..0f8d3b0a0 100644
--- a/roles/kubernetes/kubeadm_common/tasks/main.yml
+++ b/roles/kubernetes/kubeadm_common/tasks/main.yml
@@ -1,15 +1,17 @@
 ---
 - name: Kubeadm | Create directory to store kubeadm patches
   file:
-    path: "{{ kubeadm_patches.dest_dir }}"
+    path: "{{ kubeadm_patches_dir }}"
     state: directory
     mode: "0640"
-  when: kubeadm_patches is defined and kubeadm_patches.enabled
+  when: kubeadm_patches | length > 0
 
 - name: Kubeadm | Copy kubeadm patches from inventory files
   copy:
-    src: "{{ kubeadm_patches.source_dir }}/"
-    dest: "{{ kubeadm_patches.dest_dir }}"
+    content: "{{ item.patch | to_yaml }}"
+    dest: "{{ kubeadm_patches_dir }}/{{ item.target }}{{ suffix }}+{{ item.type | d('strategic') }}.yaml"
     owner: "root"
     mode: "0644"
-  when: kubeadm_patches is defined and kubeadm_patches.enabled
+  loop: "{{ kubeadm_patches }}"
+  loop_control:
+    index_var: suffix

From c87097fc35b491cfddabb4e779bab1fe3e9d0396 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 10 Sep 2024 17:34:04 +0200
Subject: [PATCH 3/4] Document how to use kubeadm patches

---
 docs/ansible/vars.md                          |  7 ++++++
 .../group_vars/k8s_cluster/k8s-cluster.yml    | 24 +++++++++++++++----
 .../kube-controller-manager+merge.yaml        |  8 -------
 .../sample/patches/kube-scheduler+merge.yaml  |  8 -------
 .../kubeadm_common/defaults/main.yml          |  6 +++++
 5 files changed, 32 insertions(+), 21 deletions(-)
 delete mode 100644 inventory/sample/patches/kube-controller-manager+merge.yaml
 delete mode 100644 inventory/sample/patches/kube-scheduler+merge.yaml

diff --git a/docs/ansible/vars.md b/docs/ansible/vars.md
index b172f4ada..f8d040e12 100644
--- a/docs/ansible/vars.md
+++ b/docs/ansible/vars.md
@@ -337,6 +337,13 @@ in the form of dicts of key-value pairs of configuration parameters that will be
 * *kube_kubeadm_controller_extra_args*
 * *kube_kubeadm_scheduler_extra_args*
 
+### Kubeadm patches
+
+When extra flags are not sufficient and there is a need to further customize kubernetes components,
+[kubeadm patches](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches)
+can be used.
+You should use the [`kubeadm_patches` variable](../../roles/kubernetes/kubeadm_common/defaults/main.yml) for that purpose.
+
 ## App variables
 
 * *helm_version* - Only supports v3.x. Existing v2 installs (with Tiller) will not be modified and need to be removed manually.
diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml
index 522ddc589..24f896818 100644
--- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml
+++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml
@@ -366,11 +366,25 @@ auto_renew_certificates: false
 # First Monday of each month
 # auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube_control_plane'].index(inventory_hostname) }}0:00"
 
-# kubeadm patches path
-kubeadm_patches:
-  enabled: false
-  source_dir: "{{ inventory_dir }}/patches"
-  dest_dir: "{{ kube_config_dir }}/patches"
+kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
+kubeadm_patches: []
+# See https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches
+# Correspondance with this link
+# patchtype = type
+# target = target
+# suffix -> managed automatically
+# extension -> always "yaml"
+# kubeadm_patches:
+# - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration
+#   type: strategic(default)|json|merge
+#   patch:
+#    metadata:
+#      annotations:
+#        example.com/test: "true"
+#      labels:
+#        example.com/prod_level: "{{ prod_level }}"
+# - ...
+# Patches are applied in the order they are specified.
 
 # Set to true to remove the role binding to anonymous users created by kubeadm
 remove_anonymous_access: false
diff --git a/inventory/sample/patches/kube-controller-manager+merge.yaml b/inventory/sample/patches/kube-controller-manager+merge.yaml
deleted file mode 100644
index 3f0fbbcd5..000000000
--- a/inventory/sample/patches/kube-controller-manager+merge.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-controller-manager
-  annotations:
-    prometheus.io/scrape: 'true'
-    prometheus.io/port: '10257'
diff --git a/inventory/sample/patches/kube-scheduler+merge.yaml b/inventory/sample/patches/kube-scheduler+merge.yaml
deleted file mode 100644
index 00f457237..000000000
--- a/inventory/sample/patches/kube-scheduler+merge.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-scheduler
-  annotations:
-    prometheus.io/scrape: 'true'
-    prometheus.io/port: '10259'
diff --git a/roles/kubernetes/kubeadm_common/defaults/main.yml b/roles/kubernetes/kubeadm_common/defaults/main.yml
index f7d70691a..acbcdcf5f 100644
--- a/roles/kubernetes/kubeadm_common/defaults/main.yml
+++ b/roles/kubernetes/kubeadm_common/defaults/main.yml
@@ -1,6 +1,12 @@
 ---
 kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
 kubeadm_patches: []
+# See https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches
+# Correspondance with this link
+# patchtype = type
+# target = target
+# suffix -> managed automatically
+# extension -> always "yaml"
 # kubeadm_patches:
 # - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration
 #   type: strategic(default)|json|merge

From d54356e113785677246fe3d3c1b017ca9e24f1db Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 10 Sep 2024 17:38:56 +0200
Subject: [PATCH 4/4] Add testcase for kubeadm_patches

---
 .../packet_ubuntu24-calico-etcd-datastore.yml   | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/files/packet_ubuntu24-calico-etcd-datastore.yml b/tests/files/packet_ubuntu24-calico-etcd-datastore.yml
index 4f35d2f87..7a27ce566 100644
--- a/tests/files/packet_ubuntu24-calico-etcd-datastore.yml
+++ b/tests/files/packet_ubuntu24-calico-etcd-datastore.yml
@@ -27,3 +27,20 @@ containerd_registries_mirrors:
         skip_verify: true
 
 calico_datastore: "etcd"
+
+# Test kubeadm patches
+kubeadm_patches:
+  - target: kube-apiserver
+    patch:
+      metadata:
+        annotations:
+          example.com/test: "true"
+        labels:
+          example.com/prod_level: "prep"
+  - target: kube-controller-manager
+    patch:
+      metadata:
+        annotations:
+          example.com/test: "false"
+        labels:
+          example.com/prod_level: "prep"