diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml index a30b96d6a..db4766de7 100644 --- a/.gitlab-ci/packet.yml +++ b/.gitlab-ci/packet.yml @@ -273,6 +273,11 @@ packet_debian11-custom-cni: extends: .packet_pr when: manual +packet_debian11-kubelet-csr-approver: + stage: deploy-part2 + extends: .packet_pr + when: manual + # ### PR JOBS PART3 # Long jobs (45min+) diff --git a/docs/hardening.md b/docs/hardening.md index 521e7d8c0..d791c4dee 100644 --- a/docs/hardening.md +++ b/docs/hardening.md @@ -117,7 +117,8 @@ Let's take a deep look to the resultant **kubernetes** configuration: * The `anonymous-auth` (on `kube-apiserver`) is set to `true` by default. This is fine, because it is considered safe if you enable `RBAC` for the `authorization-mode`. * The `enable-admission-plugins` has not the `PodSecurityPolicy` admission plugin. This because it is going to be definitely removed from **kubernetes** `v1.25`. For this reason we decided to set the newest `PodSecurity` (for more details, please take a look here: ). Then, we set the `EventRateLimit` plugin, providing additional configuration files (that are automatically created under the hood and mounted inside the `kube-apiserver` container) to make it work. * The `encryption-provider-config` provide encryption at rest. This means that the `kube-apiserver` encrypt data that is going to be stored before they reach `etcd`. So the data is completely unreadable from `etcd` (in case an attacker is able to exploit this). -* The `rotateCertificates` in `KubeletConfiguration` is set to `true` along with `serverTLSBootstrap`. This could be used in alternative to `tlsCertFile` and `tlsPrivateKeyFile` parameters. Additionally it automatically generates certificates by itself, but you need to manually approve them or at least using an operator to do this (for more details, please take a look here: ). +* The `rotateCertificates` in `KubeletConfiguration` is set to `true` along with `serverTLSBootstrap`. This could be used in alternative to `tlsCertFile` and `tlsPrivateKeyFile` parameters. Additionally it automatically generates certificates by itself. By default the CSRs are approved automatically via [kubelet-csr-approver](https://github.com/postfinance/kubelet-csr-approver). You can customize approval configuration by modifying Helm values via `kubelet_csr_approver_values`. + See for more information on the subject. * If you are installing **kubernetes** in an AppArmor-based OS (eg. Debian/Ubuntu) you can enable the `AppArmor` feature gate uncommenting the lines with the comment `# AppArmor-based OS` on top. * The `kubelet_systemd_hardening`, both with `kubelet_secure_addresses` setup a minimal firewall on the system. To better understand how these variables work, here's an explanatory image: ![kubelet hardening](img/kubelet-hardening.png) @@ -134,5 +135,3 @@ ansible-playbook -v cluster.yml \ ``` **N.B.** The `vars.yaml` contains our general cluster information (SANs, load balancer, dns, etc..) and `hardening.yaml` is the file described above. - -Once completed the cluster deployment, don't forget to approve the generated certificates (check them with `kubectl get csr`, approve with `kubectl certificate approve `). This action is necessary because the `secureTLSBootstrap` option and `RotateKubeletServerCertificate` feature gate for `kubelet` are enabled (CIS [4.2.11](https://www.tenable.com/audits/items/CIS_Kubernetes_v1.20_v1.0.0_Level_1_Worker.audit:05af3dfbca8e0c3fb3559c6c7de29191), [4.2.12](https://www.tenable.com/audits/items/CIS_Kubernetes_v1.20_v1.0.0_Level_1_Worker.audit:5351c76f8c5bff8f98c29a5200a35435)). diff --git a/docs/vars.md b/docs/vars.md index 06f1e6f6f..6ff12c3cd 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -199,9 +199,9 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m * *kubelet_rotate_server_certificates* - Auto rotate the kubelet server certificates by requesting new certificates from the kube-apiserver when the certificate expiration approaches. - **Note** that server certificates are **not** approved automatically. Approve them manually - (`kubectl get csr`, `kubectl certificate approve`) or implement custom approving controller like - [kubelet-rubber-stamp](https://github.com/kontena/kubelet-rubber-stamp). + Note that enabling this also activates *kubelet_csr_approver* which approves automatically the CSRs. + To customize its behavior, you can override the Helm values via *kubelet_csr_approver_values*. + See [kubelet-csr-approver](https://github.com/postfinance/kubelet-csr-approver) for more information. * *kubelet_streaming_connection_idle_timeout* - Set the maximum time a streaming connection can be idle before the connection is automatically closed. diff --git a/playbooks/cluster.yml b/playbooks/cluster.yml index 5f163de6a..81da5be46 100644 --- a/playbooks/cluster.yml +++ b/playbooks/cluster.yml @@ -91,6 +91,7 @@ - { role: kubernetes/kubeadm, tags: kubeadm} - { role: kubernetes/node-label, tags: node-label } - { role: network_plugin, tags: network } + - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver } - hosts: calico_rr gather_facts: False diff --git a/playbooks/upgrade_cluster.yml b/playbooks/upgrade_cluster.yml index 39dd95a01..15809e845 100644 --- a/playbooks/upgrade_cluster.yml +++ b/playbooks/upgrade_cluster.yml @@ -117,6 +117,7 @@ - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller } - { role: network_plugin, tags: network } - { role: kubernetes-apps/network_plugin, tags: network } + - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver } - { role: kubernetes-apps/policy_controller, tags: policy-controller } - name: Finally handle worker upgrades, based on given batch size diff --git a/roles/helm-apps/README.md b/roles/helm-apps/README.md index 27b480cb0..8619688f2 100644 --- a/roles/helm-apps/README.md +++ b/roles/helm-apps/README.md @@ -32,8 +32,8 @@ Playbook example: chart_ref: simple-app/simple-app wait_timeout: "10m" # override the same option in `release_common_opts` repositories: "{{ repos }}" - - repo_name: simple-app - repo_url: "https://blog.leiwang.info/simple-app" + - name: simple-app + url: "https://blog.leiwang.info/simple-app" release_common_opts: "{{ helm_params }}" wait_timeout: "5m" ``` diff --git a/roles/helm-apps/meta/main.yml b/roles/helm-apps/meta/main.yml new file mode 100644 index 000000000..32ea6d4fe --- /dev/null +++ b/roles/helm-apps/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: kubernetes-apps/helm diff --git a/roles/kubernetes-apps/kubelet-csr-approver/defaults/main.yml b/roles/kubernetes-apps/kubelet-csr-approver/defaults/main.yml new file mode 100644 index 000000000..2edce709b --- /dev/null +++ b/roles/kubernetes-apps/kubelet-csr-approver/defaults/main.yml @@ -0,0 +1,12 @@ +--- +kubelet_csr_approver_enabled: "{{ kubelet_rotate_server_certificates }}" +kubelet_csr_approver_namespace: kube-system + +kubelet_csr_approver_repository_name: kubelet-csr-approver +kubelet_csr_approver_repository_url: https://postfinance.github.io/kubelet-csr-approver +kubelet_csr_approver_chart_ref: "{{ kubelet_csr_approver_repository_name }}/kubelet-csr-approver" +kubelet_csr_approver_chart_version: 0.2.8 + +# Fill values override here +# See upstream https://github.com/postfinance/kubelet-csr-approver +kubelet_csr_approver_values: {} diff --git a/roles/kubernetes-apps/kubelet-csr-approver/meta/main.yml b/roles/kubernetes-apps/kubelet-csr-approver/meta/main.yml new file mode 100644 index 000000000..93d13830c --- /dev/null +++ b/roles/kubernetes-apps/kubelet-csr-approver/meta/main.yml @@ -0,0 +1,20 @@ +--- +dependencies: + - role: helm-apps + when: + - inventory_hostname == groups['kube_control_plane'][0] + - kubelet_csr_approver_enabled + environment: + http_proxy: "{{ http_proxy | default('') }}" + https_proxy: "{{ https_proxy | default('') }}" + release_common_opts: {} + releases: + - name: kubelet-csr-approver + namespace: "{{ kubelet_csr_approver_namespace }}" + chart_ref: "{{ kubelet_csr_approver_chart_ref }}" + chart_version: "{{ kubelet_csr_approver_chart_version }}" + wait: true + values: "{{ kubelet_csr_approver_values }}" + repositories: + - name: "{{ kubelet_csr_approver_repository_name }}" + url: "{{ kubelet_csr_approver_repository_url }}" diff --git a/tests/files/packet_centos7-flannel-addons-ha.yml b/tests/files/packet_centos7-flannel-addons-ha.yml index 4d060a7c2..26d5bbddd 100644 --- a/tests/files/packet_centos7-flannel-addons-ha.yml +++ b/tests/files/packet_centos7-flannel-addons-ha.yml @@ -25,6 +25,7 @@ metrics_server_kubelet_insecure_tls: true kube_token_auth: true enable_nodelocaldns: false kubelet_rotate_server_certificates: true +kubelet_csr_approver_enabled: false kube_oidc_url: https://accounts.google.com/.well-known/openid-configuration kube_oidc_client_id: kubespray-example diff --git a/tests/files/packet_debian11-kubelet-csr-approver.yml b/tests/files/packet_debian11-kubelet-csr-approver.yml new file mode 100644 index 000000000..d1be09843 --- /dev/null +++ b/tests/files/packet_debian11-kubelet-csr-approver.yml @@ -0,0 +1,11 @@ +--- +# Instance settings +cloud_image: debian-11 +mode: default + +# Kubespray settings +kubelet_rotate_server_certificates: true +kubelet_csr_approver_enabled: true +kubelet_csr_approver_values: + # Do not check DNS resolution in testing (not recommended in production) + bypassDnsResolution: true diff --git a/tests/files/packet_ubuntu20-calico-aio-hardening.yml b/tests/files/packet_ubuntu20-calico-aio-hardening.yml index 940c1fd8d..16cf6ff3b 100644 --- a/tests/files/packet_ubuntu20-calico-aio-hardening.yml +++ b/tests/files/packet_ubuntu20-calico-aio-hardening.yml @@ -80,6 +80,7 @@ etcd_deployment_type: kubeadm kubelet_authentication_token_webhook: true kube_read_only_port: 0 kubelet_rotate_server_certificates: true +kubelet_csr_approver_enabled: false kubelet_protect_kernel_defaults: true kubelet_event_record_qps: 1 kubelet_rotate_certificates: true diff --git a/tests/testcases/030_check-network.yml b/tests/testcases/030_check-network.yml index 499064d7f..e2287f9e4 100644 --- a/tests/testcases/030_check-network.yml +++ b/tests/testcases/030_check-network.yml @@ -15,6 +15,35 @@ bin_dir: "/usr/local/bin" when: not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + - name: Check kubelet serving certificates approved with kubelet_csr_approver + block: + + - name: Get certificate signing requests + command: "{{ bin_dir }}/kubectl get csr" + register: get_csr + changed_when: false + + - debug: # noqa unnamed-task + msg: "{{ get_csr.stdout.split('\n') }}" + + - name: Check there are csrs + assert: + that: get_csr.stdout_lines | length > 0 + fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found + + - name: Get Denied/Pending certificate signing requests + shell: "{{ bin_dir }}/kubectl get csr | grep -e Denied -e Pending || true" + register: get_csr_denied_pending + changed_when: false + + - name: Check there are Denied/Pending csrs + assert: + that: get_csr_denied_pending.stdout_lines | length == 0 + fail_msg: kubelet_csr_approver is enabled but CSRs are not approved + when: + - kubelet_rotate_server_certificates | default(false) + - kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false)) + - name: Approve kubelet serving certificates block: @@ -37,7 +66,9 @@ - debug: # noqa unnamed-task msg: "{{ certificate_approve.stdout.split('\n') }}" - when: kubelet_rotate_server_certificates | default(false) + when: + - kubelet_rotate_server_certificates | default(false) + - not (kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false))) - name: Create test namespace command: "{{ bin_dir }}/kubectl create namespace test"