Fix: check expiry before do breaking renew and container restart actions (#12194)

* Fix: check expiraty before renew

Since certificate renewal and container restarts involve higher risks,
they should be executed with extra caution.

* squash to Fix: check expiraty before renew

* squash to Fix: address more comments from VannTen

Signed-off-by: Peter Pan <Peter.Pan@daocloud.io>

---------

Signed-off-by: Peter Pan <Peter.Pan@daocloud.io>

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -346,7 +346,7 @@ event_ttl_duration: "1h0m0s"
 ## Automatically renew K8S control plane certificates on first Monday of each month
 auto_renew_certificates: false
 # First Monday of each month
-# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube_control_plane'].index(inventory_hostname) }}0:00"
+# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:00:00"
 
 kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
 kubeadm_patches: []

roles/kubernetes/control-plane/templates/k8s-certs-renew.sh.j2

@@ -1,8 +1,26 @@
 #!/bin/bash
 
-echo "## Expiration before renewal ##"
+echo "## Check Expiration before renewal ##"
+
 {{ bin_dir }}/kubeadm certs check-expiration
 
+days_buffer=7 # set a time margin, because we should not renew at the last moment
+calendar={{ auto_renew_certificates_systemd_calendar }}
+next_time=$(systemctl show k8s-certs-renew.timer  -p NextElapseUSecRealtime --value)
+
+if [ "${next_time}" == "" ]; then
+	echo "## Skip expiry comparison due to fail to parse next elapse from systemd calendar,do renewal directly ##"
+else
+	current_time=$(date +%s)
+	target_time=$(date -d "${next_time} + ${days_buffer} days" +%s) # $next_time - $days_buffer days
+	expiry_threshold=$(( ${target_time} - ${current_time} ))
+	expired_certs=$({{ bin_dir }}/kubeadm certs check-expiration -o jsonpath="{.certificates[?(@.residualTime<${expiry_threshold}.0)]}")
+    if [ "${expired_certs}" == "" ];then
+		echo "## Skip cert renew and K8S container restart, since all residualTimes are beyond threshold ##"
+		exit 0
+	fi
+fi
+
 echo "## Renewing certificates managed by kubeadm ##"
 {{ bin_dir }}/kubeadm certs renew all