From 9631b5fd44d5b9a7230a21da9879c9b5ed554df9 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 25 Apr 2025 10:10:35 +0200 Subject: [PATCH 1/2] Move etcd inventory sample doc to role defaults --- inventory/sample/group_vars/etcd.yml | 38 ------------------- roles/etcd/defaults/main.yml | 23 +++++++---- roles/etcd/templates/etcd-events.env.j2 | 6 --- roles/etcd/templates/etcd.env.j2 | 12 ------ .../templates/kubeadm-config.v1beta3.yaml.j2 | 8 ---- .../templates/kubeadm-config.v1beta4.yaml.j2 | 8 ---- 6 files changed, 15 insertions(+), 80 deletions(-) delete mode 100644 inventory/sample/group_vars/etcd.yml diff --git a/inventory/sample/group_vars/etcd.yml b/inventory/sample/group_vars/etcd.yml deleted file mode 100644 index 88d040953..000000000 --- a/inventory/sample/group_vars/etcd.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -## Etcd auto compaction retention for mvcc key value store in hour -# etcd_compaction_retention: 0 - -## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics. -# etcd_metrics: basic - -## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing. -## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM. -## This value is only relevant when deploying etcd with `etcd_deployment_type: docker` -# etcd_memory_limit: "512M" - -## Etcd has a default of 2G for its space quota. If you put a value in etcd_memory_limit which is less than -## etcd_quota_backend_bytes, you may encounter out of memory terminations of the etcd cluster. Please check -## etcd documentation for more information. -# 8G is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it. -# etcd_quota_backend_bytes: "2147483648" - -# Maximum client request size in bytes the server will accept. -# etcd is designed to handle small key value pairs typical for metadata. -# Larger requests will work, but may increase the latency of other requests -# etcd_max_request_bytes: "1572864" - -### ETCD: disable peer client cert authentication. -# This affects ETCD_PEER_CLIENT_CERT_AUTH variable -# etcd_peer_client_auth: true - -## Enable distributed tracing -## To enable this experimental feature, set the etcd_experimental_enable_distributed_tracing: true, along with the -## etcd_experimental_distributed_tracing_sample_rate to choose how many samples to collect per million spans, -## the default sampling rate is 0 https://etcd.io/docs/v3.5/op-guide/monitoring/#distributed-tracing -# etcd_experimental_enable_distributed_tracing: false -# etcd_experimental_distributed_tracing_sample_rate: 100 -# etcd_experimental_distributed_tracing_address: "localhost:4317" -# etcd_experimental_distributed_tracing_service_name: etcd - -## The interval for etcd watch progress notify events -# etcd_experimental_watch_progress_notify_interval: 5s diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index 784111be6..76f03c076 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -34,6 +34,7 @@ etcd_script_dir: "{{ bin_dir }}/etcd-scripts" etcd_heartbeat_interval: "250" etcd_election_timeout: "5000" +## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics. etcd_metrics: "basic" # Define in inventory to set a separate port for etcd to expose metrics on @@ -46,18 +47,21 @@ etcd_metrics: "basic" etcd_extra_vars: {} # Limits -# Limit memory only if <4GB memory on host. 0=unlimited -# This value is only relevant when deploying etcd with `etcd_deployment_type: docker` +## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing. +## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM. +## This value is only relevant when deploying etcd with `etcd_deployment_type: docker` etcd_memory_limit: "{% if ansible_memtotal_mb < 4096 %}512M{% else %}0{% endif %}" -# The default storage size limit is 2G. +## Etcd has a default of 2G for its space quota. If you put a value in etcd_memory_limit which is less than +## etcd_quota_backend_bytes, you may encounter out of memory terminations of the etcd cluster. Please check +## etcd documentation for more information. # 8G is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it. -# etcd_quota_backend_bytes: "2147483648" +etcd_quota_backend_bytes: "2147483648" # Maximum client request size in bytes the server will accept. # etcd is designed to handle small key value pairs typical for metadata. # Larger requests will work, but may increase the latency of other requests -# etcd_max_request_bytes: "1572864" +etcd_max_request_bytes: "1572864" # Uncomment to set CPU share for etcd # etcd_cpu_limit: 300m @@ -66,6 +70,7 @@ etcd_blkio_weight: 1000 etcd_node_cert_hosts: "{{ groups['k8s_cluster'] }}" +## Etcd auto compaction retention for mvcc key value store in hour etcd_compaction_retention: "8" # Force clients like etcdctl to use TLS certs (different than peer security) @@ -75,10 +80,10 @@ etcd_secure_client: true etcd_peer_client_auth: true # Maximum number of snapshot files to retain (0 is unlimited) -# etcd_max_snapshots: 5 +etcd_max_snapshots: 5 # Maximum number of wal files to retain (0 is unlimited) -# etcd_max_wals: 5 +etcd_max_wals: 5 # Number of loop retries etcd_retries: 4 @@ -120,9 +125,11 @@ unsafe_show_logs: false # Enable distributed tracing # https://etcd.io/docs/v3.5/op-guide/monitoring/#distributed-tracing etcd_experimental_enable_distributed_tracing: false -etcd_experimental_distributed_tracing_sample_rate: 100 +etcd_experimental_distributed_tracing_sample_rate: 100 # Per million spans etcd_experimental_distributed_tracing_address: "localhost:4317" etcd_experimental_distributed_tracing_service_name: etcd # The interval for etcd watch progress notify events etcd_experimental_watch_progress_notify_interval: 5s + +etcd_log_level: info diff --git a/roles/etcd/templates/etcd-events.env.j2 b/roles/etcd/templates/etcd-events.env.j2 index 1e576f561..1c33856cb 100644 --- a/roles/etcd/templates/etcd-events.env.j2 +++ b/roles/etcd/templates/etcd-events.env.j2 @@ -13,15 +13,9 @@ ETCD_NAME={{ etcd_member_name }}-events ETCD_PROXY=off ETCD_INITIAL_CLUSTER={{ etcd_events_peer_addresses }} ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }} -{% if etcd_snapshot_count is defined %} ETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }} -{% endif %} -{% if etcd_quota_backend_bytes is defined %} ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }} -{% endif %} -{% if etcd_max_request_bytes is defined %} ETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }} -{% endif %} # TLS settings ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem diff --git a/roles/etcd/templates/etcd.env.j2 b/roles/etcd/templates/etcd.env.j2 index 3f7d39d09..33794412c 100644 --- a/roles/etcd/templates/etcd.env.j2 +++ b/roles/etcd/templates/etcd.env.j2 @@ -19,24 +19,12 @@ ETCD_NAME={{ etcd_member_name }} ETCD_PROXY=off ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }} ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }} -{% if etcd_snapshot_count is defined %} ETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }} -{% endif %} -{% if etcd_quota_backend_bytes is defined %} ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }} -{% endif %} -{% if etcd_max_request_bytes is defined %} ETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }} -{% endif %} -{% if etcd_log_level is defined %} ETCD_LOG_LEVEL={{ etcd_log_level }} -{% endif %} -{% if etcd_max_snapshots is defined %} ETCD_MAX_SNAPSHOTS={{ etcd_max_snapshots }} -{% endif %} -{% if etcd_max_wals is defined %} ETCD_MAX_WALS={{ etcd_max_wals }} -{% endif %} # Flannel need etcd v2 API ETCD_ENABLE_V2=true diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 index 37caf3188..f5ee517ce 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 @@ -59,18 +59,10 @@ etcd: {% if etcd_listen_metrics_urls is defined %} listen-metrics-urls: "{{ etcd_listen_metrics_urls }}" {% endif %} -{% if etcd_snapshot_count is defined %} snapshot-count: "{{ etcd_snapshot_count }}" -{% endif %} -{% if etcd_quota_backend_bytes is defined %} quota-backend-bytes: "{{ etcd_quota_backend_bytes }}" -{% endif %} -{% if etcd_max_request_bytes is defined %} max-request-bytes: "{{ etcd_max_request_bytes }}" -{% endif %} -{% if etcd_log_level is defined %} log-level: "{{ etcd_log_level }}" -{% endif %} {% for key, value in etcd_extra_vars.items() %} {{ key }}: "{{ value }}" {% endfor %} diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 index ad340409c..ec256ad14 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 @@ -68,22 +68,14 @@ etcd: - name: listen-metrics-urls value: "{{ etcd_listen_metrics_urls }}" {% endif %} -{% if etcd_snapshot_count is defined %} - name: snapshot-count value: "{{ etcd_snapshot_count }}" -{% endif %} -{% if etcd_quota_backend_bytes is defined %} - name: quota-backend-bytes value: "{{ etcd_quota_backend_bytes }}" -{% endif %} -{% if etcd_max_request_bytes is defined %} - name: max-request-bytes value: "{{ etcd_max_request_bytes }}" -{% endif %} -{% if etcd_log_level is defined %} - name: log-level value: "{{ etcd_log_level }}" -{% endif %} {% for key, value in etcd_extra_vars.items() %} - name: {{ key }} value: "{{ value }}" From 9c2bdeec63114d11f219d0329bb55b61bde5b9e6 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 16 May 2025 14:51:29 +0200 Subject: [PATCH 2/2] Decouple etcd defaults in a separate role This allows us to reuse the defaults in other places without putting everything in kubespray-defaults. In that, for kubernetes/control-plane. --- roles/etcd/meta/main.yml | 1 + roles/{etcd => etcd_defaults}/defaults/main.yml | 0 roles/{etcd => etcd_defaults}/vars/main.yml | 0 roles/kubernetes/control-plane/meta/main.yml | 1 + 4 files changed, 2 insertions(+) rename roles/{etcd => etcd_defaults}/defaults/main.yml (100%) rename roles/{etcd => etcd_defaults}/vars/main.yml (100%) diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml index e99664660..eda7561e2 100644 --- a/roles/etcd/meta/main.yml +++ b/roles/etcd/meta/main.yml @@ -6,3 +6,4 @@ dependencies: - role: adduser user: "{{ addusers.kube }}" when: not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos) + - role: etcd_defaults diff --git a/roles/etcd/defaults/main.yml b/roles/etcd_defaults/defaults/main.yml similarity index 100% rename from roles/etcd/defaults/main.yml rename to roles/etcd_defaults/defaults/main.yml diff --git a/roles/etcd/vars/main.yml b/roles/etcd_defaults/vars/main.yml similarity index 100% rename from roles/etcd/vars/main.yml rename to roles/etcd_defaults/vars/main.yml diff --git a/roles/kubernetes/control-plane/meta/main.yml b/roles/kubernetes/control-plane/meta/main.yml index 9e5d86e0f..ceb05687d 100644 --- a/roles/kubernetes/control-plane/meta/main.yml +++ b/roles/kubernetes/control-plane/meta/main.yml @@ -7,3 +7,4 @@ dependencies: - etcd_deployment_type == "kubeadm" - not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos) - role: network_plugin/calico_defaults + - role: etcd_defaults