From 3d76c30354670b163f6e4047773dbb6fcf0f3262 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 12 Jan 2024 04:11:00 +0100 Subject: [PATCH] [2.23] Fix calico-node in etcd mode (#10768) * CI: Document the 'all-in-one' layout + small refactoring (#10725) * Rename aio to all-in-one and document it ADTM. Acronyms don't tell much. * Refactor vm_count in tests provisioning * Add test case for calico using etcd datastore (#10722) * Add multinode ci layout * Add test case for calico using etcd datastore * Fix calico-node in etcd mode (#10438) * Calico : add ETCD endpoints to install-cni container * Calico : remove nodename from configmap in etcd mode --------- Co-authored-by: Olivier Levitt --- .gitlab-ci/packet.yml | 19 +++++++----- docs/test_cases.md | 6 +++- .../calico/templates/calico-config.yml.j2 | 6 ---- .../calico/templates/calico-node.yml.j2 | 7 +++++ tests/cloud_playbooks/create-gce.yml | 2 +- .../roles/packet-ci/defaults/main.yml | 2 +- .../roles/packet-ci/tasks/main.yml | 4 --- .../roles/packet-ci/templates/inventory.j2 | 12 +++++++- .../roles/packet-ci/vars/main.yml | 13 +++++++++ ...l => packet_amazon-linux-2-all-in-one.yml} | 2 +- ... => packet_ubuntu20-all-in-one-docker.yml} | 2 +- ...acket_ubuntu20-calico-aio-ansible-2_11.yml | 1 - ...buntu20-calico-all-in-one-ansible-2_11.yml | 1 + ..._ubuntu20-calico-all-in-one-hardening.yml} | 2 +- ... => packet_ubuntu20-calico-all-in-one.yml} | 2 +- ... => packet_ubuntu22-all-in-one-docker.yml} | 2 +- ... => packet_ubuntu22-calico-all-in-one.yml} | 2 +- .../packet_ubuntu22-calico-etcd-datastore.yml | 29 +++++++++++++++++++ tests/templates/inventory-gce.j2 | 4 +-- 19 files changed, 88 insertions(+), 30 deletions(-) create mode 100644 tests/cloud_playbooks/roles/packet-ci/vars/main.yml rename tests/files/{packet_amazon-linux-2-aio.yml => packet_amazon-linux-2-all-in-one.yml} (75%) rename tests/files/{packet_ubuntu20-aio-docker.yml => packet_ubuntu20-all-in-one-docker.yml} (95%) delete mode 120000 tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml create mode 120000 tests/files/packet_ubuntu20-calico-all-in-one-ansible-2_11.yml rename tests/files/{packet_ubuntu20-calico-aio-hardening.yml => packet_ubuntu20-calico-all-in-one-hardening.yml} (99%) rename tests/files/{packet_ubuntu20-calico-aio.yml => packet_ubuntu20-calico-all-in-one.yml} (94%) rename tests/files/{packet_ubuntu22-aio-docker.yml => packet_ubuntu22-all-in-one-docker.yml} (96%) rename tests/files/{packet_ubuntu22-calico-aio.yml => packet_ubuntu22-calico-all-in-one.yml} (97%) create mode 100644 tests/files/packet_ubuntu22-calico-etcd-datastore.yml diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml index c6f9f2f71..c7032fc15 100644 --- a/.gitlab-ci/packet.yml +++ b/.gitlab-ci/packet.yml @@ -31,8 +31,8 @@ packet_cleanup_old: - make cleanup-packet after_script: [] -# The ubuntu20-calico-aio jobs are meant as early stages to prevent running the full CI if something is horribly broken -packet_ubuntu20-calico-aio: +# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken +packet_ubuntu20-calico-all-in-one: stage: deploy-part1 extends: .packet_pr when: on_success @@ -41,22 +41,27 @@ packet_ubuntu20-calico-aio: # ### PR JOBS PART2 -packet_ubuntu20-aio-docker: +packet_ubuntu20-all-in-one-docker: stage: deploy-part2 extends: .packet_pr when: on_success -packet_ubuntu20-calico-aio-hardening: +packet_ubuntu20-calico-all-in-one-hardening: stage: deploy-part2 extends: .packet_pr when: on_success -packet_ubuntu22-aio-docker: +packet_ubuntu22-all-in-one-docker: stage: deploy-part2 extends: .packet_pr when: on_success -packet_ubuntu22-calico-aio: +packet_ubuntu22-calico-all-in-one: + stage: deploy-part2 + extends: .packet_pr + when: on_success + +packet_ubuntu22-calico-etcd-datastore: stage: deploy-part2 extends: .packet_pr when: on_success @@ -235,7 +240,7 @@ packet_fedora37-calico-swap-selinux: extends: .packet_pr when: manual -packet_amazon-linux-2-aio: +packet_amazon-linux-2-all-in-one: stage: deploy-part2 extends: .packet_pr when: manual diff --git a/docs/test_cases.md b/docs/test_cases.md index 1fdce682c..d5aef62f2 100644 --- a/docs/test_cases.md +++ b/docs/test_cases.md @@ -1,6 +1,6 @@ # Node Layouts -There are four node layout types: `default`, `separate`, `ha`, and `scale`. +There are six node layout types: `default`, `separate`, `ha`, `scale`, `all-in-one`, and `multinode`. `default` is a non-HA two nodes setup with one separate `kube_node` and the `etcd` group merged with the `kube_control_plane`. @@ -16,6 +16,10 @@ in the Ansible inventory. This helps test TLS certificate generation at scale to prevent regressions and profile certain long-running tasks. These nodes are never actually deployed, but certificates are generated for them. +`all-in-one` layout use a single node for with `kube_control_plane`, `etcd` and `kube_node` merged. + +`multinode` layout consists of two separate `kube_node` and a merged single `etcd+kube_control_plane` node. + Note, the canal network plugin deploys flannel as well plus calico policy controller. ## Test cases diff --git a/roles/network_plugin/calico/templates/calico-config.yml.j2 b/roles/network_plugin/calico/templates/calico-config.yml.j2 index f1a3f2e76..26983ecae 100644 --- a/roles/network_plugin/calico/templates/calico-config.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-config.yml.j2 @@ -36,12 +36,6 @@ data: {% if calico_datastore == "kdd" %} "datastore_type": "kubernetes", "nodename": "__KUBERNETES_NODE_NAME__", - {% else %} - {% if cloud_provider is defined %} - "nodename": "{{ calico_kubelet_name.stdout }}", - {% else %} - "nodename": "{{ calico_baremetal_nodename }}", - {% endif %} {% endif %} "type": "calico", "log_level": "info", diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2 index 4e49f3bc4..2b95af6d4 100644 --- a/roles/network_plugin/calico/templates/calico-node.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-node.yml.j2 @@ -95,6 +95,13 @@ spec: # Prevents the container from sleeping forever. - name: SLEEP value: "false" +{% if calico_datastore == "etcd" %} + - name: ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_endpoints +{% endif %} {% if calico_datastore == "kdd" %} # Set the hostname based on the k8s node name. - name: KUBERNETES_NODE_NAME diff --git a/tests/cloud_playbooks/create-gce.yml b/tests/cloud_playbooks/create-gce.yml index 78c96b085..ad9127ff4 100644 --- a/tests/cloud_playbooks/create-gce.yml +++ b/tests/cloud_playbooks/create-gce.yml @@ -23,7 +23,7 @@ instance_names: >- {%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale'] -%} k8s-{{ test_name }}-1,k8s-{{ test_name }}-2,k8s-{{ test_name }}-3 - {%- elif mode == 'aio' -%} + {%- elif mode == 'all-in-one' -%} k8s-{{ test_name }}-1 {%- else -%} k8s-{{ test_name }}-1,k8s-{{ test_name }}-2 diff --git a/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml b/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml index 17dd3d8b9..c3e801a97 100644 --- a/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml +++ b/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml @@ -18,7 +18,7 @@ memory_allocation_ratio: 1 inventory_path: "/tmp/{{ test_name }}/inventory" # Deployment mode -mode: aio +mode: all-in-one # Cloud init config for each os type # distro: fedora -> I2Nsb3VkLWNvbmZpZwpzeXN0ZW1faW5mbzoKICBkaXN0cm86IGZlZG9yYQp1c2VyczoKIC0gbmFtZToga3ViZXNwcmF5CiAgIGdyb3Vwczogd2hlZWwKICAgc3VkbzogJ0FMTD0oQUxMKSBOT1BBU1NXRDpBTEwnCiAgIHNoZWxsOiAvYmluL2Jhc2gKICAgbG9ja19wYXNzd2Q6IEZhbHNlCiAgIGhvbWU6IC9ob21lL2t1YmVzcHJheQogICBzc2hfYXV0aG9yaXplZF9rZXlzOgogICAgIC0gc3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFCQVFDYW5UaS9lS3gwK3RIWUpBZURocStzRlMyT2JVUDEvSTY5ZjdpVjNVdGtLbFQyMEpmVzFmNkZlWHQvMDRWZjI3V1FxK05xczZ2R0JxRDlRWFNZdWYrdDAvczdFUExqVGVpOW1lMW1wcXIrdVRlK0tEdFRQMzlwZkQzL2VWQ2FlQjcyNkdQMkZrYUQwRnpwbUViNjZPM05xaHhPUTk2R3gvOVhUdXcvSzNsbGo0T1ZENkdyalIzQjdjNFh0RUJzWmNacHBNSi9vSDFtR3lHWGRoMzFtV1FTcUFSTy9QOFU4R3d0MCtIR3BVd2gvaGR5M3QrU1lvVEIyR3dWYjB6b3lWd3RWdmZEUXpzbThmcTNhdjRLdmV6OGtZdU5ESnYwNXg0bHZVWmdSMTVaRFJYc0FuZGhReXFvWGRDTEFlMCtlYUtYcTlCa1d4S0ZiOWhQZTBBVWpqYTU= diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml index 633c87253..f2dea7f6b 100644 --- a/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml +++ b/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml @@ -3,10 +3,6 @@ - name: "Include custom vars for ci job: {{ ci_job_name }}" include_vars: "../files/{{ ci_job_name }}.yml" -- name: Set VM count needed for CI test_id - set_fact: - vm_count: "{%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale', 'ha-recover', 'ha-recover-noquorum'] -%}{{ 3 | int }}{%- elif mode == 'aio' -%}{{ 1 | int }}{%- else -%}{{ 2 | int }}{%- endif -%}" - - name: Cleamup old VMs import_tasks: cleanup-old-vms.yml diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 index c49d582f4..da7e74969 100644 --- a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 +++ b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 @@ -33,7 +33,7 @@ instance-2 [etcd] instance-1 -{% elif mode == "aio" %} +{% elif mode == "all-in-one" %} [kube_control_plane] instance-1 @@ -81,6 +81,16 @@ instance-2 [broken_etcd] instance-1 etcd_member_name=etcd2 instance-2 etcd_member_name=etcd3 +{% elif mode == "multinode" %} +[kube_control_plane] +instance-1 + +[etcd] +instance-1 + +[kube_node] +instance-2 +instance-3 {% endif %} [k8s_cluster:children] diff --git a/tests/cloud_playbooks/roles/packet-ci/vars/main.yml b/tests/cloud_playbooks/roles/packet-ci/vars/main.yml new file mode 100644 index 000000000..5cfe561b7 --- /dev/null +++ b/tests/cloud_playbooks/roles/packet-ci/vars/main.yml @@ -0,0 +1,13 @@ +--- +_vm_count_dict: + separate: 3 + separate-scale: 3 + ha: 3 + ha-scale: 3 + ha-recover: 3 + ha-recover-noquorum: 3 + all-in-one: 1 + multinode: 3 + default: 2 + +vm_count: "{{ _vm_count_dict[mode | d('default')] }}" diff --git a/tests/files/packet_amazon-linux-2-aio.yml b/tests/files/packet_amazon-linux-2-all-in-one.yml similarity index 75% rename from tests/files/packet_amazon-linux-2-aio.yml rename to tests/files/packet_amazon-linux-2-all-in-one.yml index 7b2c69bb8..5faf6d305 100644 --- a/tests/files/packet_amazon-linux-2-aio.yml +++ b/tests/files/packet_amazon-linux-2-all-in-one.yml @@ -1,4 +1,4 @@ --- # Instance settings cloud_image: amazon-linux-2 -mode: aio +mode: all-in-one diff --git a/tests/files/packet_ubuntu20-aio-docker.yml b/tests/files/packet_ubuntu20-all-in-one-docker.yml similarity index 95% rename from tests/files/packet_ubuntu20-aio-docker.yml rename to tests/files/packet_ubuntu20-all-in-one-docker.yml index edc12207c..2ed6307d8 100644 --- a/tests/files/packet_ubuntu20-aio-docker.yml +++ b/tests/files/packet_ubuntu20-all-in-one-docker.yml @@ -1,7 +1,7 @@ --- # Instance settings cloud_image: ubuntu-2004 -mode: aio +mode: all-in-one # Kubespray settings auto_renew_certificates: true diff --git a/tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml b/tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml deleted file mode 120000 index 10064637f..000000000 --- a/tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml +++ /dev/null @@ -1 +0,0 @@ -packet_ubuntu20-calico-aio.yml \ No newline at end of file diff --git a/tests/files/packet_ubuntu20-calico-all-in-one-ansible-2_11.yml b/tests/files/packet_ubuntu20-calico-all-in-one-ansible-2_11.yml new file mode 120000 index 000000000..0c812884a --- /dev/null +++ b/tests/files/packet_ubuntu20-calico-all-in-one-ansible-2_11.yml @@ -0,0 +1 @@ +packet_ubuntu20-calico-all-in-one.yml \ No newline at end of file diff --git a/tests/files/packet_ubuntu20-calico-aio-hardening.yml b/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml similarity index 99% rename from tests/files/packet_ubuntu20-calico-aio-hardening.yml rename to tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml index 16cf6ff3b..e4bf63da0 100644 --- a/tests/files/packet_ubuntu20-calico-aio-hardening.yml +++ b/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml @@ -1,7 +1,7 @@ --- # Instance settings cloud_image: ubuntu-2004 -mode: aio +mode: all-in-one # Kubespray settings auto_renew_certificates: true diff --git a/tests/files/packet_ubuntu20-calico-aio.yml b/tests/files/packet_ubuntu20-calico-all-in-one.yml similarity index 94% rename from tests/files/packet_ubuntu20-calico-aio.yml rename to tests/files/packet_ubuntu20-calico-all-in-one.yml index 41d4a1315..3cfc99c96 100644 --- a/tests/files/packet_ubuntu20-calico-aio.yml +++ b/tests/files/packet_ubuntu20-calico-all-in-one.yml @@ -1,7 +1,7 @@ --- # Instance settings cloud_image: ubuntu-2004 -mode: aio +mode: all-in-one # Kubespray settings auto_renew_certificates: true diff --git a/tests/files/packet_ubuntu22-aio-docker.yml b/tests/files/packet_ubuntu22-all-in-one-docker.yml similarity index 96% rename from tests/files/packet_ubuntu22-aio-docker.yml rename to tests/files/packet_ubuntu22-all-in-one-docker.yml index b78c6b0a4..3abc4b254 100644 --- a/tests/files/packet_ubuntu22-aio-docker.yml +++ b/tests/files/packet_ubuntu22-all-in-one-docker.yml @@ -1,7 +1,7 @@ --- # Instance settings cloud_image: ubuntu-2204 -mode: aio +mode: all-in-one vm_memory: 1600Mi # Kubespray settings diff --git a/tests/files/packet_ubuntu22-calico-aio.yml b/tests/files/packet_ubuntu22-calico-all-in-one.yml similarity index 97% rename from tests/files/packet_ubuntu22-calico-aio.yml rename to tests/files/packet_ubuntu22-calico-all-in-one.yml index c9458f568..4c5b5d711 100644 --- a/tests/files/packet_ubuntu22-calico-aio.yml +++ b/tests/files/packet_ubuntu22-calico-all-in-one.yml @@ -1,7 +1,7 @@ --- # Instance settings cloud_image: ubuntu-2204 -mode: aio +mode: all-in-one vm_memory: 1600Mi # Kubespray settings diff --git a/tests/files/packet_ubuntu22-calico-etcd-datastore.yml b/tests/files/packet_ubuntu22-calico-etcd-datastore.yml new file mode 100644 index 000000000..c95a7602f --- /dev/null +++ b/tests/files/packet_ubuntu22-calico-etcd-datastore.yml @@ -0,0 +1,29 @@ +--- +# Instance settings +cloud_image: ubuntu-2204 +mode: multinode +vm_memory: 1600Mi + +# Kubespray settings +auto_renew_certificates: true + +# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko +kube_proxy_mode: iptables +enable_nodelocaldns: False + +containerd_registries: + "docker.io": "https://mirror.gcr.io" + +containerd_registries_mirrors: + - prefix: docker.io + mirrors: + - host: https://mirror.gcr.io + capabilities: ["pull", "resolve"] + skip_verify: false + - prefix: 172.19.16.11:5000 + mirrors: + - host: http://172.19.16.11:5000 + capabilities: ["pull", "resolve", "push"] + skip_verify: true + +calico_datastore: "etcd" diff --git a/tests/templates/inventory-gce.j2 b/tests/templates/inventory-gce.j2 index 33e9bbc73..0312b59f5 100644 --- a/tests/templates/inventory-gce.j2 +++ b/tests/templates/inventory-gce.j2 @@ -1,6 +1,6 @@ {% set node1 = gce.instance_data[0].name %} {{node1}} ansible_ssh_host={{gce.instance_data[0].public_ip}} -{% if mode != "aio" %} +{% if mode != "all-in-one" %} {% set node2 = gce.instance_data[1].name %} {{node2}} ansible_ssh_host={{gce.instance_data[1].public_ip}} {% endif %} @@ -46,7 +46,7 @@ [etcd] {{node1}} -{% elif mode == "aio" %} +{% elif mode == "all-in-one" %} [kube_control_plane] {{node1}}