From e24216bedc90529d0de18f95a5fa78c99d7e4bf5 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 14 Feb 2025 09:28:21 +0100 Subject: [PATCH] Automatically derive defaults versions from checksums (#11906) * Automatically derive defaults versions from checksums Currently, when updating checksums, we manually update the default versions. However, AFAICT, for all components where we have checksums, we're using the newest version out of those checksums. Codify this in the `_version` defaults variables definition to make the process automatic and reduce manual steps (as well as the diff size during reviews). We assume the versions are sorted, with newest first. This should be guaranteed by the pre-commit hooks. * Validate checksums are ordered by versions, newest first * Generalize render-readme-versions hook for other static files The pre-commit hook introduced a142f40e2 (Update versions in README.md with pre-commit, 2025-01-21) allow to update our README with new versions. It turns out other "static" files (== which don't interpret Ansible variables) also use the default version (in that case, our Dockefiles, but there might be others) The Dockerfile breaks if the variable they use (`kube_version`) is a Jinja template. For helping with automatic version upgrade, generalize the hook to deal with other static files, and make a template out of the Dockerfile. * Dockerfile: template kube_version with pre-commit instead of runtime * Validate all versions/checksums are strings in pre-commit All the ansible/python tooling for version is for version strings. YAML unhelpfully consider some stuff as number, so enforce this. * Stringify checksums versions --- .pre-commit-config.yaml | 14 +- Dockerfile | 8 +- pipeline.Dockerfile | 7 +- .../defaults/main/checksums.yml | 152 +++++++++--------- .../defaults/main/download.yml | 28 ++-- .../kubespray-defaults/defaults/main/main.yml | 4 +- scripts/Dockerfile.j2 | 50 ++++++ scripts/assert-sorted-checksums.yml | 38 +++++ scripts/pipeline.Dockerfile.j2 | 60 +++++++ ...on.yml => propagate_ansible_variables.yml} | 8 + 10 files changed, 264 insertions(+), 105 deletions(-) create mode 100644 scripts/Dockerfile.j2 create mode 100755 scripts/assert-sorted-checksums.yml create mode 100644 scripts/pipeline.Dockerfile.j2 rename scripts/{render_readme_version.yml => propagate_ansible_variables.yml} (80%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1b20ed901..c8d38246f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -79,14 +79,22 @@ repos: additional_dependencies: - jinja2 - - id: render-readme-versions - name: Update versions in README.md to match their defaults values + - id: propagate-ansible-variables + name: Update static files referencing default kubespray values language: python additional_dependencies: - ansible-core>=2.16.4 - entry: scripts/render_readme_version.yml + entry: scripts/propagate_ansible_variables.yml pass_filenames: false + - id: check-checksums-sorted + name: Check that our checksums are correctly sorted by version + entry: scripts/assert-sorted-checksums.yml + language: python + pass_filenames: false + additional_dependencies: + - ansible + - repo: https://github.com/markdownlint/markdownlint rev: v0.12.0 hooks: diff --git a/Dockerfile b/Dockerfile index a2fa98096..766cfebf2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,11 +34,9 @@ RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \ SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN --mount=type=bind,source=roles/kubespray-defaults/defaults/main/main.yml,target=roles/kubespray-defaults/defaults/main/main.yml \ - KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \ - OS_ARCHITECTURE=$(dpkg --print-architecture) \ - && curl -L "https://dl.k8s.io/release/${KUBE_VERSION}/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \ - && echo "$(curl -L "https://dl.k8s.io/release/${KUBE_VERSION}/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \ +RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \ + && curl -L "https://dl.k8s.io/release/v1.32.0/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \ + && echo "$(curl -L "https://dl.k8s.io/release/v1.32.0/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \ && chmod a+x /usr/local/bin/kubectl COPY *.yml ./ diff --git a/pipeline.Dockerfile b/pipeline.Dockerfile index 49d00ae4f..24f76c20e 100644 --- a/pipeline.Dockerfile +++ b/pipeline.Dockerfile @@ -42,16 +42,13 @@ RUN apt update -q \ WORKDIR /kubespray ADD ./requirements.txt /kubespray/requirements.txt ADD ./tests/requirements.txt /kubespray/tests/requirements.txt -ADD ./roles/kubespray-defaults/defaults/main/main.yml /kubespray/roles/kubespray-defaults/defaults/main/main.yml - RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \ && pip install --no-compile --no-cache-dir pip -U \ && pip install --no-compile --no-cache-dir -r tests/requirements.txt \ && pip install --no-compile --no-cache-dir -r requirements.txt \ - && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \ - && curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \ - && echo $(curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \ + && curl -L https://dl.k8s.io/release/v1.32.0/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \ + && echo $(curl -L https://dl.k8s.io/release/v1.32.0/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \ && chmod a+x /usr/local/bin/kubectl \ # Install Vagrant && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \ diff --git a/roles/kubespray-defaults/defaults/main/checksums.yml b/roles/kubespray-defaults/defaults/main/checksums.yml index f97fd64fe..a03355e55 100644 --- a/roles/kubespray-defaults/defaults/main/checksums.yml +++ b/roles/kubespray-defaults/defaults/main/checksums.yml @@ -603,10 +603,10 @@ runc_checksums: v1.1.8: a816cd654e804249c4f757cc6bf2aa2c128e4b8e6a993067d44c63c891c081ab crun_checksums: arm: - 1.17: 0 + '1.17': 0 1.16.1: 0 - 1.16: 0 - 1.15: 0 + '1.16': 0 + '1.15': 0 1.14.4: 0 1.14.3: 0 1.14.2: 0 @@ -615,10 +615,10 @@ crun_checksums: 1.11.1: 0 1.9.2: 0 arm64: - 1.17: 3049017b99208f5ecd15c1366f47a77dace87f42dccf317ad40a07f1a867518c + '1.17': 3049017b99208f5ecd15c1366f47a77dace87f42dccf317ad40a07f1a867518c 1.16.1: 973817340e6da12c90c751b011c797396940cca965cefa74557bd1c0939f4042 - 1.16: 4595ff16487b16d2158fa8c3452bc0e1ecdc177ab2ace40fc02cd6e49838ff67 - 1.15: 2ed5fe6def4c1d57f219747bac5e71cb22312ef026fe63ed8e3246a4dcfebe13 + '1.16': 4595ff16487b16d2158fa8c3452bc0e1ecdc177ab2ace40fc02cd6e49838ff67 + '1.15': 2ed5fe6def4c1d57f219747bac5e71cb22312ef026fe63ed8e3246a4dcfebe13 1.14.4: 308f8719055de178897f66cbb72d6a02567050ac645dd5eca52f48de347dda6c 1.14.3: 0486629e1599c3bccded279f6555ff22691958cde56203ceca099af6f2407263 1.14.2: 409ebdcb4935b004ce0efa8ada4aaf8d4dd63b77cde1d0acdf55664c168acbd9 @@ -627,10 +627,10 @@ crun_checksums: 1.11.1: c8b0d243f6ac4fb02665c157b5404e5184bdc9240dbdcdde0ccef2db352ce97a 1.9.2: 1ad8bd3c1aa693f59133c480aa13bbdf6d81e4528e72ce955612c6bae8cb1720 amd64: - 1.17: e9512a3e034e781b2396d068fd24eafcd5788e410403da886df9dc8871d504a5 + '1.17': e9512a3e034e781b2396d068fd24eafcd5788e410403da886df9dc8871d504a5 1.16.1: 7b6f1791fb9b2c49ec959b9384b3c4e2ec8c69945fd5292a179d23eb62422eb3 - 1.16: 7f53bffd6b0e216f8f6d6472bb73dc4c6c4ea2c2e7342c52d4bee2972798ce68 - 1.15: f02c66dcc38b9d06f19a92dfb5ac831aba9c33ae48dbf4ab92d7680ca1140172 + '1.16': 7f53bffd6b0e216f8f6d6472bb73dc4c6c4ea2c2e7342c52d4bee2972798ce68 + '1.15': f02c66dcc38b9d06f19a92dfb5ac831aba9c33ae48dbf4ab92d7680ca1140172 1.14.4: 4f170aaa10d2ef02560cfb60b67ddfa1a83b1b4f7018227e9cb23a6af3955ec1 1.14.3: 80c5ab9422d4672f650f2bad3da933568349b64117d055486abc3534517be2af 1.14.2: 4d3a64961ea9e6a1313ab807f86a17bc6ebcecad2df84a120322fddebff00bcf @@ -639,10 +639,10 @@ crun_checksums: 1.11.1: ca8c9cef23f4a3f7a635ee58a3d9fa35e768581fda89dc3b6baed219cc407a02 1.9.2: 2bb60bcd5652cb17e44f66f0b8ae48195434bd1d66593db97fba85c7778eac53 ppc64le: - 1.17: ca8ee0fabcac57b61b80f6c234ae20b3b9821433fdf1a6306be5defeac11930e + '1.17': ca8ee0fabcac57b61b80f6c234ae20b3b9821433fdf1a6306be5defeac11930e 1.16.1: 9590ce79697c5509731f8e58d1733b7051c36f92104925221ca8bda800afee41 - 1.16: fc7199a2faac1ca0e3e58dee4dd369b9065aa0d95f3257d8803e521213f1bd9b - 1.15: dd0aad6140175ef83792e601c8e89cf66813486e9070aac7f39cac040283d4fd + '1.16': fc7199a2faac1ca0e3e58dee4dd369b9065aa0d95f3257d8803e521213f1bd9b + '1.15': dd0aad6140175ef83792e601c8e89cf66813486e9070aac7f39cac040283d4fd 1.14.4: aa7263d3c54e478158ed5a70a435208096e434e58ccbc2a334ecbbbc384eff09 1.14.3: b3304ce1a983e4e1abd4b2bc59eedaa188299be838bdcd8b376f1f8d489bdc94 1.14.2: 1cf8f3296d1f6ab4189da565d2ac3552059e8e455cc665b913f4b5f3e484bdd7 @@ -718,78 +718,78 @@ kata_containers_binary_checksums: 3.0.1: 0 gvisor_runsc_binary_checksums: arm: - 20240305: 0 - 20240212: 0 - 20240206: 0 - 20240129: 0 - 20240122: 0 - 20240115: 0 - 20240109: 0 - 20231218: 0 + '20240305': 0 + '20240212': 0 + '20240206': 0 + '20240129': 0 + '20240122': 0 + '20240115': 0 + '20240109': 0 + '20231218': 0 arm64: - 20240305: b8b54b45fed2dd1fa14decefecc68c8da605b8abaaee97a0550deeee4afc427f - 20240212: a03fb515df9cabf1c618193e9ed7400543c0410ab7107d1ce291ebc9212521cf - 20240206: 50b637dcb7c1b2fb1c1ce189a48ca6732d4b5a5c17ac08d5dd22d33b06fd31c8 - 20240129: d2ecc989f27d40a0e7cd53f0712fa91405b1eef2cb466deccffa41a7f607bacd - 20240122: ae9507f4ff950dc315e7dea2c4b0086dce66b88b8c8bac2008d8e754bac7af7a - 20240115: 7b2ce18408212542477c31cc1bd0ddddf6fbf7439d57e56f6884091f62c81cd8 - 20240109: 51a1b299997834b902192806def688b1e23ff6b14f28a9ed3397f3f6572a189a - 20231218: 86262a78946deacc309c0f08883659ee3298c288048dc30955945e71993c81a8 + '20240305': b8b54b45fed2dd1fa14decefecc68c8da605b8abaaee97a0550deeee4afc427f + '20240212': a03fb515df9cabf1c618193e9ed7400543c0410ab7107d1ce291ebc9212521cf + '20240206': 50b637dcb7c1b2fb1c1ce189a48ca6732d4b5a5c17ac08d5dd22d33b06fd31c8 + '20240129': d2ecc989f27d40a0e7cd53f0712fa91405b1eef2cb466deccffa41a7f607bacd + '20240122': ae9507f4ff950dc315e7dea2c4b0086dce66b88b8c8bac2008d8e754bac7af7a + '20240115': 7b2ce18408212542477c31cc1bd0ddddf6fbf7439d57e56f6884091f62c81cd8 + '20240109': 51a1b299997834b902192806def688b1e23ff6b14f28a9ed3397f3f6572a189a + '20231218': 86262a78946deacc309c0f08883659ee3298c288048dc30955945e71993c81a8 amd64: - 20240305: 3b949f7fab2c7d3d75df09fe5f170b46951e62b8833dcc4abad0a4d6c12f41f3 - 20240212: da5390680d18c3f98f1e88cd7363f97de42ed63a767e61d476b1740b0918b93c - 20240206: 996a8e855c1d54a7dcf688d52ee698fd714f0fd143c42ee793707e7f4f18124d - 20240129: b7765ea92c0100fcd1d03c7b23073c9be9486350cf38ffcbb72eb7915fe26605 - 20240122: d184712583d543b8f56a28e8583a1fa55c7256e77934123fe21c621e0d9b975c - 20240115: 9ae176da972b288880e69b1a438052eea2c502b6292aea8a1a33fbcf65e135dd - 20240109: f32810820c81a4dfe570080c06c5dabfc1be74ec0d5da659f93ae5cc1fc5c098 - 20231218: c353d36a134dfc2fab8509f72a34abf6a761603975eb00a39e4077c41aeaf31b + '20240305': 3b949f7fab2c7d3d75df09fe5f170b46951e62b8833dcc4abad0a4d6c12f41f3 + '20240212': da5390680d18c3f98f1e88cd7363f97de42ed63a767e61d476b1740b0918b93c + '20240206': 996a8e855c1d54a7dcf688d52ee698fd714f0fd143c42ee793707e7f4f18124d + '20240129': b7765ea92c0100fcd1d03c7b23073c9be9486350cf38ffcbb72eb7915fe26605 + '20240122': d184712583d543b8f56a28e8583a1fa55c7256e77934123fe21c621e0d9b975c + '20240115': 9ae176da972b288880e69b1a438052eea2c502b6292aea8a1a33fbcf65e135dd + '20240109': f32810820c81a4dfe570080c06c5dabfc1be74ec0d5da659f93ae5cc1fc5c098 + '20231218': c353d36a134dfc2fab8509f72a34abf6a761603975eb00a39e4077c41aeaf31b ppc64le: - 20240305: 0 - 20240212: 0 - 20240206: 0 - 20240129: 0 - 20240122: 0 - 20240115: 0 - 20240109: 0 - 20231218: 0 + '20240305': 0 + '20240212': 0 + '20240206': 0 + '20240129': 0 + '20240122': 0 + '20240115': 0 + '20240109': 0 + '20231218': 0 gvisor_containerd_shim_binary_checksums: arm: - 20240305: 0 - 20240212: 0 - 20240206: 0 - 20240129: 0 - 20240122: 0 - 20240115: 0 - 20240109: 0 - 20231218: 0 + '20240305': 0 + '20240212': 0 + '20240206': 0 + '20240129': 0 + '20240122': 0 + '20240115': 0 + '20240109': 0 + '20231218': 0 arm64: - 20240305: 466c51e4f4bf592da0edf8c70c70ba74f026bb48f980bb28ffb582a93c88c049 - 20240212: 4b122fd5684c068d5d73189a30a8130cc5280aefadda0b8532321446c9c79c90 - 20240206: 34ded13729aeea0bee6c6d4cbc57ac19a9f4a532631b307ae975cbeb2a09a4ff - 20240129: 41c033549c24c13c776db42d212a416a2df20a6cff57cc26f70df8cdff738441 - 20240122: e5f3dbcd7f1b1fb9f46e1432656a8b07dda63a5c65fdbe639062761439df23c0 - 20240115: eae0a657656c4153db44dd51ca285b423b44c4eaad872ea56c18b6a430cdfda5 - 20240109: 40eb0a4f5f0013afb221e228fd6e71887127c4b09c7f2eb36705a0cd5c746d57 - 20231218: 5f66938de981221359a64f05a5c770b228090db3a2697d91ad622c18dd19f4b2 + '20240305': 466c51e4f4bf592da0edf8c70c70ba74f026bb48f980bb28ffb582a93c88c049 + '20240212': 4b122fd5684c068d5d73189a30a8130cc5280aefadda0b8532321446c9c79c90 + '20240206': 34ded13729aeea0bee6c6d4cbc57ac19a9f4a532631b307ae975cbeb2a09a4ff + '20240129': 41c033549c24c13c776db42d212a416a2df20a6cff57cc26f70df8cdff738441 + '20240122': e5f3dbcd7f1b1fb9f46e1432656a8b07dda63a5c65fdbe639062761439df23c0 + '20240115': eae0a657656c4153db44dd51ca285b423b44c4eaad872ea56c18b6a430cdfda5 + '20240109': 40eb0a4f5f0013afb221e228fd6e71887127c4b09c7f2eb36705a0cd5c746d57 + '20231218': 5f66938de981221359a64f05a5c770b228090db3a2697d91ad622c18dd19f4b2 amd64: - 20240305: 11a1b482e0ed6c72ea6ca72692e1cb2d0794214d142be5389e30517a96b157dc - 20240212: 48333e9b6158f8d4192a35e1d1f74319b6a083d6cbc3779c847548de6a5faf5f - 20240206: 9c88e82b71dc07f689c74f61143ea00fa8621a6d5c31c5fadb9714ad3be8465a - 20240129: 840b4b9d47bd04f3dfed6cf8fbee7c2c4a697e17461c22afb873d67499d4d9b9 - 20240122: cd7d9e4bb4cb0ac8242d15fc03580880f53eb36ebd9fb8d686e2811e86ad698e - 20240115: b95d05f667f1040cb07f262f27396d1deb23573ce4c4a31ea3568e6ca3b70c24 - 20240109: d677683326cfd42c7913636651f74ffd1a6866066877903d8a58c644422c2e18 - 20231218: a0578a357feb9320298730bf5ba683880ba35c476dc74dc82c79f0b5acc42656 + '20240305': 11a1b482e0ed6c72ea6ca72692e1cb2d0794214d142be5389e30517a96b157dc + '20240212': 48333e9b6158f8d4192a35e1d1f74319b6a083d6cbc3779c847548de6a5faf5f + '20240206': 9c88e82b71dc07f689c74f61143ea00fa8621a6d5c31c5fadb9714ad3be8465a + '20240129': 840b4b9d47bd04f3dfed6cf8fbee7c2c4a697e17461c22afb873d67499d4d9b9 + '20240122': cd7d9e4bb4cb0ac8242d15fc03580880f53eb36ebd9fb8d686e2811e86ad698e + '20240115': b95d05f667f1040cb07f262f27396d1deb23573ce4c4a31ea3568e6ca3b70c24 + '20240109': d677683326cfd42c7913636651f74ffd1a6866066877903d8a58c644422c2e18 + '20231218': a0578a357feb9320298730bf5ba683880ba35c476dc74dc82c79f0b5acc42656 ppc64le: - 20240305: 0 - 20240212: 0 - 20240206: 0 - 20240129: 0 - 20240122: 0 - 20240115: 0 - 20240109: 0 - 20231218: 0 + '20240305': 0 + '20240212': 0 + '20240206': 0 + '20240129': 0 + '20240122': 0 + '20240115': 0 + '20240109': 0 + '20231218': 0 nerdctl_archive_checksums: arm: 2.0.3: d95f238738623ae1f4fb01b6a7f287436ba85493700a9de263b3efbff57424d4 diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index b4a288598..a7e151128 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -73,13 +73,13 @@ image_info_command_on_localhost: "{{ lookup('vars', image_command_tool_on_localh image_arch: "{{ host_architecture | default('amd64') }}" # Versions -crun_version: 1.17 -runc_version: v1.2.4 -kata_containers_version: 3.1.3 -youki_version: 0.4.1 -gvisor_version: 20240305 -containerd_version: 2.0.2 -cri_dockerd_version: 0.3.11 +crun_version: "{{ (crun_checksums['amd64'] | dict2items)[0].key }}" +runc_version: "{{ (runc_checksums['amd64'] | dict2items)[0].key }}" +kata_containers_version: "{{ (kata_containers_binary_checksums['amd64'] | dict2items)[0].key }}" +youki_version: "{{ (youki_checksums['amd64'] | dict2items)[0].key }}" +gvisor_version: "{{ (gvisor_runsc_binary_checksums['amd64'] | dict2items)[0].key }}" +containerd_version: "{{ (containerd_archive_checksums['amd64'] | dict2items)[0].key }}" +cri_dockerd_version: "{{ (cri_dockerd_archive_checksums['amd64'] | dict2items)[0].key }}" # this is relevant when container_manager == 'docker' docker_containerd_version: 1.6.32 @@ -99,7 +99,7 @@ github_image_repo: "ghcr.io" # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download -calico_version: "v3.29.1" +calico_version: "{{ (calicoctl_binary_checksums['amd64'] | dict2items)[0].key }}" calico_ctl_version: "{{ calico_version }}" calico_cni_version: "{{ calico_version }}" calico_policy_version: "{{ calico_version }}" @@ -111,19 +111,19 @@ calico_apiserver_enabled: false flannel_version: "v0.22.0" flannel_cni_version: "v1.1.2" weave_version: 2.8.7 -cni_version: "v1.4.0" +cni_version: "{{ (cni_binary_checksums['amd64'] | dict2items)[0].key }}" cilium_version: "v1.15.9" -cilium_cli_version: "v0.16.0" +cilium_cli_version: "{{ (ciliumcli_binary_checksums['amd64'] | dict2items)[0].key }}" cilium_enable_hubble: false kube_ovn_version: "v1.12.21" kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}" kube_router_version: "v2.0.0" multus_version: "v4.1.0" -helm_version: "v3.16.4" -nerdctl_version: "2.0.3" -skopeo_version: "v1.16.1" +helm_version: "{{ (helm_archive_checksums['amd64'] | dict2items)[0].key }}" +nerdctl_version: "{{ (nerdctl_archive_checksums['amd64'] | dict2items)[0].key }}" +skopeo_version: "{{ (skopeo_binary_checksums['amd64'] | dict2items)[0].key }}" # Get kubernetes major version (i.e. 1.17.4 => 1.17) kube_major_version: "{{ kube_version | regex_replace('^v([0-9])+\\.([0-9]+)\\.[0-9]+', 'v\\1.\\2') }}" @@ -159,7 +159,7 @@ scheduler_plugins_supported_versions: v1.29: 0 scheduler_plugins_version: "{{ scheduler_plugins_supported_versions[kube_major_version] }}" -yq_version: "v4.42.1" +yq_version: "{{ (yq_checksums['amd64'] | dict2items)[0].key }}" github_url: https://github.com dl_k8s_io_url: https://dl.k8s.io diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index ad8234589..dee728b75 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -18,10 +18,10 @@ kubelet_fail_swap_on: true kubelet_swap_behavior: LimitedSwap ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.32.0 +kube_version: "{{ (kubelet_checksums['amd64'] | dict2items)[0].key }}" ## The minimum version working -kube_version_min_required: v1.30.0 +kube_version_min_required: "{{ (kubelet_checksums['amd64'] | dict2items)[-1].key }}" ## Kube Proxy mode One of ['iptables', 'ipvs'] kube_proxy_mode: ipvs diff --git a/scripts/Dockerfile.j2 b/scripts/Dockerfile.j2 new file mode 100644 index 000000000..c22f3fe4b --- /dev/null +++ b/scripts/Dockerfile.j2 @@ -0,0 +1,50 @@ +# syntax=docker/dockerfile:1 + +# Use imutable image tags rather than mutable tags (like ubuntu:22.04) +FROM ubuntu:22.04@sha256:149d67e29f765f4db62aa52161009e99e389544e25a8f43c8c89d4a445a7ca37 + +# Some tools like yamllint need this +# Pip needs this as well at the moment to install ansible +# (and potentially other packages) +# See: https://github.com/pypa/pip/issues/10219 +ENV LANG=C.UTF-8 \ + DEBIAN_FRONTEND=noninteractive \ + PYTHONDONTWRITEBYTECODE=1 + +WORKDIR /kubespray + +# hadolint ignore=DL3008 +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + apt-get update -q \ + && apt-get install -yq --no-install-recommends \ + curl \ + python3 \ + python3-pip \ + sshpass \ + vim \ + rsync \ + openssh-client \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /var/log/* + +RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \ + --mount=type=cache,sharing=locked,id=pipcache,mode=0777,target=/root/.cache/pip \ + pip install --no-compile --no-cache-dir -r requirements.txt \ + && find /usr -type d -name '*__pycache__' -prune -exec rm -rf {} \; + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \ + && curl -L "https://dl.k8s.io/release/{{ kube_version }}/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \ + && echo "$(curl -L "https://dl.k8s.io/release/{{ kube_version }}/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \ + && chmod a+x /usr/local/bin/kubectl + +COPY *.yml ./ +COPY *.cfg ./ +COPY roles ./roles +COPY contrib ./contrib +COPY inventory ./inventory +COPY library ./library +COPY extra_playbooks ./extra_playbooks +COPY playbooks ./playbooks +COPY plugins ./plugins diff --git a/scripts/assert-sorted-checksums.yml b/scripts/assert-sorted-checksums.yml new file mode 100755 index 000000000..36ee1efb2 --- /dev/null +++ b/scripts/assert-sorted-checksums.yml @@ -0,0 +1,38 @@ +#!/usr/bin/env ansible-playbook +--- +- name: Check all checksums are sorted by version + hosts: localhost + connection: local + gather_facts: false + vars: + fallback_ip: 'bypass tasks in kubespray-defaults' + _keys: "{{ query('ansible.builtin.varnames', '^.+_checksums$') }}" + _values: "{{ query('ansible.builtin.vars', *_keys) | map('dict2items') }}" + _components_archs_values: "{{ _keys | zip(_values) | community.general.dict | dict2items | subelements('value') }}" + _minimal_data_needed: "{{ _components_archs_values | map(attribute='0.key') | zip(_components_archs_values | map(attribute='1')) }}" + roles: + - kubespray-defaults + tasks: + - name: Check all versions are strings + assert: + that: "{{ item.1.value | reject('string') == [] }}" + quiet: true + loop: "{{ _minimal_data_needed }}" + loop_control: + label: "{{ item.0 }}:{{ item.1.key }}" + - name: Check all checksums are sorted by version + vars: + actual: "{{ item.1.value.keys() | map('string') | reverse}}" + sorted: "{{ item.1.value.keys() | map('string') | community.general.version_sort }}" + assert: + that: actual == sorted + quiet: true + msg: "{{ actual | ansible.utils.fact_diff(sorted) }}" + loop: "{{ _minimal_data_needed }}" + loop_control: + label: "{{ item.0 }}:{{ item.1.key }}" + when: + - item.1.value is not string + - (item.1.value | dict2items)[0].value is string or + (item.1.value | dict2items)[0].value is number + # only do list, the others are checksums with a different structure diff --git a/scripts/pipeline.Dockerfile.j2 b/scripts/pipeline.Dockerfile.j2 new file mode 100644 index 000000000..a6148f67b --- /dev/null +++ b/scripts/pipeline.Dockerfile.j2 @@ -0,0 +1,60 @@ +# Use imutable image tags rather than mutable tags (like ubuntu:22.04) +FROM ubuntu:jammy-20230308 +# Some tools like yamllint need this +# Pip needs this as well at the moment to install ansible +# (and potentially other packages) +# See: https://github.com/pypa/pip/issues/10219 +ENV VAGRANT_VERSION=2.4.1 \ + VAGRANT_DEFAULT_PROVIDER=libvirt \ + VAGRANT_ANSIBLE_TAGS=facts \ + LANG=C.UTF-8 \ + DEBIAN_FRONTEND=noninteractive \ + PYTHONDONTWRITEBYTECODE=1 + +RUN apt update -q \ + && apt install -yq \ + libssl-dev \ + python3-dev \ + python3-pip \ + sshpass \ + apt-transport-https \ + jq \ + moreutils \ + libvirt-dev \ + openssh-client \ + rsync \ + git \ + ca-certificates \ + curl \ + gnupg2 \ + software-properties-common \ + unzip \ + libvirt-clients \ + qemu-utils \ + qemu-kvm \ + dnsmasq \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ + && add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ + && apt update -q \ + && apt install --no-install-recommends -yq docker-ce \ + && apt autoremove -yqq --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/* + +WORKDIR /kubespray +ADD ./requirements.txt /kubespray/requirements.txt +ADD ./tests/requirements.txt /kubespray/tests/requirements.txt + +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \ + && pip install --no-compile --no-cache-dir pip -U \ + && pip install --no-compile --no-cache-dir -r tests/requirements.txt \ + && pip install --no-compile --no-cache-dir -r requirements.txt \ + && curl -L https://dl.k8s.io/release/{{ kube_version }}/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \ + && echo $(curl -L https://dl.k8s.io/release/{{ kube_version }}/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \ + && chmod a+x /usr/local/bin/kubectl \ + # Install Vagrant + && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \ + && dpkg -i vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \ + && rm vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \ + && vagrant plugin install vagrant-libvirt \ + # Install Kubernetes collections + && pip install --no-compile --no-cache-dir kubernetes \ + && ansible-galaxy collection install kubernetes.core diff --git a/scripts/render_readme_version.yml b/scripts/propagate_ansible_variables.yml similarity index 80% rename from scripts/render_readme_version.yml rename to scripts/propagate_ansible_variables.yml index 7e3791433..745c80ce2 100755 --- a/scripts/render_readme_version.yml +++ b/scripts/propagate_ansible_variables.yml @@ -20,3 +20,11 @@ marker: '' block: "\n{{ lookup('ansible.builtin.template', 'readme_versions.md.j2') }}\n\n" path: ../README.md + - name: Render Dockerfiles + template: + src: "{{ item }}.j2" + dest: "../{{ item }}" + mode: "0644" + loop: + - 'pipeline.Dockerfile' + - 'Dockerfile'