From 51069223f5fcd7e1f0540c160b38626fbce36ead Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 11 Dec 2023 16:56:17 +0100 Subject: [PATCH] Decouple kubespray-defaults from download (#10626) * Decouple role kubespray-defaults from download Avoids doing re-importing the download role on every invocation of kubespray-defaults (and skipping everything). This has a measurable effect on playbook performance. * Update docs refering to moved download defaults --- .ansible-lint-ignore | 2 +- Dockerfile | 2 +- contrib/offline/README.md | 2 +- contrib/offline/generate_list.sh | 4 ++-- docs/offline-environment.md | 2 +- extra_playbooks/migrate_openstack_provider.yml | 2 +- inventory/sample/group_vars/all/all.yml | 2 +- pipeline.Dockerfile | 4 ++-- .../defaults/main/checksums.yml | 0 .../defaults/main/download.yml} | 0 .../defaults/{main.yaml => main/main.yml} | 0 roles/kubespray-defaults/meta/main.yml | 6 ------ roles/kubespray-defaults/tasks/main.yaml | 2 +- scripts/download_hash.py | 4 ++-- scripts/download_hash.sh | 2 +- tests/scripts/check_readme_versions.sh | 4 ++-- 16 files changed, 16 insertions(+), 22 deletions(-) rename roles/{download => kubespray-defaults}/defaults/main/checksums.yml (100%) rename roles/{download/defaults/main/main.yml => kubespray-defaults/defaults/main/download.yml} (100%) rename roles/kubespray-defaults/defaults/{main.yaml => main/main.yml} (100%) delete mode 100644 roles/kubespray-defaults/meta/main.yml diff --git a/.ansible-lint-ignore b/.ansible-lint-ignore index 03a371318..730b579e7 100644 --- a/.ansible-lint-ignore +++ b/.ansible-lint-ignore @@ -5,4 +5,4 @@ roles/kubernetes/control-plane/defaults/main/main.yml jinja[spacing] roles/kubernetes/kubeadm/defaults/main.yml jinja[spacing] roles/kubernetes/node/defaults/main.yml jinja[spacing] roles/kubernetes/preinstall/defaults/main.yml jinja[spacing] -roles/kubespray-defaults/defaults/main.yaml jinja[spacing] +roles/kubespray-defaults/defaults/main/main.yml jinja[spacing] diff --git a/Dockerfile b/Dockerfile index 347c1af49..3e81ed1d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,7 @@ RUN apt update -q \ MarkupSafe==2.1.3 \ ruamel.yaml==0.17.21 \ passlib==1.7.4 \ - && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main.yaml) \ + && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \ && curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \ && echo $(curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \ && chmod a+x /usr/local/bin/kubectl \ diff --git a/contrib/offline/README.md b/contrib/offline/README.md index c059f5f8a..ecf5ffc36 100644 --- a/contrib/offline/README.md +++ b/contrib/offline/README.md @@ -27,7 +27,7 @@ manage-offline-container-images.sh register ## generate_list.sh -This script generates the list of downloaded files and the list of container images by `roles/download/defaults/main/main.yml` file. +This script generates the list of downloaded files and the list of container images by `roles/kubespray-defaults/main/download.yml` file. Run this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files, all downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template. diff --git a/contrib/offline/generate_list.sh b/contrib/offline/generate_list.sh index 646360f1d..e298c8e85 100755 --- a/contrib/offline/generate_list.sh +++ b/contrib/offline/generate_list.sh @@ -5,7 +5,7 @@ CURRENT_DIR=$(cd $(dirname $0); pwd) TEMP_DIR="${CURRENT_DIR}/temp" REPO_ROOT_DIR="${CURRENT_DIR%/contrib/offline}" -: ${DOWNLOAD_YML:="roles/download/defaults/main/main.yml"} +: ${DOWNLOAD_YML:="roles/kubespray-defaults/main/download.yml"} mkdir -p ${TEMP_DIR} @@ -19,7 +19,7 @@ sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \ | sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/\"//g' > ${TEMP_DIR}/images.list.template # add kube-* images to images list template -# Those container images are downloaded by kubeadm, then roles/download/defaults/main/main.yml +# Those container images are downloaded by kubeadm, then roles/kubespray-defaults/main/download.yml # doesn't contain those images. That is reason why here needs to put those images into the # list separately. KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy" diff --git a/docs/offline-environment.md b/docs/offline-environment.md index 743c2f741..33748a58c 100644 --- a/docs/offline-environment.md +++ b/docs/offline-environment.md @@ -95,7 +95,7 @@ If you use the settings like the one above, you'll need to define in your invent * `registry_host`: Container image registry. If you _don't_ use the same repository path for the container images that the ones defined - in [Download's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/download/defaults/main/main.yml) + in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray-defaults/main/download.yml) , you need to override the `*_image_repo` for these container images. If you want to make your life easier, use the same repository path, you won't have to override anything else. * `registry_addr`: Container image registry, but only have [domain or ip]:[port]. diff --git a/extra_playbooks/migrate_openstack_provider.yml b/extra_playbooks/migrate_openstack_provider.yml index a82a58710..bba3d918c 100644 --- a/extra_playbooks/migrate_openstack_provider.yml +++ b/extra_playbooks/migrate_openstack_provider.yml @@ -12,7 +12,7 @@ hosts: kube_control_plane[0] tasks: - name: Include kubespray-default variables - include_vars: ../roles/kubespray-defaults/defaults/main.yaml + include_vars: ../roles/kubespray-defaults/defaults/main/main.yml - name: Copy get_cinder_pvs.sh to master copy: src: get_cinder_pvs.sh diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml index b93f1a3ac..c7f762870 100644 --- a/inventory/sample/group_vars/all/all.yml +++ b/inventory/sample/group_vars/all/all.yml @@ -57,7 +57,7 @@ loadbalancer_apiserver_healthcheck_port: 8081 # https_proxy: "" # https_proxy_cert_file: "" -## Refer to roles/kubespray-defaults/defaults/main.yml before modifying no_proxy +## Refer to roles/kubespray-defaults/defaults/main/main.yml before modifying no_proxy # no_proxy: "" ## Some problems may occur when downloading files over https proxy due to ansible bug diff --git a/pipeline.Dockerfile b/pipeline.Dockerfile index eb4dcf6d6..4eba6a1a5 100644 --- a/pipeline.Dockerfile +++ b/pipeline.Dockerfile @@ -40,11 +40,11 @@ WORKDIR /kubespray RUN --mount=type=bind,target=./requirements.txt,src=./requirements.txt \ --mount=type=bind,target=./tests/requirements.txt,src=./tests/requirements.txt \ - --mount=type=bind,target=./roles/kubespray-defaults/defaults/main.yaml,src=./roles/kubespray-defaults/defaults/main.yaml \ + --mount=type=bind,target=./roles/kubespray-defaults/defaults/main/main.yml,src=./roles/kubespray-defaults/defaults/main/main.yml \ update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \ && pip install --no-compile --no-cache-dir pip -U \ && pip install --no-compile --no-cache-dir -r tests/requirements.txt \ - && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main.yaml) \ + && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \ && curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \ && echo $(curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \ && chmod a+x /usr/local/bin/kubectl \ diff --git a/roles/download/defaults/main/checksums.yml b/roles/kubespray-defaults/defaults/main/checksums.yml similarity index 100% rename from roles/download/defaults/main/checksums.yml rename to roles/kubespray-defaults/defaults/main/checksums.yml diff --git a/roles/download/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/download.yml similarity index 100% rename from roles/download/defaults/main/main.yml rename to roles/kubespray-defaults/defaults/main/download.yml diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main/main.yml similarity index 100% rename from roles/kubespray-defaults/defaults/main.yaml rename to roles/kubespray-defaults/defaults/main/main.yml diff --git a/roles/kubespray-defaults/meta/main.yml b/roles/kubespray-defaults/meta/main.yml deleted file mode 100644 index 88d70247e..000000000 --- a/roles/kubespray-defaults/meta/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -dependencies: - - role: download - skip_downloads: true - tags: - - facts diff --git a/roles/kubespray-defaults/tasks/main.yaml b/roles/kubespray-defaults/tasks/main.yaml index 650408726..e3defd27a 100644 --- a/roles/kubespray-defaults/tasks/main.yaml +++ b/roles/kubespray-defaults/tasks/main.yaml @@ -1,7 +1,7 @@ --- - name: Configure defaults debug: - msg: "Check roles/kubespray-defaults/defaults/main.yml" + msg: "Check roles/kubespray-defaults/defaults/main/main.yml" tags: - always diff --git a/scripts/download_hash.py b/scripts/download_hash.py index dfd84462b..6b76f21cc 100644 --- a/scripts/download_hash.py +++ b/scripts/download_hash.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # After a new version of Kubernetes has been released, -# run this script to update roles/download/defaults/main/main.yml +# run this script to update roles/kubespray-defaults/main/download.yml # with new hashes. import hashlib @@ -10,7 +10,7 @@ import sys import requests from ruamel.yaml import YAML -MAIN_YML = "../roles/download/defaults/main/main.yml" +MAIN_YML = "../roles/kubespray-defaults/main/download.yml" def open_main_yaml(): yaml = YAML() diff --git a/scripts/download_hash.sh b/scripts/download_hash.sh index b37e46626..94c181661 100755 --- a/scripts/download_hash.sh +++ b/scripts/download_hash.sh @@ -6,7 +6,7 @@ if [[ ${DEBUG:-false} == "true" ]]; then set -o xtrace fi -checksums_file="$(git rev-parse --show-toplevel)/roles/download/defaults/main/checksums.yml" +checksums_file="$(git rev-parse --show-toplevel)/roles/kubespray-defaults/defaults/main/checksums.yml" downloads_folder=/tmp/kubespray_binaries function get_versions { diff --git a/tests/scripts/check_readme_versions.sh b/tests/scripts/check_readme_versions.sh index d796d9c43..0f1836f9d 100755 --- a/tests/scripts/check_readme_versions.sh +++ b/tests/scripts/check_readme_versions.sh @@ -7,7 +7,7 @@ TARGET_COMPONENTS="containerd calico cilium flannel kube-ovn kube-router weave c cd $(dirname $0)/../../ echo checking kubernetes.. -version_from_default=$(grep "^kube_version:" ./roles/kubespray-defaults/defaults/main.yaml | awk '{print $2}' | sed s/\"//g) +version_from_default=$(grep "^kube_version:" ./roles/kubespray-defaults/defaults/main/main.yml | awk '{print $2}' | sed s/\"//g) version_from_readme=$(grep " \[kubernetes\]" ./README.md | awk '{print $3}') if [ "${version_from_default}" != "${version_from_readme}" ]; then echo "The version of kubernetes is different between main.yml(${version_from_default}) and README.md(${version_from_readme})." @@ -17,7 +17,7 @@ fi for component in $(echo ${TARGET_COMPONENTS}); do echo checking ${component}.. - version_from_default=$(grep "^$(echo ${component} | sed s/"-"/"_"/g)_version:" ./roles/download/defaults/main/main.yml | awk '{print $2}' | sed s/\"//g | sed s/^v//) + version_from_default=$(grep "^$(echo ${component} | sed s/"-"/"_"/g)_version:" ./roles/kubespray-defaults/defaults/main/download.yml | awk '{print $2}' | sed s/\"//g | sed s/^v//) if [ "${version_from_default}" = "" ]; then version_from_default=$(grep "^$(echo ${component} | sed s/"-"/"_"/g)_version:" ./roles/kubernetes/node/defaults/main.yml | awk '{print $2}' | sed s/\"//g | sed s/^v//) fi