Browse Source
[release-2.25] CI: rework pipeline: short/extended based on labels (#11424)
[release-2.25] CI: rework pipeline: short/extended based on labels (#11424)
* CI: reduce VM resources requests to improve scheduling * CI: Reduce default jobs; add labels(ci-full/extended) to run more test * CI: use jobs dependencies instead of stages * precommit one-job * CI: Use Kubevirt VM to run Molecule and Vagrant jobs --------- Co-authored-by: ant31 <2t.antoine@gmail.com>pull/11428/head
k8s-infra-cherrypick-robot
3 months ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
30 changed files with 419 additions and 321 deletions
Split View
Diff Options
-
30.gitlab-ci.yml
-
2.gitlab-ci/build.yml
-
43.gitlab-ci/lint.yml
-
64.gitlab-ci/molecule.yml
-
362.gitlab-ci/packet.yml
-
2.gitlab-ci/pre-commit-dynamic-stub.yml
-
22.gitlab-ci/terraform.yml
-
41.gitlab-ci/vagrant.yml
-
1Vagrantfile
-
4docs/developers/ci-setup.md
-
5pipeline.Dockerfile
-
3tests/Makefile
-
5tests/cloud_playbooks/roles/packet-ci/defaults/main.yml
-
6tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2
-
2tests/files/packet_almalinux8-calico-ha-ebpf.yml
-
2tests/files/packet_almalinux8-calico-nodelocaldns-secondary.yml
-
2tests/files/packet_almalinux8-calico.yml
-
2tests/files/packet_almalinux8-docker.yml
-
2tests/files/packet_almalinux8-kube-ovn.yml
-
2tests/files/packet_rockylinux8-calico.yml
-
2tests/files/packet_rockylinux9-calico.yml
-
2tests/files/packet_rockylinux9-cilium.yml
-
2tests/files/packet_ubuntu22-all-in-one-docker.yml
-
2tests/files/packet_ubuntu22-calico-all-in-one.yml
-
2tests/files/packet_ubuntu24-all-in-one-docker.yml
-
2tests/files/packet_ubuntu24-calico-all-in-one.yml
-
2tests/files/packet_ubuntu24-calico-etcd-datastore.yml
-
1tests/files/vagrant_ubuntu20-flannel.rb
-
1tests/files/vagrant_ubuntu20-flannel.yml
-
122tests/scripts/vagrant-install.sh
@ -1,40 +1,35 @@ |
|||
--- |
|||
generate-pre-commit: |
|||
image: 'mikefarah/yq@sha256:bcb889a1f9bdb0613c8a054542d02360c2b1b35521041be3e1bd8fbd0534d411' |
|||
stage: build |
|||
before_script: [] |
|||
pre-commit: |
|||
stage: test |
|||
tags: |
|||
- ffci |
|||
image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef' |
|||
variables: |
|||
PRE_COMMIT_HOME: /pre-commit-cache |
|||
script: |
|||
- > |
|||
yq -r < .pre-commit-config.yaml '.repos[].hooks[].id' | |
|||
sed 's/^/ - /' | |
|||
cat .gitlab-ci/pre-commit-dynamic-stub.yml - > pre-commit-generated.yml |
|||
artifacts: |
|||
- pre-commit run --all-files |
|||
cache: |
|||
key: pre-commit-all |
|||
paths: |
|||
- pre-commit-generated.yml |
|||
|
|||
run-pre-commit: |
|||
stage: unit-tests |
|||
trigger: |
|||
include: |
|||
- artifact: pre-commit-generated.yml |
|||
job: generate-pre-commit |
|||
strategy: depend |
|||
- /pre-commit-cache |
|||
needs: [] |
|||
|
|||
vagrant-validate: |
|||
extends: .job |
|||
stage: unit-tests |
|||
tags: [light] |
|||
stage: test |
|||
tags: [ffci] |
|||
variables: |
|||
VAGRANT_VERSION: 2.3.7 |
|||
script: |
|||
- ./tests/scripts/vagrant-validate.sh |
|||
- ./tests/scripts/vagrant-validate.sh |
|||
except: ['triggers', 'master'] |
|||
|
|||
|
|||
# TODO: convert to pre-commit hook |
|||
check-galaxy-version: |
|||
stage: unit-tests |
|||
tags: [light] |
|||
needs: [] |
|||
stage: test |
|||
tags: [ffci] |
|||
image: python:3 |
|||
script: |
|||
- tests/scripts/check_galaxy_version.sh |
|||
- tests/scripts/check_galaxy_version.sh |
|||
@ -1,64 +1,73 @@ |
|||
--- |
|||
|
|||
.vagrant: |
|||
extends: .testcases |
|||
needs: |
|||
- ci-not-authorized |
|||
variables: |
|||
CI_PLATFORM: "vagrant" |
|||
SSH_USER: "vagrant" |
|||
VAGRANT_DEFAULT_PROVIDER: "libvirt" |
|||
KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb |
|||
tags: [c3.small.x86] |
|||
only: [/^pr-.*$/] |
|||
except: ['triggers'] |
|||
image: $PIPELINE_IMAGE |
|||
DOCKER_NAME: vagrant |
|||
VAGRANT_ANSIBLE_TAGS: facts |
|||
tags: [ffci-vm-large] |
|||
# only: [/^pr-.*$/] |
|||
# except: ['triggers'] |
|||
image: quay.io/kubespray/vm-kubespray-ci:v6 |
|||
services: [] |
|||
before_script: |
|||
- echo $USER |
|||
- python3 -m venv citest |
|||
- source citest/bin/activate |
|||
- vagrant plugin expunge --reinstall --force --no-tty |
|||
- vagrant plugin install vagrant-libvirt |
|||
- pip install --no-compile --no-cache-dir pip -U |
|||
- pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt |
|||
- pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt |
|||
- ./tests/scripts/vagrant_clean.sh |
|||
script: |
|||
- ./tests/scripts/testcases_run.sh |
|||
after_script: |
|||
- chronic ./tests/scripts/testcases_cleanup.sh |
|||
|
|||
vagrant_ubuntu20-calico-dual-stack: |
|||
stage: deploy-part2 |
|||
stage: deploy-extended |
|||
extends: .vagrant |
|||
when: manual |
|||
# FIXME: this test if broken (perma-failing) |
|||
|
|||
vagrant_ubuntu20-weave-medium: |
|||
stage: deploy-part2 |
|||
stage: deploy-extended |
|||
extends: .vagrant |
|||
when: manual |
|||
|
|||
vagrant_ubuntu20-flannel: |
|||
stage: deploy-part2 |
|||
stage: deploy-part1 |
|||
extends: .vagrant |
|||
when: on_success |
|||
allow_failure: false |
|||
|
|||
vagrant_ubuntu20-flannel-collection: |
|||
stage: deploy-part2 |
|||
stage: deploy-extended |
|||
extends: .vagrant |
|||
when: on_success |
|||
when: manual |
|||
|
|||
vagrant_ubuntu20-kube-router-sep: |
|||
stage: deploy-part2 |
|||
stage: deploy-extended |
|||
extends: .vagrant |
|||
when: manual |
|||
|
|||
# Service proxy test fails connectivity testing |
|||
vagrant_ubuntu20-kube-router-svc-proxy: |
|||
stage: deploy-part2 |
|||
stage: deploy-extended |
|||
extends: .vagrant |
|||
when: manual |
|||
|
|||
vagrant_fedora37-kube-router: |
|||
stage: deploy-part2 |
|||
stage: deploy-extended |
|||
extends: .vagrant |
|||
when: manual |
|||
# FIXME: this test if broken (perma-failing) |
|||
|
|||
vagrant_centos7-kube-router: |
|||
stage: deploy-part2 |
|||
stage: deploy-extended |
|||
extends: .vagrant |
|||
when: manual |
|||
@ -1,3 +1,4 @@ |
|||
--- |
|||
# Kubespray settings |
|||
kube_network_plugin: flannel |
|||
ansible_ssh_private_key: .vagrant.d/insecure_private_key |
|||
@ -0,0 +1,122 @@ |
|||
#!/bin/bash |
|||
|
|||
# install_vagrant() { |
|||
# sudo apt install vagrant-libvirt vagrant -y |
|||
# sudo vagrant plugin install vagrant-libvirt |
|||
# } |
|||
|
|||
# prep(){ |
|||
# sudo apt-get update -y |
|||
# sudo apt-get install ca-certificates curl libvirt-daemon-system\ |
|||
# libvirt-clients qemu-utils qemu-kvm htop atop -y |
|||
|
|||
# sudo install -m 0755 -d /etc/apt/keyrings |
|||
# } |
|||
# install_docker() { |
|||
# VERSION_STRING=5:26.1.0-1~ubuntu.24.04~noble |
|||
# sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc |
|||
# sudo chmod a+r /etc/apt/keyrings/docker.asc |
|||
|
|||
# # Add the repository to Apt sources: |
|||
# echo \ |
|||
# "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ |
|||
# $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ |
|||
# sudo tee /etc/apt/sources.list.d/docker.list > /dev/null |
|||
# sudo apt-get update -y |
|||
|
|||
# sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y |
|||
# } |
|||
# install_docker_auto () { |
|||
# curl -fsSL https://get.docker.com -o get-docker.sh |
|||
# sudo sh ./get-docker.sh --dry-run |
|||
# } |
|||
|
|||
|
|||
|
|||
VAGRANT_VERSION=2.4.1 |
|||
VAGRANT_DEFAULT_PROVIDER=libvirt |
|||
VAGRANT_ANSIBLE_TAGS=facts |
|||
LANG=C.UTF-8 |
|||
DEBIAN_FRONTEND=noninteractive |
|||
PYTHONDONTWRITEBYTECODE=1 |
|||
KUBE_VERSION=1.29.5 |
|||
pipeline_install() { |
|||
cp /etc/apt/sources.list /etc/apt/sources.list."$(date +"%F")" |
|||
sed -i -e '/^# deb-src.*universe$/s/# //g' /etc/apt/sources.list |
|||
sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources |
|||
|
|||
apt update |
|||
# libssl-dev \ |
|||
# python3-dev \ |
|||
# # jq \ |
|||
# moreutils \ |
|||
# libvirt-dev \ |
|||
# # rsync \ |
|||
# git \ |
|||
# # htop \ |
|||
# gpg \ |
|||
# atop |
|||
|
|||
# gnupg2 \ |
|||
# software-properties-common |
|||
# |
|||
apt install --no-install-recommends -y \ |
|||
git \ |
|||
make \ |
|||
python3-pip \ |
|||
sshpass \ |
|||
apt-transport-https \ |
|||
openssh-client \ |
|||
ca-certificates \ |
|||
curl \ |
|||
libfuse2 \ |
|||
unzip \ |
|||
qemu-utils \ |
|||
libvirt-daemon-system \ |
|||
libvirt-clients \ |
|||
qemu-kvm \ |
|||
ebtables libguestfs-tools \ |
|||
ruby-fog-libvirt \ |
|||
libvirt-dev \ |
|||
gcc \ |
|||
build-essential \ |
|||
ruby-libvirt \ |
|||
libxslt-dev libxml2-dev zlib1g-dev \ |
|||
python3-venv python3-full \ |
|||
dnsmasq |
|||
|
|||
apt-get build-dep -y ruby-libvirt ruby-dev |
|||
### VAGRANT ### |
|||
# apt-get install -y unzip |
|||
curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_linux_amd64.zip |
|||
unzip vagrant_${VAGRANT_VERSION}_linux_amd64.zip |
|||
mv vagrant /usr/local/bin/vagrant |
|||
chmod a+x /usr/local/bin/vagrant |
|||
# ls -la /usr/local/bin/vagrant |
|||
/usr/local/bin/vagrant plugin install vagrant-libvirt |
|||
usermod -aG kvm kubespray |
|||
usermod -aG libvirt kubespray |
|||
|
|||
### DOCKER ### |
|||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - |
|||
add-apt-repository -y "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" |
|||
apt update |
|||
apt install --no-install-recommends -y docker-ce |
|||
apt autoremove -y --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/* |
|||
|
|||
### KUBECTL ### |
|||
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" |
|||
mv kubectl /usr/local/bin/kubectl |
|||
chmod a+x /usr/local/bin/kubectl |
|||
systemctl restart libvirtd |
|||
# Install Vagrant |
|||
# apt update -y |
|||
# echo apt-get install -y unzip libfuse2 vagrant vagrant-libvirt |
|||
# apt --fix-broken install -y |
|||
# dpkg --configure -a -y |
|||
|
|||
|
|||
} |
|||
# wrapped up in a function so that we have some protection against only getting |
|||
# half the file during "curl | sh" |
|||
pipeline_install |
|||
Write
Preview
Loading…
Cancel
Save