Browse Source
CI: rework pipeline: short/extended based on labels (#11324)
CI: rework pipeline: short/extended based on labels (#11324)
* CI: reduce VM resources requests to improve scheduling * CI: Reduce default jobs; add labels(ci-full/extended) to run more test * CI: use jobs dependencies instead of stages * precommit one-job * CI: Use Kubevirt VM to run Molecule and Vagrant jobspull/11341/head
Antoine Legrand
4 months ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
30 changed files with 419 additions and 321 deletions
Unified View
Diff Options
-
30.gitlab-ci.yml
-
2.gitlab-ci/build.yml
-
43.gitlab-ci/lint.yml
-
64.gitlab-ci/molecule.yml
-
362.gitlab-ci/packet.yml
-
2.gitlab-ci/pre-commit-dynamic-stub.yml
-
22.gitlab-ci/terraform.yml
-
41.gitlab-ci/vagrant.yml
-
1Vagrantfile
-
4docs/developers/ci-setup.md
-
5pipeline.Dockerfile
-
3tests/Makefile
-
5tests/cloud_playbooks/roles/packet-ci/defaults/main.yml
-
6tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2
-
2tests/files/packet_almalinux8-calico-ha-ebpf.yml
-
2tests/files/packet_almalinux8-calico-nodelocaldns-secondary.yml
-
2tests/files/packet_almalinux8-calico.yml
-
2tests/files/packet_almalinux8-docker.yml
-
2tests/files/packet_almalinux8-kube-ovn.yml
-
2tests/files/packet_rockylinux8-calico.yml
-
2tests/files/packet_rockylinux9-calico.yml
-
2tests/files/packet_rockylinux9-cilium.yml
-
2tests/files/packet_ubuntu22-all-in-one-docker.yml
-
2tests/files/packet_ubuntu22-calico-all-in-one.yml
-
2tests/files/packet_ubuntu24-all-in-one-docker.yml
-
2tests/files/packet_ubuntu24-calico-all-in-one.yml
-
2tests/files/packet_ubuntu24-calico-etcd-datastore.yml
-
1tests/files/vagrant_ubuntu20-flannel.rb
-
1tests/files/vagrant_ubuntu20-flannel.yml
-
122tests/scripts/vagrant-install.sh
@ -1,40 +1,35 @@ |
|||||
--- |
--- |
||||
generate-pre-commit: |
|
||||
image: 'mikefarah/yq@sha256:bcb889a1f9bdb0613c8a054542d02360c2b1b35521041be3e1bd8fbd0534d411' |
|
||||
stage: build |
|
||||
before_script: [] |
|
||||
|
pre-commit: |
||||
|
stage: test |
||||
|
tags: |
||||
|
- ffci |
||||
|
image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef' |
||||
|
variables: |
||||
|
PRE_COMMIT_HOME: /pre-commit-cache |
||||
script: |
script: |
||||
- > |
|
||||
yq -r < .pre-commit-config.yaml '.repos[].hooks[].id' | |
|
||||
sed 's/^/ - /' | |
|
||||
cat .gitlab-ci/pre-commit-dynamic-stub.yml - > pre-commit-generated.yml |
|
||||
artifacts: |
|
||||
|
- pre-commit run --all-files |
||||
|
cache: |
||||
|
key: pre-commit-all |
||||
paths: |
paths: |
||||
- pre-commit-generated.yml |
|
||||
|
|
||||
run-pre-commit: |
|
||||
stage: unit-tests |
|
||||
trigger: |
|
||||
include: |
|
||||
- artifact: pre-commit-generated.yml |
|
||||
job: generate-pre-commit |
|
||||
strategy: depend |
|
||||
|
- /pre-commit-cache |
||||
|
needs: [] |
||||
|
|
||||
vagrant-validate: |
vagrant-validate: |
||||
extends: .job |
extends: .job |
||||
stage: unit-tests |
|
||||
tags: [light] |
|
||||
|
stage: test |
||||
|
tags: [ffci] |
||||
variables: |
variables: |
||||
VAGRANT_VERSION: 2.3.7 |
VAGRANT_VERSION: 2.3.7 |
||||
script: |
script: |
||||
- ./tests/scripts/vagrant-validate.sh |
|
||||
|
- ./tests/scripts/vagrant-validate.sh |
||||
except: ['triggers', 'master'] |
except: ['triggers', 'master'] |
||||
|
|
||||
|
|
||||
# TODO: convert to pre-commit hook |
# TODO: convert to pre-commit hook |
||||
check-galaxy-version: |
check-galaxy-version: |
||||
stage: unit-tests |
|
||||
tags: [light] |
|
||||
|
needs: [] |
||||
|
stage: test |
||||
|
tags: [ffci] |
||||
image: python:3 |
image: python:3 |
||||
script: |
script: |
||||
- tests/scripts/check_galaxy_version.sh |
|
||||
|
- tests/scripts/check_galaxy_version.sh |
||||
@ -1,64 +1,73 @@ |
|||||
--- |
--- |
||||
|
|
||||
.vagrant: |
.vagrant: |
||||
extends: .testcases |
extends: .testcases |
||||
|
needs: |
||||
|
- ci-not-authorized |
||||
variables: |
variables: |
||||
CI_PLATFORM: "vagrant" |
CI_PLATFORM: "vagrant" |
||||
SSH_USER: "vagrant" |
SSH_USER: "vagrant" |
||||
VAGRANT_DEFAULT_PROVIDER: "libvirt" |
VAGRANT_DEFAULT_PROVIDER: "libvirt" |
||||
KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb |
KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb |
||||
tags: [c3.small.x86] |
|
||||
only: [/^pr-.*$/] |
|
||||
except: ['triggers'] |
|
||||
image: $PIPELINE_IMAGE |
|
||||
|
DOCKER_NAME: vagrant |
||||
|
VAGRANT_ANSIBLE_TAGS: facts |
||||
|
tags: [ffci-vm-large] |
||||
|
# only: [/^pr-.*$/] |
||||
|
# except: ['triggers'] |
||||
|
image: quay.io/kubespray/vm-kubespray-ci:v6 |
||||
services: [] |
services: [] |
||||
before_script: |
before_script: |
||||
|
- echo $USER |
||||
|
- python3 -m venv citest |
||||
|
- source citest/bin/activate |
||||
|
- vagrant plugin expunge --reinstall --force --no-tty |
||||
|
- vagrant plugin install vagrant-libvirt |
||||
|
- pip install --no-compile --no-cache-dir pip -U |
||||
|
- pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt |
||||
|
- pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt |
||||
- ./tests/scripts/vagrant_clean.sh |
- ./tests/scripts/vagrant_clean.sh |
||||
script: |
script: |
||||
- ./tests/scripts/testcases_run.sh |
- ./tests/scripts/testcases_run.sh |
||||
after_script: |
|
||||
- chronic ./tests/scripts/testcases_cleanup.sh |
|
||||
|
|
||||
vagrant_ubuntu20-calico-dual-stack: |
vagrant_ubuntu20-calico-dual-stack: |
||||
stage: deploy-part2 |
|
||||
|
stage: deploy-extended |
||||
extends: .vagrant |
extends: .vagrant |
||||
when: manual |
when: manual |
||||
# FIXME: this test if broken (perma-failing) |
# FIXME: this test if broken (perma-failing) |
||||
|
|
||||
vagrant_ubuntu20-weave-medium: |
vagrant_ubuntu20-weave-medium: |
||||
stage: deploy-part2 |
|
||||
|
stage: deploy-extended |
||||
extends: .vagrant |
extends: .vagrant |
||||
when: manual |
when: manual |
||||
|
|
||||
vagrant_ubuntu20-flannel: |
vagrant_ubuntu20-flannel: |
||||
stage: deploy-part2 |
|
||||
|
stage: deploy-part1 |
||||
extends: .vagrant |
extends: .vagrant |
||||
when: on_success |
when: on_success |
||||
allow_failure: false |
allow_failure: false |
||||
|
|
||||
vagrant_ubuntu20-flannel-collection: |
vagrant_ubuntu20-flannel-collection: |
||||
stage: deploy-part2 |
|
||||
|
stage: deploy-extended |
||||
extends: .vagrant |
extends: .vagrant |
||||
when: on_success |
|
||||
|
when: manual |
||||
|
|
||||
vagrant_ubuntu20-kube-router-sep: |
vagrant_ubuntu20-kube-router-sep: |
||||
stage: deploy-part2 |
|
||||
|
stage: deploy-extended |
||||
extends: .vagrant |
extends: .vagrant |
||||
when: manual |
when: manual |
||||
|
|
||||
# Service proxy test fails connectivity testing |
# Service proxy test fails connectivity testing |
||||
vagrant_ubuntu20-kube-router-svc-proxy: |
vagrant_ubuntu20-kube-router-svc-proxy: |
||||
stage: deploy-part2 |
|
||||
|
stage: deploy-extended |
||||
extends: .vagrant |
extends: .vagrant |
||||
when: manual |
when: manual |
||||
|
|
||||
vagrant_fedora37-kube-router: |
vagrant_fedora37-kube-router: |
||||
stage: deploy-part2 |
|
||||
|
stage: deploy-extended |
||||
extends: .vagrant |
extends: .vagrant |
||||
when: manual |
when: manual |
||||
# FIXME: this test if broken (perma-failing) |
# FIXME: this test if broken (perma-failing) |
||||
|
|
||||
vagrant_centos7-kube-router: |
vagrant_centos7-kube-router: |
||||
stage: deploy-part2 |
|
||||
|
stage: deploy-extended |
||||
extends: .vagrant |
extends: .vagrant |
||||
when: manual |
when: manual |
||||
@ -1,3 +1,4 @@ |
|||||
--- |
--- |
||||
# Kubespray settings |
# Kubespray settings |
||||
kube_network_plugin: flannel |
kube_network_plugin: flannel |
||||
|
ansible_ssh_private_key: .vagrant.d/insecure_private_key |
||||
@ -0,0 +1,122 @@ |
|||||
|
#!/bin/bash |
||||
|
|
||||
|
# install_vagrant() { |
||||
|
# sudo apt install vagrant-libvirt vagrant -y |
||||
|
# sudo vagrant plugin install vagrant-libvirt |
||||
|
# } |
||||
|
|
||||
|
# prep(){ |
||||
|
# sudo apt-get update -y |
||||
|
# sudo apt-get install ca-certificates curl libvirt-daemon-system\ |
||||
|
# libvirt-clients qemu-utils qemu-kvm htop atop -y |
||||
|
|
||||
|
# sudo install -m 0755 -d /etc/apt/keyrings |
||||
|
# } |
||||
|
# install_docker() { |
||||
|
# VERSION_STRING=5:26.1.0-1~ubuntu.24.04~noble |
||||
|
# sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc |
||||
|
# sudo chmod a+r /etc/apt/keyrings/docker.asc |
||||
|
|
||||
|
# # Add the repository to Apt sources: |
||||
|
# echo \ |
||||
|
# "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ |
||||
|
# $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ |
||||
|
# sudo tee /etc/apt/sources.list.d/docker.list > /dev/null |
||||
|
# sudo apt-get update -y |
||||
|
|
||||
|
# sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y |
||||
|
# } |
||||
|
# install_docker_auto () { |
||||
|
# curl -fsSL https://get.docker.com -o get-docker.sh |
||||
|
# sudo sh ./get-docker.sh --dry-run |
||||
|
# } |
||||
|
|
||||
|
|
||||
|
|
||||
|
VAGRANT_VERSION=2.4.1 |
||||
|
VAGRANT_DEFAULT_PROVIDER=libvirt |
||||
|
VAGRANT_ANSIBLE_TAGS=facts |
||||
|
LANG=C.UTF-8 |
||||
|
DEBIAN_FRONTEND=noninteractive |
||||
|
PYTHONDONTWRITEBYTECODE=1 |
||||
|
KUBE_VERSION=1.29.5 |
||||
|
pipeline_install() { |
||||
|
cp /etc/apt/sources.list /etc/apt/sources.list."$(date +"%F")" |
||||
|
sed -i -e '/^# deb-src.*universe$/s/# //g' /etc/apt/sources.list |
||||
|
sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources |
||||
|
|
||||
|
apt update |
||||
|
# libssl-dev \ |
||||
|
# python3-dev \ |
||||
|
# # jq \ |
||||
|
# moreutils \ |
||||
|
# libvirt-dev \ |
||||
|
# # rsync \ |
||||
|
# git \ |
||||
|
# # htop \ |
||||
|
# gpg \ |
||||
|
# atop |
||||
|
|
||||
|
# gnupg2 \ |
||||
|
# software-properties-common |
||||
|
# |
||||
|
apt install --no-install-recommends -y \ |
||||
|
git \ |
||||
|
make \ |
||||
|
python3-pip \ |
||||
|
sshpass \ |
||||
|
apt-transport-https \ |
||||
|
openssh-client \ |
||||
|
ca-certificates \ |
||||
|
curl \ |
||||
|
libfuse2 \ |
||||
|
unzip \ |
||||
|
qemu-utils \ |
||||
|
libvirt-daemon-system \ |
||||
|
libvirt-clients \ |
||||
|
qemu-kvm \ |
||||
|
ebtables libguestfs-tools \ |
||||
|
ruby-fog-libvirt \ |
||||
|
libvirt-dev \ |
||||
|
gcc \ |
||||
|
build-essential \ |
||||
|
ruby-libvirt \ |
||||
|
libxslt-dev libxml2-dev zlib1g-dev \ |
||||
|
python3-venv python3-full \ |
||||
|
dnsmasq |
||||
|
|
||||
|
apt-get build-dep -y ruby-libvirt ruby-dev |
||||
|
### VAGRANT ### |
||||
|
# apt-get install -y unzip |
||||
|
curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_linux_amd64.zip |
||||
|
unzip vagrant_${VAGRANT_VERSION}_linux_amd64.zip |
||||
|
mv vagrant /usr/local/bin/vagrant |
||||
|
chmod a+x /usr/local/bin/vagrant |
||||
|
# ls -la /usr/local/bin/vagrant |
||||
|
/usr/local/bin/vagrant plugin install vagrant-libvirt |
||||
|
usermod -aG kvm kubespray |
||||
|
usermod -aG libvirt kubespray |
||||
|
|
||||
|
### DOCKER ### |
||||
|
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - |
||||
|
add-apt-repository -y "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" |
||||
|
apt update |
||||
|
apt install --no-install-recommends -y docker-ce |
||||
|
apt autoremove -y --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/* |
||||
|
|
||||
|
### KUBECTL ### |
||||
|
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" |
||||
|
mv kubectl /usr/local/bin/kubectl |
||||
|
chmod a+x /usr/local/bin/kubectl |
||||
|
systemctl restart libvirtd |
||||
|
# Install Vagrant |
||||
|
# apt update -y |
||||
|
# echo apt-get install -y unzip libfuse2 vagrant vagrant-libvirt |
||||
|
# apt --fix-broken install -y |
||||
|
# dpkg --configure -a -y |
||||
|
|
||||
|
|
||||
|
} |
||||
|
# wrapped up in a function so that we have some protection against only getting |
||||
|
# half the file during "curl | sh" |
||||
|
pipeline_install |
||||
Write
Preview
Loading…
Cancel
Save