CI: cleanup terraform + deduplicate and simplify

1 month ago · b9e251ac7a
10 changed files with 38 additions and 170 deletions
--- a/.ansible-lint
+++ b/.ansible-lint
@ -39,5 +39,7 @@ exclude_paths:
  - .github
  - .ansible
  - .cache
+  - .gitlab-ci.yml
+  - .gitlab-ci
 mock_modules:
  - gluster.gluster.gluster_volume
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -31,12 +31,12 @@ variables:
  ANSIBLE_VERBOSITY: 2
  RECOVER_CONTROL_PLANE_TEST: "false"
  RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-  TERRAFORM_VERSION: 1.3.7
+  TF_VERSION: 1.3.7
  PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"

 before_script:
  - ./tests/scripts/rebase.sh
-  - mkdir -p /.ssh
+  - mkdir -p cluster-dump $ANSIBLE_INVENTORY

 .job: &job
  tags:
--- a/.gitlab-ci/build.yml
+++ b/.gitlab-ci/build.yml
@ -15,9 +15,8 @@
    PROJECT_DIR: $CI_PROJECT_DIR
    DOCKERFILE: Dockerfile
    GODEBUG: "http2client=0"
-  before_script:
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
  script:
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
    - /kaniko/executor --cache=true
                       --cache-dir=image-cache
                       --context $PROJECT_DIR
--- a/.gitlab-ci/kubevirt.yml
+++ b/.gitlab-ci/kubevirt.yml
@ -2,15 +2,12 @@
 .kubevirt:
  extends: .job-moderated
  interruptible: true
-  before_script:
-    - ./tests/scripts/rebase.sh
-    - mkdir -p cluster-dump
  script:
+    - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml
+                      -c local -e @"tests/files/${TESTCASE}.yml"
    - ./tests/scripts/testcases_run.sh
  variables:
    ANSIBLE_TIMEOUT: "120"
-    CI_PLATFORM: packet
-    SSH_USER: kubespray
  tags:
    - ffci
  needs:
--- a/.gitlab-ci/molecule.yml
+++ b/.gitlab-ci/molecule.yml
@ -8,8 +8,6 @@
  needs:
  - pipeline-image
  # - ci-not-authorized
-  before_script:
-  - ./tests/scripts/rebase.sh
  script:
  - ./tests/scripts/molecule_run.sh
  after_script:
--- a/.gitlab-ci/terraform.yml
+++ b/.gitlab-ci/terraform.yml
@ -5,28 +5,21 @@
  needs:
    - ci-not-authorized
    - pipeline-image
+  variables:
+    TF_VAR_public_key_path: "${ANSIBLE_PRIVATE_KEY_FILE}.pub"
+    TF_VAR_ssh_private_key_path: $ANSIBLE_PRIVATE_KEY_FILE
+    CLUSTER: $CI_COMMIT_REF_NAME
+    TERRAFORM_STATE_ROOT: $CI_PROJECT_DIR
  stage: deploy-part1
  before_script:
-    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
    - ./tests/scripts/rebase.sh
-    - ./tests/scripts/testcases_prepare.sh
+    - mkdir -p cluster-dump $ANSIBLE_INVENTORY
    - ./tests/scripts/terraform_install.sh
-    # Set Ansible config
-    - cp ansible.cfg ~/.ansible.cfg
-    # Prepare inventory
    - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
-    - ln -s contrib/terraform/$PROVIDER/hosts
+    - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts
    - terraform -chdir="contrib/terraform/$PROVIDER" init
-    # Copy SSH keypair
-    - mkdir -p ~/.ssh
-    - echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
-    - chmod 400 ~/.ssh/id_rsa
-    - echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
-    - mkdir -p contrib/terraform/$PROVIDER/group_vars
-    # Random subnet to avoid routing conflicts
-    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"

-.terraform_validate:
+terraform_validate:
  extends: .terraform_install
  tags: [ffci]
  only: ['master', /^pr-.*$/]
@ -36,6 +29,17 @@
  stage: test
  needs:
    - pipeline-image
+  parallel:
+    matrix:
+      - PROVIDER:
+          - openstack
+          - equinix
+          - aws
+          - exoscale
+          - hetzner
+          - vsphere
+          - upcloud
+          - nifcloud

 .terraform_apply:
  extends: .terraform_install
@ -43,99 +47,22 @@
  stage: deploy-extended
  when: manual
  only: [/^pr-.*$/]
-  artifacts:
-    when: always
-    paths:
-      - cluster-dump/
  variables:
    ANSIBLE_INVENTORY_UNPARSED_FAILED: "true"
-    ANSIBLE_INVENTORY: hosts
-    CI_PLATFORM: tf
-    TF_VAR_ssh_user: $SSH_USER
+    TF_VAR_ssh_user: $ANSIBLE_REMOTE_USER
    TF_VAR_cluster_name: $CI_JOB_ID
  script:
+    # Set Ansible config
+    - cp ansible.cfg ~/.ansible.cfg
+    - ssh-keygen -N '' -f $ANSIBLE_PRIVATE_KEY_FILE -t rsa
+    - mkdir -p contrib/terraform/$PROVIDER/group_vars
+    # Random subnet to avoid routing conflicts
+    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
+    - terraform -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
    - tests/scripts/testcases_run.sh
  after_script:
    # Cleanup regardless of exit code
-    - ./tests/scripts/testcases_cleanup.sh
-
-tf-validate-openstack:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: openstack
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-equinix:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: equinix
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-aws:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: aws
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-exoscale:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: exoscale
-
-tf-validate-hetzner:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: hetzner
-
-tf-validate-vsphere:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: vsphere
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-upcloud:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: upcloud
-    CLUSTER: $CI_COMMIT_REF_NAME
-
-tf-validate-nifcloud:
-  extends: .terraform_validate
-  variables:
-    TF_VERSION: $TERRAFORM_VERSION
-    PROVIDER: nifcloud
-
-# tf-packet-ubuntu20-default:
-#   extends: .terraform_apply
-#   variables:
-#     TF_VERSION: $TERRAFORM_VERSION
-#     PROVIDER: packet
-#     CLUSTER: $CI_COMMIT_REF_NAME
-#     TF_VAR_number_of_k8s_masters: "1"
-#     TF_VAR_number_of_k8s_nodes: "1"
-#     TF_VAR_plan_k8s_masters: t1.small.x86
-#     TF_VAR_plan_k8s_nodes: t1.small.x86
-#     TF_VAR_metro: am
-#     TF_VAR_public_key_path: ""
-#     TF_VAR_operating_system: ubuntu_20_04
-
-.ovh_variables: &ovh_variables
-  OS_AUTH_URL: https://auth.cloud.ovh.net/v3
-  OS_PROJECT_ID: 8d3cd5d737d74227ace462dee0b903fe
-  OS_PROJECT_NAME: "9361447987648822"
-  OS_USER_DOMAIN_NAME: Default
-  OS_PROJECT_DOMAIN_ID: default
-  OS_USERNAME: 8XuhBMfkKVrk
-  OS_REGION_NAME: UK1
-  OS_INTERFACE: public
-  OS_IDENTITY_API_VERSION: "3"
+    - terraform -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve

 # Elastx is generously donating resources for Kubespray on Openstack CI
 # Contacts: @gix @bl0m1
@ -169,11 +96,8 @@ tf-elastx_ubuntu20-calico:
  allow_failure: true
  variables:
    <<: *elastx_variables
-    TF_VERSION: $TERRAFORM_VERSION
    PROVIDER: openstack
-    CLUSTER: $CI_COMMIT_REF_NAME
    ANSIBLE_TIMEOUT: "60"
-    SSH_USER: ubuntu
    TF_VAR_number_of_k8s_masters: "1"
    TF_VAR_number_of_k8s_masters_no_floating_ip: "0"
    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
@ -194,46 +118,3 @@ tf-elastx_ubuntu20-calico:
    TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b      # v1-standard-2
    TF_VAR_image: ubuntu-20.04-server-latest
    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
-
-# OVH voucher expired, commenting job until things are sorted  out
-
-# tf-ovh_cleanup:
-#  stage: unit-tests
-#  tags: [light]
-#  image: python
-#  environment: ovh
-#  variables:
-#    <<: *ovh_variables
-#  before_script:
-#    - pip install -r scripts/openstack-cleanup/requirements.txt
-#  script:
-#    - ./scripts/openstack-cleanup/main.py
-
-# tf-ovh_ubuntu20-calico:
-#  extends: .terraform_apply
-#  when: on_success
-#  environment: ovh
-#  variables:
-#    <<: *ovh_variables
-#    TF_VERSION: $TERRAFORM_VERSION
-#    PROVIDER: openstack
-#    CLUSTER: $CI_COMMIT_REF_NAME
-#    ANSIBLE_TIMEOUT: "60"
-#    SSH_USER: ubuntu
-#    TF_VAR_number_of_k8s_masters: "0"
-#    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
-#    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
-#    TF_VAR_number_of_etcd: "0"
-#    TF_VAR_number_of_k8s_nodes: "0"
-#    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
-#    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
-#    TF_VAR_number_of_bastions: "0"
-#    TF_VAR_number_of_k8s_masters_no_etcd: "0"
-#    TF_VAR_use_neutron: "0"
-#    TF_VAR_floatingip_pool: "Ext-Net"
-#    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
-#    TF_VAR_network_name: "Ext-Net"
-#    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
-#    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
-#    TF_VAR_image: "Ubuntu 20.04"
-#    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
--- a/.gitlab-ci/vagrant.yml
+++ b/.gitlab-ci/vagrant.yml
@ -28,7 +28,10 @@ vagrant:
    - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
    - ./tests/scripts/vagrant_clean.sh
  script:
+    - vagrant up
    - ./tests/scripts/testcases_run.sh
+  after_script:
+    - vagrant destroy -f
  cache:
    key: $CI_JOB_NAME_SLUG
    paths:
--- a/tests/scripts/create-tf.sh
+++ b/tests/scripts/create-tf.sh
@ -1,5 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-
-cd ..
-terraform -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
--- a/tests/scripts/delete-tf.sh
+++ b/tests/scripts/delete-tf.sh
@ -1,5 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-
-cd ..
-terraform -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
--- a/tests/scripts/testcases_run.sh
+++ b/tests/scripts/testcases_run.sh
@ -16,8 +16,6 @@ fi
 export ANSIBLE_BECOME=true
 export ANSIBLE_BECOME_USER=root

-make -C tests create-${CI_PLATFORM} -s
-
 # Test collection build and install by installing our collection, emptying our repository, adding
 # cluster.yml, reset.yml, and remote-node.yml files that simply point to our collection's playbooks, and then
 # running the same tests as before