PHASE 2 - Enable Packet-CI in gitlab and move unit-tests and deploy-part1 (#4538)

* PHASE 2 - Enable Packet-CI in gitlab * Add gitlab files * Reset files back and only keep Packet * Include packet * Add missing Upgrade Tests * Update GCE jobs etc * Fix bug * Yaml lint all gitlab files * Remove GCE * Test * Test again * Enable GCE again * Install requirements * Cleanup the gitlab file * Cleanup runner tags * Install requirements * Test * Test variables for gce * Test again * Test again * Fix * Update
5 years ago · d4b9f15c0a
26 changed files with 429 additions and 265 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -27,42 +27,18 @@ variables:
  UPGRADE_TEST: "false"
  LOG_LEVEL: "-vv"

-# asia-east1-a
-# asia-northeast1-a
-# europe-west1-b
-# us-central1-a
-# us-east1-b
-# us-west1-a
-
 before_script:
  - /usr/bin/python -m pip install -r tests/requirements.txt
  - mkdir -p /.ssh

 .job: &job
  tags:
-    - kubernetes
-    - docker
+    - packet
  image: quay.io/kubespray/kubespray:v2.9.0

-.docker_service: &docker_service
-  services:
-    - docker:dind
-
-.create_cluster: &create_cluster
-  <<: *job
-  <<: *docker_service
-
 .testcases: &testcases
  <<: *job
-  <<: *docker_service
-  cache:
-    key: "$CI_BUILD_REF_NAME"
-    paths:
-      - downloads/
-      - $HOME/.cache
  before_script:
-    - docker info
-    - /usr/bin/python -m pip install -r requirements.txt
    - /usr/bin/python -m pip install -r tests/requirements.txt
    - mkdir -p /.ssh
    - mkdir -p $HOME/.ssh
@ -81,9 +57,6 @@ before_script:
    - test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
    # Checkout the CI vars file so it is available
    - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
-    # Workaround https://github.com/kubernetes-sigs/kubespray/issues/2021
-    - 'sh -c "echo ignore_assert_errors: true | tee -a tests/files/${CI_JOB_NAME}.yml"'
-

    # Create cluster
    - >
@ -133,76 +106,76 @@ before_script:
    ## Idempotency checks 1/5 (repeat deployment)
    - >
      if [ "${IDEMPOT_CHECK}" = "true" ]; then
-      ansible-playbook
-      -i ${ANSIBLE_INVENTORY}
-      -b --become-user=root
-      --private-key=${HOME}/.ssh/id_rsa
-      -u $SSH_USER
-      ${SSH_ARGS}
-      ${LOG_LEVEL}
-      -e @${CI_TEST_VARS}
-      -e ansible_python_interpreter=${PYPATH}
-      -e local_release_dir=${PWD}/downloads
-      --limit "all:!fake_hosts"
-      cluster.yml;
+        ansible-playbook
+        -i ${ANSIBLE_INVENTORY}
+        -b --become-user=root
+        --private-key=${HOME}/.ssh/id_rsa
+        -u $SSH_USER
+        ${SSH_ARGS}
+        ${LOG_LEVEL}
+        -e @${CI_TEST_VARS}
+        -e ansible_python_interpreter=${PYPATH}
+        -e local_release_dir=${PWD}/downloads
+        --limit "all:!fake_hosts"
+        cluster.yml;
      fi

    ## Idempotency checks 2/5 (Advanced DNS checks)
    - >
      if [ "${IDEMPOT_CHECK}" = "true" ]; then
-      ansible-playbook
-      -i ${ANSIBLE_INVENTORY}
-      -b --become-user=root
-      --private-key=${HOME}/.ssh/id_rsa
-      -u $SSH_USER
-      ${SSH_ARGS}
-      ${LOG_LEVEL}
-      -e @${CI_TEST_VARS}
-      --limit "all:!fake_hosts"
-      tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
+        ansible-playbook
+        -i ${ANSIBLE_INVENTORY}
+        -b --become-user=root
+        --private-key=${HOME}/.ssh/id_rsa
+        -u $SSH_USER
+        ${SSH_ARGS}
+        ${LOG_LEVEL}
+        -e @${CI_TEST_VARS}
+        --limit "all:!fake_hosts"
+        tests/testcases/040_check-network-adv.yml;
      fi

    ## Idempotency checks 3/5 (reset deployment)
    - >
      if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-      ansible-playbook
-      -i ${ANSIBLE_INVENTORY}
-      -b --become-user=root
-      --private-key=${HOME}/.ssh/id_rsa
-      -u $SSH_USER
-      ${SSH_ARGS}
-      ${LOG_LEVEL}
-      -e @${CI_TEST_VARS}
-      -e ansible_python_interpreter=${PYPATH}
-      -e reset_confirmation=yes
-      --limit "all:!fake_hosts"
-      reset.yml;
+        ansible-playbook
+        -i ${ANSIBLE_INVENTORY}
+        -b --become-user=root
+        --private-key=${HOME}/.ssh/id_rsa
+        -u $SSH_USER
+        ${SSH_ARGS}
+        ${LOG_LEVEL}
+        -e @${CI_TEST_VARS}
+        -e ansible_python_interpreter=${PYPATH}
+        -e reset_confirmation=yes
+        --limit "all:!fake_hosts"
+        reset.yml;
      fi

    ## Idempotency checks 4/5 (redeploy after reset)
    - >
      if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-      ansible-playbook
-      -i ${ANSIBLE_INVENTORY}
-      -b --become-user=root
-      --private-key=${HOME}/.ssh/id_rsa
-      -u $SSH_USER
-      ${SSH_ARGS}
-      ${LOG_LEVEL}
-      -e @${CI_TEST_VARS}
-      -e ansible_python_interpreter=${PYPATH}
-      -e local_release_dir=${PWD}/downloads
-      --limit "all:!fake_hosts"
-      cluster.yml;
+        ansible-playbook
+        -i ${ANSIBLE_INVENTORY}
+        -b --become-user=root
+        --private-key=${HOME}/.ssh/id_rsa
+        -u $SSH_USER
+        ${SSH_ARGS}
+        ${LOG_LEVEL}
+        -e @${CI_TEST_VARS}
+        -e ansible_python_interpreter=${PYPATH}
+        -e local_release_dir=${PWD}/downloads
+        --limit "all:!fake_hosts"
+        cluster.yml;
      fi

    ## Idempotency checks 5/5 (Advanced DNS checks)
    - >
      if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-      ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH}
-      -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root
-      --limit "all:!fake_hosts"
-      tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
+        ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH}
+        -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root
+        --limit "all:!fake_hosts"
+        tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
      fi

  after_script:
@ -213,8 +186,6 @@ before_script:
 ci-authorized:
  extends: .job
  stage: moderator
-  before_script:
-    - apt-get -y install jq
  script:
    - /bin/sh scripts/premoderator.sh
  except: ['triggers', 'master']
@ -224,3 +195,4 @@ include:
  - .gitlab-ci/gce.yml
  - .gitlab-ci/digital-ocean.yml
  - .gitlab-ci/terraform.yml
+  - .gitlab-ci/packet.yml
--- a/.gitlab-ci/digital-ocean.yml
+++ b/.gitlab-ci/digital-ocean.yml
@ -1,3 +1,4 @@
+---
 .do_variables: &do_variables
  PRIVATE_KEY: $DO_PRIVATE_KEY
  CI_PLATFORM: "do"
@ -5,6 +6,8 @@

 .do: &do
  extends: .testcases
+  tags:
+    - do

 do_ubuntu-canal-ha:
  stage: deploy-part2
--- a/.gitlab-ci/gce.yml
+++ b/.gitlab-ci/gce.yml
@ -1,3 +1,4 @@
+---
 .gce_variables: &gce_variables
  GCE_USER: travis
  SSH_USER: $GCE_USER
@ -5,113 +6,41 @@
  CI_PLATFORM: "gce"
  PRIVATE_KEY: $GCE_PRIVATE_KEY

-.gce: &gce
-  extends: .testcases
+.docker_service: &docker_service
+  services:
+    - docker:dind

-# Test matrix. Leave the comments for markup scripts.
-.coreos_calico_aio_variables: &coreos_calico_aio_variables
-  # stage: deploy-part1
-  MOVED_TO_GROUP_VARS: "true"
+.cache: &cache
+  cache:
+    key: "$CI_BUILD_REF_NAME"
+    paths:
+      - downloads/
+      - $HOME/.cache

-.ubuntu18_flannel_aio_variables: &ubuntu18_flannel_aio_variables
-  # stage: deploy-part1
-  MOVED_TO_GROUP_VARS: "true"
+.gce: &gce
+  extends: .testcases
+  <<: *docker_service
+  <<: *cache
+  variables:
+    <<: *gce_variables
+  tags:
+    - gce

 .centos_weave_kubeadm_variables: &centos_weave_kubeadm_variables
  # stage: deploy-part1
  UPGRADE_TEST: "graceful"

-.ubuntu_canal_kubeadm_variables: &ubuntu_canal_kubeadm_variables
-  # stage: deploy-part1
-  MOVED_TO_GROUP_VARS: "true"
-
-.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
-  # stage: deploy-special
-  MOVED_TO_GROUP_VARS: "true"
-
-.ubuntu_contiv_sep_variables: &ubuntu_contiv_sep_variables
-  # stage: deploy-special
-  MOVED_TO_GROUP_VARS: "true"
-
-.coreos_cilium_variables: &coreos_cilium_variables
-  # stage: deploy-special
-  MOVED_TO_GROUP_VARS: "true"
-
-.ubuntu_cilium_sep_variables: &ubuntu_cilium_sep_variables
-  # stage: deploy-special
-  MOVED_TO_GROUP_VARS: "true"
-
-.rhel7_weave_variables: &rhel7_weave_variables
-  # stage: deploy-part1
-  MOVED_TO_GROUP_VARS: "true"
-
-.centos7_flannel_addons_variables: &centos7_flannel_addons_variables
-  # stage: deploy-part2
-  MOVED_TO_GROUP_VARS: "true"
-
-.debian9_calico_variables: &debian9_calico_variables
-  # stage: deploy-part2
-  MOVED_TO_GROUP_VARS: "true"
-
-.coreos_canal_variables: &coreos_canal_variables
-  # stage: deploy-part2
-  MOVED_TO_GROUP_VARS: "true"
-
-.rhel7_canal_sep_variables: &rhel7_canal_sep_variables
-  # stage: deploy-special
-  MOVED_TO_GROUP_VARS: "true"
-
-.ubuntu_weave_sep_variables: &ubuntu_weave_sep_variables
-  # stage: deploy-special
-  MOVED_TO_GROUP_VARS: "true"
-
-.centos7_calico_ha_variables: &centos7_calico_ha_variables
-  # stage: deploy-special
-  MOVED_TO_GROUP_VARS: "true"
-
-.centos7_kube_router_variables: &centos7_kube_router_variables
-  # stage: deploy-special
-  MOVED_TO_GROUP_VARS: "true"
-
 .centos7_multus_calico_variables: &centos7_multus_calico_variables
  # stage: deploy-part2
  UPGRADE_TEST: "graceful"

-.coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables
-  # stage: deploy-special
-  MOVED_TO_GROUP_VARS: "true"
-
-.coreos_kube_router_variables: &coreos_kube_router_variables
-  # stage: deploy-special
-  MOVED_TO_GROUP_VARS: "true"
-
-.ubuntu_rkt_sep_variables: &ubuntu_rkt_sep_variables
-  # stage: deploy-part1
-  MOVED_TO_GROUP_VARS: "true"
-
-.ubuntu_flannel_variables: &ubuntu_flannel_variables
-  # stage: deploy-part2
-  MOVED_TO_GROUP_VARS: "true"
-
-.ubuntu_kube_router_variables: &ubuntu_kube_router_variables
-  # stage: deploy-special
-  MOVED_TO_GROUP_VARS: "true"
-
-.opensuse_canal_variables: &opensuse_canal_variables
-  # stage: deploy-part2
-  MOVED_TO_GROUP_VARS: "true"
-
-
 # Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
 ### PR JOBS PART1

 gce_ubuntu18-flannel-aio:
  stage: deploy-part1
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu18_flannel_aio_variables
-  when: on_success
+  when: manual
  except: ['triggers']
  only: [/^pr-.*$/]

@ -120,9 +49,6 @@ gce_ubuntu18-flannel-aio:
 gce_coreos-calico-aio:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *coreos_calico_aio_variables
  when: on_success
  except: ['triggers']
  only: [/^pr-.*$/]
@ -130,9 +56,6 @@ gce_coreos-calico-aio:
 gce_centos7-flannel-addons:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *centos7_flannel_addons_variables
  when: on_success
  except: ['triggers']
  only: [/^pr-.*$/]
@ -143,7 +66,6 @@ gce_centos-weave-kubeadm-sep:
  stage: deploy-part2
  <<: *gce
  variables:
-    <<: *gce_variables
    <<: *centos_weave_kubeadm_variables
  when: on_success
  only: ['triggers']
@ -151,45 +73,30 @@ gce_centos-weave-kubeadm-sep:
 gce_ubuntu-weave-sep:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu_weave_sep_variables
  when: manual
  only: ['triggers']

 gce_coreos-calico-sep-triggers:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *coreos_calico_aio_variables
  when: on_success
  only: ['triggers']

 gce_ubuntu-canal-ha-triggers:
  stage: deploy-special
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu_canal_ha_variables
  when: on_success
  only: ['triggers']

 gce_centos7-flannel-addons-triggers:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *centos7_flannel_addons_variables
  when: on_success
  only: ['triggers']

 gce_ubuntu-weave-sep-triggers:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu_weave_sep_variables
  when: on_success
  only: ['triggers']

@ -199,9 +106,6 @@ gce_ubuntu-weave-sep-triggers:
 gce_ubuntu-canal-ha:
  stage: deploy-special
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu_canal_ha_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -209,9 +113,6 @@ gce_ubuntu-canal-ha:
 gce_ubuntu-canal-kubeadm:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu_canal_kubeadm_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -219,18 +120,12 @@ gce_ubuntu-canal-kubeadm:
 gce_ubuntu-canal-kubeadm-triggers:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu_canal_kubeadm_variables
  when: on_success
  only: ['triggers']

 gce_ubuntu-flannel-ha:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu_flannel_variables
  when: manual
  except: ['triggers']

@ -238,7 +133,6 @@ gce_centos-weave-kubeadm-triggers:
  stage: deploy-part2
  <<: *gce
  variables:
-    <<: *gce_variables
    <<: *centos_weave_kubeadm_variables
  when: on_success
  only: ['triggers']
@ -246,9 +140,6 @@ gce_centos-weave-kubeadm-triggers:
 gce_ubuntu-contiv-sep:
  stage: deploy-special
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu_contiv_sep_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -256,9 +147,6 @@ gce_ubuntu-contiv-sep:
 gce_coreos-cilium:
  stage: deploy-special
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *coreos_cilium_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -266,9 +154,6 @@ gce_coreos-cilium:
 gce_ubuntu-cilium-sep:
  stage: deploy-special
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu_cilium_sep_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -276,9 +161,6 @@ gce_ubuntu-cilium-sep:
 gce_rhel7-weave:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *rhel7_weave_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -286,18 +168,12 @@ gce_rhel7-weave:
 gce_rhel7-weave-triggers:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *rhel7_weave_variables
  when: on_success
  only: ['triggers']

 gce_debian9-calico-upgrade:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *debian9_calico_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -305,18 +181,12 @@ gce_debian9-calico-upgrade:
 gce_debian9-calico-triggers:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *debian9_calico_variables
  when: on_success
  only: ['triggers']

 gce_coreos-canal:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *coreos_canal_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -324,18 +194,12 @@ gce_coreos-canal:
 gce_coreos-canal-triggers:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *coreos_canal_variables
  when: on_success
  only: ['triggers']

 gce_rhel7-canal-sep:
  stage: deploy-special
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *rhel7_canal_sep_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -343,18 +207,12 @@ gce_rhel7-canal-sep:
 gce_rhel7-canal-sep-triggers:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *rhel7_canal_sep_variables
  when: on_success
  only: ['triggers']

 gce_centos7-calico-ha:
  stage: deploy-special
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *centos7_calico_ha_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -362,18 +220,12 @@ gce_centos7-calico-ha:
 gce_centos7-calico-ha-triggers:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *centos7_calico_ha_variables
  when: on_success
  only: ['triggers']

 gce_centos7-kube-router:
  stage: deploy-special
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *centos7_kube_router_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -382,7 +234,6 @@ gce_centos7-multus-calico:
  stage: deploy-part2
  <<: *gce
  variables:
-    <<: *gce_variables
    <<: *centos7_multus_calico_variables
  when: manual
  except: ['triggers']
@ -391,9 +242,6 @@ gce_centos7-multus-calico:
 gce_opensuse-canal:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *opensuse_canal_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -402,9 +250,6 @@ gce_opensuse-canal:
 gce_coreos-alpha-weave-ha:
  stage: deploy-special
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *coreos_alpha_weave_ha_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -412,9 +257,6 @@ gce_coreos-alpha-weave-ha:
 gce_coreos-kube-router:
  stage: deploy-special
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *coreos_kube_router_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -422,9 +264,6 @@ gce_coreos-kube-router:
 gce_ubuntu-rkt-sep:
  stage: deploy-part2
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu_rkt_sep_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
@ -432,9 +271,6 @@ gce_ubuntu-rkt-sep:
 gce_ubuntu-kube-router-sep:
  stage: deploy-special
  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu_kube_router_variables
  when: manual
  except: ['triggers']
  only: ['master', /^pr-.*$/]
--- a/.gitlab-ci/lint.yml
+++ b/.gitlab-ci/lint.yml
@ -1,3 +1,4 @@
+---
 yamllint:
  extends: .job
  stage: unit-tests
--- a/.gitlab-ci/packet.yml
+++ b/.gitlab-ci/packet.yml
@ -0,0 +1,158 @@
+---
+.packet_variables: &packet_variables
+  CI_PLATFORM: "packet"
+  SSH_USER: "kubespray"
+
+.packet: &packet
+  extends: .testcases
+  variables:
+    <<: *packet_variables
+  tags:
+    - packet
+
+.test-upgrade: &test-upgrade
+  variables:
+    UPGRADE_TEST: "graceful"
+
+packet_ubuntu18-flannel-aio:
+  stage: deploy-part1
+  <<: *packet
+  when: on_success
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+# ### PR JOBS PART2
+
+packet_centos7-flannel-addons:
+  stage: deploy-part2
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
+# ### MANUAL JOBS
+
+packet_centos-weave-kubeadm-sep:
+  stage: deploy-part2
+  <<: *packet
+  when: on_success
+  only: ['triggers']
+
+packet_ubuntu-weave-sep:
+  stage: deploy-part2
+  <<: *packet
+  when: manual
+  only: ['triggers']
+
+# # More builds for PRs/merges (manual) and triggers (auto)
+
+packet_ubuntu-canal-ha:
+  stage: deploy-special
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_ubuntu-canal-kubeadm:
+  stage: deploy-part2
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_ubuntu-flannel-ha:
+  stage: deploy-part2
+  <<: *packet
+  when: manual
+  except: ['triggers']
+
+packet_ubuntu-contiv-sep:
+  stage: deploy-special
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_ubuntu-cilium-sep:
+  stage: deploy-special
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_rhel7-weave:
+  stage: deploy-part2
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_debian9-calico-upgrade:
+  stage: deploy-part2
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_rhel7-canal-sep:
+  stage: deploy-special
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_centos7-calico-ha:
+  stage: deploy-special
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_centos7-cilium:
+  stage: deploy-special
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_rhel7-cilium:
+  stage: deploy-special
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_centos7-kube-router:
+  stage: deploy-special
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_centos7-multus-calico:
+  stage: deploy-part2
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_opensuse-canal:
+  stage: deploy-part2
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_ubuntu-rkt-sep:
+  stage: deploy-part2
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+packet_ubuntu-kube-router-sep:
+  stage: deploy-special
+  <<: *packet
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
--- a/.gitlab-ci/terraform.yml
+++ b/.gitlab-ci/terraform.yml
@ -1,3 +1,4 @@
+---
 # Tests for contrib/terraform/
 .terraform_install:
  extends: .job
@ -123,7 +124,7 @@ tf-apply-ovh:
    TF_VAR_floatingip_pool: "Ext-Net"
    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
    TF_VAR_network_name: "Ext-Net"
-    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229" #s1-8
-    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229" #s1-8
+    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
+    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
    TF_VAR_image: "Ubuntu 18.04"
    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
--- a/tests/files/packet_centos-weave-kubeadm-sep.yml
+++ b/tests/files/packet_centos-weave-kubeadm-sep.yml
@ -0,0 +1,10 @@
+# Instance settings
+cloud_image: centos-7
+mode: ha
+
+# Kubespray settings
+kube_network_plugin: weave
+kubeadm_enabled: true
+deploy_netchecker: true
+kubernetes_audit: true
+dns_min_replicas: 1
--- a/tests/files/packet_centos7-calico-ha.yml
+++ b/tests/files/packet_centos7-calico-ha.yml
@ -0,0 +1,10 @@
+# Instance settings
+cloud_image: centos-7
+mode: ha
+
+# Kubespray settings
+kube_network_plugin: calico
+download_localhost: true
+download_run_once: true
+deploy_netchecker: true
+dns_min_replicas: 1
--- a/tests/files/packet_centos7-cilium.yml
+++ b/tests/files/packet_centos7-cilium.yml
@ -0,0 +1,9 @@
+# Instance settings
+cloud_image: centos-7
+mode: default
+
+# Kubespray settings
+kube_network_plugin: cilium
+deploy_netchecker: true
+enable_network_policy: true
+dns_min_replicas: 1
--- a/tests/files/packet_centos7-flannel-addons.yml
+++ b/tests/files/packet_centos7-flannel-addons.yml
@ -0,0 +1,23 @@
+# Instance settings
+cloud_image: centos-7
+mode: ha
+
+# Kubespray settings
+kube_network_plugin: flannel
+helm_enabled: true
+kubernetes_audit: true
+etcd_events_cluster_setup: true
+local_volume_provisioner_enabled: true
+etcd_deployment_type: host
+deploy_netchecker: true
+dns_min_replicas: 1
+cloud_provider: gce
+kube_encrypt_secret_data: true
+ingress_nginx_enabled: true
+cert_manager_enabled: true
+metrics_server_enabled: true
+kube_token_auth: true
+kube_basic_auth: true
+enable_nodelocaldns: true
+
+vm_memory: 6144Mi
--- a/tests/files/packet_centos7-kube-router.yml
+++ b/tests/files/packet_centos7-kube-router.yml
@ -0,0 +1,9 @@
+# Instance settings
+cloud_image: centos-7
+mode: default
+
+# Kubespray settings
+kube_network_plugin: kube-router
+deploy_netchecker: true
+enable_network_policy: true
+dns_min_replicas: 1
--- a/tests/files/packet_centos7-multus-calico.yml
+++ b/tests/files/packet_centos7-multus-calico.yml
@ -0,0 +1,9 @@
+# Instance settings
+cloud_image: centos-7
+mode: default
+
+# Kubespray settings
+kube_network_plugin_multus: true
+kube_network_plugin: calico
+deploy_netchecker: true
+dns_min_replicas: 1
--- a/tests/files/packet_debian9-calico-upgrade.yml
+++ b/tests/files/packet_debian9-calico-upgrade.yml
@ -0,0 +1,8 @@
+# Instance settings
+cloud_image: debian-9
+mode: default
+
+# Kubespray settings
+kube_network_plugin: calico
+deploy_netchecker: true
+dns_min_replicas: 1
--- a/tests/files/packet_opensuse-canal.yml
+++ b/tests/files/packet_opensuse-canal.yml
@ -0,0 +1,8 @@
+# Instance settings
+cloud_image: opensuse-leap-15
+mode: default
+
+# Kubespray settings
+kube_network_plugin: canal
+deploy_netchecker: true
+dns_min_replicas: 1
--- a/tests/files/packet_rhel7-canal-sep.yml
+++ b/tests/files/packet_rhel7-canal-sep.yml
@ -0,0 +1,8 @@
+# Instance settings
+cloud_image: rhel-server-7
+mode: separate
+
+# Kubespray settings
+kube_network_plugin: canal
+deploy_netchecker: true
+dns_min_replicas: 1
--- a/tests/files/packet_rhel7-cilium.yml
+++ b/tests/files/packet_rhel7-cilium.yml
@ -0,0 +1,9 @@
+# Instance settings
+cloud_image: rhel-server-7
+mode: default
+
+# Kubespray settings
+kube_network_plugin: cilium
+deploy_netchecker: true
+enable_network_policy: true
+dns_min_replicas: 1
--- a/tests/files/packet_rhel7-weave.yml
+++ b/tests/files/packet_rhel7-weave.yml
@ -0,0 +1,8 @@
+# Instance settings
+cloud_image: rhel-server-7
+mode: default
+
+# Kubespray settings
+kube_network_plugin: weave
+deploy_netchecker: true
+dns_min_replicas: 1
--- a/tests/files/packet_ubuntu-canal-ha.yml
+++ b/tests/files/packet_ubuntu-canal-ha.yml
@ -0,0 +1,8 @@
+# Instance settings
+cloud_image: ubuntu-1604
+mode: separate
+
+# Kubespray settings
+kube_network_plugin: canal
+deploy_netchecker: true
+dns_min_replicas: 1
--- a/tests/files/packet_ubuntu-canal-kubeadm.yml
+++ b/tests/files/packet_ubuntu-canal-kubeadm.yml
@ -0,0 +1,10 @@
+# Instance settings
+cloud_image: ubuntu-1604
+mode: ha
+
+# Kubespray settings
+kube_network_plugin: canal
+kubeadm_enabled: true
+dynamic_kubelet_configuration: true
+deploy_netchecker: true
+dns_min_replicas: 1
--- a/tests/files/packet_ubuntu-cilium-sep.yml
+++ b/tests/files/packet_ubuntu-cilium-sep.yml
@ -0,0 +1,9 @@
+# Instance settings
+cloud_image: ubuntu-1604
+mode: separate
+
+# Kubespray settings
+kube_network_plugin: cilium
+deploy_netchecker: true
+enable_network_policy: true
+dns_min_replicas: 1
--- a/tests/files/packet_ubuntu-contiv-sep.yml
+++ b/tests/files/packet_ubuntu-contiv-sep.yml
@ -0,0 +1,8 @@
+# Instance settings
+cloud_image: ubuntu-1604
+mode: separate
+
+# Kubespray settings
+kube_network_plugin: contiv
+deploy_netchecker: true
+dns_min_replicas: 1
--- a/tests/files/packet_ubuntu-flannel-ha.yml
+++ b/tests/files/packet_ubuntu-flannel-ha.yml
@ -0,0 +1,10 @@
+# Instance settings
+cloud_image: ubuntu-1604
+mode: ha
+
+# Kubespray settings
+kube_network_plugin: flannel
+kubeadm_enabled: true
+skip_non_kubeadm_warning: true
+deploy_netchecker: true
+dns_min_replicas: 1
--- a/tests/files/packet_ubuntu-kube-router-sep.yml
+++ b/tests/files/packet_ubuntu-kube-router-sep.yml
@ -0,0 +1,9 @@
+# Instance settings
+cloud_image: ubuntu-1604
+mode: separate
+
+# Kubespray settings
+bootstrap_os: ubuntu
+kube_network_plugin: kube-router
+deploy_netchecker: true
+dns_min_replicas: 1
--- a/tests/files/packet_ubuntu-rkt-sep.yml
+++ b/tests/files/packet_ubuntu-rkt-sep.yml
@ -0,0 +1,12 @@
+# Instance settings
+cloud_image: ubuntu-1604
+mode: separate
+
+# Kubespray settings
+kube_network_plugin: flannel
+etcd_deployment: rkt
+kubelet_deployment: rkt
+download_localhost: true
+download_run_once: true
+deploy_netchecker: true
+dns_min_replicas: 1
--- a/tests/files/packet_ubuntu-weave-sep.yml
+++ b/tests/files/packet_ubuntu-weave-sep.yml
@ -0,0 +1,8 @@
+# Instance settings
+cloud_image: ubuntu-1604
+mode: separate
+
+# Kubespray settings
+kube_network_plugin: weave
+deploy_netchecker: true
+dns_min_replicas: 1
--- a/tests/files/packet_ubuntu18-flannel-aio.yml
+++ b/tests/files/packet_ubuntu18-flannel-aio.yml
@ -0,0 +1,8 @@
+# Instance settings
+cloud_image: ubuntu-1804
+mode: aio
+
+# Kubespray settings
+kube_network_plugin: flannel
+deploy_netchecker: true
+kubedns_min_replicas: 1