Manual steps for Gitlab CI pipeline

* Reduce default testcase to 2 nodes, add HA case.
* Adjust gen_matrix script for Travis/Gitlab CIs.
* Enable netchecker deploy foro gitlab CI.
* Sync other things from travis matrix and reorder them as build steps
  for pull requests, master branch, auto/manual.
* Do auto-step1 from part1 and manual step2,3 for branches/PRs.
* Do manual steps from part2, special for master merges.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

.gitlab-ci.yml

@@ -2,7 +2,7 @@ stages:
   - unit-tests
   - deploy-gce-part1
   - deploy-gce-part2
-  - deploy-gce-separated
+  - deploy-gce-special
 
 variables:
   FAILFASTCI_NAMESPACE: 'kargo-ci'
@@ -101,6 +101,7 @@ before_script:
       -e ansible_python_interpreter=${PYPATH}
       -e download_run_once=true
       -e download_localhost=true
+      -e deploy_netchecker=true
       -e local_release_dir=${PWD}/downloads
       cluster.yml
 
@@ -109,15 +110,16 @@ before_script:
     ## Test Master API
     - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/010_check-apiserver.yml $LOG_LEVEL
 
-    ## Create a POD
-    - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL
-
     ## Ping the between 2 pod
     - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
 
+    ## Advanced DNS checks
+    - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/040_check-network-adv.yml $LOG_LEVEL
+
   after_script:
     - >
       ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local  $LOG_LEVEL
+      -e mode=${CLUSTER_MODE}
       -e test_id=${TEST_ID}
       -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       -e gce_project_id=${GCE_PROJECT_ID}
@@ -127,29 +129,32 @@ before_script:
       -e inventory_path=${PWD}/inventory/inventory.ini
       -e cloud_region=${CLOUD_REGION}
 
-debian8-flannel:
+coreos-calico:
   stage: deploy-gce-part1
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    KUBE_NETWORK_PLUGIN: canal
-    CLOUD_IMAGE: debian-8-kubespray
-    CLOUD_REGION: us-west1-a
-    CLUSTER_MODE: default
-
+    KUBE_NETWORK_PLUGIN: calico
+    CLOUD_IMAGE: coreos-stable
+    CLOUD_REGION: us-west1-b
+    CLUSTER_MODE: separated
+    BOOTSTRAP_OS: coreos
+  when: on_success
+  #only: [/^pr-.*$/]
 
-coreos-canal:
+debian8-canal:
   stage: deploy-gce-part1
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
     KUBE_NETWORK_PLUGIN: canal
-    CLOUD_IMAGE: coreos-stable
+    CLOUD_IMAGE: debian-8-kubespray
     CLOUD_REGION: us-east1-b
-    CLUSTER_MODE: default
-    BOOTSTRAP_OS: coreos
+    CLUSTER_MODE: ha
+  when: manual
+  #only: [/^pr-.*$/]
 
 rhel7-weave:
   stage: deploy-gce-part1
@@ -161,6 +166,8 @@ rhel7-weave:
     CLOUD_IMAGE: rhel-7
     CLOUD_REGION: europe-west1-b
     CLUSTER_MODE: default
+  when: manual
+  #only: [/^pr-.*$/]
 
 centos7-flannel:
   stage: deploy-gce-part2
@@ -172,8 +179,8 @@ centos7-flannel:
     CLOUD_IMAGE: centos-7
     CLOUD_REGION: us-west1-a
     CLUSTER_MODE: default
-  only:
-    - master
+  when: manual
+  only: ['master']
 
 debian8-calico:
   stage: deploy-gce-part2
@@ -185,24 +192,25 @@ debian8-calico:
     CLOUD_IMAGE: debian-8-kubespray
     CLOUD_REGION: us-central1-b
     CLUSTER_MODE: default
-  only:
-    - master
+  when: manual
+  only: ['master']
 
-centos7-calico:
+coreos-canal:
   stage: deploy-gce-part2
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    KUBE_NETWORK_PLUGIN: calico
-    CLOUD_IMAGE: centos-7
-    CLOUD_REGION: europe-west1-b
+    KUBE_NETWORK_PLUGIN: canal
+    CLOUD_IMAGE: coreos-stable
+    CLOUD_REGION: us-east1-b
     CLUSTER_MODE: default
-  only:
-    - master
+    BOOTSTRAP_OS: coreos
+  when: manual
+  only: ['master']
 
 rhel7-canal:
-  stage: deploy-gce-separated
+  stage: deploy-gce-special
   <<: *job
   <<: *gce
   variables:
@@ -211,11 +219,11 @@ rhel7-canal:
     CLOUD_IMAGE: rhel-7
     CLOUD_REGION: us-east1-b
     CLUSTER_MODE: separated
-  only:
-    - master
+  when: manual
+  only: ['master']
 
 ubuntu-weave:
-  stage: deploy-gce-separated
+  stage: deploy-gce-special
   <<: *job
   <<: *gce
   variables:
@@ -224,23 +232,21 @@ ubuntu-weave:
     CLOUD_IMAGE: ubuntu-1604-xenial
     CLOUD_REGION: us-central1-b
     CLUSTER_MODE: separated
-  only:
-    - master
+  when: manual
+  only: ['master']
 
-coreos-calico:
-  stage: deploy-gce-separated
+centos7-calico:
+  stage: deploy-gce-special
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
     KUBE_NETWORK_PLUGIN: calico
-    CLOUD_IMAGE: coreos-stable
-    CLOUD_REGION: us-west1-b
-    CLUSTER_MODE: separated
-    BOOTSTRAP_OS: coreos
-  only:
-    - master
-
+    CLOUD_IMAGE: centos-7
+    CLOUD_REGION: europe-west1-b
+    CLUSTER_MODE: ha
+  when: manual
+  only: ['master']
 
 syntax-check:
   <<: *job

.travis.yml.bak

@@ -24,24 +24,24 @@ env:
       KUBE_NETWORK_PLUGIN=canal
       CLOUD_IMAGE=debian-8-kubespray
       CLOUD_REGION=asia-east1-a
-      CLUSTER_MODE=default
+      CLUSTER_MODE=ha
     - >-
       KUBE_NETWORK_PLUGIN=calico
       CLOUD_IMAGE=debian-8-kubespray
-      CLOUD_REGION=asia-northeast1-b
+      CLOUD_REGION=europe-west1-c
       CLUSTER_MODE=default
 
     # Centos 7
     - >-
       KUBE_NETWORK_PLUGIN=flannel
       CLOUD_IMAGE=centos-7
-      CLOUD_REGION=europe-west1-c
+      CLOUD_REGION=asia-northeast1-c
       CLUSTER_MODE=default
     - >-
       KUBE_NETWORK_PLUGIN=calico
       CLOUD_IMAGE=centos-7
       CLOUD_REGION=us-central1-b
-      CLUSTER_MODE=default
+      CLUSTER_MODE=ha
 
    # Redhat 7
     - >-
@@ -55,7 +55,7 @@ env:
     #  KUBE_NETWORK_PLUGIN=weave
     #  CLOUD_IMAGE=coreos-stable
     #  CLOUD_REGION=europe-west1-b
-    #  CLUSTER_MODE=default
+    #  CLUSTER_MODE=ha
     #  BOOTSTRAP_OS=coreos
     - >-
       KUBE_NETWORK_PLUGIN=canal
@@ -68,7 +68,7 @@ env:
     - >-
       KUBE_NETWORK_PLUGIN=canal
       CLOUD_IMAGE=rhel-7
-      CLOUD_REGION=europe-west1-c
+      CLOUD_REGION=asia-northeast1-b
       CLUSTER_MODE=separate
     - >-
       KUBE_NETWORK_PLUGIN=weave
@@ -84,7 +84,7 @@ env:
 
 matrix:
   allow_failures:
-    - env: KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=coreos-stable CLOUD_REGION=europe-west1-b CLUSTER_MODE=default BOOTSTRAP_OS=coreos
+    - env: KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=coreos-stable CLOUD_REGION=europe-west1-b CLUSTER_MODE=ha BOOTSTRAP_OS=coreos
 
 before_install:
   # Install Ansible.
@@ -150,6 +150,7 @@ script:
 after_script:
   - >
     $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local  $LOG_LEVEL
+    -e mode=${CLUSTER_MODE}
     -e test_id=${TEST_ID}
     -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
     -e gce_project_id=${GCE_PROJECT_ID}

docs/test_cases.md

@@ -8,19 +8,21 @@ Here is the test matrix for the Travis CI gates:
 
 |           Network plugin|                  OS type|               GCE region|             Nodes layout|
 |-------------------------|-------------------------|-------------------------|-------------------------|
-|                    canal|       debian-8-kubespray|             asia-east1-a|                  default|
-|                   calico|       debian-8-kubespray|        asia-northeast1-b|                  default|
-|                  flannel|                 centos-7|           europe-west1-c|                  default|
-|                   calico|                 centos-7|            us-central1-b|                  default|
+|                    canal|       debian-8-kubespray|             asia-east1-a|                       ha|
+|                   calico|       debian-8-kubespray|           europe-west1-c|                  default|
+|                  flannel|                 centos-7|        asia-northeast1-c|                  default|
+|                   calico|                 centos-7|            us-central1-b|                       ha|
 |                    weave|                   rhel-7|               us-east1-c|                  default|
 |                    canal|            coreos-stable|               us-west1-b|                  default|
-|                    canal|                   rhel-7|           europe-west1-c|                 separate|
+|                    canal|                   rhel-7|        asia-northeast1-b|                 separate|
 |                    weave|       ubuntu-1604-xenial|           europe-west1-d|                 separate|
 |                   calico|            coreos-stable|            us-central1-f|                 separate|
 
-Where the nodes layout `default` is that is given in the example inventory file.
-And the `separate` layout is when there is only node of each type, which is a kube master,
-compute and etcd cluster member.
+Where the nodes layout `default` is a non-HA two nodes setup with the separate `kube-node`
+and the `etcd` group merged with the `kube-master`. The `separate` layout is when
+there is only node of each type, which is a kube master, compute and etcd cluster member.
+And the `ha` layout stands for a two etcd nodes, two masters and a single worker node,
+partially intersecting though.
 
 Note, the canal network plugin deploys flannel as well plus calico policy controller.
 
@@ -30,4 +32,23 @@ bash scripts/gen_matrix.sh
 ```
 will (hopefully) generate the CI test cases from the current ``.travis.yml``.
 
+Gitlab CI test matrix
+=====================
+
+GCE instances
+-------------
 
+|             Stage|    Network plugin|           OS type|        GCE region|      Nodes layout|              When|
+|------------------|------------------|------------------|------------------|------------------|------------------|
+|             part1|            calico|     coreos-stable|        us-west1-b|         separated|        on_success|
+|             part1|             canal|debian-8-kubespray|        us-east1-b|                ha|            manual|
+|             part1|             weave|            rhel-7|    europe-west1-b|           default|            manual|
+|             part2|           flannel|          centos-7|        us-west1-a|           default|            manual|
+|             part2|            calico|debian-8-kubespray|     us-central1-b|           default|            manual|
+|             part2|             canal|     coreos-stable|        us-east1-b|           default|            manual|
+|           special|             canal|            rhel-7|        us-east1-b|         separated|            manual|
+|           special|             weave|ubuntu-1604-xenial|     us-central1-b|         separated|            manual|
+|           special|            calico|          centos-7|    europe-west1-b|                ha|            manual|
+
+The "Stage" means a build step of the build pipeline. The steps are ordered as `part1->part2->special`.
+The "When" stands for the build step trigger condition.

scripts/gen_matrix.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 # Generates MD formatted CI matrix from the .travis.yml
-a=$(perl -ne '/^\s{6}(CLOUD_IMAGE|KUBE_NETWORK_PLUGIN|CLOUD_REGION|CLUSTER_MODE)=(\S+)$/ && print "$2\n"' .travis.yml)
+a=$(perl -ne '/^\s{6}(CLOUD_IMAGE|KUBE_NETWORK_PLUGIN|CLOUD_REGION|CLUSTER_MODE)=(\S+)$/ && print "$2\n"' .travis.yml.bak)
+echo Travis-CI
 printf "|%25s|%25s|%25s|%25s|\n" "Network plugin" "OS type" "GCE region" "Nodes layout"
 echo "|-------------------------|-------------------------|-------------------------|-------------------------|"
 c=0
@@ -9,3 +10,15 @@ for i in `echo $a`; do
   [ $(($c % 4)) -eq 3 ] && printf "|\n"
   c=$(( c + 1))
 done
+
+echo
+a=$(perl -ne '/^\s{2}(stage:\sdeploy-gce-|when:\s)(\S+)$/ && print "$2\n";/^\s{4}(CLOUD_IMAGE|KUBE_NETWORK_PLUGIN|CLOUD_REGION|CLUSTER_MODE):\s(\S+)$/ && print "$2\n"' .gitlab-ci.yml)
+echo Gitlab-CI
+printf "|%18s|%18s|%18s|%18s|%18s|%18s|\n"  "Stage" "Network plugin" "OS type" "GCE region" "Nodes layout" "When"
+echo "|------------------|------------------|------------------|------------------|------------------|------------------|"
+c=0
+for i in `echo $a`; do
+  printf "|%18s" $i
+  [ $(($c % 6)) -eq 5 ] && printf "|\n"
+  c=$(( c + 1))
+done

tests/cloud_playbooks/create-gce.yml

@@ -4,15 +4,24 @@
   gather_facts: no
   vars:
     cloud_machine_type: g1-small
+    mode: default
 
   tasks:
     - name: replace_test_id
       set_fact:
         test_name: "{{test_id |regex_replace('\\.', '-')}}"
 
+    - set_fact:
+        instance_names: >-
+          {%- if mode in ['separate', 'ha'] -%}
+          k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3
+          {%- else -%}
+          k8s-{{test_name}}-1,k8s-{{test_name}}-2
+          {%- endif -%}
+
     - name: Create gce instances
       gce:
-        instance_names: "k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3"
+        instance_names: "{{instance_names}}"
         machine_type: "{{ cloud_machine_type }}"
         image: "{{ cloud_image }}"
         service_account_email: "{{ gce_service_account_email }}"

tests/cloud_playbooks/delete-gce.yml

@@ -4,15 +4,24 @@
   gather_facts: no
   vars:
     cloud_machine_type: f1-micro
+    mode: default
 
   tasks:
     - name: replace_test_id
       set_fact:
         test_name: "{{test_id |regex_replace('\\.', '-')}}"
 
+    - set_fact:
+        instance_names: >-
+          {%- if mode in ['separate', 'ha'] -%}
+          k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3
+          {%- else -%}
+          k8s-{{test_name}}-1,k8s-{{test_name}}-2
+          {%- endif -%}
+
     - name: delete gce instances
       gce:
-        instance_names: "k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3"
+        instance_names: "{{instance_names}}"
         machine_type: "{{ cloud_machine_type }}"
         image: "{{ cloud_image }}"
         service_account_email: "{{ gce_service_account_email }}"

tests/templates/inventory-gce.j2

@@ -1,6 +1,8 @@
 node1 ansible_ssh_host={{gce.instance_data[0].public_ip}}
 node2 ansible_ssh_host={{gce.instance_data[1].public_ip}}
+{% if mode is defined and mode in ["separate", "ha"] %}
 node3 ansible_ssh_host={{gce.instance_data[2].public_ip}}
+{% endif %}
 
 {% if mode is defined and mode == "separate" %}
 [kube-master]
@@ -11,19 +13,26 @@ node2
 
 [etcd]
 node3
-{% else %}
+{% elif mode is defined and mode == "ha" %}
 [kube-master]
 node1
 node2
 
 [kube-node]
-node1
-node2
 node3
 
 [etcd]
+node2
+node3
+{% else %}
+[kube-master]
 node1
+
+[kube-node]
 node2
+
+[etcd]
+node1
 {% endif %}
 
 [k8s-cluster:children]