Improve CI test matrix

For Travis CI and GCE, add a naive generator script into a markdown table.
Add GCE/Travis CI matrix docs.
Add CoreOS test cases.
Rework existing cases w/o loosing of coverage.
Rework postinstall tests to support CoreOS as well.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

.travis.yml

@@ -14,6 +14,7 @@ env:
     GS_SECRET_ACCESS_KEY=$GS_SECRET
     ANSIBLE_KEEP_REMOTE_FILES=1
     CLUSTER_MODE=default
+    BOOTSTRAP_OS=none
   matrix:
     # Debian Jessie
     - >-
@@ -26,11 +27,6 @@ env:
       CLOUD_IMAGE=debian-8-kubespray
       CLOUD_REGION=us-central1-c
       CLUSTER_MODE=default
-    - >-
-      KUBE_NETWORK_PLUGIN=weave
-      CLOUD_IMAGE=debian-8-kubespray
-      CLOUD_REGION=us-east1-d
-      CLUSTER_MODE=default
 
     # Centos 7
     - >-
@@ -50,11 +46,6 @@ env:
       CLUSTER_MODE=default
 
    # Redhat 7
-    - >-
-      KUBE_NETWORK_PLUGIN=flannel
-      CLOUD_IMAGE=rhel-7-sudo
-      CLOUD_REGION=us-east1-d
-      CLUSTER_MODE=default
     - >-
       KUBE_NETWORK_PLUGIN=calico
       CLOUD_IMAGE=rhel-7-sudo
@@ -68,24 +59,33 @@ env:
 
     # Ubuntu 16.04
     - >-
-      KUBE_NETWORK_PLUGIN=flannel
+      KUBE_NETWORK_PLUGIN=canal
       CLOUD_IMAGE=ubuntu-1604-xenial
       CLOUD_REGION=us-central1-c
       CLUSTER_MODE=default
     - >-
-      KUBE_NETWORK_PLUGIN=calico
+      KUBE_NETWORK_PLUGIN=weave
       CLOUD_IMAGE=ubuntu-1604-xenial
-      CLOUD_REGION=us-east1-d
+      CLOUD_REGION=asia-east1-c
       CLUSTER_MODE=default
+
+    # CoreOS stable
     - >-
       KUBE_NETWORK_PLUGIN=weave
-      CLOUD_IMAGE=ubuntu-1604-xenial
-      CLOUD_REGION=asia-east1-c
+      CLOUD_IMAGE=coreos-stable
+      CLOUD_REGION=europe-west1-b
+      CLUSTER_MODE=default
+      BOOTSTRAP_OS=coreos
+    - >-
+      KUBE_NETWORK_PLUGIN=canal
+      CLOUD_IMAGE=coreos-stable
+      CLOUD_REGION=us-east1-d
       CLUSTER_MODE=default
+      BOOTSTRAP_OS=coreos
 
     # Extra cases for separated roles
     - >-
-      KUBE_NETWORK_PLUGIN=flannel
+      KUBE_NETWORK_PLUGIN=canal
       CLOUD_IMAGE=rhel-7-sudo
       CLOUD_REGION=europe-west1-b
       CLUSTER_MODE=separate
@@ -99,6 +99,12 @@ env:
       CLOUD_IMAGE=debian-8-kubespray
       CLOUD_REGION=us-east1-d
       CLUSTER_MODE=separate
+    - >-
+      KUBE_NETWORK_PLUGIN=calico
+      CLOUD_IMAGE=coreos-stable
+      CLOUD_REGION=asia-east1-c
+      CLUSTER_MODE=separate
+      BOOTSTRAP_OS=coreos
 
 
 before_install:
@@ -123,6 +129,7 @@ before_script:
   - chmod 755 $HOME/.local/bin/ansible-playbook
   - $HOME/.local/bin/ansible-playbook --version
   - cp tests/ansible.cfg .
+  - export PYPATH=$([ $BOOTSTRAP_OS = none ] && echo /usr/bin/python || echo /opt/bin/python)
 #  - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}'  $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml"
 
 script:
@@ -139,19 +146,26 @@ script:
     -e cloud_region=${CLOUD_REGION}
 
     # Create cluster
-  - "$HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e cloud_provider=gce  $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} cluster.yml"
+  - >
+    $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS
+    -b --become-user=root -e cloud_provider=gce  $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+    -e bootstrap_os=${BOOTSTRAP_OS}
+    -e ansible_python_interpreter=${PYPATH}
+    cluster.yml
+
     # Tests Cases
     ## Test Master API
-  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/testcases/010_check-apiserver.yml $LOG_LEVEL
+  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} tests/testcases/010_check-apiserver.yml $LOG_LEVEL
     ## Create a POD
-  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL
+  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL
     ## Ping the between 2 pod
-  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
+  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
 
 after_failure:
   - >
     $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER
     -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e dir=$HOME
+    -e ansible_python_interpreter=${PYPATH}
     scripts/collect-info.yaml
   - >
     $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/upload-logs-gcs.yml -i "localhost," -c local
@@ -162,6 +176,7 @@ after_failure:
     -e ostype=${CLOUD_IMAGE}
     -e commit=${TRAVIS_COMMIT}
     -e dir=${HOME}
+    -e ansible_python_interpreter=${PYPATH}
 
 after_script:
   - >

docs/test_cases.md

@@ -0,0 +1,39 @@
+Travis CI test matrix
+=====================
+
+GCE instances
+-------------
+
+Here is the test matrix for the Travis CI gates:
+
+|           Network plugin|                  OS type|               GCE region|             Nodes layout|
+|-------------------------|-------------------------|-------------------------|-------------------------|
+|                  flannel|       debian-8-kubespray|           europe-west1-b|                  default|
+|                   calico|       debian-8-kubespray|            us-central1-c|                  default|
+|                  flannel|            centos-7-sudo|             asia-east1-c|                  default|
+|                   calico|            centos-7-sudo|           europe-west1-b|                  default|
+|                    weave|            centos-7-sudo|            us-central1-c|                  default|
+|                   calico|              rhel-7-sudo|             asia-east1-c|                  default|
+|                    weave|              rhel-7-sudo|           europe-west1-b|                  default|
+|                    canal|       ubuntu-1604-xenial|            us-central1-c|                  default|
+|                    weave|       ubuntu-1604-xenial|             asia-east1-c|                  default|
+|                    weave|            coreos-stable|           europe-west1-b|                  default|
+|                    canal|            coreos-stable|               us-east1-d|                  default|
+|                    canal|              rhel-7-sudo|           europe-west1-b|                 separate|
+|                   calico|       ubuntu-1604-xenial|            us-central1-a|                 separate|
+|                    weave|       debian-8-kubespray|               us-east1-d|                 separate|
+|                   calico|            coreos-stable|             asia-east1-c|                 separate|
+
+Where the nodes layout `default` is that is given in the example inventory file.
+And the `separate` layout is when there is only node of each type, which is a kube master,
+compute and etcd cluster member.
+
+Note, the canal network plugin deploys flannel as well plus calico policy controller.
+
+Hint: the command
+```
+bash scripts/gen_matrix.sh
+```
+will (hopefully) generate the CI test cases from the current ``.travis.yml``.
+
+

scripts/gen_matrix.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+# Generates MD formatted CI matrix from the .travis.yml
+a=$(perl -ne '/^\s{6}(CLOUD_IMAGE|KUBE_NETWORK_PLUGIN|CLOUD_REGION|CLUSTER_MODE)=(\S+)$/ && print "$2\n"' .travis.yml)
+printf "|%25s|%25s|%25s|%25s|\n" "Network plugin" "OS type" "GCE region" "Nodes layout"
+echo "|-------------------------|-------------------------|-------------------------|-------------------------|"
+c=0
+for i in `echo $a`; do
+  printf "|%25s" $i
+  [ $(($c % 4)) -eq 3 ] && printf "|\n"
+  c=$(( c + 1))
+done

tests/testcases/020_check-create-pod.yml

@@ -2,11 +2,21 @@
 - hosts: node1
 
   tasks:
+
+  - name: Force binaries directory for CoreOS
+    set_fact:
+      bin_dir: "/opt/bin"
+    when: ansible_os_family == "CoreOS"
+
+  - set_fact:
+      bin_dir: "/usr/local/bin"
+    when: ansible_os_family != "CoreOS"
+
   - name: Run a replica controller composed of 2 pods
-    shell: "/usr/local/bin/kubectl run test --image=busybox --replicas=2 --command -- tail -f /dev/null"
+    shell: "{{bin_dir}}/kubectl run test --image=busybox --replicas=2 --command -- tail -f /dev/null"
 
   - name: Pods are running
-    shell: "/usr/local/bin/kubectl get pods --no-headers -o json"
+    shell: "{{bin_dir}}/kubectl get pods --no-headers -o json"
     register: run_pods_log
     until: [ '(run_pods_log.stdout | from_json)["items"] | map(attribute = "status.phase") | join(",") == "Running,Running"' ]
     retries: 24

tests/testcases/030_check-network.yml

@@ -3,8 +3,17 @@
 
   tasks:
 
+  - name: Force binaries directory for CoreOS
+    set_fact:
+      bin_dir: "/opt/bin"
+    when: ansible_os_family == "CoreOS"
+
+  - set_fact:
+      bin_dir: "/usr/local/bin"
+    when: ansible_os_family != "CoreOS"
+
   - name: Get pod names
-    shell: "/usr/local/bin/kubectl get pods -o json"
+    shell: "{{bin_dir}}/kubectl get pods -o json"
     register: pods
 
   - set_fact:
@@ -18,4 +27,4 @@
 
 
   - name: Ping between pods is working
-    shell: "/usr/local/bin/kubectl exec {{pod_names[0]}} -- ping -c 4 {{ pod_ips[1] }}"
+    shell: "{{bin_dir}}/kubectl exec {{pod_names[0]}} -- ping -c 4 {{ pod_ips[1] }}"