Merge pull request #611 from bogdando/coreos_canal

Improve CI test matrix
8 years ago · af3823bced
5 changed files with 108 additions and 24 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -14,6 +14,7 @@ env:
    GS_SECRET_ACCESS_KEY=$GS_SECRET
    ANSIBLE_KEEP_REMOTE_FILES=1
    CLUSTER_MODE=default
+    BOOTSTRAP_OS=none
  matrix:
    # Debian Jessie
    - >-
@ -26,11 +27,6 @@ env:
      CLOUD_IMAGE=debian-8-kubespray
      CLOUD_REGION=us-central1-c
      CLUSTER_MODE=default
-    - >-
-      KUBE_NETWORK_PLUGIN=weave
-      CLOUD_IMAGE=debian-8-kubespray
-      CLOUD_REGION=us-east1-d
-      CLUSTER_MODE=default

    # Centos 7
    - >-
@ -50,11 +46,6 @@ env:
      CLUSTER_MODE=default

   # Redhat 7
-    - >-
-      KUBE_NETWORK_PLUGIN=flannel
-      CLOUD_IMAGE=rhel-7-sudo
-      CLOUD_REGION=us-east1-d
-      CLUSTER_MODE=default
    - >-
      KUBE_NETWORK_PLUGIN=calico
      CLOUD_IMAGE=rhel-7-sudo
@ -68,24 +59,33 @@ env:

    # Ubuntu 16.04
    - >-
-      KUBE_NETWORK_PLUGIN=flannel
+      KUBE_NETWORK_PLUGIN=canal
      CLOUD_IMAGE=ubuntu-1604-xenial
      CLOUD_REGION=us-central1-c
      CLUSTER_MODE=default
    - >-
-      KUBE_NETWORK_PLUGIN=calico
+      KUBE_NETWORK_PLUGIN=weave
      CLOUD_IMAGE=ubuntu-1604-xenial
-      CLOUD_REGION=us-east1-d
+      CLOUD_REGION=asia-east1-c
      CLUSTER_MODE=default
+
+    # CoreOS stable
    - >-
      KUBE_NETWORK_PLUGIN=weave
-      CLOUD_IMAGE=ubuntu-1604-xenial
-      CLOUD_REGION=asia-east1-c
+      CLOUD_IMAGE=coreos-stable
+      CLOUD_REGION=europe-west1-b
+      CLUSTER_MODE=default
+      BOOTSTRAP_OS=coreos
+    - >-
+      KUBE_NETWORK_PLUGIN=canal
+      CLOUD_IMAGE=coreos-stable
+      CLOUD_REGION=us-east1-d
      CLUSTER_MODE=default
+      BOOTSTRAP_OS=coreos

    # Extra cases for separated roles
    - >-
-      KUBE_NETWORK_PLUGIN=flannel
+      KUBE_NETWORK_PLUGIN=canal
      CLOUD_IMAGE=rhel-7-sudo
      CLOUD_REGION=europe-west1-b
      CLUSTER_MODE=separate
@ -99,6 +99,12 @@ env:
      CLOUD_IMAGE=debian-8-kubespray
      CLOUD_REGION=us-east1-d
      CLUSTER_MODE=separate
+    - >-
+      KUBE_NETWORK_PLUGIN=calico
+      CLOUD_IMAGE=coreos-stable
+      CLOUD_REGION=asia-east1-c
+      CLUSTER_MODE=separate
+      BOOTSTRAP_OS=coreos


 before_install:
@ -123,6 +129,7 @@ before_script:
  - chmod 755 $HOME/.local/bin/ansible-playbook
  - $HOME/.local/bin/ansible-playbook --version
  - cp tests/ansible.cfg .
+  - export PYPATH=$([ $BOOTSTRAP_OS = none ] && echo /usr/bin/python || echo /opt/bin/python)
 #  - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}'  $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml"

 script:
@ -139,19 +146,26 @@ script:
    -e cloud_region=${CLOUD_REGION}

    # Create cluster
-  - "$HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e cloud_provider=gce  $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} cluster.yml"
+  - >
+    $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS
+    -b --become-user=root -e cloud_provider=gce  $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+    -e bootstrap_os=${BOOTSTRAP_OS}
+    -e ansible_python_interpreter=${PYPATH}
+    cluster.yml
+
    # Tests Cases
    ## Test Master API
-  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/testcases/010_check-apiserver.yml $LOG_LEVEL
+  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} tests/testcases/010_check-apiserver.yml $LOG_LEVEL
    ## Create a POD
-  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL
+  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL
    ## Ping the between 2 pod
-  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
+  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL

 after_failure:
  - >
    $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER
    -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e dir=$HOME
+    -e ansible_python_interpreter=${PYPATH}
    scripts/collect-info.yaml
  - >
    $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/upload-logs-gcs.yml -i "localhost," -c local
@ -162,6 +176,7 @@ after_failure:
    -e ostype=${CLOUD_IMAGE}
    -e commit=${TRAVIS_COMMIT}
    -e dir=${HOME}
+    -e ansible_python_interpreter=${PYPATH}

 after_script:
  - >
--- a/docs/test_cases.md
+++ b/docs/test_cases.md
@ -0,0 +1,39 @@
+Travis CI test matrix
+=====================
+
+GCE instances
+-------------
+
+Here is the test matrix for the Travis CI gates:
+
+|           Network plugin|                  OS type|               GCE region|             Nodes layout|
+|-------------------------|-------------------------|-------------------------|-------------------------|
+|                  flannel|       debian-8-kubespray|           europe-west1-b|                  default|
+|                   calico|       debian-8-kubespray|            us-central1-c|                  default|
+|                  flannel|            centos-7-sudo|             asia-east1-c|                  default|
+|                   calico|            centos-7-sudo|           europe-west1-b|                  default|
+|                    weave|            centos-7-sudo|            us-central1-c|                  default|
+|                   calico|              rhel-7-sudo|             asia-east1-c|                  default|
+|                    weave|              rhel-7-sudo|           europe-west1-b|                  default|
+|                    canal|       ubuntu-1604-xenial|            us-central1-c|                  default|
+|                    weave|       ubuntu-1604-xenial|             asia-east1-c|                  default|
+|                    weave|            coreos-stable|           europe-west1-b|                  default|
+|                    canal|            coreos-stable|               us-east1-d|                  default|
+|                    canal|              rhel-7-sudo|           europe-west1-b|                 separate|
+|                   calico|       ubuntu-1604-xenial|            us-central1-a|                 separate|
+|                    weave|       debian-8-kubespray|               us-east1-d|                 separate|
+|                   calico|            coreos-stable|             asia-east1-c|                 separate|
+
+Where the nodes layout `default` is that is given in the example inventory file.
+And the `separate` layout is when there is only node of each type, which is a kube master,
+compute and etcd cluster member.
+
+Note, the canal network plugin deploys flannel as well plus calico policy controller.
+
+Hint: the command
+```
+bash scripts/gen_matrix.sh
+```
+will (hopefully) generate the CI test cases from the current ``.travis.yml``.
+
+
--- a/scripts/gen_matrix.sh
+++ b/scripts/gen_matrix.sh
@ -0,0 +1,11 @@
+#!/bin/sh
+# Generates MD formatted CI matrix from the .travis.yml
+a=$(perl -ne '/^\s{6}(CLOUD_IMAGE|KUBE_NETWORK_PLUGIN|CLOUD_REGION|CLUSTER_MODE)=(\S+)$/ && print "$2\n"' .travis.yml)
+printf "|%25s|%25s|%25s|%25s|\n" "Network plugin" "OS type" "GCE region" "Nodes layout"
+echo "|-------------------------|-------------------------|-------------------------|-------------------------|"
+c=0
+for i in `echo $a`; do
+  printf "|%25s" $i
+  [ $(($c % 4)) -eq 3 ] && printf "|\n"
+  c=$(( c + 1))
+done
--- a/tests/testcases/020_check-create-pod.yml
+++ b/tests/testcases/020_check-create-pod.yml
@ -2,11 +2,21 @@
 - hosts: node1

  tasks:
+
+  - name: Force binaries directory for CoreOS
+    set_fact:
+      bin_dir: "/opt/bin"
+    when: ansible_os_family == "CoreOS"
+
+  - set_fact:
+      bin_dir: "/usr/local/bin"
+    when: ansible_os_family != "CoreOS"
+
  - name: Run a replica controller composed of 2 pods
-    shell: "/usr/local/bin/kubectl run test --image=busybox --replicas=2 --command -- tail -f /dev/null"
+    shell: "{{bin_dir}}/kubectl run test --image=busybox --replicas=2 --command -- tail -f /dev/null"

  - name: Pods are running
-    shell: "/usr/local/bin/kubectl get pods --no-headers -o json"
+    shell: "{{bin_dir}}/kubectl get pods --no-headers -o json"
    register: run_pods_log
    until: [ '(run_pods_log.stdout | from_json)["items"] | map(attribute = "status.phase") | join(",") == "Running,Running"' ]
    retries: 24
--- a/tests/testcases/030_check-network.yml
+++ b/tests/testcases/030_check-network.yml
@ -3,8 +3,17 @@

  tasks:

+  - name: Force binaries directory for CoreOS
+    set_fact:
+      bin_dir: "/opt/bin"
+    when: ansible_os_family == "CoreOS"
+
+  - set_fact:
+      bin_dir: "/usr/local/bin"
+    when: ansible_os_family != "CoreOS"
+
  - name: Get pod names
-    shell: "/usr/local/bin/kubectl get pods -o json"
+    shell: "{{bin_dir}}/kubectl get pods -o json"
    register: pods

  - set_fact:
@ -18,4 +27,4 @@


  - name: Ping between pods is working
-    shell: "/usr/local/bin/kubectl exec {{pod_names[0]}} -- ping -c 4 {{ pod_ips[1] }}"
+    shell: "{{bin_dir}}/kubectl exec {{pod_names[0]}} -- ping -c 4 {{ pod_ips[1] }}"