Add synthetic scale deployment mode

New deploy modes: scale, ha-scale, separate-scale Creates 200 fake hosts for deployment with fake hostvars. Useful for testing certificate generation and propagation to other master nodes. Updated test cases descriptions.
8 years ago · 9c1701f2aa
7 changed files with 71 additions and 24 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,6 +1,7 @@
 .vagrant
 *.retry
 inventory/vagrant_ansible_inventory
+inventory/group_vars/fake_hosts.yml
 temp
 .idea
 .tox
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -124,6 +124,7 @@ before_script:
      -e local_release_dir=${PWD}/downloads
      -e resolvconf_mode=${RESOLVCONF_MODE}
      -e vault_deployment_type=${VAULT_DEPLOYMENT}
+      --limit "all:!fake_hosts"
      cluster.yml

    # Repeat deployment if testing upgrade
@ -150,18 +151,19 @@ before_script:
      -e resolvconf_mode=${RESOLVCONF_MODE} 
      -e weave_cpu_requests=${WEAVE_CPU_LIMIT} 
      -e weave_cpu_limit=${WEAVE_CPU_LIMIT} 
+      --limit "all:!fake_hosts" 
      $PLAYBOOK; 
      fi

    # Tests Cases
    ## Test Master API
-    - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/010_check-apiserver.yml $LOG_LEVEL
+    - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL

    ## Ping the between 2 pod
-    - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
+    - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $LOG_LEVEL

    ## Advanced DNS checks
-    - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/040_check-network-adv.yml $LOG_LEVEL
+    - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml $LOG_LEVEL

    ## Idempotency checks 1/5 (repeat deployment)
    - >
@ -178,6 +180,7 @@ before_script:
      -e local_release_dir=${PWD}/downloads 
      -e etcd_deployment_type=${ETCD_DEPLOYMENT} 
      -e kubelet_deployment_type=${KUBELET_DEPLOYMENT} 
+      --limit "all:!fake_hosts" 
      cluster.yml;
      fi

@ -186,6 +189,7 @@ before_script:
      if [ "${IDEMPOT_CHECK}" = "true" ]; then
      ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} 
      -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root 
+      --limit "all:!fake_hosts" 
      tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
      fi

@ -197,7 +201,8 @@ before_script:
      --private-key=${HOME}/.ssh/id_rsa 
      -e bootstrap_os=${BOOTSTRAP_OS} 
      -e ansible_python_interpreter=${PYPATH} 
-      -e reset_confirmation=yes
+      -e reset_confirmation=yes 
+      --limit "all:!fake_hosts"
      reset.yml;
      fi

@ -216,6 +221,7 @@ before_script:
      -e local_release_dir=${PWD}/downloads 
      -e etcd_deployment_type=${ETCD_DEPLOYMENT} 
      -e kubelet_deployment_type=${KUBELET_DEPLOYMENT} 
+      --limit "all:!fake_hosts" 
      cluster.yml;
      fi

@ -224,6 +230,7 @@ before_script:
      if [ "${IDEMPOT_CHECK}" = "true" ]; then
      ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} 
      -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root 
+      --limit "all:!fake_hosts" 
      tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
      fi

@ -244,7 +251,7 @@ before_script:
 .coreos_calico_sep_variables: &coreos_calico_sep_variables
 # stage: deploy-gce-part1
  KUBE_NETWORK_PLUGIN: calico
-  CLOUD_IMAGE: coreos-stable-1235-6-0-v20170111        
+  CLOUD_IMAGE: coreos-stable-1235-6-0-v20170111
  CLOUD_REGION: us-west1-b
  CLUSTER_MODE: separate
  BOOTSTRAP_OS: coreos
@ -310,7 +317,7 @@ before_script:
  KUBE_NETWORK_PLUGIN: calico
  CLOUD_IMAGE: centos-7
  CLOUD_REGION: europe-west1-b
-  CLUSTER_MODE: ha
+  CLUSTER_MODE: ha-scale
  IDEMPOT_CHECK: "true"

 .coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables
--- a/docs/test_cases.md
+++ b/docs/test_cases.md
@ -4,25 +4,40 @@ Travis CI test matrix
 GCE instances
 -------------

-Here is the test matrix for the Travis CI gates:
+Here is the test matrix for the CI gates:

 |           Network plugin|                  OS type|               GCE region|             Nodes layout|
 |-------------------------|-------------------------|-------------------------|-------------------------|
-|                    canal|       debian-8-kubespray|             asia-east1-a|                       ha|
+|                    canal|       debian-8-kubespray|             asia-east1-a|                 ha-scale|
 |                   calico|       debian-8-kubespray|           europe-west1-c|                  default|
 |                  flannel|                 centos-7|        asia-northeast1-c|                  default|
 |                   calico|                 centos-7|            us-central1-b|                       ha|
 |                    weave|                   rhel-7|               us-east1-c|                  default|
-|                    canal|            coreos-stable|               us-west1-b|                  default|
+|                    canal|            coreos-stable|               us-west1-b|                 ha-scale|
 |                    canal|                   rhel-7|        asia-northeast1-b|                 separate|
 |                    weave|       ubuntu-1604-xenial|           europe-west1-d|                 separate|
 |                   calico|            coreos-stable|            us-central1-f|                 separate|

-Where the nodes layout `default` is a non-HA two nodes setup with the separate `kube-node`
-and the `etcd` group merged with the `kube-master`. The `separate` layout is when
-there is only node of each type, which is a kube master, compute and etcd cluster member.
-And the `ha` layout stands for a two etcd nodes, two masters and a single worker node,
-partially intersecting though.
+
+Node Layouts
+------------
+
+There are four node layout types: `default`, `separate`, `ha`, and `scale`.
+
+
+`default` is a non-HA two nodes setup with one separate `kube-node`
+and the `etcd` group merged with the `kube-master`.
+
+`separate` layout is when there is only node of each type, which includes
+ a kube-master, kube-node, and etcd cluster member.
+
+`ha` layout consists of two etcd nodes, two masters and a single worker node,
+with role intersection.
+
+`scale` layout can be combined with above layouts. It includes 200 fake hosts
+in the Ansible inventory. This helps test TLS certificate generation at scale
+to prevent regressions and profile certain long-running tasks. These nodes are
+never actually deployed, but certificates are generated for them.

 Note, the canal network plugin deploys flannel as well plus calico policy controller.

@ -40,15 +55,15 @@ GCE instances

 |               Stage|      Network plugin|             OS type|          GCE region|        Nodes layout
 |--------------------|--------------------|--------------------|--------------------|--------------------|
-|               part1|              calico|       coreos-stable|          us-west1-b|           separated|
+|               part1|              calico|       coreos-stable|          us-west1-b|            separate|
 |               part1|               canal|  debian-8-kubespray|          us-east1-b|                  ha|
-|               part1|               weave|              rhel-7|      europe-west1-b|             default|
+|               part1|               weave|              rhel-7|      europe-west1-b|               scale|
 |               part2|             flannel|            centos-7|          us-west1-a|             default|
 |               part2|              calico|  debian-8-kubespray|       us-central1-b|             default|
 |               part2|               canal|       coreos-stable|          us-east1-b|             default|
-|             special|               canal|              rhel-7|          us-east1-b|           separated|
-|             special|               weave|  ubuntu-1604-xenial|       us-central1-b|           separated|
-|             special|              calico|            centos-7|      europe-west1-b|                  ha|
+|             special|               canal|              rhel-7|          us-east1-b|            separate|
+|             special|               weave|  ubuntu-1604-xenial|       us-central1-b|               scale|
+|             special|              calico|            centos-7|      europe-west1-b|            ha-scale|
 |             special|               weave|        coreos-alpha|          us-west1-a|                  ha|

 The "Stage" means a build step of the build pipeline. The steps are ordered as `part1->part2->special`.
--- a/roles/etcd/tasks/check_certs.yml
+++ b/roles/etcd/tasks/check_certs.yml
@ -2,6 +2,7 @@
 - name: "Check_certs | check if all certs have already been generated on first master"
  stat:
    path: "{{ etcd_cert_dir }}/{{ item }}"
+    get_md5: no
  delegate_to: "{{groups['etcd'][0]}}"
  register: etcdcert_master
  run_once: true
--- a/tests/cloud_playbooks/create-gce.yml
+++ b/tests/cloud_playbooks/create-gce.yml
@ -13,7 +13,7 @@

    - set_fact:
        instance_names: >-
-          {%- if mode in ['separate', 'ha'] -%}
+          {%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale'] -%}
          k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3
          {%- else -%}
          k8s-{{test_name}}-1,k8s-{{test_name}}-2
@ -39,6 +39,18 @@
        src: ../templates/inventory-gce.j2
        dest: "{{ inventory_path }}"

+    - name: Make group_vars directory
+      file:
+        path: "{{ inventory_path|dirname }}/group_vars"
+        state: directory
+      when: mode in ['scale', 'separate-scale', 'ha-scale']
+
+    - name: Template fake hosts group vars
+      template:
+        src: ../templates/fake_hosts.yml.j2
+        dest: "{{ inventory_path|dirname }}/group_vars/fake_hosts.yml"
+      when: mode in ['scale', 'separate-scale', 'ha-scale']
+
    - name: Wait for SSH to come up
      wait_for: host={{item.public_ip}} port=22 delay=10 timeout=180 state=started
      with_items: "{{gce.instance_data}}"
--- a/tests/templates/fake_hosts.yml.j2
+++ b/tests/templates/fake_hosts.yml.j2
@ -0,0 +1,3 @@
+ansible_default_ipv4:
+  address: 255.255.255.255
+ansible_hostname: "{{ '{{' }}inventory_hostname}}"
--- a/tests/templates/inventory-gce.j2
+++ b/tests/templates/inventory-gce.j2
@ -2,12 +2,11 @@
 {% set node2 = gce.instance_data[1].name %}
 {{node1}} ansible_ssh_host={{gce.instance_data[0].public_ip}}
 {{node2}} ansible_ssh_host={{gce.instance_data[1].public_ip}}
-{% if mode is defined and mode in ["separate", "ha"] %}
+{% if mode is defined and mode in ["separate", "separate-scale"] %}
 {% set node3 = gce.instance_data[2].name %}
 {{node3}} ansible_ssh_host={{gce.instance_data[2].public_ip}}
 {% endif %}
-
-{% if mode is defined and mode == "separate" %}
+{% if mode is defined and mode in ["separate", "separate-scale"] %}
 [kube-master]
 {{node1}}

@ -19,7 +18,7 @@

 [vault]
 {{node3}}
-{% elif mode is defined and mode == "ha" %}
+{% elif mode is defined and mode in ["ha", "ha-scale"] %}
 [kube-master]
 {{node1}}
 {{node2}}
@ -51,3 +50,12 @@
 [k8s-cluster:children]
 kube-node
 kube-master
+
+{% if mode is defined and mode in ["scale", "separate-scale", "ha-scale"] %}
+[fake_hosts]
+fake_scale_host[1:200]
+
+[kube-node:children]
+fake_hosts
+{% endif %}
+