PHASE 1 - Add Packet-CI playbook and configuration (#4537)

5 years ago · b834a28891
10 changed files with 268 additions and 1 deletions
--- a/tests/Makefile
+++ b/tests/Makefile
@ -12,6 +12,10 @@ init-gce: $(HOME)/.ssh/id_rsa
 init-do: $(HOME)/.ssh/id_rsa
 	echo $(DO_PRIVATE_KEY) | base64 -d > $(HOME)/.ssh/id_rsa

+init-packet:
+	echo $(PACKET_VM_SSH_PRIVATE_KEY) | base64 -d > $(HOME)/.ssh/id_rsa
+	chmod 400 $(HOME)/.ssh/id_rsa
+
 create-gce: init-gce
 	ansible-playbook cloud_playbooks/create-gce.yml -i local_inventory/hosts.cfg -c local \
 	$(LOG_LEVEL) \
@ -41,7 +45,6 @@ create-do: init-do
 	-e inventory_path=$(INVENTORY) \
 	-e test_id=${TEST_ID}

-
 delete-do:
 	ansible-playbook -i $(INVENTORY) cloud_playbooks/create-do.yml -c local \
 	$(LOG_LEVEL) \
@ -49,3 +52,17 @@ delete-do:
 	-e state=absent \
 	-e test_id=${TEST_ID} \
 	-e inventory_path=$(INVENTORY)
+
+create-packet: init-packet
+	ansible-playbook cloud_playbooks/create-packet.yml -c local \
+	$(LOG_LEVEL) \
+	-e @"files/${CI_JOB_NAME}.yml" \
+	-e test_id=$(TEST_ID) \
+	-e inventory_path=$(INVENTORY)
+
+delete-packet:
+	ansible-playbook cloud_playbooks/delete-packet.yml -c local \
+	$(LOG_LEVEL) \
+	-e @"files/${CI_JOB_NAME}.yml" \
+	-e test_id=$(TEST_ID) \
+	-e inventory_path=$(INVENTORY)
--- a/tests/cloud_playbooks/create-packet.yml
+++ b/tests/cloud_playbooks/create-packet.yml
@ -0,0 +1,10 @@
+---
+
+- hosts: localhost
+  gather_facts: no
+  become: true
+  vars:
+    ci_job_name: "{{ lookup('env', 'CI_JOB_NAME') }}"
+    test_name: "{{ test_id | regex_replace('\\.', '-') }}"
+  roles:
+    - { role: packet-ci, vm_cleanup: false }
--- a/tests/cloud_playbooks/delete-packet.yml
+++ b/tests/cloud_playbooks/delete-packet.yml
@ -0,0 +1,10 @@
+---
+
+- hosts: localhost
+  gather_facts: no
+  become: true
+  vars:
+    ci_job_name: "{{ lookup('env', 'CI_JOB_NAME') }}"
+    test_name: "{{ test_id | regex_replace('\\.', '-') }}"
+  roles:
+    - { role: packet-ci, vm_cleanup: true }
--- a/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml
+++ b/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml
@ -0,0 +1,23 @@
+---
+
+# VM sizing
+vm_cpu_cores: 2
+vm_cpu_sockets: 1
+vm_cpu_threads: 2
+vm_memory: 4096Mi
+
+# Default path for inventory
+inventory_path: "/tmp/{{ test_name }}/inventory"
+
+# Deployment mode
+mode: aio
+
+# Cloud init config for each os type
+cloud_init:
+  centos-7: "I2Nsb3VkLWNvbmZpZwpzeXN0ZW1faW5mbzoKICBkaXN0cm86IHJoZWwKdXNlcnM6CiAtIG5hbWU6IGt1YmVzcHJheQogICBncm91cHM6IHdoZWVsCiAgIHN1ZG86ICdBTEw9KEFMTCkgTk9QQVNTV0Q6QUxMJwogICBzaGVsbDogL2Jpbi9iYXNoCiAgIGxvY2tfcGFzc3dkOiBGYWxzZQogICBob21lOiAvaG9tZS9rdWJlc3ByYXkKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1Cgo="
+  debian-9: "I2Nsb3VkLWNvbmZpZwp1c2VyczoKIC0gbmFtZToga3ViZXNwcmF5CiAgIHN1ZG86IEFMTD0oQUxMKSBOT1BBU1NXRDpBTEwKICAgc2hlbGw6IC9iaW4vYmFzaAogICBob21lOiAvaG9tZS91YnVudHUKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1"
+  fedora-28: "I2Nsb3VkLWNvbmZpZwp1c2VyczoKIC0gbmFtZToga3ViZXNwcmF5CiAgIHN1ZG86IEFMTD0oQUxMKSBOT1BBU1NXRDpBTEwKICAgc2hlbGw6IC9iaW4vYmFzaAogICBob21lOiAvaG9tZS91YnVudHUKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1"
+  opensuse-leap-15: "I2Nsb3VkLWNvbmZpZwpzeXN0ZW1faW5mbzoKICBkaXN0cm86IHJoZWwKdXNlcnM6CiAtIG5hbWU6IGt1YmVzcHJheQogICBncm91cHM6IHdoZWVsCiAgIHN1ZG86ICdBTEw9KEFMTCkgTk9QQVNTV0Q6QUxMJwogICBzaGVsbDogL2Jpbi9iYXNoCiAgIGxvY2tfcGFzc3dkOiBGYWxzZQogICBob21lOiAvaG9tZS9rdWJlc3ByYXkKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1Cgo="
+  rhel-server-7: "I2Nsb3VkLWNvbmZpZwpzeXN0ZW1faW5mbzoKICBkaXN0cm86IHJoZWwKdXNlcnM6CiAtIG5hbWU6IGt1YmVzcHJheQogICBncm91cHM6IHdoZWVsCiAgIHN1ZG86ICdBTEw9KEFMTCkgTk9QQVNTV0Q6QUxMJwogICBzaGVsbDogL2Jpbi9iYXNoCiAgIGxvY2tfcGFzc3dkOiBGYWxzZQogICBob21lOiAvaG9tZS9rdWJlc3ByYXkKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1Cgo="
+  ubuntu-1604: "I2Nsb3VkLWNvbmZpZwogdXNlcnM6CiAgLSBuYW1lOiBrdWJlc3ByYXkKICAgIHN1ZG86IEFMTD0oQUxMKSBOT1BBU1NXRDpBTEwKICAgIHNoZWxsOiAvYmluL2Jhc2gKICAgIGxvY2tfcGFzc3dkOiBGYWxzZQogICAgaG9tZTogL2hvbWUva3ViZXNwcmF5CiAgICBzc2hfYXV0aG9yaXplZF9rZXlzOgogICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1"
+  ubuntu-1804: "I2Nsb3VkLWNvbmZpZwogdXNlcnM6CiAgLSBuYW1lOiBrdWJlc3ByYXkKICAgIHN1ZG86IEFMTD0oQUxMKSBOT1BBU1NXRDpBTEwKICAgIHNoZWxsOiAvYmluL2Jhc2gKICAgIGxvY2tfcGFzc3dkOiBGYWxzZQogICAgaG9tZTogL2hvbWUva3ViZXNwcmF5CiAgICBzc2hfYXV0aG9yaXplZF9rZXlzOgogICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1"
--- a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml
+++ b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml
@ -0,0 +1,58 @@
+---
+
+- name: "Create CI namespace {{ test_name }} for test vms"
+  k8s:
+    name: "{{ test_name }}"
+    kind: Namespace
+    state: present
+  register: create_ci_ns
+  failed_when: create_ci_ns.changed == false
+
+- name: "Create temp dir /tmp/{{ test_name }} for CI files"
+  file:
+    path: "/tmp/{{ test_name }}"
+    state: directory
+
+- name: Template vm files for CI job
+  template:
+    src: "vm.yml.j2"
+    dest: "/tmp/{{ test_name }}/instance-{{ vm_id }}.yml"
+  loop: "{{ range(0, vm_count|int, 1) | list }}"
+  loop_control:
+    index_var: vm_id
+
+- name: Start vms for CI job
+  k8s:
+    state: present
+    src: "/tmp/{{ test_name }}/instance-{{ vm_id }}.yml"
+  loop: "{{ range(0, vm_count|int, 1) | list }}"
+  loop_control:
+    index_var: vm_id
+
+- name: Wait for vms to have ipaddress assigned
+  shell: "kubectl get vmis -n {{ test_name }} instance-{{ vm_id }} -o json | jq '.status.interfaces[].ipAddress' | tr -d '\"'"
+  register: vm_ips
+  loop: "{{ range(0, vm_count|int, 1) | list }}"
+  loop_control:
+    index_var: vm_id
+  retries: 10
+  delay: 15
+  until:
+    - vm_ips.stdout | ipaddr
+
+- name: Wait for SSH to become available on vms
+  wait_for:
+    host: "{{ item.stdout }}"
+    port: 22
+    delay: 30
+    timeout: 240
+    state: started
+  with_items:
+    - "{{ vm_ips.results }}"
+
+- name: "Create inventory for CI test in file /tmp/{{ test_name }}/inventory"
+  template:
+    src: "inventory.j2"
+    dest: "{{ inventory_path }}"
+  vars:
+    vms: "{{ vm_ips }}"
--- a/tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml
+++ b/tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml
@ -0,0 +1,29 @@
+---
+
+- name: Check if temp directory for {{ test_name }} exists
+  stat:
+    path: "/tmp/{{ test_name }}"
+  register: temp_dir_details
+
+- name: "Cleanup temp directory for {{ test_name }}"
+  file:
+    path: "/tmp/{{ test_name }}"
+    state: absent
+
+- name: "Cleanup namespace for {{ test_name }}"
+  k8s:
+    kind: Namespace
+    state: absent
+    name: "{{ test_name }}"
+
+- name: Wait for namespace {{ test_name }} to be fully deleted
+  shell: kubectl get ns {{ test_name }}
+  register: delete_namespace
+  failed_when:
+    - delete_namespace.rc == 0
+  changed_when:
+    - delete_namespace.rc == 0
+  retries: 12
+  delay: "10"
+  until:
+    - delete_namespace.rc != 0
--- a/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml
+++ b/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml
@ -0,0 +1,16 @@
+---
+
+- name: "Include custom vars for ci job: {{ ci_job_name }}"
+  include_vars: "../files/{{ ci_job_name }}.yml"
+
+- name: Set VM count needed for CI test_id
+  set_fact:
+    vm_count: "{%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale'] -%}{{ 3|int }}{%- elif mode == 'aio' -%}{{ 1|int }}{%- else -%}{{ 2|int }}{%- endif -%}"
+
+- import_tasks: create-vms.yml
+  when:
+    - not vm_cleanup
+
+- import_tasks: delete-vms.yml
+  when:
+    - vm_cleanup | default(false)
--- a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2
+++ b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2
@ -0,0 +1,54 @@
+[all]
+{% for instance in vms.results %}
+instance-{{ loop.index }} ansible_ssh_host={{instance.stdout}}
+{% endfor %}
+
+{% if mode is defined and mode in ["separate", "separate-scale"] %}
+[kube-master]
+instance-1
+
+[kube-node]
+instance-2
+
+[etcd]
+instance-3
+{% elif mode is defined and mode in ["ha", "ha-scale"] %}
+[kube-master]
+instance-1
+instance-2
+
+[kube-node]
+instance-3
+
+[etcd]
+instance-1
+instance-2
+instance-3
+{% elif mode == "default" %}
+[kube-master]
+instance-1
+
+[kube-node]
+instance-2
+
+[etcd]
+instance-1
+{% elif mode == "aio" %}
+[kube-master]
+instance-1
+
+[kube-node]
+instance-1
+
+[etcd]
+instance-1
+
+[vault]
+instance-1
+{% endif %}
+
+[k8s-cluster:children]
+kube-node
+kube-master
+
+[fake_hosts]
--- a/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2
+++ b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2
@ -0,0 +1,49 @@
+---
+apiVersion: kubevirt.io/v1alpha3
+kind: VirtualMachine
+metadata:
+  name: "instance-{{ vm_id }}"
+  namespace: "{{ test_name }}"
+  labels:
+    kubevirt.io/os: {{ cloud_image }}
+spec:
+  running: true
+  template:
+    metadata:
+      labels:
+        kubevirt.io/size: small
+        kubevirt.io/domain: "{{ test_name }}"
+    spec:
+      domain:
+        devices:
+          blockMultiQueue: true
+          disks:
+            - disk:
+                bus: virtio
+              name: containervolume
+              cache: writethrough
+            - disk:
+                bus: virtio
+              name: cloudinitvolume
+          interfaces:
+          - name: default
+            bridge: {}
+        cpu:
+            cores: {{ vm_cpu_cores }}
+            sockets: {{ vm_cpu_sockets }}
+            threads: {{ vm_cpu_cores }}
+        resources:
+          requests:
+            memory: {{ vm_memory }}
+            cpu: {{ vm_cpu_cores }}
+      networks:
+      - name: default
+        pod: {}
+      terminationGracePeriodSeconds: 0
+      volumes:
+        - name: containervolume
+          containerDisk:
+            image: quay.io/kubespray/vm-{{ cloud_image }}
+        - name: cloudinitvolume
+          cloudInitNoCloud:
+            userDataBase64: {{ cloud_init[cloud_image] }}
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@ -6,3 +6,4 @@ tox
 dopy
 PyCrypto
 ansible-lint==4.1.0
+openshift