Merge pull request #580 from bogdando/gcs_logs

Upload logs to GCS after failure

.travis.yml

@@ -10,6 +10,8 @@ env:
     TEST_ID=$TRAVIS_JOB_NUMBER
     CONTAINER_ENGINE=docker
     PRIVATE_KEY=$GCE_PRIVATE_KEY
+    GS_ACCESS_KEY_ID=$GS_KEY
+    GS_SECRET_ACCESS_KEY=$GS_SECRET
     ANSIBLE_KEEP_REMOTE_FILES=1
     CLUSTER_MODE=default
   matrix:
@@ -122,8 +124,6 @@ before_script:
   - $HOME/.local/bin/ansible-playbook --version
   - cp tests/ansible.cfg .
 #  - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}'  $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml"
-    ## Configure ansible deployment logs to be collected as an artifact. Enable when GCS configured, see https://docs.travis-ci.com/user/deployment/gcs
-#  - $HOME/.local/bin/ansible-playbook -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scripts/configure-logs.yaml
 
 script:
   - >
@@ -147,8 +147,14 @@ script:
   - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL
     ## Ping the between 2 pod
   - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
-    ## Collect env info, enable it once GCS configured, see https://docs.travis-ci.com/user/deployment/gcs
-#  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scripts/collect-info.yaml
+
+after_failure:
+  - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scripts/collect-info.yaml >/dev/null
+  - >
+    $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/upload-logs-gcs.yml -i "localhost," -c local
+    -e test_id=${TEST_ID}
+    -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+    >/dev/null
 
 after_script:
   - >

scripts/collect-info.yaml

@@ -6,16 +6,10 @@
   vars:
     debug: false
     commands:
-      - name: git_info
-        cmd: find . -type d -name .git -execdir sh -c 'gen-gitinfos.sh global|head -12' \;
       - name: timedate_info
         cmd: timedatectl status
-      - name: space_info
-        cmd: df -h
       - name: kernel_info
         cmd: uname -r
-      - name: distro_info
-        cmd: cat /etc/issue.net
       - name: docker_info
         cmd: docker info
       - name: ip_info
@@ -24,23 +18,26 @@
         cmd: ip ro
       - name: proc_info
         cmd: ps auxf | grep -v ]$
-      - name: systemctl_info
-        cmd: systemctl status
       - name: systemctl_failed_info
         cmd: systemctl --state=failed --no-pager
       - name: k8s_info
         cmd: kubectl get all --all-namespaces -o wide
       - name: errors_info
         cmd: journalctl -p err --utc --no-pager
+      - name: etcd_info
+        cmd: etcdctl --debug cluster-health
 
     logs:
-      - /var/log/ansible.log
-      - /var/log/ansible/ansible.log
       - /var/log/syslog
       - /var/log/daemon.log
       - /var/log/kern.log
-      - inventory/inventory.ini
-      - cluster.yml
+      - /var/log/dpkg.log
+      - /var/log/apt/history.log
+      - /var/log/yum.log
+      - /var/log/calico/bird/current
+      - /var/log/calico/bird6/current
+      - /var/log/calico/felix/current
+      - /var/log/calico/confd/current
 
   tasks:
     - name: Storing commands output
@@ -50,7 +47,7 @@
       with_items: "{{commands}}"
 
     - debug: var=item
-      with_items: output.results
+      with_items: "{{output.results}}"
       when: debug
 
     - name: Fetch results

scripts/configure-logs.yaml

@@ -1,39 +0,0 @@
----
-- hosts: localhost
-  become: true
-  gather_facts: no
-
-  vars:
-    log_path: /var/log/ansible/
-    conf_file: /etc/ansible/ansible.cfg
-    human_readable_plugin: false
-    callback_plugin_path: /usr/share/ansible/plugins/callback
-
-  tasks:
-    - name: LOGS | ensure log path
-      file: path="{{log_path}}" state=directory owner={{ansible_ssh_user}}
-
-    - name: LOGS | ensure plugin path
-      file: path="{{callback_plugin_path}}" state=directory owner={{ansible_ssh_user}}
-      when: human_readable_plugin
-
-    - name: LOGS | get plugin
-      git: repo=https://gist.github.com/cd706de198c85a8255f6.git dest=/tmp/cd706de198c85a8255f6
-      when: human_readable_plugin
-
-    - name: LOGS | install plugin
-      copy: src=/tmp/cd706de198c85a8255f6/human_log.py dest="{{callback_plugin_path}}"
-      when: human_readable_plugin
-
-    - name: LOGS | config
-      lineinfile:
-        line: "log_path={{log_path}}/ansible.log"
-        regexp: "^#log_path|^log_path"
-        dest: "{{conf_file}}"
-
-    - name: LOGS | callback plugin
-      lineinfile:
-        line: "callback_plugins={{callback_plugin_path}}"
-        regexp: "^#callback_plugins|^callback_plugins"
-        dest: "{{conf_file}}"
-      when: human_readable_plugin

tests/cloud_playbooks/upload-logs-gcs.yml

@@ -0,0 +1,28 @@
+---
+- hosts: localhost
+  become: false
+  gather_facts: no
+
+  vars:
+    expire: 72000
+
+  tasks:
+    - name: replace_test_id
+      set_fact:
+        test_name: "{{ test_id | regex_replace('\\.', '-') }}"
+
+    - name: Create a bucket
+      gc_storage:
+        bucket: "{{ test_name }}"
+        mode: create
+        expiration: "{{ expire }}"
+        permission: private
+
+    - name: Upload collected diagnostic info
+      gc_storage:
+        bucket: "{{ test_name }}"
+        mode: put
+        permission: private
+        expiration: "{{ expire }}"
+        object: "build-{{ test_name }}-{{ kube_network_plugin }}-logs.tar.gz"
+        src: logs.tar.gz