Browse Source

Merge pull request #580 from bogdando/gcs_logs

Upload logs to GCS after failure
pull/583/head
Bogdan Dobrelya 8 years ago
committed by GitHub
parent
commit
047f098660
4 changed files with 48 additions and 56 deletions
  1. 14
      .travis.yml
  2. 23
      scripts/collect-info.yaml
  3. 39
      scripts/configure-logs.yaml
  4. 28
      tests/cloud_playbooks/upload-logs-gcs.yml

14
.travis.yml

@ -10,6 +10,8 @@ env:
TEST_ID=$TRAVIS_JOB_NUMBER
CONTAINER_ENGINE=docker
PRIVATE_KEY=$GCE_PRIVATE_KEY
GS_ACCESS_KEY_ID=$GS_KEY
GS_SECRET_ACCESS_KEY=$GS_SECRET
ANSIBLE_KEEP_REMOTE_FILES=1
CLUSTER_MODE=default
matrix:
@ -122,8 +124,6 @@ before_script:
- $HOME/.local/bin/ansible-playbook --version
- cp tests/ansible.cfg .
# - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}' $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml"
## Configure ansible deployment logs to be collected as an artifact. Enable when GCS configured, see https://docs.travis-ci.com/user/deployment/gcs
# - $HOME/.local/bin/ansible-playbook -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scripts/configure-logs.yaml
script:
- >
@ -147,8 +147,14 @@ script:
- $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL
## Ping the between 2 pod
- $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
## Collect env info, enable it once GCS configured, see https://docs.travis-ci.com/user/deployment/gcs
# - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scripts/collect-info.yaml
after_failure:
- $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scripts/collect-info.yaml >/dev/null
- >
$HOME/.local/bin/ansible-playbook tests/cloud_playbooks/upload-logs-gcs.yml -i "localhost," -c local
-e test_id=${TEST_ID}
-e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
>/dev/null
after_script:
- >

23
scripts/collect-info.yaml

@ -6,16 +6,10 @@
vars:
debug: false
commands:
- name: git_info
cmd: find . -type d -name .git -execdir sh -c 'gen-gitinfos.sh global|head -12' \;
- name: timedate_info
cmd: timedatectl status
- name: space_info
cmd: df -h
- name: kernel_info
cmd: uname -r
- name: distro_info
cmd: cat /etc/issue.net
- name: docker_info
cmd: docker info
- name: ip_info
@ -24,23 +18,26 @@
cmd: ip ro
- name: proc_info
cmd: ps auxf | grep -v ]$
- name: systemctl_info
cmd: systemctl status
- name: systemctl_failed_info
cmd: systemctl --state=failed --no-pager
- name: k8s_info
cmd: kubectl get all --all-namespaces -o wide
- name: errors_info
cmd: journalctl -p err --utc --no-pager
- name: etcd_info
cmd: etcdctl --debug cluster-health
logs:
- /var/log/ansible.log
- /var/log/ansible/ansible.log
- /var/log/syslog
- /var/log/daemon.log
- /var/log/kern.log
- inventory/inventory.ini
- cluster.yml
- /var/log/dpkg.log
- /var/log/apt/history.log
- /var/log/yum.log
- /var/log/calico/bird/current
- /var/log/calico/bird6/current
- /var/log/calico/felix/current
- /var/log/calico/confd/current
tasks:
- name: Storing commands output
@ -50,7 +47,7 @@
with_items: "{{commands}}"
- debug: var=item
with_items: output.results
with_items: "{{output.results}}"
when: debug
- name: Fetch results

39
scripts/configure-logs.yaml

@ -1,39 +0,0 @@
---
- hosts: localhost
become: true
gather_facts: no
vars:
log_path: /var/log/ansible/
conf_file: /etc/ansible/ansible.cfg
human_readable_plugin: false
callback_plugin_path: /usr/share/ansible/plugins/callback
tasks:
- name: LOGS | ensure log path
file: path="{{log_path}}" state=directory owner={{ansible_ssh_user}}
- name: LOGS | ensure plugin path
file: path="{{callback_plugin_path}}" state=directory owner={{ansible_ssh_user}}
when: human_readable_plugin
- name: LOGS | get plugin
git: repo=https://gist.github.com/cd706de198c85a8255f6.git dest=/tmp/cd706de198c85a8255f6
when: human_readable_plugin
- name: LOGS | install plugin
copy: src=/tmp/cd706de198c85a8255f6/human_log.py dest="{{callback_plugin_path}}"
when: human_readable_plugin
- name: LOGS | config
lineinfile:
line: "log_path={{log_path}}/ansible.log"
regexp: "^#log_path|^log_path"
dest: "{{conf_file}}"
- name: LOGS | callback plugin
lineinfile:
line: "callback_plugins={{callback_plugin_path}}"
regexp: "^#callback_plugins|^callback_plugins"
dest: "{{conf_file}}"
when: human_readable_plugin

28
tests/cloud_playbooks/upload-logs-gcs.yml

@ -0,0 +1,28 @@
---
- hosts: localhost
become: false
gather_facts: no
vars:
expire: 72000
tasks:
- name: replace_test_id
set_fact:
test_name: "{{ test_id | regex_replace('\\.', '-') }}"
- name: Create a bucket
gc_storage:
bucket: "{{ test_name }}"
mode: create
expiration: "{{ expire }}"
permission: private
- name: Upload collected diagnostic info
gc_storage:
bucket: "{{ test_name }}"
mode: put
permission: private
expiration: "{{ expire }}"
object: "build-{{ test_name }}-{{ kube_network_plugin }}-logs.tar.gz"
src: logs.tar.gz
Loading…
Cancel
Save