diff --git a/playbooks/README.rst b/playbooks/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..bfb4ea59432c0145fe3a788d1fe0ac3caf290bbd --- /dev/null +++ b/playbooks/README.rst @@ -0,0 +1,5 @@ +Zuul Ansible Playbooks +====================== + +This directory contains playbooks for use by Zuul. These playbooks are not used +by kayobe - see ``ansible/``. diff --git a/playbooks/kayobe-overcloud-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-base/overrides.yml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..db327943dcedb2972c45ef0e11409c5226469385 --- /dev/null +++ b/playbooks/kayobe-overcloud-base/overrides.yml.j2 @@ -0,0 +1,4 @@ +--- +# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as +# Ansible is run directly on the controller. +disable_selinux_do_reboot: false diff --git a/playbooks/kayobe-overcloud-base/post.yml b/playbooks/kayobe-overcloud-base/post.yml new file mode 100644 index 0000000000000000000000000000000000000000..5de02240ad3a19f05c217e271f1251edaf3cca5f --- /dev/null +++ b/playbooks/kayobe-overcloud-base/post.yml @@ -0,0 +1,7 @@ +--- +- hosts: all + roles: + - role: kayobe-diagnostics + kayobe_diagnostics_phase: "post" + kayobe_diagnostics_log_dir: "/tmp/logs" + kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" diff --git a/playbooks/kayobe-overcloud-base/pre.yml b/playbooks/kayobe-overcloud-base/pre.yml new file mode 100644 index 0000000000000000000000000000000000000000..fdc39e023d61bb8daf6cbf4cc08717f8ec98f7e1 --- /dev/null +++ b/playbooks/kayobe-overcloud-base/pre.yml @@ -0,0 +1,60 @@ +--- +- hosts: primary + vars: + logs_dir: "/tmp/logs" + kayobe_src_dir: "{{ zuul.project.src_dir }}" + roles: + - role: kayobe-diagnostics + kayobe_diagnostics_phase: "pre" + kayobe_diagnostics_log_dir: "{{ logs_dir }}" + tasks: + - name: Install dbus for debian system + apt: + name: dbus + when: + - ansible_os_family == 'Debian' + become: true + + - block: + # NOTE(mgoddard): The CentOS image used in CI has epel-release installed, + # but the configure-mirrors role used by Zuul disables epel. Since we + # install epel-release and expect epel to be enabled, enable it here. + - name: Ensure yum-utils is installed + yum: + name: yum-utils + state: installed + + - name: Enable the EPEL yum repository + command: yum-config-manager --enable epel + when: ansible_os_family == 'RedHat' + become: true + + - name: Ensure kayobe-config directory exists + file: + path: "{{ kayobe_src_dir }}/config/src" + state: directory + + - name: Ensure kayobe-config repository is cloned + git: + repo: https://github.com/stackhpc/dev-kayobe-config + dest: "{{ kayobe_src_dir }}/config/src/kayobe-config" + + - name: Ensure kayobe-config override config file exists + template: + src: overrides.yml.j2 + dest: "{{ kayobe_src_dir }}/config/src/kayobe-config/etc/kayobe/overrides.yml" + + # NOTE(mgoddard): The kayobe dev config by default expects a bridge - + # breth1 - to exist with an IP address of 192.168.33.3. + - name: Ensure all-in-one network bridge interface exists + command: "{{ item }}" + become: true + with_items: + - "ip l add breth1 type bridge" + - "ip l set breth1 up" + - "ip a add 192.168.33.3/24 dev breth1" + + - name: Ensure kayobe is installed + shell: + cmd: dev/install.sh > {{ logs_dir }}/ansible/install + chdir: "{{ kayobe_src_dir }}" diff --git a/playbooks/kayobe-overcloud-base/run.yml b/playbooks/kayobe-overcloud-base/run.yml new file mode 100644 index 0000000000000000000000000000000000000000..3d09ea500b8d4e81b89a2568a4f7dab51eed4d7f --- /dev/null +++ b/playbooks/kayobe-overcloud-base/run.yml @@ -0,0 +1,10 @@ +--- +- hosts: primary + vars: + kayobe_src_dir: "{{ zuul.project.src_dir }}" + logs_dir: "/tmp/logs" + tasks: + - name: Ensure overcloud is deployed + shell: + cmd: dev/overcloud-deploy.sh > {{ logs_dir }}/ansible/overcloud-deploy + chdir: "{{ kayobe_src_dir }}" diff --git a/roles/README.rst b/roles/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..843d4a5ebd05d9ef7fe89f2ee66732a2662a25e0 --- /dev/null +++ b/roles/README.rst @@ -0,0 +1,5 @@ +Zuul Ansible Roles +================== + +This directory contains roles for use by Zuul. These roles are not used by +kayobe - see ``ansible/roles/``. diff --git a/roles/kayobe-diagnostics/README.rst b/roles/kayobe-diagnostics/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..40549cc826e519bf5945e0c543da865f20af59c2 --- /dev/null +++ b/roles/kayobe-diagnostics/README.rst @@ -0,0 +1,6 @@ +================== +Kayobe Diagnostics +================== + +Ansible role to collect diagnostic information following a CI job performing +integration testing. diff --git a/roles/kayobe-diagnostics/defaults/main.yml b/roles/kayobe-diagnostics/defaults/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..659ce29273d3e3f5ea634640cb4918eb102a6b5f --- /dev/null +++ b/roles/kayobe-diagnostics/defaults/main.yml @@ -0,0 +1,9 @@ +--- +# Job phase - one of 'pre' or 'post'. +kayobe_diagnostics_phase: + +# Directory on the remote host in which to save diagnostics. +kayobe_diagnostics_logs_dir: + +# Directory on the executor in which to save logs. +kayobe_diagnostics_executor_log_dir: diff --git a/roles/kayobe-diagnostics/files/get_logs.sh b/roles/kayobe-diagnostics/files/get_logs.sh new file mode 100644 index 0000000000000000000000000000000000000000..2a08f2907c4556118fa9727bbb47913763bdb19c --- /dev/null +++ b/roles/kayobe-diagnostics/files/get_logs.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# NOTE(mgoddard): This has been adapted from tests/get_logs.sh in Kolla +# Ansible. + +set +o errexit + +copy_logs() { + LOG_DIR=/tmp/logs + + cp -rnL /var/lib/docker/volumes/kolla_logs/_data/* ${LOG_DIR}/kolla/ + # TODO(mgoddard): Copy kayobe config + cp -rvnL /var/log/* ${LOG_DIR}/system_logs/ + + + if [[ -x "$(command -v journalctl)" ]]; then + journalctl --no-pager > ${LOG_DIR}/system_logs/syslog.txt + journalctl --no-pager -u docker.service > ${LOG_DIR}/system_logs/docker.log + else + cp /var/log/upstart/docker.log ${LOG_DIR}/system_logs/docker.log + fi + + cp -r /etc/sudoers.d ${LOG_DIR}/system_logs/ + cp /etc/sudoers ${LOG_DIR}/system_logs/sudoers.txt + + df -h > ${LOG_DIR}/system_logs/df.txt + free > ${LOG_DIR}/system_logs/free.txt + parted -l > ${LOG_DIR}/system_logs/parted-l.txt + mount > ${LOG_DIR}/system_logs/mount.txt + env > ${LOG_DIR}/system_logs/env.txt + + if [ `command -v dpkg` ]; then + dpkg -l > ${LOG_DIR}/system_logs/dpkg-l.txt + fi + if [ `command -v rpm` ]; then + rpm -qa > ${LOG_DIR}/system_logs/rpm-qa.txt + fi + + # final memory usage and process list + ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ${LOG_DIR}/system_logs/ps.txt + + # docker related information + (docker info && docker images && docker ps -a) > ${LOG_DIR}/system_logs/docker-info.txt + + for container in $(docker ps -a --format "{{.Names}}"); do + docker logs --tail all ${container} > ${LOG_DIR}/docker_logs/${container}.txt + done + + # Rename files to .txt; this is so that when displayed via + # logs.openstack.org clicking results in the browser shows the + # files, rather than trying to send it to another app or make you + # download it, etc. + + # Rename files to .txt; this is so that when displayed via + # logs.openstack.org clicking results in the browser shows the + # files, rather than trying to send it to another app or make you + # download it, etc. + + # Rename all .log files to .txt files + for f in $(find ${LOG_DIR}/{system_logs,kolla,docker_logs} -name "*.log"); do + mv $f ${f/.log/.txt} + done + + chmod -R 777 ${LOG_DIR} + find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.txt' -execdir gzip -f -9 {} \+ + find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.json' -execdir gzip -f -9 {} \+ +} + +copy_logs diff --git a/roles/kayobe-diagnostics/tasks/main.yml b/roles/kayobe-diagnostics/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..8cc20231bd4dca45fe806031e66ba1a5cb80fed8 --- /dev/null +++ b/roles/kayobe-diagnostics/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include: "{{ kayobe_diagnostics_phase }}.yml" diff --git a/roles/kayobe-diagnostics/tasks/post.yml b/roles/kayobe-diagnostics/tasks/post.yml new file mode 100644 index 0000000000000000000000000000000000000000..cc42f13d18bef9ea7c7b9d1b8450de388e1f00df --- /dev/null +++ b/roles/kayobe-diagnostics/tasks/post.yml @@ -0,0 +1,21 @@ +--- +- name: Write host variables to a file + copy: + content: "{{ hostvars[inventory_hostname] | to_nice_json }}" + dest: "{{ kayobe_diagnostics_log_dir }}/facts.json" + +- name: Run diagnostics script + script: get_logs.sh + register: get_logs_result + become: true + failed_when: false + +- name: Print get_logs.sh output + debug: + msg: "{{ get_logs_result.stdout }}" + +- name: Download logs to executor + synchronize: + src: "{{ kayobe_diagnostics_log_dir }}" + dest: "{{ kayobe_diagnostics_executor_log_dir }}/" + mode: pull diff --git a/roles/kayobe-diagnostics/tasks/pre.yml b/roles/kayobe-diagnostics/tasks/pre.yml new file mode 100644 index 0000000000000000000000000000000000000000..6d51e6674898974e86174ce18d74e9a8b3621e24 --- /dev/null +++ b/roles/kayobe-diagnostics/tasks/pre.yml @@ -0,0 +1,17 @@ +--- +- name: Ensure node log directory exists + file: + path: "{{ kayobe_diagnostics_log_dir }}" + state: "directory" + +- name: Ensure node log subdirectories exist + file: + path: "{{ kayobe_diagnostics_log_dir }}/{{ item }}" + state: "directory" + mode: 0777 + with_items: + - "docker_logs" + - "kolla_configs" + - "system_logs" + - "kolla" + - "ansible" diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 973f80e0ece2184c7e2d1d452feb21ec0714c28b..b5a91e0ed8dcf35c82b27d2bfcafd1b81c311611 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -12,3 +12,20 @@ parent: openstack-tox-with-sudo vars: tox_envlist: ansible + +# Base job for testing overcloud deployment. +- job: + name: kayobe-overcloud-base + pre-run: playbooks/kayobe-overcloud-base/pre.yml + run: playbooks/kayobe-overcloud-base/run.yml + post-run: playbooks/kayobe-overcloud-base/post.yml + attempts: 1 + timeout: 5400 + irrelevant-files: + - ^.*\.rst$ + - ^doc/.* + +- job: + name: kayobe-overcloud-centos + parent: kayobe-overcloud-base + nodeset: kayobe-centos diff --git a/zuul.d/nodesets.yaml b/zuul.d/nodesets.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04067f11945daca736905f2b9fb506dca3b74781 --- /dev/null +++ b/zuul.d/nodesets.yaml @@ -0,0 +1,6 @@ +--- +- nodeset: + name: kayobe-centos + nodes: + - name: primary + label: centos-7 diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 905258dba866b3d6877ba62e17406dd7a8cafb29..39e17448a14ba63d57b6603388bed79bfceb3ff2 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -8,6 +8,7 @@ - build-openstack-sphinx-docs - kayobe-tox-ansible-syntax - kayobe-tox-ansible + - kayobe-overcloud-centos gate: queue: kayobe @@ -18,3 +19,4 @@ - build-openstack-sphinx-docs - kayobe-tox-ansible-syntax - kayobe-tox-ansible + - kayobe-overcloud-centos