From 4cdf71f5a117d4f8df84e1771ee9d39605cc527e Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 12 Mar 2018 12:35:00 +0000
Subject: [PATCH] Add overcloud deployment job
Adds a Zuul job to perform deployment of an OpenStack control plane
consisting of a single node. Currently no tests are run against the
resulting control plane.
Change-Id: I2728421cb5dbd1dc370a3aa7d37d913c90ba38a5
Story: #2001655
Task: #6682
---
playbooks/README.rst | 5 ++
.../kayobe-overcloud-base/overrides.yml.j2 | 4 ++
playbooks/kayobe-overcloud-base/post.yml | 7 ++
playbooks/kayobe-overcloud-base/pre.yml | 60 ++++++++++++++++
playbooks/kayobe-overcloud-base/run.yml | 10 +++
roles/README.rst | 5 ++
roles/kayobe-diagnostics/README.rst | 6 ++
roles/kayobe-diagnostics/defaults/main.yml | 9 +++
roles/kayobe-diagnostics/files/get_logs.sh | 69 +++++++++++++++++++
roles/kayobe-diagnostics/tasks/main.yml | 2 +
roles/kayobe-diagnostics/tasks/post.yml | 21 ++++++
roles/kayobe-diagnostics/tasks/pre.yml | 17 +++++
zuul.d/jobs.yaml | 17 +++++
zuul.d/nodesets.yaml | 6 ++
zuul.d/project.yaml | 2 +
15 files changed, 240 insertions(+)
create mode 100644 playbooks/README.rst
create mode 100644 playbooks/kayobe-overcloud-base/overrides.yml.j2
create mode 100644 playbooks/kayobe-overcloud-base/post.yml
create mode 100644 playbooks/kayobe-overcloud-base/pre.yml
create mode 100644 playbooks/kayobe-overcloud-base/run.yml
create mode 100644 roles/README.rst
create mode 100644 roles/kayobe-diagnostics/README.rst
create mode 100644 roles/kayobe-diagnostics/defaults/main.yml
create mode 100644 roles/kayobe-diagnostics/files/get_logs.sh
create mode 100644 roles/kayobe-diagnostics/tasks/main.yml
create mode 100644 roles/kayobe-diagnostics/tasks/post.yml
create mode 100644 roles/kayobe-diagnostics/tasks/pre.yml
create mode 100644 zuul.d/nodesets.yaml
diff --git a/playbooks/README.rst b/playbooks/README.rst
new file mode 100644
index 00000000..bfb4ea59
--- /dev/null
+++ b/playbooks/README.rst
@@ -0,0 +1,5 @@
+Zuul Ansible Playbooks
+======================
+
+This directory contains playbooks for use by Zuul. These playbooks are not used
+by kayobe - see ``ansible/``.
diff --git a/playbooks/kayobe-overcloud-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-base/overrides.yml.j2
new file mode 100644
index 00000000..db327943
--- /dev/null
+++ b/playbooks/kayobe-overcloud-base/overrides.yml.j2
@@ -0,0 +1,4 @@
+---
+# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as
+# Ansible is run directly on the controller.
+disable_selinux_do_reboot: false
diff --git a/playbooks/kayobe-overcloud-base/post.yml b/playbooks/kayobe-overcloud-base/post.yml
new file mode 100644
index 00000000..5de02240
--- /dev/null
+++ b/playbooks/kayobe-overcloud-base/post.yml
@@ -0,0 +1,7 @@
+---
+- hosts: all
+ roles:
+ - role: kayobe-diagnostics
+ kayobe_diagnostics_phase: "post"
+ kayobe_diagnostics_log_dir: "/tmp/logs"
+ kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
diff --git a/playbooks/kayobe-overcloud-base/pre.yml b/playbooks/kayobe-overcloud-base/pre.yml
new file mode 100644
index 00000000..fdc39e02
--- /dev/null
+++ b/playbooks/kayobe-overcloud-base/pre.yml
@@ -0,0 +1,60 @@
+---
+- hosts: primary
+ vars:
+ logs_dir: "/tmp/logs"
+ kayobe_src_dir: "{{ zuul.project.src_dir }}"
+ roles:
+ - role: kayobe-diagnostics
+ kayobe_diagnostics_phase: "pre"
+ kayobe_diagnostics_log_dir: "{{ logs_dir }}"
+ tasks:
+ - name: Install dbus for debian system
+ apt:
+ name: dbus
+ when:
+ - ansible_os_family == 'Debian'
+ become: true
+
+ - block:
+ # NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
+ # but the configure-mirrors role used by Zuul disables epel. Since we
+ # install epel-release and expect epel to be enabled, enable it here.
+ - name: Ensure yum-utils is installed
+ yum:
+ name: yum-utils
+ state: installed
+
+ - name: Enable the EPEL yum repository
+ command: yum-config-manager --enable epel
+ when: ansible_os_family == 'RedHat'
+ become: true
+
+ - name: Ensure kayobe-config directory exists
+ file:
+ path: "{{ kayobe_src_dir }}/config/src"
+ state: directory
+
+ - name: Ensure kayobe-config repository is cloned
+ git:
+ repo: https://github.com/stackhpc/dev-kayobe-config
+ dest: "{{ kayobe_src_dir }}/config/src/kayobe-config"
+
+ - name: Ensure kayobe-config override config file exists
+ template:
+ src: overrides.yml.j2
+ dest: "{{ kayobe_src_dir }}/config/src/kayobe-config/etc/kayobe/overrides.yml"
+
+ # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
+ # breth1 - to exist with an IP address of 192.168.33.3.
+ - name: Ensure all-in-one network bridge interface exists
+ command: "{{ item }}"
+ become: true
+ with_items:
+ - "ip l add breth1 type bridge"
+ - "ip l set breth1 up"
+ - "ip a add 192.168.33.3/24 dev breth1"
+
+ - name: Ensure kayobe is installed
+ shell:
+ cmd: dev/install.sh > {{ logs_dir }}/ansible/install
+ chdir: "{{ kayobe_src_dir }}"
diff --git a/playbooks/kayobe-overcloud-base/run.yml b/playbooks/kayobe-overcloud-base/run.yml
new file mode 100644
index 00000000..3d09ea50
--- /dev/null
+++ b/playbooks/kayobe-overcloud-base/run.yml
@@ -0,0 +1,10 @@
+---
+- hosts: primary
+ vars:
+ kayobe_src_dir: "{{ zuul.project.src_dir }}"
+ logs_dir: "/tmp/logs"
+ tasks:
+ - name: Ensure overcloud is deployed
+ shell:
+ cmd: dev/overcloud-deploy.sh > {{ logs_dir }}/ansible/overcloud-deploy
+ chdir: "{{ kayobe_src_dir }}"
diff --git a/roles/README.rst b/roles/README.rst
new file mode 100644
index 00000000..843d4a5e
--- /dev/null
+++ b/roles/README.rst
@@ -0,0 +1,5 @@
+Zuul Ansible Roles
+==================
+
+This directory contains roles for use by Zuul. These roles are not used by
+kayobe - see ``ansible/roles/``.
diff --git a/roles/kayobe-diagnostics/README.rst b/roles/kayobe-diagnostics/README.rst
new file mode 100644
index 00000000..40549cc8
--- /dev/null
+++ b/roles/kayobe-diagnostics/README.rst
@@ -0,0 +1,6 @@
+==================
+Kayobe Diagnostics
+==================
+
+Ansible role to collect diagnostic information following a CI job performing
+integration testing.
diff --git a/roles/kayobe-diagnostics/defaults/main.yml b/roles/kayobe-diagnostics/defaults/main.yml
new file mode 100644
index 00000000..659ce292
--- /dev/null
+++ b/roles/kayobe-diagnostics/defaults/main.yml
@@ -0,0 +1,9 @@
+---
+# Job phase - one of 'pre' or 'post'.
+kayobe_diagnostics_phase:
+
+# Directory on the remote host in which to save diagnostics.
+kayobe_diagnostics_logs_dir:
+
+# Directory on the executor in which to save logs.
+kayobe_diagnostics_executor_log_dir:
diff --git a/roles/kayobe-diagnostics/files/get_logs.sh b/roles/kayobe-diagnostics/files/get_logs.sh
new file mode 100644
index 00000000..2a08f290
--- /dev/null
+++ b/roles/kayobe-diagnostics/files/get_logs.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# NOTE(mgoddard): This has been adapted from tests/get_logs.sh in Kolla
+# Ansible.
+
+set +o errexit
+
+copy_logs() {
+ LOG_DIR=/tmp/logs
+
+ cp -rnL /var/lib/docker/volumes/kolla_logs/_data/* ${LOG_DIR}/kolla/
+ # TODO(mgoddard): Copy kayobe config
+ cp -rvnL /var/log/* ${LOG_DIR}/system_logs/
+
+
+ if [[ -x "$(command -v journalctl)" ]]; then
+ journalctl --no-pager > ${LOG_DIR}/system_logs/syslog.txt
+ journalctl --no-pager -u docker.service > ${LOG_DIR}/system_logs/docker.log
+ else
+ cp /var/log/upstart/docker.log ${LOG_DIR}/system_logs/docker.log
+ fi
+
+ cp -r /etc/sudoers.d ${LOG_DIR}/system_logs/
+ cp /etc/sudoers ${LOG_DIR}/system_logs/sudoers.txt
+
+ df -h > ${LOG_DIR}/system_logs/df.txt
+ free > ${LOG_DIR}/system_logs/free.txt
+ parted -l > ${LOG_DIR}/system_logs/parted-l.txt
+ mount > ${LOG_DIR}/system_logs/mount.txt
+ env > ${LOG_DIR}/system_logs/env.txt
+
+ if [ `command -v dpkg` ]; then
+ dpkg -l > ${LOG_DIR}/system_logs/dpkg-l.txt
+ fi
+ if [ `command -v rpm` ]; then
+ rpm -qa > ${LOG_DIR}/system_logs/rpm-qa.txt
+ fi
+
+ # final memory usage and process list
+ ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ${LOG_DIR}/system_logs/ps.txt
+
+ # docker related information
+ (docker info && docker images && docker ps -a) > ${LOG_DIR}/system_logs/docker-info.txt
+
+ for container in $(docker ps -a --format "{{.Names}}"); do
+ docker logs --tail all ${container} > ${LOG_DIR}/docker_logs/${container}.txt
+ done
+
+ # Rename files to .txt; this is so that when displayed via
+ # logs.openstack.org clicking results in the browser shows the
+ # files, rather than trying to send it to another app or make you
+ # download it, etc.
+
+ # Rename files to .txt; this is so that when displayed via
+ # logs.openstack.org clicking results in the browser shows the
+ # files, rather than trying to send it to another app or make you
+ # download it, etc.
+
+ # Rename all .log files to .txt files
+ for f in $(find ${LOG_DIR}/{system_logs,kolla,docker_logs} -name "*.log"); do
+ mv $f ${f/.log/.txt}
+ done
+
+ chmod -R 777 ${LOG_DIR}
+ find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.txt' -execdir gzip -f -9 {} \+
+ find ${LOG_DIR}/{system_logs,kolla,docker_logs} -iname '*.json' -execdir gzip -f -9 {} \+
+}
+
+copy_logs
diff --git a/roles/kayobe-diagnostics/tasks/main.yml b/roles/kayobe-diagnostics/tasks/main.yml
new file mode 100644
index 00000000..8cc20231
--- /dev/null
+++ b/roles/kayobe-diagnostics/tasks/main.yml
@@ -0,0 +1,2 @@
+---
+- include: "{{ kayobe_diagnostics_phase }}.yml"
diff --git a/roles/kayobe-diagnostics/tasks/post.yml b/roles/kayobe-diagnostics/tasks/post.yml
new file mode 100644
index 00000000..cc42f13d
--- /dev/null
+++ b/roles/kayobe-diagnostics/tasks/post.yml
@@ -0,0 +1,21 @@
+---
+- name: Write host variables to a file
+ copy:
+ content: "{{ hostvars[inventory_hostname] | to_nice_json }}"
+ dest: "{{ kayobe_diagnostics_log_dir }}/facts.json"
+
+- name: Run diagnostics script
+ script: get_logs.sh
+ register: get_logs_result
+ become: true
+ failed_when: false
+
+- name: Print get_logs.sh output
+ debug:
+ msg: "{{ get_logs_result.stdout }}"
+
+- name: Download logs to executor
+ synchronize:
+ src: "{{ kayobe_diagnostics_log_dir }}"
+ dest: "{{ kayobe_diagnostics_executor_log_dir }}/"
+ mode: pull
diff --git a/roles/kayobe-diagnostics/tasks/pre.yml b/roles/kayobe-diagnostics/tasks/pre.yml
new file mode 100644
index 00000000..6d51e667
--- /dev/null
+++ b/roles/kayobe-diagnostics/tasks/pre.yml
@@ -0,0 +1,17 @@
+---
+- name: Ensure node log directory exists
+ file:
+ path: "{{ kayobe_diagnostics_log_dir }}"
+ state: "directory"
+
+- name: Ensure node log subdirectories exist
+ file:
+ path: "{{ kayobe_diagnostics_log_dir }}/{{ item }}"
+ state: "directory"
+ mode: 0777
+ with_items:
+ - "docker_logs"
+ - "kolla_configs"
+ - "system_logs"
+ - "kolla"
+ - "ansible"
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index 973f80e0..b5a91e0e 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -12,3 +12,20 @@
parent: openstack-tox-with-sudo
vars:
tox_envlist: ansible
+
+# Base job for testing overcloud deployment.
+- job:
+ name: kayobe-overcloud-base
+ pre-run: playbooks/kayobe-overcloud-base/pre.yml
+ run: playbooks/kayobe-overcloud-base/run.yml
+ post-run: playbooks/kayobe-overcloud-base/post.yml
+ attempts: 1
+ timeout: 5400
+ irrelevant-files:
+ - ^.*\.rst$
+ - ^doc/.*
+
+- job:
+ name: kayobe-overcloud-centos
+ parent: kayobe-overcloud-base
+ nodeset: kayobe-centos
diff --git a/zuul.d/nodesets.yaml b/zuul.d/nodesets.yaml
new file mode 100644
index 00000000..04067f11
--- /dev/null
+++ b/zuul.d/nodesets.yaml
@@ -0,0 +1,6 @@
+---
+- nodeset:
+ name: kayobe-centos
+ nodes:
+ - name: primary
+ label: centos-7
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index 905258db..39e17448 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -8,6 +8,7 @@
- build-openstack-sphinx-docs
- kayobe-tox-ansible-syntax
- kayobe-tox-ansible
+ - kayobe-overcloud-centos
gate:
queue: kayobe
@@ -18,3 +19,4 @@
- build-openstack-sphinx-docs
- kayobe-tox-ansible-syntax
- kayobe-tox-ansible
+ - kayobe-overcloud-centos
--
GitLab