diff --git a/dev/functions b/dev/functions
index beec19087c433e4ecbe502fffb7d9a169ae383dc..60565bc49b209fe919b2936f2370b1b37bd5b4fe 100644
--- a/dev/functions
+++ b/dev/functions
@@ -153,6 +153,38 @@ function run_kayobe {
kayobe ${KAYOBE_EXTRA_ARGS} $*
}
+function control_host_bootstrap {
+ echo "Bootstrapping the Ansible control host"
+ for i in $(seq 1 3); do
+ if run_kayobe control host bootstrap; then
+ chb_success=1
+ break
+ fi
+ echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying"
+ done
+ if [[ -z ${chb_success+x} ]]; then
+ die $LINENO "Failed to bootstrap control host"
+ exit 1
+ fi
+ echo "Bootstrapped control host after $i attempts"
+}
+
+function control_host_upgrade {
+ echo "Upgrading the Ansible control host"
+ for i in $(seq 1 3); do
+ if run_kayobe control host upgrade; then
+ chu_success=1
+ break
+ fi
+ echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying"
+ done
+ if [[ -z ${chu_success+x} ]]; then
+ die $LINENO "Failed to upgrade control host"
+ exit 1
+ fi
+ echo "Upgraded control host after $i attempts"
+}
+
function seed_hypervisor_deploy {
# Deploy a seed hypervisor.
environment_setup
@@ -168,8 +200,7 @@ function seed_deploy {
# Deploy a kayobe seed in a VM.
environment_setup
- echo "Bootstrapping the Ansible control host"
- run_kayobe control host bootstrap
+ control_host_bootstrap
if [[ ${KAYOBE_SEED_VM_PROVISION} = 1 ]]; then
echo "Provisioning the seed VM"
@@ -179,7 +210,7 @@ function seed_deploy {
echo "Configuring the seed host"
run_kayobe seed host configure
- # Note: This must currently be before host configure, because host
+ # Note: This must currently be done before host configure, because host
# configure runs kolla-ansible.yml, which validates the presence of the
# built deploy images.
if is_deploy_image_built_locally; then
@@ -201,6 +232,37 @@ function seed_deploy {
run_kayobe seed service deploy
}
+function seed_upgrade {
+ # Upgrade a kayobe seed in a VM.
+ echo "Upgrading Kayobe"
+ upgrade_kayobe_venv
+
+ environment_setup
+
+ control_host_upgrade
+
+ echo "Upgrading the seed host"
+ run_kayobe seed host upgrade
+
+ if is_deploy_image_built_locally; then
+ echo "Building seed deployment images"
+ run_kayobe seed deployment image build --force-rebuild
+ else
+ echo "Not building seed deployment images"
+ fi
+
+ if [[ ${KAYOBE_SEED_CONTAINER_IMAGE_BUILD} = 1 ]]; then
+ echo "Building seed container images"
+ run_kayobe seed container image build
+ else
+ echo "Not pulling seed container images - no such command yet"
+ #run_kayobe seed container image pull
+ fi
+
+ echo "Upgrading containerised seed services"
+ run_kayobe seed service upgrade
+}
+
function overcloud_deploy {
# Deploy a kayobe control plane.
echo "Deploying a kayobe development environment. This consists of a "
@@ -208,19 +270,7 @@ function overcloud_deploy {
environment_setup
- echo "Bootstrapping the Ansible control host"
- for i in $(seq 1 3); do
- if run_kayobe control host bootstrap; then
- chb_success=1
- break
- fi
- echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying"
- done
- if [[ -z ${chb_success+x} ]]; then
- echo "Failed to bootstrap control host"
- exit 1
- fi
- echo "Bootstrapped control host after $i attempts"
+ control_host_bootstrap
echo "Configuring the controller host"
run_kayobe overcloud host configure
@@ -269,19 +319,7 @@ function overcloud_upgrade {
environment_setup
- echo "Upgrading the Ansible control host"
- for i in $(seq 1 3); do
- if run_kayobe control host upgrade; then
- chu_success=1
- break
- fi
- echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying"
- done
- if [[ -z ${chu_success+x} ]]; then
- echo "Failed to upgrade control host"
- exit 1
- fi
- echo "Upgraded control host after $i attempts"
+ control_host_upgrade
echo "Upgrading the controller host"
run_kayobe overcloud host upgrade
diff --git a/dev/seed-upgrade.sh b/dev/seed-upgrade.sh
new file mode 100755
index 0000000000000000000000000000000000000000..7f51cf5a84ded8b50c7ec7198dfe05f4f8a414c6
--- /dev/null
+++ b/dev/seed-upgrade.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# Simple script to upgrade a development environment for a seed VM using
+# kayobe. This should be executed from the hypervisor.
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${PARENT}/functions"
+
+
+function main {
+ config_init
+ seed_upgrade
+}
+
+main
diff --git a/doc/source/development/automated.rst b/doc/source/development/automated.rst
index 59e6bf9ddbba19903b531cb86178acad094781d8..aa9b4e01c6034b4979bb0a4246db344a5c2e4f81 100644
--- a/doc/source/development/automated.rst
+++ b/doc/source/development/automated.rst
@@ -193,3 +193,8 @@ Upon successful completion of this script, the seed VM will be active. The
seed VM may be accessed via SSH as the ``stack`` user::
ssh stack@192.168.33.5
+
+It is possible to test an upgrade by running the ``dev/seed-upgrade.sh``
+script::
+
+ ./dev/seed-upgrade.sh
diff --git a/playbooks/kayobe-seed-upgrade-base/bifrost-overrides.yml.j2 b/playbooks/kayobe-seed-upgrade-base/bifrost-overrides.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..0ada56a2f95d87114694231fa5029bc8b10d67e3
--- /dev/null
+++ b/playbooks/kayobe-seed-upgrade-base/bifrost-overrides.yml.j2
@@ -0,0 +1,7 @@
+---
+# Don't build an IPA deployment image, instead download upstream images.
+create_ipa_image: false
+download_ipa: true
+
+# Don't build a disk image. It takes time and can be unreliable.
+use_cirros: true
diff --git a/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2 b/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..a9dd6ea76ffc56439c808d9336c8e47d8d593caf
--- /dev/null
+++ b/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2
@@ -0,0 +1,33 @@
+---
+# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as
+# Ansible is run directly on the controller.
+disable_selinux_do_reboot: false
+
+{% if not is_previous_release | default(false) %}
+kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla'].src_dir }}"
+kolla_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla'].checkout }}"
+kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla-ansible'].src_dir }}"
+kolla_ansible_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla-ansible'].checkout }}"
+kolla_upper_constraints_file: "/tmp/upper-constraints.txt"
+{% endif %}
+
+# NOTE(mgoddard): We're using a cirros image, which doesn't require the
+# resolv.conf work around used for CentOS.
+overcloud_host_image_workaround_resolv_enabled: false
+
+# Use the CI infra's PyPI mirror.
+pip_local_mirror: true
+pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple"
+
+{% if previous_release == "pike" %}
+# kayobe-config-dev in queens changed to use overlay by default. Specify
+# devicemapper explicitly to avoid changing.
+docker_storage_driver: devicemapper
+docker_storage_volume_thinpool_size: 45%VG
+
+# NOTE(mgoddard): Use a loopback-mounted LVM volume for docker storage since
+# the overlay driver doesn't work with the ansible template module until
+# ansible 2.4.0, and this is required by bifrost.
+seed_lvm_group_data_disks:
+ - /dev/loop0
+{% endif %}
diff --git a/playbooks/kayobe-seed-upgrade-base/post.yml b/playbooks/kayobe-seed-upgrade-base/post.yml
new file mode 100644
index 0000000000000000000000000000000000000000..be286660b0d7ec193f43c1571ed2356ce4c45de6
--- /dev/null
+++ b/playbooks/kayobe-seed-upgrade-base/post.yml
@@ -0,0 +1,9 @@
+---
+- hosts: all
+ roles:
+ - role: kayobe-diagnostics
+ kayobe_diagnostics_phase: "post"
+ kayobe_diagnostics_log_dir: "/tmp/logs"
+ kayobe_diagnostics_config_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}"
+ kayobe_diagnostics_previous_config_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
+ kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
diff --git a/playbooks/kayobe-seed-upgrade-base/pre.yml b/playbooks/kayobe-seed-upgrade-base/pre.yml
new file mode 100644
index 0000000000000000000000000000000000000000..dc472ab4bcb8fc5048b0e5437465581f76363ead
--- /dev/null
+++ b/playbooks/kayobe-seed-upgrade-base/pre.yml
@@ -0,0 +1,85 @@
+---
+- hosts: primary
+ vars:
+ logs_dir: "/tmp/logs"
+ previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
+ previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
+ roles:
+ - role: kayobe-diagnostics
+ kayobe_diagnostics_phase: "pre"
+ kayobe_diagnostics_log_dir: "{{ logs_dir }}"
+
+ - role: kayobe-ci-prep
+ tasks:
+ # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both
+ # the zuul and stack users.
+ - name: Ensure upper-contraints.txt exists
+ copy:
+ src: "{{ zuul.projects['git.openstack.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}"
+ dest: "/tmp"
+ mode: 0644
+ remote_src: true
+
+ - name: Ensure previous kayobe directory exists
+ file:
+ path: "{{ previous_kayobe_src_dir }}"
+ state: directory
+
+ - name: Ensure previous kayobe repository is cloned
+ git:
+ repo: https://git.openstack.org/openstack/kayobe
+ dest: "{{ previous_kayobe_src_dir }}"
+ version: "stable/{{ previous_release | lower }}"
+
+ - name: Ensure previous kayobe-config directory exists
+ file:
+ path: "{{ previous_kayobe_config_src_dir }}"
+ state: directory
+
+ - name: Ensure kayobe-config repository is cloned
+ git:
+ repo: https://git.openstack.org/openstack/kayobe-config-dev
+ dest: "{{ previous_kayobe_config_src_dir }}"
+ version: "stable/{{ previous_release | lower }}"
+
+ # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+ # precedence over the standard config files.
+ - name: Ensure kayobe-config override config file exists
+ template:
+ src: overrides.yml.j2
+ dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+ vars:
+ is_previous_release: true
+
+ - name: Ensure bifrost overrides directory exists
+ file:
+ path: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost"
+ state: "directory"
+
+ - name: Ensure bifrost overrides file exists
+ template:
+ src: bifrost-overrides.yml.j2
+ dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/bifrost.yml"
+
+ - block:
+ # NOTE(mgoddard): Create a loopback device backed by a file for docker
+ # storage. We do this since the overlay driver doesn't work with the
+ # ansible template module until ansible 2.4.0, and this is required by
+ # bifrost.
+ - name: Ensure a docker storage backing file exists
+ command: truncate -s 20G /tmp/docker-storage
+
+ - name: Ensure the docker storage loopback device is created
+ command: losetup /dev/loop0 /tmp/docker-storage
+ become: true
+ when: previous_release == "pike"
+
+ # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
+ # breth1 - to exist on the seed with an IP address of 192.168.33.5.
+ - name: Ensure all-in-one network bridge interface exists
+ command: "{{ item }}"
+ become: true
+ with_items:
+ - "ip l add breth1 type bridge"
+ - "ip l set breth1 up"
+ - "ip a add 192.168.33.5/24 dev breth1"
diff --git a/playbooks/kayobe-seed-upgrade-base/run.yml b/playbooks/kayobe-seed-upgrade-base/run.yml
new file mode 100644
index 0000000000000000000000000000000000000000..d1d960f328831cf60472bda0b0365f357d40560d
--- /dev/null
+++ b/playbooks/kayobe-seed-upgrade-base/run.yml
@@ -0,0 +1,64 @@
+---
+- hosts: primary
+ vars:
+ kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}"
+ kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}"
+ previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
+ previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
+ logs_dir: "/tmp/logs"
+ tasks:
+
+ # Install the previous release of Kayobe, and use it to deploy a control
+ # plane.
+
+ - block:
+ - name: Ensure kayobe is installed
+ shell:
+ cmd: dev/install.sh > {{ logs_dir }}/ansible/install-pre-upgrade
+ chdir: "{{ previous_kayobe_src_dir }}"
+
+ - name: Ensure seed is deployed
+ shell:
+ cmd: dev/seed-deploy.sh > {{ logs_dir }}/ansible/seed-deploy-pre-upgrade
+ chdir: "{{ previous_kayobe_src_dir }}"
+ environment:
+ KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}"
+ # Don't provision a seed VM - use the Zuul VM as the seed host.
+ KAYOBE_SEED_VM_PROVISION: 0
+
+ # Update the Kayobe configuration to the current release.
+
+ - name: Ensure bifrost overrides directory exists
+ file:
+ path: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost"
+ state: "directory"
+
+ - name: Copy across relevant kayobe-config files
+ copy:
+ src: "{{ previous_kayobe_config_src_dir }}/{{ item }}"
+ dest: "{{ kayobe_config_src_dir }}/{{ item }}"
+ remote_src: true
+ with_items:
+ - etc/kayobe/kolla/passwords.yml
+ - etc/kayobe/kolla/config/bifrost/bifrost.yml
+
+ # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+ # precedence over the standard config files.
+ - name: Ensure kayobe-config override config file exists
+ template:
+ src: overrides.yml.j2
+ dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+
+ # TODO(mgoddard): Perform a smoke test against the previous release.
+
+ - block:
+ # Upgrade Kayobe, and use it to perform an upgrade of the seed.
+
+ - name: Ensure seed is upgraded
+ shell:
+ cmd: dev/seed-upgrade.sh > {{ logs_dir }}/ansible/seed-upgrade
+ chdir: "{{ kayobe_src_dir }}"
+
+ # TODO(mgoddard): Perform a smoke test against the upgraded current release.
+ environment:
+ KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index e4689dcc8233d359ffc1997187bcd5748a69de81..23fb52e1176e0adec35733f411a8a0b966714560 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -150,3 +150,41 @@
name: kayobe-seed-centos
parent: kayobe-seed-base
nodeset: kayobe-centos
+
+- job:
+ name: kayobe-seed-upgrade-base
+ description: |
+ Base job for testing seed upgrades.
+
+ Configures the primary VM as a seed using the previous OpenStack release,
+ and upgrades it to the current release.
+ pre-run: playbooks/kayobe-seed-upgrade-base/pre.yml
+ run: playbooks/kayobe-seed-upgrade-base/run.yml
+ post-run: playbooks/kayobe-seed-upgrade-base/post.yml
+ attempts: 1
+ timeout: 5400
+ required-projects:
+ # Include kayobe to ensure other projects can use this job.
+ - name: openstack/kayobe
+ - name: openstack/kayobe-config-dev
+ - name: openstack/kolla
+ override-checkout: stable/queens
+ - name: openstack/kolla-ansible
+ override-checkout: stable/queens
+ - name: openstack/requirements
+ override-checkout: stable/queens
+ vars:
+ # Name of the release to upgrade from.
+ previous_release: pike
+ irrelevant-files:
+ - ^.*\.rst$
+ - ^doc/.*
+ - ^releasenotes/.*
+ - ^setup.cfg$
+ - ^tools/.*$
+ - ^tox.ini$
+
+- job:
+ name: kayobe-seed-upgrade-centos
+ parent: kayobe-seed-upgrade-base
+ nodeset: kayobe-centos
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index b9a69e8cd55c29798f026d8259f6fc04c922a049..76394b6530761b0d23c6d8bd0c80675b57b821ca 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -17,6 +17,7 @@
- kayobe-overcloud-centos
- kayobe-overcloud-upgrade-centos
- kayobe-seed-centos
+ - kayobe-seed-upgrade-centos
gate:
queue: kayobe
@@ -28,3 +29,4 @@
- kayobe-overcloud-centos
- kayobe-overcloud-upgrade-centos
- kayobe-seed-centos
+ - kayobe-seed-upgrade-centos