CI: add Infra VM jobs

Depends-On: https://review.opendev.org/c/openstack/kayobe/+/812670
Depends-On: https://review.opendev.org/c/openstack/kayobe-config-dev/+/805239
Depends-On: https://review.opendev.org/c/openstack/kayobe/+/813212

Change-Id: I7dbd643fec72223bfda20cbbc37809ae78073804

dev/functions

@@ -40,6 +40,15 @@ function config_defaults {
     # Whether to deploy seed services.
     export KAYOBE_SEED_SERVICE_DEPLOY=${KAYOBE_SEED_SERVICE_DEPLOY:-1}
 
+    # Whether to provision a VM for the infra VM host.
+    export KAYOBE_INFRA_VM_PROVISION=${KAYOBE_INFRA_VM_PROVISION:-1}
+
+    # Whether to configure the infra VM host.
+    export KAYOBE_INFRA_VM_HOST_CONFIGURE=${KAYOBE_INFRA_VM_HOST_CONFIGURE:-1}
+
+    # Whether to deploy infra VM services.
+    export KAYOBE_INFRA_VM_SERVICE_DEPLOY=${KAYOBE_INFRA_VM_SERVICE_DEPLOY:-1}
+
     # Whether to use the 'kolla-ansible certificates' command to generate X.509
     # certificates.
     export KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES=${KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES:-0}
@@ -340,6 +349,28 @@ function seed_upgrade {
     run_kayobe seed service upgrade
 }
 
+function infra_vm_deploy {
+    # Deploy a kayobe infra VM.
+    environment_setup
+
+    control_host_bootstrap
+
+    if [[ ${KAYOBE_INFRA_VM_PROVISION} = 1 ]]; then
+        echo "Provisioning the infra VM"
+        run_kayobe infra vm provision
+    fi
+
+    if [[ ${KAYOBE_INFRA_VM_HOST_CONFIGURE} = 1 ]]; then
+        echo "Configuring the infra VM host"
+        run_kayobe infra vm host configure
+    fi
+
+    if [[ ${KAYOBE_INFRA_VM_SERVICE_DEPLOY} = 1 ]]; then
+        echo "Deploying containerised infra VM services"
+        run_kayobe infra vm service deploy
+    fi
+}
+
 function overcloud_deploy {
     # Deploy a kayobe control plane.
     echo "Deploying a kayobe development environment. This consists of a "

dev/infra-vm-deploy.sh

+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# Simple script to stand up a development environment for infra VMs using
+# kayobe.  This should be executed from the hypervisor.
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${PARENT}/functions"
+
+
+function main {
+    config_init
+    infra_vm_deploy
+}
+
+main

playbooks/kayobe-infra-vm-base/infra-vms-group-vars.j2

+---
+aio_interface: eth0
+
+# Route via the seed-hypervisor to the outside world.
+aio_gateway: 192.168.33.4

playbooks/kayobe-infra-vm-base/overrides.yml.j2

+---
+# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as
+# Ansible is run directly on the controller.
+disable_selinux_do_reboot: false
+
+# Use the OpenStack infra's Dockerhub mirror.
+docker_registry_mirrors:
+  - "http://{{ zuul_site_mirror_fqdn }}:8082/"
+
+kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla'].src_dir }}"
+kolla_source_version: "{{ zuul.projects['opendev.org/openstack/kolla'].checkout }}"
+kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla-ansible'].src_dir }}"
+kolla_ansible_source_version: "{{ zuul.projects['opendev.org/openstack/kolla-ansible'].checkout }}"
+kolla_openstack_logging_debug: True
+pip_upper_constraints_file: "/tmp/upper-constraints.txt"
+
+# The hosts used by Zuul may or may not have Virtualization Technology (VT)
+# enabled. Don't fail if it's disabled.
+libvirt_host_require_vt: false
+
+# Nested virtualisation is not working well in CI currently. Force the use of
+# QEMU.
+libvirt_vm_engine: "qemu"
+
+# Use the CI infra's PyPI mirror.
+pip_local_mirror: true
+pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple"
+pip_trusted_hosts:
+  - "{{ zuul_site_mirror_fqdn }}"
+
+# Try with only a single VCPU, word on the street is that QEMU doesn't play
+# nicely with more than one.
+infra_vm_vcpus: 1
+
+# Reduce the memory footprint of the infra VM.
+infra_vm_memory_mb: "{{ 1 * 1024 }}"
+
+# Use cirros rather than CentOS for the VM.
+infra_vm_bootstrap_user: cirros
+infra_vm_root_image: /opt/cache/files/cirros-0.5.2-x86_64-disk.img
+
+# Cirros doesn't load cdom drivers by default.
+vm_configdrive_device: disk
+
+# Cirros is Debian family, but doesn't support path globs in
+# /etc/network/interfaces.
+configdrive_os_family: Debian
+configdrive_debian_network_interfaces_supports_glob: false
+
+# NOTE(mgoddard): CentOS 8 removes interfaces from their bridge during ifdown,
+# and removes the bridge if there are no interfaces left. When Kayobe bounces
+# veth links plugged into the bridge, it causes the bridge which has the IP we
+# are using for SSH to be removed. Use a dummy interface.
+aio_bridge_ports:
+  - dummy1

playbooks/kayobe-infra-vm-base/pre.yml

+---
+- hosts: primary
+  environment:
+    KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
+  tasks:
+    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
+    # braio - to exist with an IP address of 192.168.33.4.
+    - import_role:
+        name: kayobe-network-bootstrap
+      vars:
+        bridge_interface: braio
+        bridge_ip: 192.168.33.4
+        bridge_prefix: 24
+        bridge_port_interface: dummy1
+
+    # NOTE(mgoddard): Configure IP forwarding and NAT to allow communication
+    # from the infra VM to the outside world.
+
+    # FIXME(mgoddard): use a libvirt network?
+    - name: Ensure NAT is configured
+      iptables:
+        chain: POSTROUTING
+        table: nat
+        out_interface: "{{ ansible_default_ipv4.interface }}"
+        jump: MASQUERADE
+      become: true
+
+    # FIXME(mgoddard): use a libvirt network?
+    - name: Ensure IP forwarding is enabled
+      sysctl:
+        name: net.ipv4.conf.all.forwarding
+        value: 1
+      become: true
+
+    - name: Ensure SELinux is disabled
+      selinux:
+        state: disabled
+      become: True
+      when: ansible_os_family == 'RedHat'
+
+    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+    # precedence over the standard config files.
+    - name: Ensure kayobe-config override config file exists
+      template:
+        src: overrides.yml.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+
+    - name: Ensure infra-vms group variables exist
+      template:
+        src: infra-vms-group-vars.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/inventory/group_vars/infra-vms/network-interfaces"
+
+    - name: Ensure kayobe is installed
+      shell:
+        cmd: dev/install.sh &> {{ logs_dir }}/ansible/install
+        chdir: "{{ kayobe_src_dir }}"
+        executable: /bin/bash

playbooks/kayobe-infra-vm-base/run.yml

+---
+- hosts: primary
+  environment:
+    KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
+    # The CirrOS image does not support much beyond logging in.
+    KAYOBE_INFRA_VM_HOST_CONFIGURE: 0
+    KAYOBE_INFRA_VM_SERVICE_DEPLOY: 0
+  tasks:
+    - name: Ensure seed hypervisor is deployed
+      shell:
+        cmd: "{{ kayobe_src_dir }}/dev/seed-hypervisor-deploy.sh &> {{ logs_dir }}/ansible/seed-hypervisor-deploy"
+        executable: /bin/bash
+
+    - name: Ensure infra VM is deployed
+      shell:
+        cmd: "{{ kayobe_src_dir }}/dev/infra-vm-deploy.sh &> {{ logs_dir }}/ansible/infra-vm-deploy"
+        executable: /bin/bash

zuul.d/jobs.yaml

@@ -223,3 +223,25 @@
     name: kayobe-seed-vm-ubuntu-focal
     parent: kayobe-seed-vm-base
     nodeset: kayobe-ubuntu-focal
+
+- job:
+    name: kayobe-infra-vm-base
+    parent: kayobe-base
+    description: |
+      Base job for testing infra VM provisioning.
+
+      Configures the primary VM as a libvirt hypervisor, and provisions an
+      infra VM.
+    pre-run: playbooks/kayobe-infra-vm-base/pre.yml
+    run: playbooks/kayobe-infra-vm-base/run.yml
+    timeout: 5400
+
+- job:
+    name: kayobe-infra-vm-centos8s
+    parent: kayobe-infra-vm-base
+    nodeset: kayobe-centos8s
+
+- job:
+    name: kayobe-infra-vm-ubuntu-focal
+    parent: kayobe-infra-vm-base
+    nodeset: kayobe-ubuntu-focal

zuul.d/project.yaml

@@ -23,6 +23,8 @@
         - kayobe-seed-upgrade-ubuntu-focal
         - kayobe-seed-vm-centos8s
         - kayobe-seed-vm-ubuntu-focal
+        - kayobe-infra-vm-centos8s
+        - kayobe-infra-vm-ubuntu-focal
 
     gate:
       queue: kayobe
@@ -41,3 +43,5 @@
         - kayobe-seed-upgrade-centos8s
         - kayobe-seed-vm-centos8s
         - kayobe-seed-vm-ubuntu-focal
+        - kayobe-infra-vm-centos8s
+        - kayobe-infra-vm-ubuntu-focal