diff --git a/ansible/compute-libvirt-host.yml b/ansible/compute-libvirt-host.yml new file mode 100644 index 0000000000000000000000000000000000000000..7e5501938f34a5e7aa0600b9bb6cdf5efe2e984c --- /dev/null +++ b/ansible/compute-libvirt-host.yml @@ -0,0 +1,56 @@ +--- +- name: Ensure the libvirt daemon is configured + hosts: compute + tags: + - libvirt-host + tasks: + - name: Ensure Ceph package repository is available + package: + name: "centos-release-ceph-{{ compute_libvirt_ceph_repo_release }}" + state: present + when: + - compute_libvirt_enabled | bool + - ansible_facts.distribution in ['CentOS', 'Rocky'] + - compute_libvirt_ceph_repo_install | bool + become: true + + - name: Include stackhpc.libvirt-host role + include_role: + name: stackhpc.libvirt-host + vars: + libvirt_host_libvirtd_conf: "{{ compute_libvirt_conf }}" + libvirt_host_qemu_conf: "{{ compute_qemu_conf }}" + libvirt_host_tcp_listen: "{{ not compute_libvirt_enable_tls | bool }}" + libvirt_host_tcp_listen_address: "{{ internal_net_name | net_ip }}:16509" + libvirt_host_tls_listen: "{{ compute_libvirt_enable_tls | bool }}" + libvirt_host_tls_listen_address: "{{ internal_net_name | net_ip }}:16514" + # TLS server and client certificates. + libvirt_host_tls_server_cert: >- + {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['servercert.pem']}))) + if libvirt_host_tls_listen | default(False) | bool else '' }} + libvirt_host_tls_server_key: >- + {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['serverkey.pem']}))) + if libvirt_host_tls_listen | default(False) | bool else '' }} + libvirt_host_tls_client_cert: >- + {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['clientcert.pem']}))) + if libvirt_host_tls_listen | default(False) | bool else '' }} + libvirt_host_tls_client_key: >- + {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['clientkey.pem']}))) + if libvirt_host_tls_listen | default(False) | bool else '' }} + libvirt_host_tls_cacert: >- + {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['cacert.pem']}))) + if libvirt_host_tls_listen | default(False) | bool else '' }} + lookup_params: + paths: "{{ libvirt_tls_cert_paths }}" + skip: true + # Support loading libvirt TLS certificates & keys from per-host and + # global locations. + libvirt_tls_cert_paths: >- + {{ (libvirt_tls_cert_dirs | unique | product([inventory_hostname]) | map('path_join') | list + + libvirt_tls_cert_dirs | unique | list) | list }} + libvirt_tls_cert_dirs: + - "{{ kayobe_env_config_path }}/certificates/libvirt" + - "{{ kayobe_config_path }}/certificates/libvirt" + libvirt_host_enable_efi_support: true + when: + - compute_libvirt_enabled | bool diff --git a/ansible/group_vars/all/compute b/ansible/group_vars/all/compute index dba12a94304a77e3b4b4c0b36a8939efb546fcd4..b8b1e8161006950670bcf66bd53538027cbdcf77 100644 --- a/ansible/group_vars/all/compute +++ b/ansible/group_vars/all/compute @@ -161,3 +161,54 @@ compute_firewalld_default_zone: # - permanent: true # - state: enabled compute_firewalld_rules: [] + +############################################################################### +# Compute node host libvirt configuration. + +# Whether to enable a host libvirt daemon. Default is true if kolla_enable_nova +# is true and kolla_enable_nova_libvirt_container is false. +compute_libvirt_enabled: "{{ kolla_enable_nova | bool and not kolla_enable_nova_libvirt_container | bool }}" + +# A dict of default configuration options to write to +# /etc/libvirt/libvirtd.conf. +compute_libvirt_conf_default: + auth_tcp: "none" + log_level: "{{ compute_libvirtd_log_level }}" + +# A dict of additional configuration options to write to +# /etc/libvirt/libvirtd.conf. +compute_libvirt_conf_extra: {} + +# A dict of configuration options to write to /etc/libvirt/libvirtd.conf. +# Default is a combination of compute_libvirt_conf_default and +# compute_libvirt_conf_extra. +compute_libvirt_conf: "{{ compute_libvirt_conf_default | combine(compute_libvirt_conf_extra) }}" + +# Numerical log level for libvirtd. Default is 3. +compute_libvirtd_log_level: 3 + +# A dict of default configuration options to write to +# /etc/libvirt/qemu.conf. +compute_qemu_conf_default: + max_files: 32768 + max_processes: 131072 + +# A dict of additional configuration options to write to +# /etc/libvirt/qemu.conf. +compute_qemu_conf_extra: {} + +# A dict of configuration options to write to /etc/libvirt/qemu.conf. +# Default is a combination of compute_qemu_conf_default and +# compute_qemu_conf_extra. +compute_qemu_conf: "{{ compute_qemu_conf_default | combine(compute_qemu_conf_extra) }}" + +# Whether to enable a libvirt TLS listener. Default is false. +compute_libvirt_enable_tls: false + +# Whether to install a Ceph package repository on CentOS and Rocky hosts. +# Default is true. +compute_libvirt_ceph_repo_install: true + +# Ceph package repository release to install on CentOS and Rocky hosts when +# compute_libvirt_ceph_repo_install is true. Default is 'pacific'. +compute_libvirt_ceph_repo_release: pacific diff --git a/ansible/group_vars/all/kolla b/ansible/group_vars/all/kolla index f10fd789c5a62dbb370a452ff05e49a6e70cc52b..ede0e5b33271feb457560bdda5d22cca846dcd25 100644 --- a/ansible/group_vars/all/kolla +++ b/ansible/group_vars/all/kolla @@ -553,6 +553,7 @@ kolla_enable_murano: "no" kolla_enable_neutron_mlnx: "no" kolla_enable_neutron_provider_networks: "no" kolla_enable_neutron_sriov: "no" +kolla_enable_nova_libvirt_container: "yes" kolla_enable_octavia: "no" kolla_enable_openvswitch: "{{ kolla_enable_neutron | bool }}" kolla_enable_ovn: "no" diff --git a/ansible/kolla-ansible.yml b/ansible/kolla-ansible.yml index f2ccc343248b2df4a04e5b0095a4e7e9eae870de..a469e474d4b9b036bd559bb54f3524e176d43d74 100644 --- a/ansible/kolla-ansible.yml +++ b/ansible/kolla-ansible.yml @@ -103,6 +103,7 @@ kolla_inspector_netmask: "{{ inspection_net_name | net_mask }}" kolla_inspector_default_gateway: "{{ inspection_net_name | net_inspection_gateway or inspection_net_name | net_gateway }}" kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}" + kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}" kolla_enable_host_ntp: false docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}" kolla_globals_paths_extra: diff --git a/ansible/kolla-openstack.yml b/ansible/kolla-openstack.yml index 36cf4daa171f89098d051faffabb5d456fd5cbdf..6941067ee724ea8855db2ed90341bc9ce686321e 100644 --- a/ansible/kolla-openstack.yml +++ b/ansible/kolla-openstack.yml @@ -249,3 +249,5 @@ kolla_extra_sahara: "{{ kolla_extra_config.sahara | default }}" kolla_extra_zookeeper: "{{ kolla_extra_config.zookeeper | default }}" kolla_extra_config_path: "{{ kayobe_env_config_path }}/kolla/config" + kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}" + kolla_nova_libvirt_certificates_src: "{{ kayobe_env_config_path }}/certificates/libvirt" diff --git a/ansible/roles/kolla-ansible/defaults/main.yml b/ansible/roles/kolla-ansible/defaults/main.yml index 8a3ff899efc8f28a11f785628b6c5c475ded4b43..9e077448228d86ff31bbfe6620bf83e006af35e0 100644 --- a/ansible/roles/kolla-ansible/defaults/main.yml +++ b/ansible/roles/kolla-ansible/defaults/main.yml @@ -236,6 +236,8 @@ kolla_openstack_logging_debug: # controllers. kolla_nova_compute_ironic_host: +kolla_libvirt_tls: + ############################################################################### # Extra free-form configuraton. diff --git a/ansible/roles/kolla-ansible/templates/kolla/globals.yml b/ansible/roles/kolla-ansible/templates/kolla/globals.yml index 8e0784541a842da4c8a8ecb8fd5b5870c96d44fe..04d5e33b31d05d71e348dde1c66439b4de8a96bb 100644 --- a/ansible/roles/kolla-ansible/templates/kolla/globals.yml +++ b/ansible/roles/kolla-ansible/templates/kolla/globals.yml @@ -393,6 +393,10 @@ enable_{{ feature_flag }}: {{ hostvars[inventory_hostname]['kolla_enable_' ~ fea # Valid options are [ none, novnc, spice, rdp ] #nova_console: "novnc" +{% if kolla_libvirt_tls is not none %} +libvirt_tls: {{ kolla_libvirt_tls | bool }} +{% endif %} + ################# # Hyper-V options ################# diff --git a/ansible/roles/kolla-ansible/vars/main.yml b/ansible/roles/kolla-ansible/vars/main.yml index 1f3fd0d5e446ed5d8b4e22c136e5cbbfe3cecc24..968c2222b4ee4fd9fd3d0e6cccd108a36f9437ef 100644 --- a/ansible/roles/kolla-ansible/vars/main.yml +++ b/ansible/roles/kolla-ansible/vars/main.yml @@ -184,6 +184,7 @@ kolla_feature_flags: - nova - nova_fake - nova_horizon_policy_file + - nova_libvirt_container - nova_serialconsole_proxy - nova_ssh - octavia diff --git a/ansible/roles/kolla-openstack/defaults/main.yml b/ansible/roles/kolla-openstack/defaults/main.yml index ae32392a7fe206a1a62daff636721dde2d5d932e..6d2ad45de65afeb359377261db3b0b0b496a66a6 100644 --- a/ansible/roles/kolla-openstack/defaults/main.yml +++ b/ansible/roles/kolla-openstack/defaults/main.yml @@ -447,9 +447,19 @@ kolla_extra_neutron_ml2: # Whether to enable Nova. kolla_enable_nova: +# Whether to enable Nova libvirt container. +kolla_enable_nova_libvirt_container: + # Free form extra configuration to append to nova.conf. kolla_extra_nova: +# Whether libvirt TLS is enabled. +kolla_libvirt_tls: + +# Directory containing libvirt certificates for nova-compute when running +# libvirt on the host. +kolla_nova_libvirt_certificates_src: + ############################################################################### # Octavia configuration. diff --git a/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml b/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml index 14b0bbc863b091e2385f3113277d5f5d235e0962..634455c1963a2e349a19549f4dd97c6826b90bf9 100644 --- a/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml +++ b/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml @@ -15,7 +15,7 @@ provisioner: inventory: group_vars: all: - kolla_extra_config_path: + kolla_extra_config_path: ${MOLECULE_TEMP_PATH:-/tmp}/molecule/kolla/config kolla_enable_aodh: true kolla_extra_aodh: | [extra-aodh.conf] @@ -116,9 +116,12 @@ provisioner: [extra-ml2_conf.ini] foo=bar kolla_enable_nova: true + kolla_enable_nova_libvirt_container: false kolla_extra_nova: | [extra-nova.conf] foo=bar + kolla_libvirt_tls: true + kolla_nova_libvirt_certificates_src: ${MOLECULE_TEMP_PATH:-/tmp}/molecule/nova-libvirt/certificates kolla_enable_octavia: true kolla_extra_octavia: | [extra-octavia.conf] diff --git a/ansible/roles/kolla-openstack/molecule/enable-everything/prepare.yml b/ansible/roles/kolla-openstack/molecule/enable-everything/prepare.yml index d78cc694036b91c1e23761b835087b95281312d8..8514e90f3a0467eb90c1e7ca180247e5dfc6e7a4 100644 --- a/ansible/roles/kolla-openstack/molecule/enable-everything/prepare.yml +++ b/ansible/roles/kolla-openstack/molecule/enable-everything/prepare.yml @@ -25,3 +25,23 @@ with_items: - "{{ kolla_inspector_ipa_kernel_path }}" - "{{ kolla_inspector_ipa_ramdisk_path }}" + + - name: Ensure nova libvirt certificates directory exists + local_action: + module: file + path: "{{ kolla_nova_libvirt_certificates_src }}" + state: directory + + # NOTE(mgoddard): Previously we were creating empty files for the kernel + # and ramdisk, but this was found to cause ansible to hang on recent + # versions of docker. Using non-empty files seems to resolve the issue. + # See https://github.com/ansible/ansible/issues/36725. + - name: Ensure nova libvirt certificates exist + local_action: + module: copy + content: fake cert + dest: "{{ kolla_nova_libvirt_certificates_src }}/{{ item }}" + with_items: + - "cacert.pem" + - "clientcert.pem" + - "clientkey.pem" diff --git a/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py b/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py index 6d5f67fe5e0a64c18c18ab2e58fca8e747f31ae9..0badbfefdf4339d48538c7cd173c80fee2f01c00 100644 --- a/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py +++ b/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py @@ -50,6 +50,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( 'murano', 'neutron', 'nova', + 'nova/nova-libvirt', 'octavia', 'placement', 'prometheus', @@ -100,7 +101,10 @@ def test_service_ini_file(host, path): @pytest.mark.parametrize( 'path', ['ironic/ironic-agent.initramfs', - 'ironic/ironic-agent.kernel']) + 'ironic/ironic-agent.kernel', + 'nova/nova-libvirt/cacert.pem', + 'nova/nova-libvirt/clientcert.pem', + 'nova/nova-libvirt/clientkey.pem']) def test_service_non_ini_file(host, path): # TODO(mgoddard): Check config file contents. path = os.path.join('/etc/kolla/config', path) diff --git a/ansible/roles/kolla-openstack/tasks/config.yml b/ansible/roles/kolla-openstack/tasks/config.yml index 5123bcda036f81c4d9c7632cc3ab9b3d7a787e96..734b5a88bb025dc32087aef1dc45390c6c6e3b90 100644 --- a/ansible/roles/kolla-openstack/tasks/config.yml +++ b/ansible/roles/kolla-openstack/tasks/config.yml @@ -80,6 +80,7 @@ recurse: true with_items: "{{ kolla_openstack_custom_config }}" register: find_src_result + delegate_to: localhost - name: Find previously generated extra configuration files find: @@ -91,7 +92,6 @@ - name: Ensure extra configuration parent directories are present file: path: "{{ item.0.item.dest }}/{{ item.1.path | relpath(item.0.item.src) | dirname }}" - recurse: true state: directory mode: 0750 with_subelements: diff --git a/ansible/roles/kolla-openstack/vars/main.yml b/ansible/roles/kolla-openstack/vars/main.yml index e7252a1373dedd17345b57ea1d2f4e57c11db132..8033b354f5dfc29720532503829d78d30b144691 100644 --- a/ansible/roles/kolla-openstack/vars/main.yml +++ b/ansible/roles/kolla-openstack/vars/main.yml @@ -178,6 +178,27 @@ kolla_openstack_custom_config: dest: "{{ kolla_node_custom_config_path }}/nova" patterns: "*" enabled: "{{ kolla_enable_nova }}" + # Nova. + - src: "{{ kolla_nova_libvirt_certificates_src }}" + dest: "{{ kolla_node_custom_config_path }}/nova/nova-libvirt" + patterns: + - clientcert.pem + - clientkey.pem + - cacert.pem + enabled: "{{ kolla_enable_nova | bool and kolla_libvirt_tls | bool }}" + untemplated: + - clientcert.pem + - clientkey.pem + - cacert.pem + - src: "{{ kolla_nova_libvirt_certificates_src }}" + dest: "{{ kolla_node_custom_config_path }}/nova/nova-libvirt" + patterns: + - servercert.pem + - serverkey.pem + enabled: "{{ kolla_enable_nova | bool and kolla_enable_nova_libvirt_container | bool and kolla_libvirt_tls | bool }}" + untemplated: + - servercert.pem + - serverkey.pem # Octavia. - src: "{{ kolla_extra_config_path }}/octavia" dest: "{{ kolla_node_custom_config_path }}/octavia" diff --git a/dev/functions b/dev/functions index 3ece5f23fb6da06ad9f7a923d5d532e7e6966525..436d33e570ee648d7f5e1af7036e8111fbe1b5e8 100644 --- a/dev/functions +++ b/dev/functions @@ -10,12 +10,8 @@ set -o pipefail function config_defaults { # Set default values for kayobe development configuration. - # Try to detect if we are running in a vagrant VM. - if [[ -e /vagrant ]]; then - KAYOBE_SOURCE_PATH_DEFAULT=/vagrant - else - KAYOBE_SOURCE_PATH_DEFAULT="$(pwd)" - fi + PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + KAYOBE_SOURCE_PATH_DEFAULT="$(dirname ${PARENT})" # Path to the kayobe source code repository. Typically this will be the # Vagrant shared directory. @@ -392,18 +388,26 @@ function overcloud_deploy { control_host_bootstrap - echo "Configuring the controller host" - run_kayobe overcloud host configure - - # FIXME(mgoddard): Perform host upgrade workarounds to ensure hostname - # resolves to IP address of API interface for RabbitMQ. This seems to be - # required since https://review.openstack.org/#/c/584427 was merged. - echo "Workaround: upgrading the controller host" - run_kayobe overcloud host upgrade - if [[ ${KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES} = 1 ]]; then echo "Generate TLS certificates" - run_kayobe kolla ansible run certificates --kolla-extra kolla_certificates_dir=${KAYOBE_CONFIG_PATH}/kolla/certificates + run_kayobe playbook run $KAYOBE_SOURCE_PATH/ansible/kolla-ansible.yml -t config + # NOTE(mgoddard): There is a chicken and egg when generating libvirt + # TLS certificates using the kolla-ansible certificates command, and + # host libvirt. The certificates command needs to be able to gather + # facts for all hosts, but since the host configure step hasn't been + # run, we don't have SSH or the kolla user configured yet. However, we + # can't run host configure without the libvirt TLS certificates. + # Workaround: add the host to SSH known hosts and SSH as $USER. + run_kayobe playbook run $KAYOBE_SOURCE_PATH/ansible/ssh-known-host.yml -l overcloud + + # Avoid populating the fact cache with this weird setup. + export ANSIBLE_CACHE_PLUGIN=memory + run_kayobe kolla ansible run certificates \ + --kolla-extra kolla_certificates_dir=${KAYOBE_CONFIG_PATH}/kolla/certificates \ + --kolla-extra ansible_user=$USER \ + --kolla-extra ansible_python_interpreter=/usr/bin/python3 + unset ANSIBLE_CACHE_PLUGIN + # Add CA cert to trust store. ca_cert=${KAYOBE_CONFIG_PATH}/kolla/certificates/ca/root.crt if [[ -e /etc/debian_version ]]; then @@ -417,6 +421,15 @@ function overcloud_deploy { fi fi + echo "Configuring the controller host" + run_kayobe overcloud host configure + + # FIXME(mgoddard): Perform host upgrade workarounds to ensure hostname + # resolves to IP address of API interface for RabbitMQ. This seems to be + # required since https://review.openstack.org/#/c/584427 was merged. + echo "Workaround: upgrading the controller host" + run_kayobe overcloud host upgrade + # Note: This must currently be before host configure, because host # configure runs kolla-ansible.yml, which validates the presence of the # built deploy images. diff --git a/dev/tenks-deploy-config-compute-libvirt-on-host.yml b/dev/tenks-deploy-config-compute-libvirt-on-host.yml new file mode 100644 index 0000000000000000000000000000000000000000..d8cd1a14f30de0012e704686d2bb3969f1784634 --- /dev/null +++ b/dev/tenks-deploy-config-compute-libvirt-on-host.yml @@ -0,0 +1,56 @@ +--- +# This file holds the config given to Tenks when running `tenks-deploy.sh`. It +# assumes the existence of the bridge `breth1`. + +node_types: + type0: + memory_mb: 1024 + vcpus: 1 + volumes: + # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent: + # https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290 + - capacity: 4GiB + physical_networks: + - physnet1 + console_log_enabled: true + # We seem to hit issues with missing cpu features in CI as a result of using host-model, e.g: + # https://zuul.opendev.org/t/openstack/build/02c33ab51664419a88a5a54ad22852a9/log/primary/system_logs/libvirt/qemu/tk0.txt.gz#38 + cpu_mode: + +specs: + - type: type0 + count: 2 + ironic_config: + resource_class: test-rc + network_interface: flat + +nova_flavors: + - resource_class: test-rc + node_type: type0 + +physnet_mappings: + physnet1: breth1 + +deploy_kernel: ipa.kernel +deploy_ramdisk: ipa.initramfs + +default_boot_mode: "bios" + +# Use the libvirt daemon deployed by Kayobe. Tenks will install libvirt client +# packages. +libvirt_host_install_daemon: false + +# Configure AppArmor for the pool on Ubuntu. +libvirt_host_configure_apparmor: true + +# Nested virtualisation is not working well in CI currently. Force the use of +# QEMU. +libvirt_vm_engine: "qemu" + +# QEMU may not be installed on the host, so set the path and avoid +# autodetection. +libvirt_vm_emulator: "{% if ansible_facts.os_family == 'RedHat' %}/usr/libexec/qemu-kvm{% else %}/usr/bin/qemu-system-x86_64{% endif %}" + +# Specify a log path in the kolla_logs Docker volume. It is accessible on the +# host at the same path. +libvirt_vm_default_console_log_dir: "/var/log/kolla/tenks" diff --git a/doc/source/configuration/reference/hosts.rst b/doc/source/configuration/reference/hosts.rst index 710d0e195238634a5dac9734c79bdb469a65f36c..7cda51e2b6025a853b6eb98ec51db16be15b676b 100644 --- a/doc/source/configuration/reference/hosts.rst +++ b/doc/source/configuration/reference/hosts.rst @@ -1044,3 +1044,154 @@ Ansible's containers do), but may be necessary when building images. Docker's live restore feature can be configured via ``docker_daemon_live_restore``, although it is disabled by default due to issues observed. + +Compute libvirt daemon +====================== +*tags:* + | ``libvirt-host`` + +.. note:: + + This section is about the libvirt daemon on compute nodes, as opposed to the + seed hypervisor. + +Since Yoga, Kayobe provides support for deploying and configuring a libvirt +host daemon, as an alternative to the ``nova_libvirt`` container support by +Kolla Ansible. The host daemon is not used by default, but it is possible to +enable it by setting ``kolla_enable_nova_libvirt_container`` to ``false`` in +``$KAYOBE_CONFIG_PATH/kolla.yml``. + +Migration of hosts from a containerised libvirt to host libvirt is currently +not supported. + +The following options are available in ``$KAYOBE_CONFIG_PATH/compute.yml`` and +are relevant only when using the libvirt daemon rather than the +``nova_libvirt`` container: + +``compute_libvirt_enabled`` + Whether to enable a host libvirt daemon. Default is true if + ``kolla_enable_nova`` is ``true`` and + ``kolla_enable_nova_libvirt_container`` is ``false``. +``compute_libvirt_conf_default`` + A dict of default configuration options to write to + ``/etc/libvirt/libvirtd.conf``. +``compute_libvirt_conf_extra`` + A dict of additional configuration options to write to + ``/etc/libvirt/libvirtd.conf``. +``compute_libvirt_conf`` + A dict of configuration options to write to ``/etc/libvirt/libvirtd.conf``. + Default is a combination of ``compute_libvirt_conf_default`` and + ``compute_libvirt_conf_extra``. +``compute_libvirtd_log_level`` + Numerical log level for libvirtd. Default is 3. +``compute_qemu_conf_default`` + A dict of default configuration options to write to + ``/etc/libvirt/qemu.conf``. +``compute_qemu_conf_extra`` + A dict of additional configuration options to write to + ``/etc/libvirt/qemu.conf``. +``compute_qemu_conf`` + A dict of configuration options to write to ``/etc/libvirt/qemu.conf``. + Default is a combination of ``compute_qemu_conf_default`` and + ``compute_qemu_conf_extra``. +``compute_libvirt_enable_tls`` + Whether to enable a libvirt TLS listener. Default is false. +``compute_libvirt_ceph_repo_install`` + Whether to install a Ceph package repository on CentOS and Rocky hosts. + Default is ``true``. +``compute_libvirt_ceph_repo_release`` + Ceph package repository release to install on CentOS and Rocky hosts when + ``compute_libvirt_ceph_repo_install`` is ``true``. Default is ``pacific``. + +Example: custom libvirtd.conf +----------------------------- + +To customise the libvirt daemon log output to send level 3 to the journal: + +.. code-block:: yaml + :caption: ``compute.yml`` + + compute_libvirt_conf_extra: + log_outputs: "3:journald" + +Example: custom qemu.conf +------------------------- + +To customise QEMU to avoid adding timestamps to logs: + +.. code-block:: yaml + :caption: ``compute.yml`` + + compute_qemu_conf_extra: + log_timestamp: 0 + +Example: enabling libvirt TLS listener +-------------------------------------- + +To enable the libvirt TLS listener: + +.. code-block:: yaml + :caption: ``compute.yml`` + + compute_libvirt_enable_tls: true + +When the TLS listener is enabled, it is necessary to provide client, server and +CA certificates. The following files should be provided: + +``cacert.pem`` + CA certificate used to sign client and server certificates. +``clientcert.pem`` + Client certificate. +``clientkey.pem`` + Client key. +``servercert.pem`` + Server certificate. +``serverkey.pem`` + Server key. + +It is recommended to encrypt the key files using Ansible Vault. + +The following paths are searched for these files: + +* ``$KAYOBE_CONFIG_PATH/certificates/libvirt/{{ inventory_hostname }}/`` +* ``$KAYOBE_CONFIG_PATH/certificates/libvirt/`` + +In this way, certificates may be generated for each host, or shared using +wildcard certificates. + +If using Kayobe environments, certificates in the environment take precedence. + +Kayobe makes the CA certificate and client certificate and key available to +Kolla Ansible, for use by the ``nova_compute`` service. + +Example: disabling Ceph repository installation +----------------------------------------------- + +On CentOS and Rocky hosts, a CentOS Storage SIG Ceph repository is installed +that provides more recent Ceph libraries than those available in CentOS/Rocky +AppStream. This may be necessary when using Ceph for Cinder volumes or Nova +ephemeral block devices. In some cases, such as when using local package +mirrors, the upstream repository may not be appropriate. The installation of +the repository may be disabled as follows: + +.. code-block:: yaml + :caption: ``compute.yml`` + + compute_libvirt_ceph_repo_install: false + +Example: installing additional packages +--------------------------------------- + +In some cases it may be useful to install additional packages on compute hosts +for use by libvirt. The `stackhpc.libvirt-host +<https://galaxy.ansible.com/stackhpc/libvirt-host>`__ Ansible role supports +this via the ``libvirt_host_extra_daemon_packages`` variable. The variable +should be defined via group variables in the Ansible inventory, to avoid +applying the change to the seed hypervisor. For example, to install the +``trousers`` package used for accessing TPM hardware: + +.. code-block:: yaml + :caption: ``inventory/group_vars/compute/libvirt`` + + libvirt_host_extra_daemon_packages: + - trousers diff --git a/doc/source/contributor/automated.rst b/doc/source/contributor/automated.rst index 8e71d1cf783f2759d67739bdb984f7b0663f379d..c0e7cc65753293df8479d01a7db99060d9e1a802 100644 --- a/doc/source/contributor/automated.rst +++ b/doc/source/contributor/automated.rst @@ -131,11 +131,6 @@ For a control plane with Ironic enabled, a "bare metal" instance can be deployed. We can use the `Tenks <https://tenks.readthedocs.io/en/latest/>`__ project to create fake bare metal nodes. -On Ubuntu, the ``nova_libvirt`` image does not contain the ``qemu-utils`` -package necessary for image operations used by Tenks. Install it as follows:: - - sudo docker exec -u root nova_libvirt bash -c 'apt update && apt -y install qemu-utils' - Clone the tenks repository:: git clone https://opendev.org/openstack/tenks.git diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml index cd0ceb2478037e9d5d25320a6de32c587add4360..b1d8d6562d604c5e1e7cf0324a10efe33ab2e7be 100644 --- a/etc/kayobe/compute.yml +++ b/etc/kayobe/compute.yml @@ -143,6 +143,53 @@ # - state: enabled #compute_firewalld_rules: +############################################################################### +# Compute node host libvirt configuration. + +# Whether to enable a host libvirt daemon. Default is true if kolla_enable_nova +# is true and kolla_enable_nova_libvirt_container is false. +#compute_libvirt_enabled: + +# A dict of default configuration options to write to +# /etc/libvirt/libvirtd.conf. +#compute_libvirt_conf_default: + +# A dict of additional configuration options to write to +# /etc/libvirt/libvirtd.conf. +#compute_libvirt_conf_extra: + +# A dict of configuration options to write to /etc/libvirt/libvirtd.conf. +# Default is a combination of compute_libvirt_conf_default and +# compute_libvirt_conf_extra. +#compute_libvirt_conf: + +# Numerical log level for libvirtd. Default is 3. +#compute_libvirtd_log_level: + +# A dict of default configuration options to write to +# /etc/libvirt/qemu.conf. +#compute_qemu_conf_default: + +# A dict of additional configuration options to write to +# /etc/libvirt/qemu.conf. +#compute_qemu_conf_extra: + +# A dict of configuration options to write to /etc/libvirt/qemu.conf. +# Default is a combination of compute_qemu_conf_default and +# compute_qemu_conf_extra. +#compute_qemu_conf: + +# Whether to enable a libvirt TLS listener. Default is false. +#compute_libvirt_enable_tls: + +# Whether to install a Ceph package repository on CentOS and Rocky hosts. +# Default is true. +#compute_libvirt_ceph_repo_install: + +# Ceph package repository release to install on CentOS and Rocky hosts when +# compute_libvirt_ceph_repo_install is true. Default is 'pacific'. +#compute_libvirt_ceph_repo_release: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 788d493b5cdd248bb2440fabe1df4437658c2542..325a67aeaae3b24470519396507e898be0acf74c 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -1125,6 +1125,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, * Optionally, create a virtualenv for kolla-ansible. * Configure a user account for kolla-ansible. * Configure Docker engine. + * Configure libvirt. """ def get_parser(self, prog_name): @@ -1157,7 +1158,8 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, self.run_kolla_ansible_overcloud(parsed_args, "bootstrap-servers") # Further kayobe playbooks. - playbooks = _build_playbook_list("docker", "swift-block-devices") + playbooks = _build_playbook_list( + "docker", "swift-block-devices", "compute-libvirt-host") self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud") diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index dbb402e1d268dcf7bbfcbb3d000639f05947a696..5986de76b5d29a13d6c3afd79c9809c0da0f11ab 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -1322,6 +1322,8 @@ class TestCase(unittest.TestCase): utils.get_data_files_path("ansible", "docker.yml"), utils.get_data_files_path( "ansible", "swift-block-devices.yml"), + utils.get_data_files_path( + "ansible", "compute-libvirt-host.yml"), ], limit="overcloud", ), @@ -1376,6 +1378,8 @@ class TestCase(unittest.TestCase): utils.get_data_files_path("ansible", "docker.yml"), utils.get_data_files_path( "ansible", "swift-block-devices.yml"), + utils.get_data_files_path( + "ansible", "compute-libvirt-host.yml"), ], limit="overcloud", ), diff --git a/playbooks/kayobe-overcloud-base/globals.yml.j2 b/playbooks/kayobe-overcloud-base/globals.yml.j2 index 7de112f61a2f694271c34dfe0060d619c06ed109..58c4c7ec91807814cd362261a10ceb43d6a42dd9 100644 --- a/playbooks/kayobe-overcloud-base/globals.yml.j2 +++ b/playbooks/kayobe-overcloud-base/globals.yml.j2 @@ -22,4 +22,5 @@ kolla_enable_tls_backend: "yes" openstack_cacert: "/etc/pki/tls/certs/ca-bundle.crt" kolla_admin_openrc_cacert: "/etc/pki/tls/certs/ca-bundle.crt" libvirt_tls: "yes" +certificates_libvirt_output_dir: "{% raw %}{{ kayobe_env_config_path }}{% endraw %}/certificates/libvirt" {% endif %} diff --git a/playbooks/kayobe-overcloud-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-base/overrides.yml.j2 index 58462fb525a1e255652f44d1999826ee39b0f375..864e29b960face9179f6f6a4573045b40a2c2836 100644 --- a/playbooks/kayobe-overcloud-base/overrides.yml.j2 +++ b/playbooks/kayobe-overcloud-base/overrides.yml.j2 @@ -42,6 +42,9 @@ kolla_ironic_default_boot_interface: ipxe {% endif %} {% if tls_enabled %} +kolla_enable_nova_libvirt_container: false +compute_libvirt_enable_tls: true + kolla_enable_tls_external: "yes" kolla_enable_tls_internal: "yes" diff --git a/playbooks/kayobe-overcloud-base/run.yml b/playbooks/kayobe-overcloud-base/run.yml index 3d77e86aa7b0084e8cc47d1b8052abfb855ba50d..350b47dfd502ae5253e80867ce9e2d164c0455da 100644 --- a/playbooks/kayobe-overcloud-base/run.yml +++ b/playbooks/kayobe-overcloud-base/run.yml @@ -3,6 +3,8 @@ environment: KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}" KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES: "{{ tls_enabled | ternary(1, 0) }}" + # TODO(mgoddard): Remove this when libvirt on host is used by default. + TENKS_CONFIG_PATH: "dev/tenks-deploy-config-compute{% if tls_enabled %}-libvirt-on-host{% endif %}.yml" tasks: - name: Ensure overcloud is deployed shell: @@ -18,8 +20,6 @@ executable: /bin/bash - name: Perform testing of the virtualized machines - # We must do this before tenks-deploy as that will stop the nova_libvirt - # container shell: cmd: dev/overcloud-test-vm.sh &> {{ logs_dir }}/ansible/overcloud-test-vm chdir: "{{ kayobe_src_dir }}" diff --git a/releasenotes/notes/libvirt-on-host-ff83f12923cc1f58.yaml b/releasenotes/notes/libvirt-on-host-ff83f12923cc1f58.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a16a9cfd201926769792cca3a1ba30c89f16c952 --- /dev/null +++ b/releasenotes/notes/libvirt-on-host-ff83f12923cc1f58.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds support for running a libvirt daemon on the host, rather than in a + container. This is done by setting ``kolla_enable_nova_libvirt_container`` + to ``false``. See `story 2009858 + <https://storyboard.openstack.org/#!/story/2009858>`__ for details. diff --git a/requirements.yml b/requirements.yml index bd8629ebd30248e6abf8b5cf63e2ffb3bc914a2a..33629a1af7973170494b91e7e3de4e4dc3572fa8 100644 --- a/requirements.yml +++ b/requirements.yml @@ -32,7 +32,7 @@ roles: - src: stackhpc.grafana-conf version: 1.1.1 - src: stackhpc.libvirt-host - version: v1.8.3 + version: v1.10.0 - src: stackhpc.libvirt-vm version: v1.14.2 - src: stackhpc.luks