Add support for libvirt+tls

To securely support live migration between computenodes we should enable
tls, with cert auth, instead of TCP with no auth support.

Implements: blueprint libvirt-tls

Change-Id: I22ea6233933c840b853fdcc8e03400b2bf577271

ansible/roles/nova/defaults/main.yml

@@ -362,7 +362,7 @@ nova_compute_host_rp_filter_mode: 0
 nova_enable_rolling_upgrade: "yes"
 nova_safety_upgrade: "no"
 
-nova_libvirt_port: "16509"
+nova_libvirt_port: "{{'16514' if libvirt_tls | bool  else '16509'}}"
 nova_ssh_port: "8022"
 
 nova_services_require_nova_conf:
@@ -410,6 +410,17 @@ ovs_bridge: "nsx-managed"
 qemu_max_files: 32768
 # The number of max processes qemu can open
 qemu_max_processes: 131072
+# Use TLS for libvirt connections and live migration
+libvirt_tls: false
+# Should kolla-ansible manage/copy the certs.  False, assumes the deployer is
+# responsible for making the TLS certs show up in the config directories
+# also means the deployer is responsible for restarting the nova_compute and
+# nova_libvirt containers when the key changes, as we can't know when to do that
+libvirt_tls_manage_certs: true
+# When using tls we are verfiying the hostname we are connected to matches the
+# libvirt cert we are presented.  As such we can't use IP's here, but keep the
+# ability for people to override the hostname to use.
+migration_hostname: "{{ ansible_nodename }}"
 
 ####################
 # Kolla

ansible/roles/nova/tasks/config-libvirt-tls.yml

+---
+- name: Copying over libvirt TLS keys {{ file }}
+  become: true
+  copy:
+    src: "{{ first_found }}"
+    dest: "{{ node_config_directory }}/{{ service_name }}/{{ file }}"
+    mode: "0600"
+  with_first_found:
+    - "{{ node_custom_config }}/nova/nova-libvirt/{{ inventory_hostname }}/{{ file }}"
+    - "{{ node_custom_config }}/nova/nova-libvirt/{{ file }}"
+  loop_control:
+    loop_var: first_found
+  notify:
+    - Restart {{ service_name }} container

ansible/roles/nova/tasks/config.yml

@@ -114,6 +114,40 @@
   notify:
     - Restart nova-libvirt container
 
+- name: Copying over libvirt TLS keys (nova-libvirt)
+  include_tasks: "config-libvirt-tls.yml"
+  vars:
+    service: "{{ nova_services['nova-libvirt'] }}"
+    service_name: nova-libvirt
+    file: "{{ item }}"
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - libvirt_tls | bool
+    - libvirt_tls_manage_certs | bool
+  with_items:
+    - cacert.pem
+    - servercert.pem
+    - serverkey.pem
+    - clientcert.pem
+    - clientkey.pem
+
+- name: Copying over libvirt TLS keys (nova-compute)
+  include_tasks: "config-libvirt-tls.yml"
+  vars:
+    service: "{{ nova_services['nova-compute'] }}"
+    service_name: nova-compute
+    file: "{{ item }}"
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - libvirt_tls | bool
+    - libvirt_tls_manage_certs | bool
+  with_items:
+    - cacert.pem
+    - clientcert.pem
+    - clientkey.pem
+
 - name: Copying files for nova-ssh
   become: true
   vars:

ansible/roles/nova/templates/libvirtd.conf.j2

+{% if libvirt_tls | bool %}
+listen_tls = 1
+listen_tcp = 0
+tls_port = "{{ nova_libvirt_port }}"
+key_file = "/etc/pki/libvirt/private/serverkey.pem"
+cert_file = "/etc/pki/libvirt/servercert.pem"
+ca_file = "/etc/pki/CA/cacert.pem"
+{% else %}
 listen_tcp = 1
 listen_tls = 0
 auth_tcp = "none"
+tcp_port = "{{ nova_libvirt_port }}"
 ca_file = ""
+{% endif %}
 log_level = 3
 log_outputs = "3:file:/var/log/kolla/libvirt/libvirtd.log"
 listen_addr = "{{ migration_interface_address }}"
-tcp_port = "{{ nova_libvirt_port }}"

ansible/roles/nova/templates/nova-compute.json.j2

@@ -24,6 +24,24 @@
             "dest": "/etc/nova/vmware_ca",
             "owner": "nova",
             "perm": "0600"
+        }{% endif %}{% if libvirt_tls | bool %},
+        {
+            "source": "{{ container_config_directory }}/clientkey.pem",
+            "dest": "/etc/pki/libvirt/private/clientkey.pem",
+            "owner": "root:nova",
+            "perm": "0640"
+        },
+        {
+            "source": "{{ container_config_directory }}/clientcert.pem",
+            "dest": "/etc/pki/libvirt/clientcert.pem",
+            "owner": "root:nova",
+            "perm": "0640"
+        },
+        {
+            "source": "{{ container_config_directory }}/cacert.pem",
+            "dest": "/etc/pki/CA/cacert.pem",
+            "owner": "root:nova",
+            "perm": "0640"
         }{% endif %},
         {
             "source": "{{ container_config_directory }}/release",

ansible/roles/nova/templates/nova-libvirt.json.j2

@@ -12,7 +12,37 @@
             "dest": "/etc/libvirt/qemu.conf",
             "owner": "root",
             "perm": "0600"
-        }{% if nova_backend == "rbd" or cinder_backend_ceph | bool %},
+        }{% if libvirt_tls | bool %},
+        {
+            "source": "{{ container_config_directory }}/serverkey.pem",
+            "dest": "/etc/pki/libvirt/private/serverkey.pem",
+            "owner": "root",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/servercert.pem",
+            "dest": "/etc/pki/libvirt/servercert.pem",
+            "owner": "root",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/clientkey.pem",
+            "dest": "/etc/pki/libvirt/private/clientkey.pem",
+            "owner": "root",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/clientcert.pem",
+            "dest": "/etc/pki/libvirt/clientcert.pem",
+            "owner": "root",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/cacert.pem",
+            "dest": "/etc/pki/CA/cacert.pem",
+            "owner": "root",
+            "perm": "0600"
+        }{% endif %}{% if nova_backend == "rbd" or cinder_backend_ceph | bool %},
         {
             "source": "{{ container_config_directory }}/secrets",
             "dest": "/etc/libvirt/secrets",

ansible/roles/nova/templates/nova.conf.d/libvirt.conf.j2

 [libvirt]
+{% if libvirt_tls | bool %}
+connection_uri = "qemu+tls://{{ migration_hostname }}/system"
+live_migration_uri = "qemu+tls://%s/system"
+{% else %}
 connection_uri = "qemu+tcp://{{ migration_interface_address }}/system"
+{% endif %}
 {% if enable_ceph | bool and nova_backend == "rbd" %}
 images_type = rbd
 images_rbd_pool = {{ ceph_nova_pool_name }}

doc/source/reference/compute/index.rst

@@ -9,6 +9,7 @@ compute services like HyperV, XenServer and so on.
    :maxdepth: 1
 
    hyperv-guide
+   libvirt-tls-guide
    masakari-guide
    nova-fake-driver
    qinling-guide

doc/source/reference/compute/libvirt-tls-guide.rst

+.. libvirt-tls-guide:
+
+===========
+Libvirt TLS
+===========
+
+The default configuration of Kolla Ansible is to run libvirt over TCP, with
+authentication disabled. As long as one takes steps to protect who can access
+the port this works well. However, in the case where you want live-migration to
+be allowed across hypervisors one may want to either add some level of
+authentication to the connections or make sure VM data is passed between
+hypervisors in a secure manner. To do this we can enable TLS for libvirt and
+make nova use it.
+
+Using libvirt TLS
+~~~~~~~~~~~~~~~~~
+
+Libvirt TLS can be enabled in Kolla Ansible by setting the following option in
+``/etc/kolla/globals.yml``:
+
+.. code-block:: yaml
+
+   libvirt_tls: "yes"
+
+Creation of the TLS certificates is currently out-of-scope for Kolla Ansible.
+You will need to either use an existing Internal CA or you will need to
+generate your own offline CA. For the TLS communication to work correctly you
+will have to supply Kolla Ansible the following pieces of information:
+
+* cacert.pem
+
+  - This is the CA's public certificate that all of the client and server
+    certificates are signed with. Libvirt and nova-compute will need this so
+    they can verify that all the certificates being used were signed by the CA
+    and should be trusted.
+
+* serverkey.pem
+
+  - This is the private key for the server, and is no different than the
+    private key of a TLS certificate. It should be carefully protected, just
+    like the private key of a TLS certificate.
+
+* servercert.pem
+
+  - This is the public certificate for the server. Libvirt will present this
+    certificate to any connection made to the TLS port. This is no different
+    than the public certificate part of a standard TLS certificate/key bundle.
+
+* clientkey.pem
+
+  - This is the client private key, which nova-compute/libvirt will use
+    when it is connecting to libvirt. Think of this as an SSH private key
+    and protect it in a similar manner.
+
+* clientcert.pem
+
+  - This is the client certificate that nova-compute/libvirt will present when
+    it is connecting to libvirt. Think of this as the public side of an SSH
+    key.
+
+Kolla Ansible will search for these files for each compute node in the
+following locations and order on the host where Kolla Ansible is executed:
+
+- ``/etc/kolla/config/nova/nova-libvirt/<hostname>/``
+- ``/etc/kolla/config/nova/nova-libvirt/``
+
+In most cases you will want to have a unique set of server and client
+certificates and keys per hypervisor and with a common CA certificate. In this
+case you would place each of the server/client certificate and key PEM files
+under ``/etc/kolla/config/nova/nova-libvirt/<hostname>/`` and the CA
+certificate under ``/etc/kolla/config/nova/nova-libvirt/``.
+
+However, it is possible to make use of wildcard server certificate and a single
+client certificate that is shared by all servers. This will allow you to
+generate a single client certificate and a single server certificate that is
+shared across every hypervisor. In this case you would store everything under
+``/etc/kolla/config/nova/nova-libvirt/``.
+
+Externally managed certificates
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+One more option for deployers who already have automation to get TLS certs onto
+servers is to disable certificate management under ``/etc/kolla/globals.yaml``:
+
+.. code-block:: yaml
+
+   libvirt_tls_manage_certs: "no"
+
+With this option disabled Kolla Ansible will simply assume that certificates
+and keys are already installed in their correct locations. Deployers will be
+responsible for making sure that the TLS certificates/keys get placed in to the
+correct container configuration directories on the servers so that they can get
+copied into the nova-compute and nova-libvirt containers. With this option
+disabled you will also be responsible for restarting the nova-compute and
+nova-libvirt containers when the certs are updated, as kolla-ansible will not
+be able to tell when the files have changed.

releasenotes/notes/libvirt-tls-support-4ab81fbdbf5519d3.yaml

+---
+features:
+  - |
+    Adds support for configuring libvirt with TLS support.  This allows for
+    secure communication between nova-compute and libvirt as well as between
+    libvirt on different hypervisors, during live-migration.  The default
+    configuration passes data in plain text, over TCP, without authentication.