Stop using kolla-ansible bootstrap-servers

The 'kayobe * host configure' commands no longer use the 'kolla-ansible
bootstrap-servers' command, and associated 'baremetal' role in Kolla
Ansible. The functionality provided by the 'baremetal' role has been
extracted into the openstack.kolla Ansible collection, and split
into separate roles. This allows Kayobe to use it directly, and only the
necessary parts.

This change improves failure handling in these Kayobe commands, and aims
to reduce confusion over which '--limit' and '--tags' arguments to
provide.  This ensures that if a host fails during a host configuration
command, other hosts are able to continue to completion. Previously, if
any host failed during the Kayobe playbooks, the 'kolla-ansible
bootstrap-servers' command would not run. This is useful at scale, where
host failures occur more frequently.

This change has implications for configuration of Kayobe, since some
variables that were previously in Kolla Ansible are now in Kayobe.

Several parts of the baremetal role have been split out and used here:

* apparmor-libvirt: disable AppArmor rules for libvirt on Ubuntu.
* docker: Docker installation & configuration. The docker role in
  openstack.kolla combines functionality from kolla-ansible and kayobe.
* etc-hosts: it proved difficult to generalise this, so we have some
  almost duplicated the code from kolla-ansible here. Requires delegated
  fact gathering for the case when --limit is used.
* firewall: support to disable UFW, for feature parity.
* kolla-packages: miscellaneous package installs & removals.

The addition of the stack user to the docker group has been moved to the
user bootstrapping playbook, and the docker SDK installation has been
moved to the virtualenv setup playbook.

Depends-On: https://review.opendev.org/c/openstack/ansible-collection-kolla/+/829587

Story: 2009854
Task: 44505

Change-Id: I61a61ca59652b13687c2247d5881012b51f666a7

.gitignore

@@ -56,8 +56,9 @@ ChangeLog
 ansible/*.retry
 ansible/roles/*/tests/*.retry
 
-# Ansible Galaxy roles
+# Ansible Galaxy roles & collections
 ansible/roles/*\.*/
+ansible/collections/
 
 # Virtualenvs
 ansible/kolla-venv/

ansible/apparmor-libvirt.yml

+---
+- name: Ensure AppArmor is disabled for containerised libvirt
+  hosts: compute
+  tags:
+    - apparmor-libvirt
+  vars:
+    # kolla_overcloud_inventory_top_level_group_map looks like:
+    # kolla_overcloud_inventory_top_level_group_map:
+    #  control:
+    #    groups:
+    #      - controllers
+    hosts_in_kolla_inventory: >-
+      {{ kolla_overcloud_inventory_top_level_group_map.values() |
+         map(attribute='groups') | flatten | unique | join(':') }}
+  tasks:
+    - name: Include openstack.kolla.apparmor_libvirt role
+      include_role:
+        name: openstack.kolla.apparmor_libvirt
+      when:
+        - inventory_hostname in query('inventory_hostnames', hosts_in_kolla_inventory)
+        - ansible_facts.distribution == "Ubuntu"

ansible/docker.yml

@@ -3,7 +3,12 @@
   hosts: docker
   tags:
     - docker
-  vars:
-    - docker_upper_constraints_file: "{{ pip_upper_constraints_file }}"
-  roles:
-    - role: docker
+  tasks:
+    - import_role:
+        name: docker
+      vars:
+        docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}"
+        docker_configure_for_zun: "{{ kolla_enable_zun | bool }}"
+        docker_http_proxy: "{{ kolla_http_proxy }}"
+        docker_https_proxy: "{{ kolla_https_proxy }}"
+        docker_no_proxy: "{{ kolla_no_proxy | select | join(',') }}"

ansible/etc-hosts.yml

+---
+- name: Ensure /etc/hosts is configured
+  hosts: overcloud
+  tags:
+    - etc-hosts
+  tasks:
+    # NOTE(mgoddard): Need to ensure that all hosts have facts available.
+    - import_role:
+        name: gather-facts-delegated
+      tags:
+        - gather-facts-delegated
+      when: etc_hosts_gather_facts | default(true)
+
+    - import_role:
+        name: etc-hosts

ansible/firewall.yml

@@ -6,6 +6,5 @@
     - firewall
   tasks:
     - name: Configure firewalld
-      include_role:
+      import_role:
         name: "firewalld"
-

ansible/inventory/group_vars/all/docker

@@ -26,3 +26,9 @@ docker_registry:
 
 # CA of docker registry
 docker_registry_ca:
+
+# List of Docker registry mirrors.
+docker_registry_mirrors: []
+
+# Enable live-restore on docker daemon
+docker_daemon_live_restore: false

ansible/kayobe-ansible-user.yml

@@ -70,9 +70,11 @@
     ansible_python_interpreter: /usr/bin/python3
   roles:
     - role: singleplatform-eng.users
+      groups_to_create: "{{ [{'name': 'docker'}] if 'docker' in group_names else [] }}"
       users:
         - username: "{{ kayobe_ansible_user }}"
           name: Kayobe deployment user
+          groups: "{{ ['docker'] if 'docker' in group_names else [] }}"
           append: True
           ssh_key:
             - "{{ lookup('file', ssh_public_key_path) }}"

ansible/kayobe-target-venv.yml

@@ -100,3 +100,14 @@
             state: present
           become: True
       when: virtualenv is not defined
+
+    - name: Ensure kolla-ansible virtualenv has docker SDK for python installed
+      pip:
+        name: docker
+        state: latest
+        virtualenv: "{{ virtualenv | default(omit) }}"
+        extra_args: "{% if docker_upper_constraints_file %}-c {{ docker_upper_constraints_file }}{% endif %}"
+      become: "{{ virtualenv is not defined }}"
+      vars:
+        docker_upper_constraints_file: "{{ pip_upper_constraints_file }}"
+      when: "'docker' in group_names"

ansible/kolla-ansible.yml

@@ -107,7 +107,6 @@
         kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}"
         kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}"
         kolla_enable_host_ntp: false
-        docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}"
         kolla_globals_paths_extra:
           - "{{ kayobe_config_path }}"
           - "{{ kayobe_env_config_path }}"

ansible/kolla-packages.yml

+---
+- name: Ensure Kolla Ansible packages are installed
+  hosts: overcloud
+  tags:
+    - kolla-packages
+  vars:
+    # kolla_overcloud_inventory_top_level_group_map looks like:
+    # kolla_overcloud_inventory_top_level_group_map:
+    #  control:
+    #    groups:
+    #      - controllers
+    hosts_in_kolla_inventory: >-
+      {{ kolla_overcloud_inventory_top_level_group_map.values() |
+         map(attribute='groups') | flatten | unique | join(':') }}
+  tasks:
+    - name: Include openstack.kolla.packages role
+      include_role:
+        name: openstack.kolla.packages
+      vars:
+        enable_multipathd: "{{ kolla_enable_multipathd | bool }}"
+      when:
+        - inventory_hostname in query('inventory_hostnames', hosts_in_kolla_inventory)

ansible/overcloud-docker-sdk-upgrade.yml

----
-- name: Ensure docker SDK for python is installed
-  hosts: overcloud
-  tags:
-    - docker-sdk-upgrade
-  tasks:
-    # Docker renamed their python SDK from docker-py to docker in the 2.0.0
-    # release, and also broke backwards compatibility. Kolla-ansible requires
-    # docker, so ensure it is installed.
-    - name: Set a fact about the virtualenv on the remote system
-      set_fact:
-        virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}"
-      when:
-        - ansible_python_interpreter is defined
-        - not ansible_python_interpreter.startswith('/bin/')
-        - not ansible_python_interpreter.startswith('/usr/bin/')
-
-    - name: Ensure legacy docker-py python package is uninstalled
-      pip:
-        name: docker-py
-        state: absent
-        virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}"
-      become: "{{ virtualenv is not defined }}"
-
-    - name: Ensure docker SDK for python is installed
-      pip:
-        name: docker
-        state: latest
-        extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}"
-        virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}"
-      become: "{{ virtualenv is not defined }}"

ansible/overcloud-host-configure.yml

@@ -12,6 +12,7 @@
 - import_playbook: "selinux.yml"
 - import_playbook: "network.yml"
 - import_playbook: "firewall.yml"
+- import_playbook: "etc-hosts.yml"
 - import_playbook: "tuned.yml"
 - import_playbook: "sysctl.yml"
 - import_playbook: "disable-glean.yml"
@@ -25,3 +26,8 @@
 - import_playbook: "kolla-ansible-user.yml"
 - import_playbook: "kolla-pip.yml"
 - import_playbook: "kolla-target-venv.yml"
+- import_playbook: "kolla-packages.yml"
+- import_playbook: "docker.yml"
+- import_playbook: "apparmor-libvirt.yml"
+- import_playbook: "swift-block-devices.yml"
+- import_playbook: "compute-libvirt-host.yml"

ansible/overcloud-host-upgrade.yml

 ---
 - import_playbook: "kayobe-target-venv.yml"
 - import_playbook: "kolla-target-venv.yml"
-- import_playbook: "overcloud-docker-sdk-upgrade.yml"
 - import_playbook: "overcloud-etc-hosts-fixup.yml"

ansible/roles/docker/defaults/main.yml

 ---
-# URL of docker registry
-docker_registry:
-
-# CA of docker registry
-docker_registry_ca:
-
-# Upper constraints file which is passed to pip when installing packages
-# into a venv.
-docker_upper_constraints_file:
+docker_storage_driver: overlay2
+docker_storage_volume_group:
+docker_storage_volume_thinpool:
+docker_registry_mirrors: []
+docker_daemon_mtu: 1500
+docker_daemon_live_restore: false

ansible/roles/docker/handlers/main.yml

----
-- name: reload docker service
-  service:
-    name: docker
-    state: reloaded
-  become: True

ansible/roles/docker/tasks/main.yml

 ---
-- name: Set a fact about the virtualenv on the remote system
-  set_fact:
-    virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}"
-  when:
-    - ansible_python_interpreter is defined
-    - not ansible_python_interpreter.startswith('/bin/')
-    - not ansible_python_interpreter.startswith('/usr/bin/')
-
-- name: Ensure docker SDK for python is installed
-  pip:
-    name: docker
-    state: latest
-    extra_args: "{% if docker_upper_constraints_file %}-c {{ docker_upper_constraints_file }}{% endif %}"
-    virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}"
-  become: "{{ virtualenv is not defined }}"
-
-- name: Ensure user is in the docker group
-  user:
-    name: "{{ ansible_facts.user_id }}"
-    groups: docker
-    append: yes
-  register: group_result
-  become: True
-
-# After adding the user to the docker group, we need to log out and in again to
-# pick up the group membership. We do this by resetting the SSH connection.
-
-- name: Reset connection to activate new group membership
-  meta: reset_connection
-  when: group_result is changed
-
-- name: Ensure Docker daemon is started
-  service:
-    name: docker
-    state: started
-  become: True
-
-- name: Ensure the path for CA file for private registry exists
-  file:
-    path: "/etc/docker/certs.d/{{ docker_registry }}"
-    state: directory
-  become: True
-  when: docker_registry is not none and docker_registry_ca is not none
-
-- name: Ensure the CA file for private registry exists
-  copy:
-    src: "{{ docker_registry_ca }}"
-    dest: "/etc/docker/certs.d/{{ docker_registry }}/ca.crt"
-  become: True
-  when: docker_registry is not none and docker_registry_ca is not none
-  notify: reload docker service
+- import_role:
+    name: openstack.kolla.docker
+  vars:
+    docker_custom_config: "{{ lookup('template', 'daemon.json.j2') | to_nice_json | indent(2) }}"

ansible/roles/etc-hosts/defaults/main.yml

+---
+# Whether to add entries to /etc/hosts.
+customize_etc_hosts: true
+
+# List of hosts to add to /etc/hosts.
+etc_hosts_hosts: "{{ ansible_play_hosts_all }}"

ansible/roles/etc-hosts/tasks/etc-hosts.yml

+---
+- name: Ensure localhost in /etc/hosts
+  lineinfile:
+    dest: /etc/hosts
+    regexp: "^127.0.0.1.*"
+    line: "127.0.0.1 localhost"
+    state: present
+  become: True
+
+# NOTE(mgoddard): Ubuntu may include a line in /etc/hosts that makes the local
+# hostname and fqdn point to 127.0.1.1. This can break
+# RabbitMQ, which expects the hostname to resolve to the API network address.
+# Remove the troublesome entry.
+# see https://bugs.launchpad.net/kolla-ansible/+bug/1837699
+# and https://bugs.launchpad.net/kolla-ansible/+bug/1862739
+- name: Ensure hostname does not point to 127.0.1.1 in /etc/hosts
+  lineinfile:
+    dest: /etc/hosts
+    regexp: "^127.0.1.1\\b.*\\s{{ ansible_facts.hostname }}\\b"
+    state: absent
+  become: True
+
+- name: Generate /etc/hosts for all of the nodes
+  blockinfile:
+    dest: /etc/hosts
+    marker: "# {mark} ANSIBLE GENERATED HOSTS"
+    block: |
+        {% for host in etc_hosts_hosts %}
+        {% if hostvars[host].internal_net_name in hostvars[host].network_interfaces %}
+        {% set hostnames = [hostvars[host].ansible_facts.nodename, hostvars[host].ansible_facts.hostname] %}
+        {{ hostvars[host].internal_net_name | net_ip(inventory_hostname=host) }} {{ hostnames | unique | join(' ') }}
+        {% endif %}
+        {% endfor %}
+  become: True
+  when:
+    # Skip hosts that do not have a valid internal network interface.
+    - internal_net_name in network_interfaces
+
+# NOTE(osmanlicilegi): The distribution might come with cloud-init installed, and manage_etc_hosts
+# configuration enabled. If so, it will override the file /etc/hosts from cloud-init templates at
+# every boot, which will break RabbitMQ. To prevent this happens, first we check whether cloud-init
+# has been installed, and then set manage_etc_hosts to false.
+- name: Check whether cloud-init has been installed, and ensure manage_etc_hosts is disabled
+  block:
+    - name: Ensure /etc/cloud/cloud.cfg exists
+      stat:
+        path: /etc/cloud/cloud.cfg
+      register: cloud_init
+
+    - name: Disable cloud-init manage_etc_hosts
+      copy:
+        content: "manage_etc_hosts: false"
+        dest: /etc/cloud/cloud.cfg.d/99-kolla.cfg
+        mode: "0660"
+      when: cloud_init.stat.exists
+  become: True

ansible/roles/etc-hosts/tasks/main.yml

+---
+- include_tasks: etc-hosts.yml
+  when: customize_etc_hosts | bool