Introduce max fail percentage to playbooks

This allows us to continue execution until a certain proportion of hosts
fail. This can be useful at scale, where failures are common, and
restarting a deployment is time-consuming.

The default max failure percentage is 100, keeping the default
behaviour. A global max failure percentage may be set via
kayobe_max_fail_percentage, and individual playbooks may define a max
failure percentage via <playbook>_max_fail_percentage.

Related Kolla Ansible patch:
https://review.opendev.org/c/openstack/kolla-ansible/+/805598

Change-Id: Ib81c72b63be5765cca664c38141ffc769640cf07

ansible/apparmor-libvirt.yml

 ---
 - name: Ensure AppArmor is disabled for containerised libvirt
   hosts: compute
+  max_fail_percentage: >-
+    {{ apparmor_libvirt_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - apparmor-libvirt
   vars:

ansible/apt.yml

 ---
 - name: Ensure APT is configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ apt_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_python_interpreter: /usr/bin/python3
   tags:

ansible/baremetal-compute-inspect.yml

@@ -20,6 +20,11 @@
 - name: Ensure baremetal compute nodes are inspected in ironic
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_inspect_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     controller_host: "{{ groups['controllers'][0] }}"
     venv: "{{ virtualenv_path }}/openstacksdk"

ansible/baremetal-compute-introspection-data-save.yml

@@ -18,6 +18,11 @@
 - name: Ensure the baremetal compute nodes' hardware introspection data is saved
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_introspection_data_save_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstack-cli"
     controller_host: "{{ groups['controllers'][0] }}"

ansible/baremetal-compute-manage.yml

@@ -20,6 +20,11 @@
 - name: Ensure baremetal compute nodes are manageable in ironic
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_manage_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstacksdk"
     ansible_python_interpreter: "{{ venv }}/bin/python"

ansible/baremetal-compute-provide.yml

@@ -20,6 +20,11 @@
 - name: Ensure baremetal compute nodes are available in ironic
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_provide_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstacksdk"
     ansible_python_interpreter: "{{ venv }}/bin/python"

ansible/baremetal-compute-register.yml

@@ -18,6 +18,11 @@
 - name: Ensure baremetal compute nodes are registered in ironic
   hosts: baremetal-compute
   gather_facts: false
+  max_fail_percentage: >-
+    {{ baremetal_compute_register_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - baremetal
   vars:

ansible/baremetal-compute-rename.yml

@@ -21,6 +21,11 @@
 - name: Rename baremetal compute nodes
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_rename_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstack-cli"
     controller_host: "{{ groups['controllers'][0] }}"

ansible/baremetal-compute-serial-console.yml

@@ -53,6 +53,11 @@
 - name: Enable serial console
   hosts: "{{ console_compute_node_limit | default('baremetal-compute') }}"
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_serial_console_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstack-cli"
     controller_host: "{{ groups['controllers'][0] }}"

ansible/compute-libvirt-host.yml

 ---
 - name: Ensure the libvirt daemon is configured
   hosts: compute
+  max_fail_percentage: >-
+    {{ compute_libvirt_host_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - libvirt-host
   tasks:

ansible/compute-node-discovery.yml

@@ -4,6 +4,11 @@
 - name: Ensure baremetal compute nodes are PXE booted
   hosts: baremetal-compute
   gather_facts: no
+  max_fail_percentage: >-
+    {{ compute_node_discovery_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     controller_host: "{{ groups['controllers'][0] }}"
   tasks:

ansible/dell-compute-node-boot-mode.yml

@@ -7,6 +7,11 @@
 - name: Ensure Dell baremetal compute nodes boot mode is set
   hosts: baremetal-compute
   gather_facts: no
+  max_fail_percentage: >-
+    {{ dell_compute_node_boot_mode_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Set this to the required boot mode. One of 'bios' or 'uefi'.
     drac_boot_mode: "bios"

ansible/dell-compute-node-discovery.yml

@@ -7,6 +7,11 @@
 - name: Ensure Dell baremetal compute nodes are PXE booted
   hosts: baremetal-compute
   gather_facts: no
+  max_fail_percentage: >-
+    {{ dell_compute_node_discovery_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Set this to the index of the inteface on which to enable PXE.
     drac_pxe_interface: 1

ansible/dell-compute-node-inventory.yml

@@ -28,6 +28,11 @@
 - name: Ensure Dell baremetal compute nodes are present in the Ansible inventory
   hosts: baremetal-compute
   gather_facts: no
+  max_fail_percentage: >-
+    {{ dell_compute_node_inventory_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     compute_node_limit: ""
     compute_node_limit_list: "{{ compute_node_limit.split(':') }}"

ansible/dev-tools.yml

 ---
 - name: Ensure development tools are installed
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ dev_tools_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - dev-tools
   roles:

ansible/disable-cloud-init.yml

@@ -5,6 +5,11 @@
 # and cause some issues in network configuration
 - name: Disable Cloud-init service
   hosts: overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ disable_cloud_init_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - disable-cloud-init
   roles:

ansible/disable-glean.yml

@@ -4,6 +4,11 @@
 # interfaces. In some cases this can lead to timeouts.
 - name: Ensure Glean is disabled and its artifacts are removed
   hosts: seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ disable_glean_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - disable-glean
   roles:

ansible/dnf.yml

 ---
 - name: Ensure DNF repos are configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ dnf_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_python_interpreter: /usr/bin/python3
   tags:

ansible/docker.yml

 ---
 - name: Ensure docker is configured
   hosts: docker
+  max_fail_percentage: >-
+    {{ docker_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - docker
   tasks:

ansible/drac-bios.yml

@@ -2,6 +2,10 @@
 - name: Ensure that overcloud nodes' BIOS are configured
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ drac_bios_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     bios_config:
       OneTimeBootMode: "OneTimeBootSeq"