From 6c54ce4d3bb4f5036e5b4f3ad18ef79733b680f5 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 9 Nov 2021 17:23:24 +0000
Subject: [PATCH] Introduce max fail percentage to playbooks

This allows us to continue execution until a certain proportion of hosts
fail. This can be useful at scale, where failures are common, and
restarting a deployment is time-consuming.

The default max failure percentage is 100, keeping the default
behaviour. A global max failure percentage may be set via
kayobe_max_fail_percentage, and individual playbooks may define a max
failure percentage via <playbook>_max_fail_percentage.

Related Kolla Ansible patch:
https://review.opendev.org/c/openstack/kolla-ansible/+/805598

Change-Id: Ib81c72b63be5765cca664c38141ffc769640cf07
---
 ansible/apparmor-libvirt.yml                  |  5 ++
 ansible/apt.yml                               |  5 ++
 ansible/baremetal-compute-inspect.yml         |  5 ++
 ...emetal-compute-introspection-data-save.yml |  5 ++
 ansible/baremetal-compute-manage.yml          |  5 ++
 ansible/baremetal-compute-provide.yml         |  5 ++
 ansible/baremetal-compute-register.yml        |  5 ++
 ansible/baremetal-compute-rename.yml          |  5 ++
 ansible/baremetal-compute-serial-console.yml  |  5 ++
 ansible/compute-libvirt-host.yml              |  5 ++
 ansible/compute-node-discovery.yml            |  5 ++
 ansible/dell-compute-node-boot-mode.yml       |  5 ++
 ansible/dell-compute-node-discovery.yml       |  5 ++
 ansible/dell-compute-node-inventory.yml       |  5 ++
 ansible/dev-tools.yml                         |  5 ++
 ansible/disable-cloud-init.yml                |  5 ++
 ansible/disable-glean.yml                     |  5 ++
 ansible/dnf.yml                               |  5 ++
 ansible/docker.yml                            |  5 ++
 ansible/drac-bios.yml                         |  4 ++
 ansible/drac-boot-order.yml                   |  4 ++
 ansible/drac-facts.yml                        |  4 ++
 ansible/dump-config.yml                       |  4 ++
 ansible/etc-hosts.yml                         |  5 ++
 ansible/firewall.yml                          |  5 ++
 ansible/host-command-run.yml                  |  6 ++-
 ansible/host-package-update.yml               |  4 ++
 ansible/infra-vm-deprovision.yml              |  8 +++
 ansible/infra-vm-provision.yml                |  8 +++
 ansible/ip-allocation.yml                     |  5 ++
 ansible/ip-routing.yml                        |  5 ++
 ansible/kayobe-ansible-user.yml               | 53 +++++++++++--------
 ansible/kayobe-target-venv.yml                |  5 ++
 ansible/kolla-ansible-user.yml                |  5 ++
 ansible/kolla-bifrost-hostvars.yml            |  4 ++
 ansible/kolla-packages.yml                    |  5 ++
 ansible/kolla-pip.yml                         |  5 ++
 ansible/kolla-target-venv.yml                 |  5 ++
 ansible/logging.yml                           |  5 ++
 ansible/luks.yml                              |  5 ++
 ansible/lvm.yml                               |  5 ++
 ansible/mdadm.yml                             |  5 ++
 ansible/network-connectivity.yml              |  4 ++
 ansible/network.yml                           |  5 ++
 ansible/overcloud-bios-raid.yml               | 12 +++++
 ansible/overcloud-deprovision.yml             |  4 ++
 ansible/overcloud-etc-hosts-fixup.yml         |  8 +++
 ansible/overcloud-facts-gather.yml            |  4 ++
 ansible/overcloud-hardware-inspect.yml        |  4 ++
 ansible/overcloud-introspection-data-save.yml |  4 ++
 ansible/overcloud-provision.yml               |  4 ++
 ansible/overcloud-service-config-save.yml     |  4 ++
 ansible/physical-network.yml                  | 32 +++++++++++
 ansible/pip.yml                               |  5 ++
 ansible/proxy.yml                             |  5 ++
 ansible/selinux.yml                           |  5 ++
 ansible/ssh-known-host.yml                    |  5 ++
 ansible/swap.yml                              |  5 ++
 ansible/swift-block-devices.yml               |  4 ++
 ansible/sysctl.yml                            |  5 ++
 ansible/time.yml                              | 10 ++++
 ansible/tuned.yml                             |  5 ++
 ansible/users.yml                             |  5 ++
 ansible/vgpu.yml                              | 15 ++++++
 ansible/wipe-disks.yml                        |  5 ++
 .../configuration/reference/ansible.rst       | 27 ++++++++++
 etc/kayobe/globals.yml                        |  4 ++
 .../max-fail-percentage-5f1d21bdcd138695.yaml |  6 +++
 68 files changed, 432 insertions(+), 23 deletions(-)
 create mode 100644 releasenotes/notes/max-fail-percentage-5f1d21bdcd138695.yaml

diff --git a/ansible/apparmor-libvirt.yml b/ansible/apparmor-libvirt.yml
index 59cc6f39..b5146057 100644
--- a/ansible/apparmor-libvirt.yml
+++ b/ansible/apparmor-libvirt.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure AppArmor is disabled for containerised libvirt
   hosts: compute
+  max_fail_percentage: >-
+    {{ apparmor_libvirt_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - apparmor-libvirt
   vars:
diff --git a/ansible/apt.yml b/ansible/apt.yml
index 08146bce..0172249e 100644
--- a/ansible/apt.yml
+++ b/ansible/apt.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure APT is configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ apt_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_python_interpreter: /usr/bin/python3
   tags:
diff --git a/ansible/baremetal-compute-inspect.yml b/ansible/baremetal-compute-inspect.yml
index 87e3eb58..c6ec1558 100644
--- a/ansible/baremetal-compute-inspect.yml
+++ b/ansible/baremetal-compute-inspect.yml
@@ -20,6 +20,11 @@
 - name: Ensure baremetal compute nodes are inspected in ironic
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_inspect_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     controller_host: "{{ groups['controllers'][0] }}"
     venv: "{{ virtualenv_path }}/openstacksdk"
diff --git a/ansible/baremetal-compute-introspection-data-save.yml b/ansible/baremetal-compute-introspection-data-save.yml
index 4c63d10c..85f17ce3 100644
--- a/ansible/baremetal-compute-introspection-data-save.yml
+++ b/ansible/baremetal-compute-introspection-data-save.yml
@@ -18,6 +18,11 @@
 - name: Ensure the baremetal compute nodes' hardware introspection data is saved
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_introspection_data_save_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstack-cli"
     controller_host: "{{ groups['controllers'][0] }}"
diff --git a/ansible/baremetal-compute-manage.yml b/ansible/baremetal-compute-manage.yml
index 66eb05a7..265f964e 100644
--- a/ansible/baremetal-compute-manage.yml
+++ b/ansible/baremetal-compute-manage.yml
@@ -20,6 +20,11 @@
 - name: Ensure baremetal compute nodes are manageable in ironic
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_manage_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstacksdk"
     ansible_python_interpreter: "{{ venv }}/bin/python"
diff --git a/ansible/baremetal-compute-provide.yml b/ansible/baremetal-compute-provide.yml
index 5fb1d801..e69a3d4d 100644
--- a/ansible/baremetal-compute-provide.yml
+++ b/ansible/baremetal-compute-provide.yml
@@ -20,6 +20,11 @@
 - name: Ensure baremetal compute nodes are available in ironic
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_provide_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstacksdk"
     ansible_python_interpreter: "{{ venv }}/bin/python"
diff --git a/ansible/baremetal-compute-register.yml b/ansible/baremetal-compute-register.yml
index f233e09b..0dedfe52 100644
--- a/ansible/baremetal-compute-register.yml
+++ b/ansible/baremetal-compute-register.yml
@@ -18,6 +18,11 @@
 - name: Ensure baremetal compute nodes are registered in ironic
   hosts: baremetal-compute
   gather_facts: false
+  max_fail_percentage: >-
+    {{ baremetal_compute_register_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - baremetal
   vars:
diff --git a/ansible/baremetal-compute-rename.yml b/ansible/baremetal-compute-rename.yml
index 3184fa25..d1ec5ddf 100644
--- a/ansible/baremetal-compute-rename.yml
+++ b/ansible/baremetal-compute-rename.yml
@@ -21,6 +21,11 @@
 - name: Rename baremetal compute nodes
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_rename_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstack-cli"
     controller_host: "{{ groups['controllers'][0] }}"
diff --git a/ansible/baremetal-compute-serial-console.yml b/ansible/baremetal-compute-serial-console.yml
index 9b5e1abc..d6fedccb 100644
--- a/ansible/baremetal-compute-serial-console.yml
+++ b/ansible/baremetal-compute-serial-console.yml
@@ -53,6 +53,11 @@
 - name: Enable serial console
   hosts: "{{ console_compute_node_limit | default('baremetal-compute') }}"
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_serial_console_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstack-cli"
     controller_host: "{{ groups['controllers'][0] }}"
diff --git a/ansible/compute-libvirt-host.yml b/ansible/compute-libvirt-host.yml
index 9b3900c2..99f8988e 100644
--- a/ansible/compute-libvirt-host.yml
+++ b/ansible/compute-libvirt-host.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure the libvirt daemon is configured
   hosts: compute
+  max_fail_percentage: >-
+    {{ compute_libvirt_host_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - libvirt-host
   tasks:
diff --git a/ansible/compute-node-discovery.yml b/ansible/compute-node-discovery.yml
index 539d6ae4..84f8bca3 100644
--- a/ansible/compute-node-discovery.yml
+++ b/ansible/compute-node-discovery.yml
@@ -4,6 +4,11 @@
 - name: Ensure baremetal compute nodes are PXE booted
   hosts: baremetal-compute
   gather_facts: no
+  max_fail_percentage: >-
+    {{ compute_node_discovery_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     controller_host: "{{ groups['controllers'][0] }}"
   tasks:
diff --git a/ansible/dell-compute-node-boot-mode.yml b/ansible/dell-compute-node-boot-mode.yml
index 67152a37..89a0d098 100644
--- a/ansible/dell-compute-node-boot-mode.yml
+++ b/ansible/dell-compute-node-boot-mode.yml
@@ -7,6 +7,11 @@
 - name: Ensure Dell baremetal compute nodes boot mode is set
   hosts: baremetal-compute
   gather_facts: no
+  max_fail_percentage: >-
+    {{ dell_compute_node_boot_mode_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Set this to the required boot mode. One of 'bios' or 'uefi'.
     drac_boot_mode: "bios"
diff --git a/ansible/dell-compute-node-discovery.yml b/ansible/dell-compute-node-discovery.yml
index 68365944..dbd12549 100644
--- a/ansible/dell-compute-node-discovery.yml
+++ b/ansible/dell-compute-node-discovery.yml
@@ -7,6 +7,11 @@
 - name: Ensure Dell baremetal compute nodes are PXE booted
   hosts: baremetal-compute
   gather_facts: no
+  max_fail_percentage: >-
+    {{ dell_compute_node_discovery_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Set this to the index of the inteface on which to enable PXE.
     drac_pxe_interface: 1
diff --git a/ansible/dell-compute-node-inventory.yml b/ansible/dell-compute-node-inventory.yml
index bc6f852d..68423d1c 100644
--- a/ansible/dell-compute-node-inventory.yml
+++ b/ansible/dell-compute-node-inventory.yml
@@ -28,6 +28,11 @@
 - name: Ensure Dell baremetal compute nodes are present in the Ansible inventory
   hosts: baremetal-compute
   gather_facts: no
+  max_fail_percentage: >-
+    {{ dell_compute_node_inventory_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     compute_node_limit: ""
     compute_node_limit_list: "{{ compute_node_limit.split(':') }}"
diff --git a/ansible/dev-tools.yml b/ansible/dev-tools.yml
index 88617af1..5f6f3ed3 100644
--- a/ansible/dev-tools.yml
+++ b/ansible/dev-tools.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure development tools are installed
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ dev_tools_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - dev-tools
   roles:
diff --git a/ansible/disable-cloud-init.yml b/ansible/disable-cloud-init.yml
index 033c7a60..7b5bf853 100644
--- a/ansible/disable-cloud-init.yml
+++ b/ansible/disable-cloud-init.yml
@@ -5,6 +5,11 @@
 # and cause some issues in network configuration
 - name: Disable Cloud-init service
   hosts: overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ disable_cloud_init_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - disable-cloud-init
   roles:
diff --git a/ansible/disable-glean.yml b/ansible/disable-glean.yml
index bf583ebd..e2f35181 100644
--- a/ansible/disable-glean.yml
+++ b/ansible/disable-glean.yml
@@ -4,6 +4,11 @@
 # interfaces. In some cases this can lead to timeouts.
 - name: Ensure Glean is disabled and its artifacts are removed
   hosts: seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ disable_glean_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - disable-glean
   roles:
diff --git a/ansible/dnf.yml b/ansible/dnf.yml
index bb5aafb3..73999b91 100644
--- a/ansible/dnf.yml
+++ b/ansible/dnf.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure DNF repos are configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ dnf_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_python_interpreter: /usr/bin/python3
   tags:
diff --git a/ansible/docker.yml b/ansible/docker.yml
index efcdd3f8..3ebffde9 100644
--- a/ansible/docker.yml
+++ b/ansible/docker.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure docker is configured
   hosts: docker
+  max_fail_percentage: >-
+    {{ docker_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - docker
   tasks:
diff --git a/ansible/drac-bios.yml b/ansible/drac-bios.yml
index a832f2ea..e7ce85e9 100644
--- a/ansible/drac-bios.yml
+++ b/ansible/drac-bios.yml
@@ -2,6 +2,10 @@
 - name: Ensure that overcloud nodes' BIOS are configured
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ drac_bios_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     bios_config:
       OneTimeBootMode: "OneTimeBootSeq"
diff --git a/ansible/drac-boot-order.yml b/ansible/drac-boot-order.yml
index 52d12ba1..2ba51182 100644
--- a/ansible/drac-boot-order.yml
+++ b/ansible/drac-boot-order.yml
@@ -2,6 +2,10 @@
 - name: Ensure that overcloud nodes' boot order is configured
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ drac_boot_order_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_host: "{{ ipmi_address }}"
     ansible_user: "{{ ipmi_username }}"
diff --git a/ansible/drac-facts.yml b/ansible/drac-facts.yml
index 23507951..41bf7e48 100644
--- a/ansible/drac-facts.yml
+++ b/ansible/drac-facts.yml
@@ -2,6 +2,10 @@
 - name: Gather and display BIOS and RAID facts from iDRACs
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ drac_facts_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     # The role simply pulls in the drac_facts module.
     - role: stackhpc.drac-facts
diff --git a/ansible/dump-config.yml b/ansible/dump-config.yml
index 94d134ad..46ebc089 100644
--- a/ansible/dump-config.yml
+++ b/ansible/dump-config.yml
@@ -8,6 +8,10 @@
 - name: Dump configuration from one or more hosts
   hosts: "{{ dump_hosts }}"
   gather_facts: "{{ dump_facts }}"
+  max_fail_percentage: >-
+    {{ dump_config_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - dump-config
   vars:
diff --git a/ansible/etc-hosts.yml b/ansible/etc-hosts.yml
index 270ac64d..941e6c47 100644
--- a/ansible/etc-hosts.yml
+++ b/ansible/etc-hosts.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure /etc/hosts is configured
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ etc_hosts_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - etc-hosts
   tasks:
diff --git a/ansible/firewall.yml b/ansible/firewall.yml
index fd51e383..8455d05f 100644
--- a/ansible/firewall.yml
+++ b/ansible/firewall.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure firewall is configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ firewall_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - config
     - firewall
diff --git a/ansible/host-command-run.yml b/ansible/host-command-run.yml
index b074ca1e..ba5497db 100644
--- a/ansible/host-command-run.yml
+++ b/ansible/host-command-run.yml
@@ -1,7 +1,11 @@
 ---
 - name: Run a command
-  gather_facts: False
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  gather_facts: False
+  max_fail_percentage: >-
+    {{ host_command_run_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tasks:
     - name: Run a command
       shell: "{{ host_command_to_run }}"
diff --git a/ansible/host-package-update.yml b/ansible/host-package-update.yml
index 06bc7307..94b014ba 100644
--- a/ansible/host-package-update.yml
+++ b/ansible/host-package-update.yml
@@ -1,6 +1,10 @@
 ---
 - name: Update host packages
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ host_package_update_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Optionally set this to a list of packages to update. Default behaviour is
     # to update all packages.
diff --git a/ansible/infra-vm-deprovision.yml b/ansible/infra-vm-deprovision.yml
index f350f9e0..b0f978e4 100644
--- a/ansible/infra-vm-deprovision.yml
+++ b/ansible/infra-vm-deprovision.yml
@@ -15,6 +15,10 @@
 
 - name: Ensure defined infra VMs are destroyed
   hosts: hypervisors
+  max_fail_percentage: >-
+    {{ infra_vm_deprovision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - infra-vm-deprovision
   tasks:
@@ -27,6 +31,10 @@
 - name: Set facts about infra VMs
   gather_facts: false
   hosts: "{{ infra_vm_limit | default('infra-vms') }}"
+  max_fail_percentage: >-
+    {{ infra_vm_deprovision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - infra-vm-deprovision
   tasks:
diff --git a/ansible/infra-vm-provision.yml b/ansible/infra-vm-provision.yml
index 28d4e325..d1375472 100644
--- a/ansible/infra-vm-provision.yml
+++ b/ansible/infra-vm-provision.yml
@@ -15,6 +15,10 @@
 
 - name: Ensure defined infra VMs are deployed
   hosts: hypervisors
+  max_fail_percentage: >-
+    {{ infra_vm_provision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - infra-vm-provision
   tasks:
@@ -26,6 +30,10 @@
 - name: Wait for infra VMs to be accessible
   hosts: "{{ infra_vm_limit | default('infra-vms') }}"
   gather_facts: false
+  max_fail_percentage: >-
+    {{ infra_vm_provision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - infra-vm-provision
   tasks:
diff --git a/ansible/ip-allocation.yml b/ansible/ip-allocation.yml
index 81038ed5..b82d934a 100644
--- a/ansible/ip-allocation.yml
+++ b/ansible/ip-allocation.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure IP addresses are allocated
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ ip_allocation_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - ip-allocation
   gather_facts: no
diff --git a/ansible/ip-routing.yml b/ansible/ip-routing.yml
index 452914cc..9aee6ba2 100644
--- a/ansible/ip-routing.yml
+++ b/ansible/ip-routing.yml
@@ -3,6 +3,11 @@
 
 - name: Ensure IP routing is enabled
   hosts: seed-hypervisor:seed
+  max_fail_percentage: >-
+    {{ ip_routing_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - ip-routing
   roles:
diff --git a/ansible/kayobe-ansible-user.yml b/ansible/kayobe-ansible-user.yml
index deb15f35..4f9895f0 100644
--- a/ansible/kayobe-ansible-user.yml
+++ b/ansible/kayobe-ansible-user.yml
@@ -9,6 +9,11 @@
 - name: Determine whether user bootstrapping is required
   hosts: seed-hypervisor:seed:overcloud:infra-vms
   gather_facts: false
+  max_fail_percentage: >-
+    {{ kayobe_ansible_user_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kayobe-ansible-user
   tasks:
@@ -31,15 +36,24 @@
           attempting bootstrap
       when: ssh_result.unreachable | default(false)
 
-- name: Ensure python is installed
+- name: Ensure python is installed and the Kayobe Ansible user account exists
   hosts: kayobe_user_bootstrap_required_True
   gather_facts: no
+  max_fail_percentage: >-
+    {{ kayobe_ansible_user_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_user: "{{ bootstrap_user }}"
+    # We can't assume that a virtualenv exists at this point, so use the system
+    # python interpreter.
+    ansible_python_interpreter: /usr/bin/python3
     dnf_options:
       - "-y"
       - "{% if 'proxy' in dnf_config %}--setopt=proxy={{ dnf_config['proxy'] }}{% endif %}"
   tags:
+    - kayobe-ansible-user
     - ensure-python
   tasks:
     - name: Check if python is installed
@@ -53,29 +67,19 @@
       raw: "test -e /usr/bin/apt && (sudo apt -y update && sudo apt install -y python3-minimal) || (sudo dnf {{ dnf_options | select | join(' ') }} install python3)"
       when: check_python.rc != 0
 
-- name: Ensure the Kayobe Ansible user account exists
-  hosts: kayobe_user_bootstrap_required_True
-  gather_facts: false
-  tags:
-    - kayobe-ansible-user
-  vars:
-    ansible_user: "{{ bootstrap_user }}"
-    # We can't assume that a virtualenv exists at this point, so use the system
-    # python interpreter.
-    ansible_python_interpreter: /usr/bin/python3
-  roles:
-    - role: singleplatform-eng.users
-      groups_to_create: "{{ [{'name': 'docker'}] if 'docker' in group_names else [] }}"
-      users:
-        - username: "{{ kayobe_ansible_user }}"
-          name: Kayobe deployment user
-          groups: "{{ ['docker'] if 'docker' in group_names else [] }}"
-          append: True
-          ssh_key:
-            - "{{ lookup('file', ssh_public_key_path) }}"
+    - import_role:
+        name: singleplatform-eng.users
+      vars:
+        groups_to_create: "{{ [{'name': 'docker'}] if 'docker' in group_names else [] }}"
+        users:
+          - username: "{{ kayobe_ansible_user }}"
+            name: Kayobe deployment user
+            groups: "{{ ['docker'] if 'docker' in group_names else [] }}"
+            append: True
+            ssh_key:
+              - "{{ lookup('file', ssh_public_key_path) }}"
       become: True
 
-  post_tasks:
     - name: Ensure the Kayobe Ansible user has passwordless sudo
       copy:
         content: "{{ kayobe_ansible_user }} ALL=(ALL) NOPASSWD: ALL"
@@ -86,6 +90,11 @@
 - name: Verify that the Kayobe Ansible user account is accessible
   hosts: seed-hypervisor:seed:overcloud:infra-vms
   gather_facts: false
+  max_fail_percentage: >-
+    {{ kayobe_ansible_user_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kayobe-ansible-user
   vars:
diff --git a/ansible/kayobe-target-venv.yml b/ansible/kayobe-target-venv.yml
index 8d51ee44..2fc5a942 100644
--- a/ansible/kayobe-target-venv.yml
+++ b/ansible/kayobe-target-venv.yml
@@ -5,6 +5,11 @@
 - name: Ensure a virtualenv exists for kayobe
   hosts: seed:seed-hypervisor:overcloud:infra-vms
   gather_facts: False
+  max_fail_percentage: >-
+    {{ kayobe_target_venv_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kayobe-target-venv
   tasks:
diff --git a/ansible/kolla-ansible-user.yml b/ansible/kolla-ansible-user.yml
index e9802d87..90c87b1f 100644
--- a/ansible/kolla-ansible-user.yml
+++ b/ansible/kolla-ansible-user.yml
@@ -2,6 +2,11 @@
 - name: Ensure the Kolla Ansible user account exists
   hosts: seed:overcloud
   gather_facts: false
+  max_fail_percentage: >-
+    {{ kolla_ansible_user_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kolla-ansible
     - kolla-ansible-user
diff --git a/ansible/kolla-bifrost-hostvars.yml b/ansible/kolla-bifrost-hostvars.yml
index b4ab4fb4..9fbed026 100644
--- a/ansible/kolla-bifrost-hostvars.yml
+++ b/ansible/kolla-bifrost-hostvars.yml
@@ -4,6 +4,10 @@
 - name: Ensure the Bifrost overcloud inventory is populated
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ kolla_bifrost_hostvars_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kolla-bifrost-hostvars
   become: true
diff --git a/ansible/kolla-packages.yml b/ansible/kolla-packages.yml
index 9b746998..54425e09 100644
--- a/ansible/kolla-packages.yml
+++ b/ansible/kolla-packages.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure Kolla Ansible packages are installed
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ kolla_packages_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kolla-packages
   vars:
diff --git a/ansible/kolla-pip.yml b/ansible/kolla-pip.yml
index 35a33805..e6c74c38 100644
--- a/ansible/kolla-pip.yml
+++ b/ansible/kolla-pip.yml
@@ -2,6 +2,11 @@
 - name: Configure local PyPi mirror for Kolla Ansible
   hosts: seed:overcloud
   gather_facts: false
+  max_fail_percentage: >-
+    {{ kolla_pip_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kolla-ansible
     - kolla-pip
diff --git a/ansible/kolla-target-venv.yml b/ansible/kolla-target-venv.yml
index 37305d3a..760813f2 100644
--- a/ansible/kolla-target-venv.yml
+++ b/ansible/kolla-target-venv.yml
@@ -5,6 +5,11 @@
 - name: Ensure a virtualenv exists for kolla-ansible
   hosts: seed:overcloud
   gather_facts: False
+  max_fail_percentage: >-
+    {{ kolla_target_venv_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # kolla_overcloud_inventory_top_level_group_map looks like:
     # kolla_overcloud_inventory_top_level_group_map:
diff --git a/ansible/logging.yml b/ansible/logging.yml
index ddc584af..326fe66d 100644
--- a/ansible/logging.yml
+++ b/ansible/logging.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure Logging configuration is applied
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ logging_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   gather_facts: false
   vars:
     # NOTE(wszumski): Kayobe target env does not yet exist.
diff --git a/ansible/luks.yml b/ansible/luks.yml
index 8dea1146..57e4796f 100644
--- a/ansible/luks.yml
+++ b/ansible/luks.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure encryption configuration is applied
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ luks_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - luks
   tasks:
diff --git a/ansible/lvm.yml b/ansible/lvm.yml
index 53a176ed..3d46edad 100644
--- a/ansible/lvm.yml
+++ b/ansible/lvm.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure LVM configuration is applied
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ lvm_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - lvm
     - upgrade-check
diff --git a/ansible/mdadm.yml b/ansible/mdadm.yml
index 6b907777..14338dd0 100644
--- a/ansible/mdadm.yml
+++ b/ansible/mdadm.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure software RAID configuration is applied
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ mdadm_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - mdadm
   roles:
diff --git a/ansible/network-connectivity.yml b/ansible/network-connectivity.yml
index ccd95230..745032aa 100644
--- a/ansible/network-connectivity.yml
+++ b/ansible/network-connectivity.yml
@@ -1,6 +1,10 @@
 ---
 - name: Check network connectivity between hosts
   hosts: seed:seed-hypervisor:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ network_connectivity_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Set this to an external IP address to check.
     nc_external_ip: 8.8.8.8
diff --git a/ansible/network.yml b/ansible/network.yml
index 51d93b91..c5b99ad0 100644
--- a/ansible/network.yml
+++ b/ansible/network.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure networking is configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ network_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - config
     - network
diff --git a/ansible/overcloud-bios-raid.yml b/ansible/overcloud-bios-raid.yml
index efa25ad4..4f4199e1 100644
--- a/ansible/overcloud-bios-raid.yml
+++ b/ansible/overcloud-bios-raid.yml
@@ -9,6 +9,10 @@
 - name: Group overcloud nodes by their BMC type
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ overcloud_bios_raid_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - bios
     - raid
@@ -35,6 +39,10 @@
 - name: Check whether any changes to nodes' BIOS and RAID configuration are required
   hosts: overcloud_with_bmcs_of_type_idrac
   gather_facts: no
+  max_fail_percentage: >-
+    {{ overcloud_bios_raid_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - bios
     - raid
@@ -57,6 +65,10 @@
 - name: Ensure that overcloud BIOS and RAID volumes are configured
   hosts: overcloud_with_bmcs_of_type_idrac
   gather_facts: no
+  max_fail_percentage: >-
+    {{ overcloud_bios_raid_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - bios
     - raid
diff --git a/ansible/overcloud-deprovision.yml b/ansible/overcloud-deprovision.yml
index 824f8ccb..40e467f9 100644
--- a/ansible/overcloud-deprovision.yml
+++ b/ansible/overcloud-deprovision.yml
@@ -3,6 +3,10 @@
 
 - name: Ensure the overcloud nodes are deprovisioned
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_deprovision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - deprovision
   vars:
diff --git a/ansible/overcloud-etc-hosts-fixup.yml b/ansible/overcloud-etc-hosts-fixup.yml
index 49b0bb3b..c20ffab7 100644
--- a/ansible/overcloud-etc-hosts-fixup.yml
+++ b/ansible/overcloud-etc-hosts-fixup.yml
@@ -9,6 +9,10 @@
 
 - name: Ensure overcloud hosts' /etc/hosts does not contain incorrect IPs
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_etc_hosts_fixup_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - etc-hosts-fixup
   tasks:
@@ -25,6 +29,10 @@
 
 - name: Ensure rabbitmq containers' /etc/hosts does not contain incorrect IPs
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_etc_hosts_fixup_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - etc-hosts-fixup
   vars:
diff --git a/ansible/overcloud-facts-gather.yml b/ansible/overcloud-facts-gather.yml
index a8f101af..98beb7d6 100644
--- a/ansible/overcloud-facts-gather.yml
+++ b/ansible/overcloud-facts-gather.yml
@@ -1,6 +1,10 @@
 ---
 - name: Gather facts
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_facts_gather_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   gather_facts: false
   tasks:
     - name: Gather facts
diff --git a/ansible/overcloud-hardware-inspect.yml b/ansible/overcloud-hardware-inspect.yml
index 3477b7c1..dd383411 100644
--- a/ansible/overcloud-hardware-inspect.yml
+++ b/ansible/overcloud-hardware-inspect.yml
@@ -3,6 +3,10 @@
 
 - name: Ensure the overcloud nodes' hardware is inspected
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_hardware_inspect_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - hardware-inspect
   vars:
diff --git a/ansible/overcloud-introspection-data-save.yml b/ansible/overcloud-introspection-data-save.yml
index 945567b8..1a95752d 100644
--- a/ansible/overcloud-introspection-data-save.yml
+++ b/ansible/overcloud-introspection-data-save.yml
@@ -1,6 +1,10 @@
 ---
 - name: Ensure the overcloud nodes' hardware introspection data is saved
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_introspection_data_save_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     seed_host: "{{ groups['seed'][0] }}"
     # Override this to save results to another location.
diff --git a/ansible/overcloud-provision.yml b/ansible/overcloud-provision.yml
index 20ef8be9..72a54613 100644
--- a/ansible/overcloud-provision.yml
+++ b/ansible/overcloud-provision.yml
@@ -3,6 +3,10 @@
 
 - name: Ensure the overcloud nodes are provisioned
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_provision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - provision
   vars:
diff --git a/ansible/overcloud-service-config-save.yml b/ansible/overcloud-service-config-save.yml
index 3062fcdb..7b1772da 100644
--- a/ansible/overcloud-service-config-save.yml
+++ b/ansible/overcloud-service-config-save.yml
@@ -1,6 +1,10 @@
 ---
 - name: Save overcloud service configuration
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_service_config_save_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - service-config-save
   vars:
diff --git a/ansible/physical-network.yml b/ansible/physical-network.yml
index fed79857..45600e5d 100644
--- a/ansible/physical-network.yml
+++ b/ansible/physical-network.yml
@@ -5,6 +5,10 @@
 - name: Group hosts by their switch type and apply configuration filters
   hosts: switches
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Set this variable to True to configure the network for hardware
     # discovery.
@@ -90,6 +94,10 @@
 - name: Display switch configuration
   hosts: switches_in_display_mode_True
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tasks:
     - name: Display the candidate global switch configuration
       debug:
@@ -102,6 +110,10 @@
 - name: Ensure Arista physical switches are configured
   hosts: switches_of_type_arista:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -114,6 +126,10 @@
 - name: Ensure DellOS physical switches are configured
   hosts: switches_of_type_dellos6:switches_of_type_dellos9:switches_of_type_dellos10:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -127,6 +143,10 @@
 - name: Ensure Dell PowerConnect physical switches are configured
   hosts: switches_of_type_dell-powerconnect:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -139,6 +159,10 @@
 - name: Ensure Juniper physical switches are configured
   hosts: switches_of_type_junos:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -152,6 +176,10 @@
 - name: Ensure Mellanox physical switches are configured
   hosts: switches_of_type_mellanox:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -164,6 +192,10 @@
 - name: Ensure Cumulus physical switches are configured with NCLU
   hosts: switches_of_type_nclu:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
diff --git a/ansible/pip.yml b/ansible/pip.yml
index f9ca0377..3bea4a70 100644
--- a/ansible/pip.yml
+++ b/ansible/pip.yml
@@ -1,6 +1,11 @@
 ---
 - name: Configure local PyPi mirror
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ pip_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - pip
   vars:
diff --git a/ansible/proxy.yml b/ansible/proxy.yml
index e618b9c0..1f39eb6d 100644
--- a/ansible/proxy.yml
+++ b/ansible/proxy.yml
@@ -1,5 +1,10 @@
 - name: Configure HTTP(S) proxy settings
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ proxy_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_python_interpreter: /usr/bin/python3
   tags:
diff --git a/ansible/selinux.yml b/ansible/selinux.yml
index a03e67a2..aa00d4ce 100644
--- a/ansible/selinux.yml
+++ b/ansible/selinux.yml
@@ -1,6 +1,11 @@
 ---
 - name: Configure SELinux state and reboot if required
   hosts: seed:seed-hypervisor:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ selinux_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - selinux
   roles:
diff --git a/ansible/ssh-known-host.yml b/ansible/ssh-known-host.yml
index bb4ae311..a13ffba3 100644
--- a/ansible/ssh-known-host.yml
+++ b/ansible/ssh-known-host.yml
@@ -2,6 +2,11 @@
 - name: Ensure known hosts are configured
   hosts: all
   gather_facts: no
+  max_fail_percentage: >-
+    {{ ssh_known_host_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - ssh-known-host
   roles:
diff --git a/ansible/swap.yml b/ansible/swap.yml
index 5402e38e..82ccbba8 100644
--- a/ansible/swap.yml
+++ b/ansible/swap.yml
@@ -2,6 +2,11 @@
 - name: Configure swap
   hosts: seed-hypervisor:seed:overcloud:infra-vms
   become: true
+  max_fail_percentage: >-
+    {{ swap_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - swap
   tasks:
diff --git a/ansible/swift-block-devices.yml b/ansible/swift-block-devices.yml
index c476a7e3..306a13d1 100644
--- a/ansible/swift-block-devices.yml
+++ b/ansible/swift-block-devices.yml
@@ -1,6 +1,10 @@
 ---
 - name: Ensure Swift block devices are prepared
   hosts: "{{ swift_hosts }}"
+  max_fail_percentage: >-
+    {{ swift_block_devices_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     swift_hosts: storage
   tags:
diff --git a/ansible/sysctl.yml b/ansible/sysctl.yml
index 0f4ad743..cf2a2793 100644
--- a/ansible/sysctl.yml
+++ b/ansible/sysctl.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure sysctl parameters are configured
   hosts: seed:seed-hypervisor:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ sysctl_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - sysctl
   roles:
diff --git a/ansible/time.yml b/ansible/time.yml
index 18a18d94..2c02e5bb 100644
--- a/ansible/time.yml
+++ b/ansible/time.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure timezone is configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ time_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - timezone
   tasks:
@@ -24,6 +29,11 @@
 
 - name: Ensure Chrony is installed and configured
   hosts: ntp
+  max_fail_percentage: >-
+    {{ time_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - ntp
   tasks:
diff --git a/ansible/tuned.yml b/ansible/tuned.yml
index 9553da7c..69c5ba8d 100644
--- a/ansible/tuned.yml
+++ b/ansible/tuned.yml
@@ -1,6 +1,11 @@
 ---
 - name: Configure tuned profile
   hosts: seed:seed-hypervisor:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ tuned_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - tuned
   roles:
diff --git a/ansible/users.yml b/ansible/users.yml
index 6a89b769..6afc1fd5 100644
--- a/ansible/users.yml
+++ b/ansible/users.yml
@@ -1,6 +1,11 @@
 ---
 - name: Ensure users exist
   hosts: seed:seed-hypervisor:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ users_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - users
   roles:
diff --git a/ansible/vgpu.yml b/ansible/vgpu.yml
index ebda83fa..58a424eb 100644
--- a/ansible/vgpu.yml
+++ b/ansible/vgpu.yml
@@ -1,6 +1,11 @@
 ---
 - name: Configure IOMMU
   hosts: iommu
+  max_fail_percentage: >-
+    {{ vgpu_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - iommu
     - vgpu
@@ -15,6 +20,11 @@
 
 - name: Configure NVIDIA VGPUs
   hosts: vgpu
+  max_fail_percentage: >-
+    {{ vgpu_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - vgpu
   tasks:
@@ -28,6 +38,11 @@
 
 - name: Reboot when required
   hosts: iommu:vgpu
+  max_fail_percentage: >-
+    {{ vgpu_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - reboot
     - iommu
diff --git a/ansible/wipe-disks.yml b/ansible/wipe-disks.yml
index a833f3c1..79fe2edf 100644
--- a/ansible/wipe-disks.yml
+++ b/ansible/wipe-disks.yml
@@ -9,6 +9,11 @@
 
 - name: Ensure that all unmounted block devices are wiped
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ wipe_disks_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - wipe-disks
   tasks:
diff --git a/doc/source/configuration/reference/ansible.rst b/doc/source/configuration/reference/ansible.rst
index 49372428..1d8a6521 100644
--- a/doc/source/configuration/reference/ansible.rst
+++ b/doc/source/configuration/reference/ansible.rst
@@ -146,3 +146,30 @@ Similarly, for Kolla Ansible (notice the similar but different file names):
    :caption: ``$KAYOBE_CONFIG_PATH/kolla/globals.yml``
 
    kolla_ansible_setup_gather_subset: "all,!facter"
+
+Max failure percentage
+======================
+
+It is possible to specify a `maximum failure percentage
+<https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_error_handling.html#setting-a-maximum-failure-percentage>`__
+using ``kayobe_max_fail_percentage``. By default this is undefined, which is
+equivalent to a value of 100, meaning that Ansible will continue execution
+until all hosts have failed or completed. For example:
+
+.. code-block:: yaml
+   :caption: ``$KAYOBE_CONFIG_PATH/globals.yml``
+
+   kayobe_max_fail_percentage: 50
+
+A max fail percentage may be set for the ``kayobe * host configure`` commands
+using ``host_configure_max_fail_percentage``, or for a specific playbook using
+``<playbook>_max_fail_percentage`` where ``<playbook>`` is the playbook name
+with dashes replaced with underscores and without the ``.yml`` extension. For
+example:
+
+.. code-block:: yaml
+   :caption: ``$KAYOBE_CONFIG_PATH/globals.yml``
+
+   kayobe_max_fail_percentage: 50
+   host_configure_max_fail_percentage: 25
+   time_max_fail_percentage: 100
diff --git a/etc/kayobe/globals.yml b/etc/kayobe/globals.yml
index 50af1cb1..a97f74d1 100644
--- a/etc/kayobe/globals.yml
+++ b/etc/kayobe/globals.yml
@@ -64,6 +64,10 @@
 # to not specify a gather subset.
 #kayobe_ansible_setup_gather_subset:
 
+# Global maximum failure percentage. By default this is undefined, which is
+# equivalent to a value of 100.
+#kayobe_max_fail_percentage:
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes
diff --git a/releasenotes/notes/max-fail-percentage-5f1d21bdcd138695.yaml b/releasenotes/notes/max-fail-percentage-5f1d21bdcd138695.yaml
new file mode 100644
index 00000000..c0768f2c
--- /dev/null
+++ b/releasenotes/notes/max-fail-percentage-5f1d21bdcd138695.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds support for setting the max fail percentage for Ansible plays via
+    ``kayobe_max_fail_percentage``. It can also be set on a per-playbook basis,
+    e.g. ``time_max_fail_percentage``.
-- 
GitLab