Introduce max fail percentage to playbooks

This allows us to continue execution until a certain proportion of hosts
fail. This can be useful at scale, where failures are common, and
restarting a deployment is time-consuming.

The default max failure percentage is 100, keeping the default
behaviour. A global max failure percentage may be set via
kayobe_max_fail_percentage, and individual playbooks may define a max
failure percentage via <playbook>_max_fail_percentage.

Related Kolla Ansible patch:
https://review.opendev.org/c/openstack/kolla-ansible/+/805598

Change-Id: Ib81c72b63be5765cca664c38141ffc769640cf07

ansible/lvm.yml

 ---
 - name: Ensure LVM configuration is applied
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ lvm_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - lvm
     - upgrade-check

ansible/mdadm.yml

 ---
 - name: Ensure software RAID configuration is applied
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ mdadm_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - mdadm
   roles:

ansible/network-connectivity.yml

 ---
 - name: Check network connectivity between hosts
   hosts: seed:seed-hypervisor:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ network_connectivity_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Set this to an external IP address to check.
     nc_external_ip: 8.8.8.8

ansible/network.yml

 ---
 - name: Ensure networking is configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ network_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - config
     - network

ansible/overcloud-bios-raid.yml

@@ -9,6 +9,10 @@
 - name: Group overcloud nodes by their BMC type
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ overcloud_bios_raid_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - bios
     - raid
@@ -35,6 +39,10 @@
 - name: Check whether any changes to nodes' BIOS and RAID configuration are required
   hosts: overcloud_with_bmcs_of_type_idrac
   gather_facts: no
+  max_fail_percentage: >-
+    {{ overcloud_bios_raid_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - bios
     - raid
@@ -57,6 +65,10 @@
 - name: Ensure that overcloud BIOS and RAID volumes are configured
   hosts: overcloud_with_bmcs_of_type_idrac
   gather_facts: no
+  max_fail_percentage: >-
+    {{ overcloud_bios_raid_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - bios
     - raid

ansible/overcloud-deprovision.yml

@@ -3,6 +3,10 @@
 
 - name: Ensure the overcloud nodes are deprovisioned
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_deprovision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - deprovision
   vars:

ansible/overcloud-etc-hosts-fixup.yml

@@ -9,6 +9,10 @@
 
 - name: Ensure overcloud hosts' /etc/hosts does not contain incorrect IPs
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_etc_hosts_fixup_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - etc-hosts-fixup
   tasks:
@@ -25,6 +29,10 @@
 
 - name: Ensure rabbitmq containers' /etc/hosts does not contain incorrect IPs
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_etc_hosts_fixup_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - etc-hosts-fixup
   vars:

ansible/overcloud-facts-gather.yml

 ---
 - name: Gather facts
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_facts_gather_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   gather_facts: false
   tasks:
     - name: Gather facts

ansible/overcloud-hardware-inspect.yml

@@ -3,6 +3,10 @@
 
 - name: Ensure the overcloud nodes' hardware is inspected
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_hardware_inspect_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - hardware-inspect
   vars:

ansible/overcloud-introspection-data-save.yml

 ---
 - name: Ensure the overcloud nodes' hardware introspection data is saved
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_introspection_data_save_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     seed_host: "{{ groups['seed'][0] }}"
     # Override this to save results to another location.

ansible/overcloud-provision.yml

@@ -3,6 +3,10 @@
 
 - name: Ensure the overcloud nodes are provisioned
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_provision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - provision
   vars:

ansible/overcloud-service-config-save.yml

 ---
 - name: Save overcloud service configuration
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_service_config_save_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - service-config-save
   vars:

ansible/physical-network.yml

@@ -5,6 +5,10 @@
 - name: Group hosts by their switch type and apply configuration filters
   hosts: switches
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Set this variable to True to configure the network for hardware
     # discovery.
@@ -90,6 +94,10 @@
 - name: Display switch configuration
   hosts: switches_in_display_mode_True
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tasks:
     - name: Display the candidate global switch configuration
       debug:
@@ -102,6 +110,10 @@
 - name: Ensure Arista physical switches are configured
   hosts: switches_of_type_arista:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -114,6 +126,10 @@
 - name: Ensure DellOS physical switches are configured
   hosts: switches_of_type_dellos6:switches_of_type_dellos9:switches_of_type_dellos10:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -127,6 +143,10 @@
 - name: Ensure Dell PowerConnect physical switches are configured
   hosts: switches_of_type_dell-powerconnect:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -139,6 +159,10 @@
 - name: Ensure Juniper physical switches are configured
   hosts: switches_of_type_junos:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -152,6 +176,10 @@
 - name: Ensure Mellanox physical switches are configured
   hosts: switches_of_type_mellanox:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -164,6 +192,10 @@
 - name: Ensure Cumulus physical switches are configured with NCLU
   hosts: switches_of_type_nclu:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 

ansible/pip.yml

 ---
 - name: Configure local PyPi mirror
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ pip_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - pip
   vars:

ansible/proxy.yml

 - name: Configure HTTP(S) proxy settings
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ proxy_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_python_interpreter: /usr/bin/python3
   tags:

ansible/selinux.yml

 ---
 - name: Configure SELinux state and reboot if required
   hosts: seed:seed-hypervisor:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ selinux_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - selinux
   roles:

ansible/ssh-known-host.yml

@@ -2,6 +2,11 @@
 - name: Ensure known hosts are configured
   hosts: all
   gather_facts: no
+  max_fail_percentage: >-
+    {{ ssh_known_host_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - ssh-known-host
   roles:

ansible/swap.yml

@@ -2,6 +2,11 @@
 - name: Configure swap
   hosts: seed-hypervisor:seed:overcloud:infra-vms
   become: true
+  max_fail_percentage: >-
+    {{ swap_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - swap
   tasks:

ansible/swift-block-devices.yml

 ---
 - name: Ensure Swift block devices are prepared
   hosts: "{{ swift_hosts }}"
+  max_fail_percentage: >-
+    {{ swift_block_devices_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     swift_hosts: storage
   tags:

ansible/sysctl.yml

 ---
 - name: Ensure sysctl parameters are configured
   hosts: seed:seed-hypervisor:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ sysctl_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - sysctl
   roles: