Introduce max fail percentage to playbooks

This allows us to continue execution until a certain proportion of hosts
fail. This can be useful at scale, where failures are common, and
restarting a deployment is time-consuming.

The default max failure percentage is 100, keeping the default
behaviour. A global max failure percentage may be set via
kayobe_max_fail_percentage, and individual playbooks may define a max
failure percentage via <playbook>_max_fail_percentage.

Related Kolla Ansible patch:
https://review.opendev.org/c/openstack/kolla-ansible/+/805598

Change-Id: Ib81c72b63be5765cca664c38141ffc769640cf07

ansible/drac-boot-order.yml

@@ -2,6 +2,10 @@
 - name: Ensure that overcloud nodes' boot order is configured
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ drac_boot_order_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_host: "{{ ipmi_address }}"
     ansible_user: "{{ ipmi_username }}"

ansible/drac-facts.yml

@@ -2,6 +2,10 @@
 - name: Gather and display BIOS and RAID facts from iDRACs
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ drac_facts_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     # The role simply pulls in the drac_facts module.
     - role: stackhpc.drac-facts

ansible/dump-config.yml

@@ -8,6 +8,10 @@
 - name: Dump configuration from one or more hosts
   hosts: "{{ dump_hosts }}"
   gather_facts: "{{ dump_facts }}"
+  max_fail_percentage: >-
+    {{ dump_config_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - dump-config
   vars:

ansible/etc-hosts.yml

 ---
 - name: Ensure /etc/hosts is configured
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ etc_hosts_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - etc-hosts
   tasks:

ansible/firewall.yml

 ---
 - name: Ensure firewall is configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ firewall_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - config
     - firewall

ansible/host-command-run.yml

 ---
 - name: Run a command
-  gather_facts: False
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  gather_facts: False
+  max_fail_percentage: >-
+    {{ host_command_run_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tasks:
     - name: Run a command
       shell: "{{ host_command_to_run }}"

ansible/host-package-update.yml

 ---
 - name: Update host packages
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ host_package_update_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Optionally set this to a list of packages to update. Default behaviour is
     # to update all packages.

ansible/infra-vm-deprovision.yml

@@ -15,6 +15,10 @@
 
 - name: Ensure defined infra VMs are destroyed
   hosts: hypervisors
+  max_fail_percentage: >-
+    {{ infra_vm_deprovision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - infra-vm-deprovision
   tasks:
@@ -27,6 +31,10 @@
 - name: Set facts about infra VMs
   gather_facts: false
   hosts: "{{ infra_vm_limit | default('infra-vms') }}"
+  max_fail_percentage: >-
+    {{ infra_vm_deprovision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - infra-vm-deprovision
   tasks:

ansible/infra-vm-provision.yml

@@ -15,6 +15,10 @@
 
 - name: Ensure defined infra VMs are deployed
   hosts: hypervisors
+  max_fail_percentage: >-
+    {{ infra_vm_provision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - infra-vm-provision
   tasks:
@@ -26,6 +30,10 @@
 - name: Wait for infra VMs to be accessible
   hosts: "{{ infra_vm_limit | default('infra-vms') }}"
   gather_facts: false
+  max_fail_percentage: >-
+    {{ infra_vm_provision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - infra-vm-provision
   tasks:

ansible/ip-allocation.yml

 ---
 - name: Ensure IP addresses are allocated
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ ip_allocation_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - ip-allocation
   gather_facts: no

ansible/ip-routing.yml

@@ -3,6 +3,11 @@
 
 - name: Ensure IP routing is enabled
   hosts: seed-hypervisor:seed
+  max_fail_percentage: >-
+    {{ ip_routing_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - ip-routing
   roles:

ansible/kayobe-ansible-user.yml

@@ -9,6 +9,11 @@
 - name: Determine whether user bootstrapping is required
   hosts: seed-hypervisor:seed:overcloud:infra-vms
   gather_facts: false
+  max_fail_percentage: >-
+    {{ kayobe_ansible_user_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kayobe-ansible-user
   tasks:
@@ -31,15 +36,24 @@
           attempting bootstrap
       when: ssh_result.unreachable | default(false)
 
-- name: Ensure python is installed
+- name: Ensure python is installed and the Kayobe Ansible user account exists
   hosts: kayobe_user_bootstrap_required_True
   gather_facts: no
+  max_fail_percentage: >-
+    {{ kayobe_ansible_user_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_user: "{{ bootstrap_user }}"
+    # We can't assume that a virtualenv exists at this point, so use the system
+    # python interpreter.
+    ansible_python_interpreter: /usr/bin/python3
     dnf_options:
       - "-y"
       - "{% if 'proxy' in dnf_config %}--setopt=proxy={{ dnf_config['proxy'] }}{% endif %}"
   tags:
+    - kayobe-ansible-user
     - ensure-python
   tasks:
     - name: Check if python is installed
@@ -53,29 +67,19 @@
       raw: "test -e /usr/bin/apt && (sudo apt -y update && sudo apt install -y python3-minimal) || (sudo dnf {{ dnf_options | select | join(' ') }} install python3)"
       when: check_python.rc != 0
 
-- name: Ensure the Kayobe Ansible user account exists
-  hosts: kayobe_user_bootstrap_required_True
-  gather_facts: false
-  tags:
-    - kayobe-ansible-user
-  vars:
-    ansible_user: "{{ bootstrap_user }}"
-    # We can't assume that a virtualenv exists at this point, so use the system
-    # python interpreter.
-    ansible_python_interpreter: /usr/bin/python3
-  roles:
-    - role: singleplatform-eng.users
-      groups_to_create: "{{ [{'name': 'docker'}] if 'docker' in group_names else [] }}"
-      users:
-        - username: "{{ kayobe_ansible_user }}"
-          name: Kayobe deployment user
-          groups: "{{ ['docker'] if 'docker' in group_names else [] }}"
-          append: True
-          ssh_key:
-            - "{{ lookup('file', ssh_public_key_path) }}"
+    - import_role:
+        name: singleplatform-eng.users
+      vars:
+        groups_to_create: "{{ [{'name': 'docker'}] if 'docker' in group_names else [] }}"
+        users:
+          - username: "{{ kayobe_ansible_user }}"
+            name: Kayobe deployment user
+            groups: "{{ ['docker'] if 'docker' in group_names else [] }}"
+            append: True
+            ssh_key:
+              - "{{ lookup('file', ssh_public_key_path) }}"
       become: True
 
-  post_tasks:
     - name: Ensure the Kayobe Ansible user has passwordless sudo
       copy:
         content: "{{ kayobe_ansible_user }} ALL=(ALL) NOPASSWD: ALL"
@@ -86,6 +90,11 @@
 - name: Verify that the Kayobe Ansible user account is accessible
   hosts: seed-hypervisor:seed:overcloud:infra-vms
   gather_facts: false
+  max_fail_percentage: >-
+    {{ kayobe_ansible_user_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kayobe-ansible-user
   vars:

ansible/kayobe-target-venv.yml

@@ -5,6 +5,11 @@
 - name: Ensure a virtualenv exists for kayobe
   hosts: seed:seed-hypervisor:overcloud:infra-vms
   gather_facts: False
+  max_fail_percentage: >-
+    {{ kayobe_target_venv_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kayobe-target-venv
   tasks:

ansible/kolla-ansible-user.yml

@@ -2,6 +2,11 @@
 - name: Ensure the Kolla Ansible user account exists
   hosts: seed:overcloud
   gather_facts: false
+  max_fail_percentage: >-
+    {{ kolla_ansible_user_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kolla-ansible
     - kolla-ansible-user

ansible/kolla-bifrost-hostvars.yml

@@ -4,6 +4,10 @@
 - name: Ensure the Bifrost overcloud inventory is populated
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ kolla_bifrost_hostvars_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kolla-bifrost-hostvars
   become: true

ansible/kolla-packages.yml

 ---
 - name: Ensure Kolla Ansible packages are installed
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ kolla_packages_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kolla-packages
   vars:

ansible/kolla-pip.yml

@@ -2,6 +2,11 @@
 - name: Configure local PyPi mirror for Kolla Ansible
   hosts: seed:overcloud
   gather_facts: false
+  max_fail_percentage: >-
+    {{ kolla_pip_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kolla-ansible
     - kolla-pip

ansible/kolla-target-venv.yml

@@ -5,6 +5,11 @@
 - name: Ensure a virtualenv exists for kolla-ansible
   hosts: seed:overcloud
   gather_facts: False
+  max_fail_percentage: >-
+    {{ kolla_target_venv_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # kolla_overcloud_inventory_top_level_group_map looks like:
     # kolla_overcloud_inventory_top_level_group_map:

ansible/logging.yml

 ---
 - name: Ensure Logging configuration is applied
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ logging_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   gather_facts: false
   vars:
     # NOTE(wszumski): Kayobe target env does not yet exist.

ansible/luks.yml

 ---
 - name: Ensure encryption configuration is applied
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ luks_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - luks
   tasks: