MariaDB lights out recovery

This playbook only matters for multinode since AIO can recover from
power outage without additional configuration.

DocImpact
Implements: blueprint mariadb-lights-out
Change-Id: I903c3bcd069af39814bcabcef37684b1f043391f

ansible/mariadb_recovery.yml

+---
+- hosts: mariadb
+  roles:
+    - { role: mariadb,
+        tags: mariadb,
+        when: enable_mariadb | bool }
+  vars:
+    mariadb_recover: true

ansible/roles/mariadb/tasks/bootstrap.yml

 ---
-- name: Cleaning up temp file on localhost
-  local_action: file path=/tmp/kolla_mariadb_cluster state=absent
-  changed_when: False
-  always_run: True
-  run_once: True
+- include: lookup_cluster.yml
 
-- name: Creating temp file on localhost
-  local_action: copy content=None dest=/tmp/kolla_mariadb_cluster mode=0600
-  changed_when: False
-  always_run: True
-  run_once: True
-
-- name: Creating mariadb volume
-  kolla_docker:
-    action: "create_volume"
-    common_options: "{{ docker_common_options }}"
-    name: "mariadb"
-  register: mariadb_volume
-
-- name: Writing hostname of host with existing cluster files to temp file
-  local_action: copy content={{ ansible_hostname }} dest=/tmp/kolla_mariadb_cluster mode=0600
-  changed_when: False
-  always_run: True
-  when: not mariadb_volume | changed
-
-- name: Registering host from temp file
-  set_fact:
-    delegate_host: "{{ lookup('file', '/tmp/kolla_mariadb_cluster') }}"
-
-- name: Cleaning up temp file on localhost
-  local_action: file path=/tmp/kolla_mariadb_cluster state=absent
-  changed_when: False
-  always_run: True
-  run_once: True
-
-- name: Starting MariaDB bootstrap container
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    environment:
-      KOLLA_BOOTSTRAP:
-      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
-      DB_ROOT_PASSWORD: "{{ database_password }}"
-      DB_MAX_TIMEOUT: "{{ database_max_timeout }}"
-    image: "{{ mariadb_image_full }}"
-    labels:
-      BOOTSTRAP:
-    name: "mariadb"
-    restart_policy: "never"
-    volumes:
-      - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro"
-      - "mariadb:/var/lib/mysql"
-      - "kolla_logs:/var/log/kolla/"
+- include: bootstrap_cluster.yml
   when:
     - delegate_host == 'None'
     - inventory_hostname == groups['mariadb'][0]
 
-- name: Waiting for MariaDB service to be ready
-  command: "docker exec mariadb ls /var/lib/mysql/{{ item }}"
-  register: bootstrap_result
-  when:
-    - delegate_host == 'None'
-    - inventory_hostname == groups['mariadb'][0]
-  until: bootstrap_result | success
-  changed_when: False
-  retries: 6
-  delay: 10
-  with_items:
-    - cluster.exists
-    - mariadb.pid
+- include: recover_cluster.yml
+  when: mariadb_recover | default(False)

ansible/roles/mariadb/tasks/bootstrap_cluster.yml

+---
+- name: Starting MariaDB bootstrap container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    environment:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      DB_ROOT_PASSWORD: "{{ database_password }}"
+      DB_MAX_TIMEOUT: "{{ database_max_timeout }}"
+      ARGS: "--wsrep-new-cluster"
+    image: "{{ mariadb_image_full }}"
+    labels:
+      BOOTSTRAP:
+    name: "mariadb"
+    restart_policy: "never"
+    volumes:
+      - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro"
+      - "kolla_logs:/var/log/kolla/"
+      - "mariadb:/var/lib/mysql"
+
+- name: Waiting for MariaDB service to be ready
+  command: "docker exec mariadb ls /var/lib/mysql/{{ item }}"
+  register: bootstrap_result
+  until: bootstrap_result | success
+  changed_when: False
+  retries: 6
+  delay: 10
+  with_items:
+    - cluster.exists
+    - mariadb.pid

ansible/roles/mariadb/tasks/lookup_cluster.yml

+---
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_mariadb_cluster state=absent
+  changed_when: False
+  always_run: True
+  run_once: True
+
+- name: Creating temp file on localhost
+  local_action: copy content=None dest=/tmp/kolla_mariadb_cluster mode=0600
+  changed_when: False
+  always_run: True
+  run_once: True
+
+- name: Creating mariadb volume
+  kolla_docker:
+    action: "create_volume"
+    common_options: "{{ docker_common_options }}"
+    name: "mariadb"
+  register: mariadb_volume
+
+- name: Writing hostname of host with existing cluster files to temp file
+  local_action: copy content={{ ansible_hostname }} dest=/tmp/kolla_mariadb_cluster mode=0600
+  changed_when: False
+  always_run: True
+  when: not mariadb_volume | changed
+
+- name: Registering host from temp file
+  set_fact:
+    delegate_host: "{{ lookup('file', '/tmp/kolla_mariadb_cluster') }}"
+
+- name: Cleaning up temp file on localhost
+  local_action: file path=/tmp/kolla_mariadb_cluster state=absent
+  changed_when: False
+  always_run: True
+  run_once: True

ansible/roles/mariadb/tasks/recover_cluster.yml

+---
+- fail:
+    msg: "MariaDB cluster was not found. Is your inventory correct?"
+  when: delegate_host == 'None'
+
+- name: Checking if and mariadb containers are running
+  kolla_docker:
+    name: "mariadb"
+    action: "get_container_state"
+  register: container_state
+
+- fail:
+    msg: "There are running MariaDB nodes, please stop them first."
+  when: container_state.Running | bool
+  any_errors_fatal: True
+
+- name: Starting first MariaDB container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    environment:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      ARGS: "--wsrep-new-cluster"
+    image: "{{ mariadb_image_full }}"
+    labels:
+      BOOTSTRAP:
+    name: "mariadb"
+    restart_policy: "never"
+    volumes:
+      - "{{ node_config_directory }}/mariadb/:{{ container_config_directory }}/:ro"
+      - "kolla_logs:/var/log/kolla/"
+      - "mariadb:/var/lib/mysql"
+  when:
+    - (mariadb_recover_inventory_name is not defined and inventory_hostname == groups['mariadb'][0]) or
+      (mariadb_recover_inventory_name is defined and inventory_hostname == mariadb_recover_inventory_name)
+
+- name: Reset bootstrap fact
+  set_fact:
+    delegate_host: "None"

docker/mariadb/extend_start.sh

@@ -33,7 +33,6 @@ fi
 
 # This catches all cases of the BOOTSTRAP variable being set, including empty
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]] && [[ ! -e /var/lib/mysql/cluster.exists ]]; then
-    ARGS="--wsrep-new-cluster"
     mysql_install_db
     bootstrap_db
     touch /var/lib/mysql/cluster.exists

tools/kolla-ansible

@@ -38,6 +38,7 @@ Options:
 
 Commands:
     prechecks    Do pre-deployment checks for hosts
+    mariadb_recovery    Recover a completely stopped mariadb cluster
     deploy       Deploy and start all kolla containers
     post-deploy  Do post deploy on deploy node
     pull         Pull all images for containers (only pulls, no runnnig container changes)
@@ -119,6 +120,11 @@ case "$1" in
         ACTION="Pre-deployment checking"
         PLAYBOOK="${BASEDIR}/ansible/prechecks.yml"
         ;;
+(mariadb_recovery)
+        ACTION="Attempting to restart mariadb cluster"
+        EXTRA_OPTS="$EXTRA_OPTS -e action=deploy"
+        PLAYBOOK="${BASEDIR}/ansible/mariadb_recovery.yml"
+        ;;
 (deploy)
         ACTION="Deploying Playbooks"
         EXTRA_OPTS="$EXTRA_OPTS -e action=deploy"