Support configuring persistent systemd journal

This is useful to ensure logs are persisted across reboots.

Change-Id: Ie5ff7536b160196dc57448c3ed982d15826b2cbe
Closes-Bug: #2055299
Co-Authored-By: Piotr Parczewski <piotr@stackhpc.com>

ansible/infra-vm-host-configure.yml

 ---
 - import_playbook: "ssh-known-host.yml"
 - import_playbook: "kayobe-ansible-user.yml"
+- import_playbook: "logging.yml"
 - import_playbook: "proxy.yml"
 - import_playbook: "apt.yml"
 - import_playbook: "dnf.yml"

ansible/inventory/group_vars/all/logging

+---
+###############################################################################
+# Logging configuration
+
+# Journald storage. One of: volatile, persistent, auto, or none. Defaults to
+# `persistent`.
+journald_storage: persistent
+
+###############################################################################

ansible/logging.yml

+---
+- name: Ensure Logging configuration is applied
+  hosts: seed-hypervisor:seed:overcloud:infra-vms
+  gather_facts: false
+  vars:
+    # NOTE(wszumski): Kayobe target env does not yet exist.
+    ansible_python_interpreter: /usr/bin/python3
+  tags:
+    - logging
+  tasks:
+    - name: Create /var/log/journal
+      become: true
+      file:
+        path: /var/log/journal
+        owner: "root"
+        group: "root"
+        mode: 0755
+        state: directory
+      notify: Flush journal to disk
+      when: journald_storage == "persistent"
+
+    - name: Ensure journald drop in directory exists
+      file:
+        path: "/etc/systemd/journald.conf.d/"
+        owner: "root"
+        group: "root"
+        mode: 0770
+        state: directory
+      become: true
+
+    - name: Ensure journald.conf overrides are set
+      copy:
+        content: |
+           [Journal]
+           Storage={{ journald_storage }}
+        dest: /etc/systemd/journald.conf.d/kayobe.conf
+        owner: root
+        group: root
+        mode: 0660
+      become: true
+      notify: Restart journald
+
+  handlers:
+    - name: Restart journald
+      service:
+        name: systemd-journald
+        state: restarted
+      become: true
+    - name: Flush journal to disk
+      become: true
+      command: journalctl --flush

ansible/overcloud-host-configure.yml

 ---
 - import_playbook: "ssh-known-host.yml"
 - import_playbook: "kayobe-ansible-user.yml"
+- import_playbook: "logging.yml"
 - import_playbook: "proxy.yml"
 - import_playbook: "apt.yml"
 - import_playbook: "dnf.yml"

ansible/seed-host-configure.yml

 ---
 - import_playbook: "ssh-known-host.yml"
 - import_playbook: "kayobe-ansible-user.yml"
+- import_playbook: "logging.yml"
 - import_playbook: "proxy.yml"
 - import_playbook: "apt.yml"
 - import_playbook: "dnf.yml"

ansible/seed-hypervisor-host-configure.yml

 ---
 - import_playbook: "ssh-known-host.yml"
 - import_playbook: "kayobe-ansible-user.yml"
+- import_playbook: "logging.yml"
 - import_playbook: "proxy.yml"
 - import_playbook: "apt.yml"
 - import_playbook: "dnf.yml"

doc/source/configuration/reference/hosts.rst

@@ -1393,3 +1393,25 @@ Installing packages required by Kolla Ansible
 
 A small number of packages are required to be installed on the hosts for Kolla
 Ansible and the services that it deploys, while some others must be removed.
+
+Logging
+=======
+*tags:*
+  | ``logging``
+
+Kayobe will configure persistent logging for nodes in the following ansible groups:
+
+- seed-hypervisor
+- seed
+- overcloud
+- infra-vms
+
+This means that the systemd journal will be written to local storage (instead
+of to memory) and will allow you to view the journal from previous boots. The
+storage limit defaults to 10% of the filesystem with a 4GiB hard limit (when
+using journald defaults). See `journald documentation
+<https://www.freedesktop.org/software/systemd/man/latest/journald.conf.html#SystemMaxUse=>`__
+for more details.
+
+Should you wish to disable this feature, you can set ``journald_storage`` to
+``volatile``.
\ No newline at end of file

etc/kayobe/logging.yml

+---
+###############################################################################
+# Logging configuration
+
+# Journald storage. One of: volatile, persistent, auto, or none. Defaults to
+# `persistent`.
+#journald_storage:
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes

releasenotes/notes/adds-persistent-journal-99951ac651dc4a23.yaml

+---
+features:
+  - |
+    Configures journald to use a persistent storage by default. This allows you
+    to keep journald logs across reboots and is controlled by the
+    ``journald_storage`` variable. See :kayobe-doc:`Kayobe documentation
+    <configuration/reference/hosts#logging>` for more details.