Support setting Kafka storage volume

Not everyone wants Kafka data stored on a Docker volume. This
change allows a user to flexibly control where the data is stored.

Change-Id: I2ba8c7a85c7bf2564f954a43c6e6dbb3257fe902

ansible/group_vars/all.yml

@@ -1042,6 +1042,11 @@ enable_vitrage_prometheus_datasource: "{{ enable_prometheus | bool }}"
 influxdb_address: "{{ kolla_internal_fqdn }}"
 influxdb_datadir_volume: "influxdb"
 
+#################
+# Kafka options
+#################
+kafka_datadir_volume: "kafka"
+
 #########################
 # Internal Image options
 #########################

ansible/roles/destroy/tasks/cleanup_host.yml

@@ -10,6 +10,7 @@
     nova_instance_datadir_volume: "{{ nova_instance_datadir_volume }}"
     gnocchi_metric_datadir_volume: "{{ gnocchi_metric_datadir_volume }}"
     influxdb_datadir_volume: "{{ influxdb_datadir_volume }}"
+    kafka_datadir_volume: "{{ kafka_datadir_volume }}"
     kolla_internal_vip_address: "{{ kolla_internal_vip_address }}"
     kolla_external_vip_address: "{{ kolla_external_vip_address }}"
     kolla_dev_repos_directory: "{{ kolla_dev_repos_directory }}"

ansible/roles/kafka/defaults/main.yml

@@ -34,6 +34,6 @@ kafka_dimensions: "{{ default_container_dimensions }}"
 kafka_default_volumes:
   - "{{ node_config_directory }}/kafka/:{{ container_config_directory }}/"
   - "/etc/localtime:/etc/localtime:ro"
-  - "kafka:/var/lib/kafka/data"
+  - "{{ kafka_datadir_volume }}:/var/lib/kafka/data"
   - "kolla_logs:/var/log/kolla/"
 kafka_extra_volumes: "{{ default_extra_volumes }}"

doc/source/reference/logging-and-monitoring/index.rst

@@ -10,6 +10,7 @@ logging and monitoring services available in kolla.
 
    central-logging-guide
    influxdb-guide
+   kafka-guide
    monasca-guide
    osprofiler-guide
    prometheus-guide

doc/source/reference/logging-and-monitoring/kafka-guide.rst

+.. _kafka-guide:
+
+============
+Apache Kafka
+============
+
+Overview
+~~~~~~~~
+
+`Kafka <https://kafka.apache.org/intro>`_ is a distributed stream processing
+system. It forms the central component of Monasca and in an OpenStack context
+can also be used as an experimental messaging backend in `Oslo messaging
+<https://docs.openstack.org/oslo.messaging/latest/admin/kafka.html>`_.
+
+Kafka
+~~~~~
+
+A spinning disk array is normally sufficient for Kafka. The data directory
+defaults to a docker volume, ``kafka``. Since it can use a lot of disk space,
+you may wish to store the data on a dedicated device. This can be achieved by
+setting ``kafka_datadir_volume`` in ``/etc/kolla/globals.yml``:
+
+.. code-block:: yaml
+
+    kafka_datadir_volume: /mnt/spinning_array/kafka/

releasenotes/notes/add-variable-kafka-datadir-volume-382015a42sc57636.yaml

+---
+features:
+  - |
+    Adds a new variable, ``kafka_datadir_volume``. This allows you to control
+    where the Kafka data is stored. Generally you will want this to be a
+    spinning disk, or an array of spinning disks.

tools/cleanup-host

@@ -68,6 +68,11 @@ if [[ "$influxdb_datadir_volume" != "influxdb" && -d "$influxdb_datadir_volume"
     rm -rfv  $influxdb_datadir_volume
 fi
 
+if [[ "$kafka_datadir_volume" != "kafka" && -d "$kafka_datadir_volume" ]]; then
+    echo "Removing kafka volume if it is customised"
+    rm -rfv  $kafka_datadir_volume
+fi
+
 FOLDER_PATH="/etc/kolla/"
 
 if [[ -e "$FOLDER_PATH/ovsdpdk-db/ovs-dpdkctl.sh" ]]; then