Add support for deploying Apache Kafka

This change supports deploying Apache Kafka.

Depends-On: Iab8d200c2450238f3c0c708d1f4184490f6e6284
for support in Kolla-Ansible.
Task: 6697
Story: 2001627

Change-Id: I22c79f75e9cfed647c03216b94bab5ee0790cbf2

ansible/group_vars/all/kolla

@@ -144,6 +144,8 @@ overcloud_container_image_regex_map:
     enabled: "{{ kolla_enable_ironic | bool }}"
   - regex: iscsid
     enabled: "{{ kolla_enable_cinder | bool or kolla_enable_ironic | bool }}"
+  - regex: kafka
+    enabled: "{{ kolla_enable_kafka | bool }}"
   - regex: keepalived
     enabled: "{{ kolla_enable_haproxy | bool }}"
   - regex: keystone
@@ -301,7 +303,6 @@ kolla_enable_haproxy: "yes"
 kolla_enable_heat: "yes"
 kolla_enable_horizon: "yes"
 kolla_enable_ironic: "yes"
-# Kafka is not currently supported
 kolla_enable_kafka: "no"
 kolla_enable_kibana: "{{ 'yes' if kolla_enable_central_logging | bool else 'no' }}"
 kolla_enable_magnum: "no"
@@ -316,7 +317,7 @@ kolla_enable_osprofiler: "no"
 kolla_enable_sahara: "no"
 kolla_enable_skydive: "no"
 kolla_enable_swift: "no"
-kolla_enable_zookeeper: "no"
+kolla_enable_zookeeper: "{{ 'yes' if kolla_enable_kafka | bool else 'no' }}"
 
 ###############################################################################
 # Passwords and credentials.

ansible/kolla-openstack.yml

@@ -107,6 +107,7 @@
             - { name: inspector, file: ironic-inspector.conf }
             - { name: ironic, file: ironic.conf }
             - { name: ironic_dnsmasq, file: ironic/ironic-dnsmasq.conf }
+            - { name: kafka, file: kafka.server.properties }
             - { name: magnum, file: magnum.conf }
             - { name: murano, file: murano.conf }
             - { name: neutron, file: neutron.conf }
@@ -208,6 +209,7 @@
       kolla_extra_inspector: "{{ kolla_extra_config.inspector | default }}"
       kolla_extra_ironic: "{{ kolla_extra_config.ironic | default }}"
       kolla_extra_ironic_dnsmasq: "{{ kolla_extra_config.ironic_dnsmasq | default }}"
+      kolla_extra_kafka: "{{ kolla_extra_config.kafka | default }}"
       kolla_extra_magnum: "{{ kolla_extra_config.magnum | default }}"
       kolla_extra_murano: "{{ kolla_extra_config.murano | default }}"
       kolla_extra_neutron: "{{ kolla_extra_config.neutron | default }}"

ansible/roles/kolla-ansible/templates/overcloud-components.j2

@@ -36,6 +36,9 @@ control
 [influxdb:children]
 monitoring
 
+[kafka:children]
+control
+
 [karbor:children]
 control
 

ansible/roles/kolla-ansible/tests/test-defaults.yml

@@ -116,6 +116,7 @@
               - neutron_tenant_network_types
               - enable_glance
               - enable_ironic
+              - enable_kafka
               - enable_neutron
               - enable_nova
               - enable_zookeeper

ansible/roles/kolla-ansible/tests/test-extras.yml

@@ -85,6 +85,7 @@
             kolla_enable_horizon: True
             kolla_enable_influxdb: True
             kolla_enable_ironic: True
+            kolla_enable_kafka: True
             kolla_enable_karbor: True
             kolla_enable_kibana: True
             kolla_enable_kuryr: True
@@ -219,6 +220,7 @@
               #enable_horizon: True
               #enable_influxdb: True
               #enable_ironic: True
+              #enable_kafka: True
               #enable_karbor: True
               #enable_kibana: True
               #enable_kuryr: True

ansible/roles/kolla-ansible/vars/main.yml

@@ -82,6 +82,7 @@ kolla_feature_flags:
   - horizon
   - influxdb
   - ironic
+  - kafka
   - karbor
   - kibana
   - kuryr

ansible/roles/kolla-openstack/defaults/main.yml

@@ -246,6 +246,15 @@ kolla_inspector_default_gateway:
 # Free form extra configuration to append to dnsmasq.conf.
 kolla_extra_ironic_dnsmasq:
 
+###############################################################################
+# Kafka configuration.
+
+# Whether to enable Kafka.
+kolla_enable_kafka:
+
+# Free form extra configuration to append to kafka.server.properties.
+kolla_extra_kafka:
+
 ###############################################################################
 # Magnum configuration.
 

ansible/roles/kolla-openstack/molecule/default/tests/test_default.py

@@ -45,6 +45,7 @@ def test_service_config_directory(host, path):
      'heat',
      'horizon',
      'ironic',
+     'kafka',
      'magnum',
      'manila',
      'murano',

ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml

@@ -52,6 +52,10 @@ provisioner:
         kolla_inspector_ipa_ramdisk_path: ${MOLECULE_TEMP_PATH:-/tmp}/ironic-agent.initramfs
         kolla_extra_ironic_dnsmasq: |
           extra=bar
+        kolla_enable_kafka: True
+        kolla_extra_kafka: |
+          [extra-kafka.server.properties]
+          foo=bar
         kolla_enable_magnum: True
         kolla_extra_magnum: |
           [extra-magnum.conf]

ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py

@@ -37,6 +37,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
      'heat',
      'horizon',
      'ironic',
+     'kafka',
      'keystone',
      'magnum',
      'manila',
@@ -61,6 +62,7 @@ def test_service_config_directory(host, path):
      'heat.conf',
      'ironic.conf',
      'ironic-inspector.conf',
+     'kafka.server.properties',
      'magnum.conf',
      'neutron/ml2_conf.ini',
      'murano.conf',

ansible/roles/kolla-openstack/tasks/config.yml

@@ -22,6 +22,7 @@
     - { src: ironic.conf.j2, dest: ironic.conf, enabled: "{{ kolla_enable_ironic }}" }
     - { src: ironic-dnsmasq.conf.j2, dest: ironic/ironic-dnsmasq.conf, enabled: "{{ kolla_enable_ironic }}" }
     - { src: ironic-inspector.conf.j2, dest: ironic-inspector.conf, enabled: "{{ kolla_enable_ironic }}" }
+    - { src: kafka.server.properties.j2, dest: kafka.server.properties, enabled: "{{ kolla_enable_kafka }}" }
     - { src: magnum.conf.j2, dest: magnum.conf, enabled: "{{ kolla_enable_magnum }}" }
     - { src: ml2_conf.ini.j2, dest: neutron/ml2_conf.ini, enabled: "{{ kolla_enable_neutron }}" }
     - { src: murano.conf.j2, dest: murano.conf, enabled: "{{ kolla_enable_murano }}" }

ansible/roles/kolla-openstack/templates/kafka.server.properties.j2

+# {{ ansible_managed }}
+
+{% if kolla_extra_kafka %}
+#######################
+# Extra configuration
+#######################
+
+{{ kolla_extra_kafka }}
+{% endif %}

ansible/roles/kolla-openstack/vars/main.yml

@@ -65,6 +65,11 @@ kolla_openstack_custom_config:
       - ironic-agent.kernel
       - ironic-dnsmasq.conf
       - pxelinux.default
+  # Kafka.
+  - src: "{{ kolla_extra_config_path }}/kafka"
+    dest: "{{ kolla_node_custom_config_path }}/kafka"
+    patterns: "*"
+    enabled: "{{ kolla_enable_kafka }}"
   # Keystone.
   - src: "{{ kolla_extra_config_path }}/keystone"
     dest: "{{ kolla_node_custom_config_path }}/keystone"