Allow custom fluentd input configurations

Co-authored-by: Mark Goddard <mark@stackhpc.com>
Change-Id: I2e5ecf5b01cc842ec480fc4d883a7d2283fc1c31

ansible/roles/common/handlers/main.yml

@@ -19,6 +19,7 @@
     - service.enabled | bool
     - config_json.changed | bool
       or fluentd_input.changed | bool
+      or fluentd_input_custom.changed | bool
       or fluentd_output.changed | bool
       or fluentd_output_custom.changed | bool
       or fluentd_format.changed | bool

ansible/roles/common/tasks/config.yml

@@ -58,6 +58,27 @@
   notify:
     - Restart fluentd container
 
+- name: Find custom fluentd input config files
+  local_action:
+    module: find
+    path: "{{ node_custom_config }}/fluentd/input"
+    pattern: "*.conf"
+  run_once: True
+  register: find_custom_fluentd_inputs
+  when:
+    - enable_fluentd | bool
+
+- name: Copying over custom fluentd input config files
+  template:
+    src: "{{ item.path }}"
+    dest: "{{ node_config_directory }}/fluentd/input/{{ item.path | basename }}"
+  register: fluentd_input_custom
+  when:
+    - enable_fluentd | bool
+  with_items: "{{ find_custom_fluentd_inputs.files }}"
+  notify:
+    - Restart fluentd container
+
 - name: Copying over fluentd output config files
   template:
     src: "conf/output/{{ item.name }}.conf.j2"

ansible/roles/common/templates/fluentd.json.j2

@@ -11,39 +11,11 @@
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },
+        {# Copy all configuration files in input/ directory to include #}
+        {# custom input configs. #}
         {
-            "source": "{{ container_config_directory }}/input/00-global.conf",
-            "dest": "{{ fluentd_dir }}/input/00-global.conf",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        },
-        {
-            "source": "{{ container_config_directory }}/input/01-syslog.conf",
-            "dest": "{{ fluentd_dir }}/input/01-syslog.conf",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        },
-        {
-            "source": "{{ container_config_directory }}/input/02-mariadb.conf",
-            "dest": "{{ fluentd_dir }}/input/02-mariadb.conf",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        },
-        {
-            "source": "{{ container_config_directory }}/input/03-rabbitmq.conf",
-            "dest": "{{ fluentd_dir }}/input/03-rabbitmq.conf",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        },
-        {
-            "source": "{{ container_config_directory }}/input/04-openstack-wsgi.conf",
-            "dest": "{{ fluentd_dir }}/input/04-openstack-wsgi.conf",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        },
-        {
-            "source": "{{ container_config_directory }}/input/05-libvirt.conf",
-            "dest": "{{ fluentd_dir }}/input/05-libvirt.conf",
+            "source": "{{ container_config_directory }}/input/*.conf",
+            "dest": "{{ fluentd_dir }}/input/",
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },

doc/source/reference/central-logging-guide.rst

@@ -164,8 +164,16 @@ can also be exported at once by choosing *export everything* option.
 In the same tab (Settings - Objects) one can also import saved items by
 choosing *import* option.
 
+Custom log rules
+~~~~~~~~~~~~~~~~
+
+Kolla-Ansible automatically deploys Fluentd for forwarding OpenStack logs
+from across the control plane to a central logging repository. The Fluentd
+configuration is split into four parts: Input, forwarding, filtering and
+formatting. The following can be customised:
+
 Custom log filtering
-====================
+--------------------
 
 In some scenarios it may be useful to apply custom filters to logs before
 forwarding them.  This may be useful to add additional tags to the messages
@@ -177,7 +185,7 @@ configuration files in ``/etc/kolla/config/fluentd/filter/*.conf`` on the
 control host.
 
 Custom log forwarding
-~~~~~~~~~~~~~~~~~~~~~
+---------------------
 
 In some scenarios it may be useful to forward logs to a logging service other
 than elasticsearch.  This can be done by configuring custom fluentd outputs.
@@ -185,3 +193,13 @@ than elasticsearch.  This can be done by configuring custom fluentd outputs.
 Configuration of custom fluentd outputs is possible by placing output
 configuration files in ``/etc/kolla/config/fluentd/output/*.conf`` on the
 control host.
+
+Custom log inputs
+-----------------
+
+In some scenarios it may be useful to input logs from other services, e.g.
+network equipment. This can be done by configuring custom fluentd inputs.
+
+Configuration of custom fluentd inputs is possible by placing input
+configuration files in ``/etc/kolla/config/fluentd/input/*.conf`` on the
+control host.

releasenotes/notes/fluentd-custom-input-config-db2675a1356717e6.yaml

+---
+features:
+  - |
+    Adds ability to configure custom fluentd inputs.
+
+    Configuration of custom fluentd inputs is possible by placing
+    input configuration files in /etc/kolla/config/fluentd/input/\*.conf.