From 1b4430314cea3bded83875e613668a0eeec5bbe1 Mon Sep 17 00:00:00 2001
From: Christian Berendt <berendt@betacloud-solutions.de>
Date: Tue, 6 Mar 2018 17:29:27 +0100
Subject: [PATCH] Allow custom fluentd input configurations
Co-authored-by: Mark Goddard <mark@stackhpc.com>
Change-Id: I2e5ecf5b01cc842ec480fc4d883a7d2283fc1c31
---
ansible/roles/common/handlers/main.yml | 1 +
ansible/roles/common/tasks/config.yml | 21 +++++++++++
.../roles/common/templates/fluentd.json.j2 | 36 +++----------------
.../reference/central-logging-guide.rst | 22 ++++++++++--
...-custom-input-config-db2675a1356717e6.yaml | 7 ++++
5 files changed, 53 insertions(+), 34 deletions(-)
create mode 100644 releasenotes/notes/fluentd-custom-input-config-db2675a1356717e6.yaml
diff --git a/ansible/roles/common/handlers/main.yml b/ansible/roles/common/handlers/main.yml
index 96a36d1926..0887c18b92 100644
--- a/ansible/roles/common/handlers/main.yml
+++ b/ansible/roles/common/handlers/main.yml
@@ -19,6 +19,7 @@
- service.enabled | bool
- config_json.changed | bool
or fluentd_input.changed | bool
+ or fluentd_input_custom.changed | bool
or fluentd_output.changed | bool
or fluentd_output_custom.changed | bool
or fluentd_format.changed | bool
diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index cc663e2755..e9934164df 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -58,6 +58,27 @@
notify:
- Restart fluentd container
+- name: Find custom fluentd input config files
+ local_action:
+ module: find
+ path: "{{ node_custom_config }}/fluentd/input"
+ pattern: "*.conf"
+ run_once: True
+ register: find_custom_fluentd_inputs
+ when:
+ - enable_fluentd | bool
+
+- name: Copying over custom fluentd input config files
+ template:
+ src: "{{ item.path }}"
+ dest: "{{ node_config_directory }}/fluentd/input/{{ item.path | basename }}"
+ register: fluentd_input_custom
+ when:
+ - enable_fluentd | bool
+ with_items: "{{ find_custom_fluentd_inputs.files }}"
+ notify:
+ - Restart fluentd container
+
- name: Copying over fluentd output config files
template:
src: "conf/output/{{ item.name }}.conf.j2"
diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2
index 55a3f37c0d..b8d16f47da 100644
--- a/ansible/roles/common/templates/fluentd.json.j2
+++ b/ansible/roles/common/templates/fluentd.json.j2
@@ -11,39 +11,11 @@
"owner": "{{ fluentd_user }}",
"perm": "0600"
},
+ {# Copy all configuration files in input/ directory to include #}
+ {# custom input configs. #}
{
- "source": "{{ container_config_directory }}/input/00-global.conf",
- "dest": "{{ fluentd_dir }}/input/00-global.conf",
- "owner": "{{ fluentd_user }}",
- "perm": "0600"
- },
- {
- "source": "{{ container_config_directory }}/input/01-syslog.conf",
- "dest": "{{ fluentd_dir }}/input/01-syslog.conf",
- "owner": "{{ fluentd_user }}",
- "perm": "0600"
- },
- {
- "source": "{{ container_config_directory }}/input/02-mariadb.conf",
- "dest": "{{ fluentd_dir }}/input/02-mariadb.conf",
- "owner": "{{ fluentd_user }}",
- "perm": "0600"
- },
- {
- "source": "{{ container_config_directory }}/input/03-rabbitmq.conf",
- "dest": "{{ fluentd_dir }}/input/03-rabbitmq.conf",
- "owner": "{{ fluentd_user }}",
- "perm": "0600"
- },
- {
- "source": "{{ container_config_directory }}/input/04-openstack-wsgi.conf",
- "dest": "{{ fluentd_dir }}/input/04-openstack-wsgi.conf",
- "owner": "{{ fluentd_user }}",
- "perm": "0600"
- },
- {
- "source": "{{ container_config_directory }}/input/05-libvirt.conf",
- "dest": "{{ fluentd_dir }}/input/05-libvirt.conf",
+ "source": "{{ container_config_directory }}/input/*.conf",
+ "dest": "{{ fluentd_dir }}/input/",
"owner": "{{ fluentd_user }}",
"perm": "0600"
},
diff --git a/doc/source/reference/central-logging-guide.rst b/doc/source/reference/central-logging-guide.rst
index 4ec165817c..540cd63436 100644
--- a/doc/source/reference/central-logging-guide.rst
+++ b/doc/source/reference/central-logging-guide.rst
@@ -164,8 +164,16 @@ can also be exported at once by choosing *export everything* option.
In the same tab (Settings - Objects) one can also import saved items by
choosing *import* option.
+Custom log rules
+~~~~~~~~~~~~~~~~
+
+Kolla-Ansible automatically deploys Fluentd for forwarding OpenStack logs
+from across the control plane to a central logging repository. The Fluentd
+configuration is split into four parts: Input, forwarding, filtering and
+formatting. The following can be customised:
+
Custom log filtering
-====================
+--------------------
In some scenarios it may be useful to apply custom filters to logs before
forwarding them. This may be useful to add additional tags to the messages
@@ -177,7 +185,7 @@ configuration files in ``/etc/kolla/config/fluentd/filter/*.conf`` on the
control host.
Custom log forwarding
-~~~~~~~~~~~~~~~~~~~~~
+---------------------
In some scenarios it may be useful to forward logs to a logging service other
than elasticsearch. This can be done by configuring custom fluentd outputs.
@@ -185,3 +193,13 @@ than elasticsearch. This can be done by configuring custom fluentd outputs.
Configuration of custom fluentd outputs is possible by placing output
configuration files in ``/etc/kolla/config/fluentd/output/*.conf`` on the
control host.
+
+Custom log inputs
+-----------------
+
+In some scenarios it may be useful to input logs from other services, e.g.
+network equipment. This can be done by configuring custom fluentd inputs.
+
+Configuration of custom fluentd inputs is possible by placing input
+configuration files in ``/etc/kolla/config/fluentd/input/*.conf`` on the
+control host.
diff --git a/releasenotes/notes/fluentd-custom-input-config-db2675a1356717e6.yaml b/releasenotes/notes/fluentd-custom-input-config-db2675a1356717e6.yaml
new file mode 100644
index 0000000000..3323e84463
--- /dev/null
+++ b/releasenotes/notes/fluentd-custom-input-config-db2675a1356717e6.yaml
@@ -0,0 +1,7 @@
+---
+features:
+ - |
+ Adds ability to configure custom fluentd inputs.
+
+ Configuration of custom fluentd inputs is possible by placing
+ input configuration files in /etc/kolla/config/fluentd/input/\*.conf.
--
GitLab