diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index b42f2e2fae8aee10f90b7b19281542b22d701346..4d79d42a988e5046ddeff35910cd9780b8fae83b 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -452,6 +452,7 @@ prometheus_memcached_exporter_port: "9150" prometheus_rabbitmq_exporter_port: "15692" # Default cadvisor port of 8080 already in use prometheus_cadvisor_port: "18080" +prometheus_fluentd_integration_port: "24231" # Prometheus alertmanager ports prometheus_alertmanager_port: "9093" @@ -1099,6 +1100,7 @@ enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}" enable_prometheus_node_exporter: "{{ enable_prometheus | bool }}" enable_prometheus_memcached_exporter: "{{ enable_memcached | bool }}" enable_prometheus_cadvisor: "{{ enable_prometheus | bool }}" +enable_prometheus_fluentd_integration: "{{ enable_prometheus | bool and enable_fluentd | bool }}" enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}" enable_prometheus_ceph_mgr_exporter: "no" enable_prometheus_openstack_exporter: "{{ enable_prometheus | bool }}" diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index 022570bff3361f556902e686d89da57d66c07b51..d03fdd5993fd53dcb47e8247fb45cc37a1aa7b91 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -115,17 +115,29 @@ ( not enable_monasca | bool or not monasca_ingest_control_plane_logs | bool ) }} fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}" # Inputs - fluentd_input_files: "{{ default_input_files | customise_fluentd(customised_input_files) }}" + fluentd_input_files: "{{ default_input_files_enabled | customise_fluentd(customised_input_files) }}" + default_input_files_enabled: "{{ default_input_files | selectattr('enabled') | map(attribute='name') | list }}" default_input_files: - - "conf/input/00-global.conf.j2" - - "conf/input/01-syslog.conf.j2" - - "conf/input/02-mariadb.conf.j2" - - "conf/input/03-rabbitmq.conf.j2" - - "conf/input/04-openstack-wsgi.conf.j2" - - "conf/input/05-libvirt.conf.j2" - - "conf/input/06-zookeeper.conf.j2" - - "conf/input/07-kafka.conf.j2" - - "conf/input/09-monasca.conf.j2" + - name: "conf/input/00-global.conf.j2" + enabled: true + - name: "conf/input/01-syslog.conf.j2" + enabled: true + - name: "conf/input/02-mariadb.conf.j2" + enabled: true + - name: "conf/input/03-rabbitmq.conf.j2" + enabled: true + - name: "conf/input/04-openstack-wsgi.conf.j2" + enabled: true + - name: "conf/input/05-libvirt.conf.j2" + enabled: true + - name: "conf/input/06-zookeeper.conf.j2" + enabled: true + - name: "conf/input/07-kafka.conf.j2" + enabled: true + - name: "conf/input/08-prometheus.conf.j2" + enabled: "{{ enable_prometheus_fluentd_integration | bool }}" + - name: "conf/input/09-monasca.conf.j2" + enabled: true customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | list }}" # Filters fluentd_filter_files: "{{ default_filter_files | customise_fluentd(customised_filter_files) }}" diff --git a/ansible/roles/common/templates/conf/input/08-prometheus.conf.j2 b/ansible/roles/common/templates/conf/input/08-prometheus.conf.j2 new file mode 100644 index 0000000000000000000000000000000000000000..1aa5da895e1317a9a7afd54320f79dd858c90232 --- /dev/null +++ b/ansible/roles/common/templates/conf/input/08-prometheus.conf.j2 @@ -0,0 +1,14 @@ +<source> + @type prometheus + bind {{ api_interface_address }} + port {{ prometheus_fluentd_integration_port }} + metrics_path /metrics +</source> + +<source> + @type prometheus_output_monitor + interval 10 + <labels> + Hostname ${hostname} + </labels> +</source> diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2 index 44e5226f5f3850b4a676669c0b463dfdb32301ef..4dbce1852f0359e9d27d961b4549a9268e92ccda 100644 --- a/ansible/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.j2 @@ -74,6 +74,15 @@ scrape_configs: {% endfor %} {% endif %} +{% if enable_prometheus_fluentd_integration | bool %} + - job_name: fluentd + static_configs: + - targets: +{% for host in groups['fluentd'] %} + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}' +{% endfor %} +{% endif %} + {% if enable_prometheus_ceph_mgr_exporter | bool %} - job_name: ceph_mgr_exporter honor_labels: true diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index a528d01bfaeafdd59e759e382ecc8cb872159e6e..74d8e67d86a139cd791b35e24e9ca164f1610f94 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -694,6 +694,7 @@ #enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}" #enable_prometheus_node_exporter: "{{ enable_prometheus | bool }}" #enable_prometheus_cadvisor: "{{ enable_prometheus | bool }}" +#enable_prometheus_fluentd_integration: "{{ enable_prometheus | bool and enable fluentd | bool }}" #enable_prometheus_memcached: "{{ enable_prometheus | bool }}" #enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}" #enable_prometheus_ceph_mgr_exporter: "no" diff --git a/releasenotes/notes/add-support-for-fluentd-exporter-ca02ff9182039e3e.yaml b/releasenotes/notes/add-support-for-fluentd-exporter-ca02ff9182039e3e.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2896fd2e724db1a6e62cb4d8b594785e77ae405e --- /dev/null +++ b/releasenotes/notes/add-support-for-fluentd-exporter-ca02ff9182039e3e.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Adds support for integrating Fluentd metrics into Prometheus. By + default this is now enabled when Prometheus is enabled. This behaviour + can be overridden via the `enable_prometheus_fluentd_integration` flag. By + default the integration provides metrics relating to the processing of logs + by Fluentd. These metrics can be useful for monitoring the status of the + Fluentd service. Additional metrics can also be extracted from logs via + custom Fluentd config.