Fluentd cleanup

We no longer have to support multiple fluentd versions and rely
on the workaround.

Change-Id: Id4083a9f07317fc04467e52176850f3909703169

ansible/roles/common/tasks/config.yml

@@ -23,38 +23,6 @@
     - paths
   when: service | service_enabled_and_mapped_to_host
 
-- name: Ensure fluentd image is present for label check
-  vars:
-    service_name: "fluentd"
-    service: "{{ common_services[service_name] }}"
-  become: true
-  kolla_docker:
-    action: "ensure_image"
-    common_options: "{{ docker_common_options }}"
-    image: "{{ service.image }}"
-  when:
-    - fluentd_version is not defined or fluentd_binary is not defined
-    - service | service_enabled_and_mapped_to_host
-
-- name: Fetch fluentd image labels
-  vars:
-    service_name: "fluentd"
-    service: "{{ common_services[service_name] }}"
-  become: true
-  docker_image_info:
-    name: "{{ service.image }}"
-  register: fluentd_labels
-  when:
-    - fluentd_version is not defined or fluentd_binary is not defined
-    - service | service_enabled_and_mapped_to_host
-
-- name: Set fluentd facts
-  set_fact:
-    fluentd_binary: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary }}"
-  when:
-    - fluentd_binary is not defined
-    - common_services.fluentd | service_enabled_and_mapped_to_host
-
 - include_tasks: copy-certs.yml
   when:
     - kolla_copy_ca_into_containers | bool
@@ -112,7 +80,6 @@
       {{ ( enable_elasticsearch | bool or
            ( elasticsearch_address != kolla_internal_fqdn )) and
          ( not enable_monasca | bool or not monasca_ingest_control_plane_logs | bool ) }}
-    fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
     # Inputs
     fluentd_input_files: "{{ default_input_files_enabled | customise_fluentd(customised_input_files) }}"
     default_input_files_enabled: "{{ default_input_files | selectattr('enabled') | map(attribute='name') | list }}"
@@ -142,7 +109,7 @@
     fluentd_filter_files: "{{ default_filter_files | customise_fluentd(customised_filter_files) }}"
     default_filter_files:
       - "conf/filter/00-record_transformer.conf.j2"
-      - "conf/filter/{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}.conf.j2"
+      - "conf/filter/01-rewrite.conf.j2"
       - "conf/filter/02-parser.conf.j2"
     customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | list }}"
     # Formats

ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2

-<match kolla.var.log.kolla.*.*.log>
-    @type rewrite_tag_filter
-    capitalize_regex_backreference yes
-    rewriterule1 programname ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|octavia-api-access|placement-api-access)$ apache_access
-    rewriterule2 programname ^(aodh_wsgi_access|barbican_api_uwsgi_access|zun_api_wsgi_access|vitrage_wsgi_access)$ wsgi_access
-    rewriterule3 programname ^(nova-api|nova-compute|nova-compute-ironic|nova-conductor|nova-manage|nova-novncproxy|nova-scheduler|nova-placement-api|placement-api|privsep-helper)$ openstack_python
-    rewriterule4 programname ^(sahara-api|sahara-engine)$ openstack_python
-    rewriterule5 programname ^(neutron-server|neutron-openvswitch-agent|neutron-ns-metadata-proxy|neutron-metadata-agent|neutron-l3-agent|neutron-dhcp-agent)$ openstack_python
-    rewriterule6 programname ^(magnum-conductor|magnum-api)$ openstack_python
-    rewriterule7 programname ^(keystone)$ openstack_python
-    rewriterule8 programname ^(heat-engine|heat-api|heat-api-cfn)$ openstack_python
-    rewriterule9 programname ^(glance-api)$ openstack_python
-    rewriterule10 programname ^(cloudkitty-storage-init|cloudkitty-processor|cloudkitty-dbsync|cloudkitty-api)$ openstack_python
-    rewriterule11 programname ^(ceilometer-polling|ceilometer-agent-notification)$ openstack_python
-    rewriterule12 programname ^(barbican-api|barbican-worker|barbican-keystone-listener|barbican-db-manage|app)$ openstack_python
-    rewriterule13 programname ^(aodh-notifier|aodh-listener|aodh-evaluator|aodh-dbsync)$ openstack_python
-    rewriterule14 programname ^(cinder-api|cinder-scheduler|cinder-manage|cinder-volume|cinder-backup|privsep-helper)$ openstack_python
-    rewriterule15 programname ^(mistral-server|mistral-engine|mistral-executor)$ openstack_python
-    rewriterule16 programname ^(designate-api|designate-central|designate-manage|designate-mdns|designate-sink|designate-worker)$ openstack_python
-    rewriterule17 programname ^(manila-api|manila-data|manila-manage|manila-share|manila-scheduler)$ openstack_python
-    rewriterule18 programname ^(trove-api|trove-conductor|trove-manage|trove-taskmanager)$ openstack_python
-    rewriterule20 programname ^(murano-api|murano-engine)$ openstack_python
-    rewriterule21 programname ^(senlin-api|senlin-conductor|senlin-engine|senlin-health-manager)$ openstack_python
-    rewriterule22 programname ^(watcher-api|watcher-applier|watcher-db-manage|watcher-decision-engine)$ openstack_python
-    rewriterule23 programname ^(freezer-api|freezer-api_access|freezer-manage)$ openstack_python
-    rewriterule24 programname ^(octavia-api|octavia-health-manager|octavia-housekeeping|octavia-worker)$ openstack_python
-    rewriterule25 programname ^(zun-api|zun-compute|zun-cni-daemon)$ openstack_python
-    rewriterule26 programname ^(kuryr-server)$ openstack_python
-    rewriterule27 programname ^(gnocchi-api|gnocchi-statsd|gnocchi-metricd|gnocchi-upgrade)$ openstack_python
-    rewriterule28 programname ^(ironic-api|ironic-conductor|ironic-inspector)$ openstack_python
-    rewriterule30 programname ^(tacker-server|tacker-conductor)$ openstack_python
-    rewriterule31 programname ^(vitrage-ml|vitrage-notifier|vitrage-graph|vitrage-persistor)$ openstack_python
-    rewriterule32 programname ^(blazar-api|blazar-manager)$ openstack_python
-    rewriterule33 programname ^(cyborg-api|cyborg-conductor|cyborg-agent)$ openstack_python
-    rewriterule34 programname ^(monasca-api|monasca-notification|monasca-persister|agent-collector|agent-forwarder|agent-statsd)$ openstack_python
-    rewriterule35 programname .+ unmatched
-</match>
-
-# Retag log messages from MariaDB according to log format
-<match infra.mariadb>
-    @type rewrite_tag_filter
-    rewriterule1 Payload ^\d{6} infra.mariadb.mysqld_safe
-    rewriterule2 Payload ^\d{4}-\d{2}-\d{2} infra.mariadb.mysqld
-</match>

ansible/roles/common/templates/conf/input/00-global.conf.j2

@@ -12,7 +12,7 @@
                 "/var/log/kolla/*/*-error.log",
                 "/var/log/kolla/*/*_access.log",
                 "/var/log/kolla/*/*_error.log"]
-  pos_file /var/run/{{ fluentd_binary }}/kolla-openstack.pos
+  pos_file /var/run/td-agent/kolla-openstack.pos
   tag kolla.*
   format multiline
   format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/

ansible/roles/common/templates/conf/input/02-mariadb.conf.j2

@@ -3,7 +3,7 @@
 <source>
   @type tail
   path /var/log/kolla/mariadb/mariadb.log
-  pos_file /var/run/{{ fluentd_binary }}/mariadb.pos
+  pos_file /var/run/td-agent/mariadb.pos
   tag infra.mariadb
   format multiline
   format_firstline /^(\d{4}-\d{2}-\d{2}|\d{6}) /
@@ -13,7 +13,7 @@
 <source>
   @type tail
   path /var/log/kolla/mariadb/xinetd.log
-  pos_file /var/run/{{ fluentd_binary }}/mariadb-xinetd.pos
+  pos_file /var/run/td-agent/mariadb-xinetd.pos
   tag infra.mariadb-xinetd
   format multiline
   format_firstline /^\d{2}/\d{1,2}/\d{1,2}@\d{1,2}:\d{1,2}:\d{1,2}\: (START|EXIT)\: /

ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2

 <source>
   @type tail
   path /var/log/kolla/rabbitmq/rabbit@{{ ansible_facts.hostname }}.log
-  pos_file /var/run/{{ fluentd_binary }}/rabbit.pos
+  pos_file /var/run/td-agent/rabbit.pos
   tag infra.rabbit
   format multiline
   format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}/

ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2

@@ -2,7 +2,7 @@
 <source>
   @type tail
   path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log,/var/log/kolla/*/*_access.log,/var/log/kolla/*/*_error.log
-  pos_file /var/run/{{ fluentd_binary }}/kolla-openstack-wsgi.pos
+  pos_file /var/run/td-agent/kolla-openstack-wsgi.pos
   tag kolla.*
   format /^(?<Payload>.*)$/
   enable_watch_timer false

ansible/roles/common/templates/conf/input/05-libvirt.conf.j2

 <source>
   @type tail
   path /var/log/kolla/libvirt/libvirtd.log
-  pos_file /var/run/{{ fluentd_binary }}/libvirt.pos
+  pos_file /var/run/td-agent/libvirt.pos
   tag infra.libvirt
   format /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?<Pid>\d+): (?<log_level>\S+) : (?<Payload>.*)?$/
   time_key Timestamp

ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2

 <source>
   @type tail
   path /var/log/kolla/zookeeper/zookeeper.log
-  pos_file /var/run/{{ fluentd_binary }}/zookeeper.pos
+  pos_file /var/run/td-agent/zookeeper.pos
   tag infra.*
   format multiline
   format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/

ansible/roles/common/templates/conf/input/07-kafka.conf.j2

 <source>
   @type tail
   path /var/log/kolla/kafka/controller.log, /var/log/kolla/kafka/server.log, /var/log/kolla/kafka/state-change.log
-  pos_file /var/run/{{ fluentd_binary }}/kafka.pos
+  pos_file /var/run/td-agent/kafka.pos
   tag infra.*
   format multiline
   format_firstline /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] \S+ .*$/

ansible/roles/common/templates/conf/input/09-monasca.conf.j2

 <source>
   @type tail
   path /var/log/kolla/monasca/agent*.log
-  pos_file /var/run/{{ fluentd_binary }}/monasca-agent.pos
+  pos_file /var/run/td-agent/monasca-agent.pos
   tag kolla.*
   format multiline
   format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/
@@ -12,7 +12,7 @@
 <source>
   @type tail
   path /var/log/kolla/monasca/grafana.log
-  pos_file /var/run/{{ fluentd_binary }}/monasca-grafana.pos
+  pos_file /var/run/td-agent/monasca-grafana.pos
   tag infra.*
   format multiline
   format_firstline /^t=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{4} lvl=\S+ msg=.*$/

ansible/roles/common/templates/fluentd.json.j2

-{% set fluentd_user = fluentd_binary %}
-{% set fluentd_dir = '/etc/' ~ fluentd_binary %}
-
-{% if fluentd_binary == 'fluentd' %}
-    {% set fluentd_conf = 'fluent.conf' %}
-    {% if kolla_base_distro in ['ubuntu', 'debian'] %}
-        {% set fluentd_cmd = '/usr/local/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %}
-    {% else %}
-        {% set fluentd_cmd = '/usr/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %}
-    {% endif %}
-{% elif fluentd_binary == 'td-agent' %}
-    {% set fluentd_conf = fluentd_binary ~ '.conf' %}
-    {% set fluentd_cmd = '/usr/sbin/td-agent' %}
-{% endif %}
+{% set fluentd_user = 'td-agent' %}
+{% set fluentd_dir = '/etc/td-agent' %}
+{% set fluentd_conf = 'td-agent.conf' %}
+{% set fluentd_cmd = '/usr/sbin/td-agent' %}
 
 {
     "command": "{{ fluentd_cmd }} -o /var/log/kolla/fluentd/fluentd.log",

releasenotes/notes/fluentd-cleanup-efe1a668e36b0872.yaml

+---
+upgrade:
+  - |
+    ``fluentd_binary`` and ``fluentd_version`` variables are no longer in use
+    as Kolla Ansible supports a single fluentd version across all supported
+    Kolla image flavours.