Merge "Ingest non-standard Monasca logs"

ansible/roles/common/tasks/config.yml

@@ -56,6 +56,7 @@
     - "06-zookeeper"
     - "07-kafka"
     - "08-opendaylight"
+    - "09-monasca"
   notify:
     - Restart fluentd container
 

ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2

@@ -34,4 +34,5 @@
     rewriterule31 programname ^(vitrage-ml|vitrage-notifier|vitrage-graph)$ openstack_python
     rewriterule32 programname ^(blazar-api|blazar-manager)$ openstack_python
     rewriterule33 programname ^(cyborg-api|cyborg-conductor|cyborg-agent)$ openstack_python
+    rewriterule34 programname ^(agent-collector|agent-forwarder|agent-statsd)$ openstack_python
 </match>

ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2

@@ -171,4 +171,9 @@
     pattern ^(qinling-engine|qinling-api)$
     tag openstack_python
   </rule>
+  <rule>
+    key     programname
+    pattern ^(agent-collector|agent-forwarder|agent-statsd)$
+    tag openstack_python
+  </rule>
 </match>

ansible/roles/common/templates/conf/input/00-global.conf.j2

@@ -44,7 +44,9 @@
 <source>
   @type tail
   path {% for service, enabled in services if enabled | bool %}/var/log/kolla/{{ service }}/*.log{% if not loop.last %},{% endif %}{% endfor %}
-  exclude_path ["/var/log/kolla/neutron/dnsmasq.log",
+  exclude_path ["/var/log/kolla/monasca/agent*.log",
+                "/var/log/kolla/monasca/grafana.log",
+                "/var/log/kolla/neutron/dnsmasq.log",
                 "/var/log/kolla/*/*-access.log",
                 "/var/log/kolla/*/*-error.log"]
   pos_file /var/run/{{ fluentd_dir }}/kolla-openstack.pos

ansible/roles/common/templates/conf/input/09-monasca.conf.j2

+<source>
+  @type tail
+  path /var/log/kolla/monasca/agent*.log
+  pos_file /var/run/fluentd/monasca-agent.pos
+  tag kolla.*
+  format multiline
+  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/
+  format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+) \| (?<log_level>\S+) \| (?<programname>\S+) \| (?<Payload>.*)$/
+  time_key Timestamp
+</source>
+
+<source>
+  @type tail
+  path /var/log/kolla/monasca/grafana.log
+  pos_file /var/run/fluentd/monasca-grafana.pos
+  tag infra.*
+  format multiline
+  format_firstline /^t=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{4} lvl=\S+ msg=.*$/
+  format1 /^t=(?<Timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{4}) lvl=(?<log_level>\S+) msg=(?<Payload>.*)$/
+  time_key Timestamp
+</source>
+