Merge "Add fluentd role"

ansible/group_vars/all.yml

@@ -228,6 +228,8 @@ grafana_server_port: "3000"
 
 tacker_server_port: "9890"
 
+fluentd_syslog_port: "5140"
+
 public_protocol: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}"
 internal_protocol: "http"
 admin_protocol: "http"

ansible/roles/common/defaults/main.yml

@@ -10,10 +10,6 @@ kolla_toolbox_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ d
 kolla_toolbox_tag: "{{ openstack_release }}"
 kolla_toolbox_image_full: "{{ kolla_toolbox_image }}:{{ kolla_toolbox_tag }}"
 
-heka_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-heka"
-heka_tag: "{{ openstack_release }}"
-heka_image_full: "{{ heka_image }}:{{ heka_tag }}"
-
 cron_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-cron"
 cron_tag: "{{ openstack_release }}"
 cron_image_full: "{{ cron_image }}:{{ cron_tag }}"

ansible/roles/common/tasks/clean_heka.yml

+---
+- name: Get container facts
+  kolla_container_facts:
+    name:
+      - heka
+  register: container_facts
+
+- name: Removing heka container
+  kolla_docker:
+    action: "remove_container"
+    name: "heka"
+  when:
+    - container_facts['heka'] is defined

ansible/roles/common/tasks/config.yml

@@ -5,7 +5,11 @@
     state: "directory"
     recurse: yes
   with_items:
-    - "heka"
+    - "fluentd"
+    - "fluentd/input"
+    - "fluentd/output"
+    - "fluentd/format"
+    - "fluentd/filter"
     - "kolla-toolbox"
     - "cron"
     - "cron/logrotate"
@@ -15,56 +19,50 @@
     src: "{{ item }}.json.j2"
     dest: "{{ node_config_directory }}/{{ item }}/config.json"
   with_items:
-    - "heka"
+    - "fluentd"
     - "kolla-toolbox"
     - "cron"
 
-- name: Copying over heka config files
+- name: Copying over fluentd input config files
   template:
-    src: "heka-{{ item.src|default(item.name) }}.toml.j2"
-    dest: "{{ node_config_directory }}/heka/heka-{{ item.name }}.toml"
-  when: item.enabled | bool
+    src: "conf/input/{{ item }}.conf.j2"
+    dest: "{{ node_config_directory }}/fluentd/input/{{ item }}.conf"
   with_items:
-    - { name: "aodh", enabled: "{{ enable_aodh }}" }
-    - { name: "barbican", enabled: "{{ enable_barbican }}"}
-    - { name: "ceilometer", enabled: "{{ enable_ceilometer }}" }
-    - { name: "elasticsearch", enabled: "{{ enable_elasticsearch }}" }
-    - { name: "global", enabled: "yes" }
-    - { name: "gnocchi", enabled: "{{ enable_gnocchi }}" }
-    - { name: "grafana", enabled: "{{ enable_grafana }}" }
-    - { name: "haproxy", enabled: "{{ enable_haproxy }}" }
-    - { name: "horizon", enabled: "{{ enable_horizon }}" }
-    - { name: "keepalived", enabled: "{{ enable_haproxy }}" }
-    - { name: "keystone", enabled: "{{ enable_keystone }}" }
-    - { name: "mariadb", enabled: "{{ enable_mariadb }}" }
-    - { name: "openstack", enabled: "yes" }
-    - { name: "panko", enabled: "{{ enable_panko }}" }
-    - { name: "rabbitmq", enabled: "{{ enable_rabbitmq }}" }
-    - { name: "swift-account-auditor", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-account-reaper", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-account-replicator", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-account-server", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-container-auditor", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-container-replicator", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-container-server", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-container-updater", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-object-auditor", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-object-expirer", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-object-replicator", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-object-server", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-object-updater", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-proxy-server", src: "swift", enabled: "{{ enable_swift }}" }
-    - { name: "swift-rsyncd", src: "swift", enabled: "{{ enable_swift }}" }
+    - "00-global"
+    - "01-syslog"
+    - "02-mariadb"
+    - "03-rabbitmq"
+
+- name: Copying over fluentd ouput config files
+  template:
+    src: "conf/output/{{ item }}.conf.j2"
+    dest: "{{ node_config_directory }}/fluentd/output/{{ item }}.conf"
+  with_items:
+    - "00-local"
+    - "01-es"
+- name: Copying over fluentd format config files
+  template:
+    src: "conf/format/{{ item }}.conf.j2"
+    dest: "{{ node_config_directory }}/fluentd/format/{{ item }}.conf"
+  with_items:
+    - "apache_access"
+    - "wsgi_access"
+    - "wsgi_python"
 
-- name: Heka custom config file exists
-  local_action: stat path="{{ node_custom_config }}/heka/heka-custom.toml"
-  register: heka_custom_stat_result
+- name: Copying over fluentd filter config files
+  template:
+    src: "conf/filter/{{ item }}.conf.j2"
+    dest: "{{ node_config_directory }}/fluentd/filter/{{ item }}.conf"
+  with_items:
+    - "00-record_transformer"
+    - "01-rewrite"
 
-- name: Copying over heka custom config file
+- name: Copying over tg-agent.conf
   template:
-    src: "{{ node_custom_config }}/heka/heka-custom.toml"
-    dest: "{{ node_config_directory }}/heka/heka-custom.toml"
-  when: heka_custom_stat_result.stat.exists == true
+    src: "td-agent.conf.j2"
+    dest: "{{ node_config_directory }}/{{ item }}/td-agent.conf"
+  with_items:
+    - "fluentd"
 
 - name: Copying over cron logrotate config files
   template:

ansible/roles/common/tasks/pull.yml

@@ -5,11 +5,11 @@
     common_options: "{{ docker_common_options }}"
     image: "{{ kolla_toolbox_image_full }}"
 
-- name: Pulling heka image
+- name: Pulling fluentd image
   kolla_docker:
     action: "pull_image"
     common_options: "{{ docker_common_options }}"
-    image: "{{ heka_image_full }}"
+    image: "{{ fluentd_image_full }}"
 
 - name: Pulling cron image
   kolla_docker:

ansible/roles/common/tasks/reconfigure.yml

 ---
-- name: Ensuring the heka container is up
+- name: Ensuring the fluentd container is up
   kolla_docker:
-    name: "heka"
+    name: "fluentd"
     action: "get_container_state"
   register: container_state
   failed_when: container_state.Running == false
 
 - include: config.yml
 
-- name: Checking the heka config
-  command: docker exec heka /usr/local/bin/kolla_set_configs --check
+- name: Checking the fluentd config
+  command: docker exec fluentd /usr/local/bin/kolla_set_configs --check
   changed_when: false
   failed_when: false
   register: check_result
 
-- name: Getting the heka container config strategy
+- name: Getting the fluentd container config strategy
   kolla_docker:
-    name: "heka"
+    name: "fluentd"
     action: "get_container_env"
   register: container_env
 
-- name: Removing the heka container
+- name: Removing the fluentd container
   kolla_docker:
-    name: "heka"
+    name: "fluentd"
     action: "remove_container"
   register: remove_container
   when:
@@ -32,9 +32,9 @@
 - include: start.yml
   when: remove_container.changed
 
-- name: Restarting the heka container
+- name: Restarting the fluentd container
   kolla_docker:
-    name: "heka"
+    name: "fluentd"
     action: "restart_container"
   when:
     - config_strategy == "COPY_ALWAYS"

ansible/roles/common/tasks/start.yml

 ---
-- name: Starting heka container
+- name: Starting fluentd container
   kolla_docker:
     action: "start_container"
     common_options: "{{ docker_common_options }}"
     environment:
       KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
       SKIP_LOG_SETUP: "true"
-    image: "{{ heka_image_full }}"
-    name: "heka"
+    image: "{{ fluentd_image_full }}"
+    name: "fluentd"
     volumes:
-      - "{{ node_config_directory }}/heka/:{{ container_config_directory }}/:ro"
+      - "{{ node_config_directory }}/fluentd/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla/"
-      - "heka:/var/cache/hekad"
-      - "heka_socket:/var/lib/kolla/heka/"
 
 - name: Starting kolla-toolbox container
   kolla_docker:
@@ -45,5 +43,4 @@
     volumes:
       - "{{ node_config_directory }}/cron/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
-      - "heka_socket:/var/lib/kolla/heka/"
       - "kolla_logs:/var/log/kolla/"

ansible/roles/common/tasks/upgrade.yml

 ---
 - include: config.yml
 
+- include: clean_heka.yml
+
 - include: start.yml

ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2

+<filter *.var.log.kolla.*.*.log>
+    @type record_transformer
+    <record>
+        Hostname ${hostname}
+        Logger openstack.${tag_parts[4]}
+        programname ${tag_parts[5]}
+    </record>
+</filter>

ansible/roles/common/templates/conf/filter/01-rewrite.conf.j2

+<match kolla.var.log.kolla.*.*.log>
+    @type rewrite_tag_filter
+    capitalize_regex_backreference yes
+    rewriterule1 programname ^(horizon-access|ceilometer-api-access|keystone-apache-admin-access|keystone-apache-public-access)$ apache_access
+    rewriterule2 programname ^aodh_wsgi_access$ wsgi_access
+    rewriterule3 programname ^(nova-api|nova-compute|nova-conductor|nova-consoleauth|nova-manage|nova-novncproxy|nova-scheduler|privsep-helper).* openstack_python
+    rewriterule4 programname ^(sahara-api|sahara-engine).* openstack_python
+    rewriterule5 programname ^(neutron-server|neutron-openvswitch-agent|neutron-ns-metadata-proxy|neutron-metadata-agent|neutron-l3-agent|neutron-dhcp-agent).* openstack_python
+    rewriterule6 programname ^(magnum-conductor|magnum-api).* openstack_python
+    rewriterule7 programname ^(keystone).* openstack_python
+    rewriterule8 programname ^(heat-engine|heat-api|heat-api-cfn).* openstack_python
+    rewriterule9 programname ^(registry|api).* openstack_python
+    rewriterule10 programname ^(cloudkitty-storage-init|cloudkitty-processor|cloudkitty-dbsync|cloudkitty-api).* openstack_python
+    rewriterule11 programname ^(ceilometer-polling|ceilometer-collector|ceilometer-agent-notification|loadwsgi).* openstack_python
+    rewriterule12 programname ^(barbican-worker|barbican-keystone-listener|barbican-db-manage|barbican-api|app).* openstack_python
+    rewriterule13 programname ^(aodh-notifier|aodh-listener|aodh-evaluator|aodh-dbsync).* openstack_python
+</match>

ansible/roles/common/templates/conf/format/apache_access.conf.j2

+<filter apache_access>
+  @type parser
+  reserve_data true
+  key_name message
+  <parse>
+  @type "grok"
+  grok_pattern \[%{HTTPDATE:Timestamp}\] "(?:%{WORD:http_method} %{NOTSPACE:http_url}(?: HTTP/%{NUMBER:http_version})?|%{DATA:rawrequest})" %{NUMBER:http_status} (?:\d+|-)
+  time_key Timestamp
+  time_format %d/%b/%Y:%H:%M:%S %z
+  keep_time_key true
+  </parse>
+</filter>
+
+<filter apache_access>
+   @type record_transformer
+   <record>
+     severity_label INFO
+     Severity 6
+   </record>
+</filter>

ansible/roles/common/templates/conf/format/wsgi_access.conf.j2

+<filter wsgi_access>
+  @type parser
+  reserve_data true
+  key_name message
+  <parse>
+  @type "grok"
+  grok_pattern %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:Timestamp}\] "(?:%{WORD:http_method} %{NOTSPACE:http_url}(?: HTTP/%{NUMBER:http_version})?|%{DATA:rawrequest})" %{NUMBER:http_status} (?:%{NUMBER:http_bytes}|-) (?:%{NUMBER:http_response_time_us}|-) %{QS:referrer} %{QS:agent}
+  time_key Timestamp
+  time_format %d/%b/%Y:%H:%M:%S %z
+  keep_time_key true
+  </parse>
+</filter>

ansible/roles/common/templates/conf/format/wsgi_python.conf.j2

+<filter openstack_python>
+  @type parser
+  reserve_data true
+  key_name message
+  <parse>
+  @type "grok"
+  grok_pattern %{TIMESTAMP_ISO8601:Timestamp} %{NUMBER:Pid} %{LOGLEVEL:severity_label} %{NOTSPACE:python_module} \[req-(?:%{NOTSPACE:request_id}) (?:%{NOTSPACE:user_id}) (?:%{NOTSPACE:tenant_id}) .*\] %{GREEDYDATA:Payload}
+  time_format "%Y-%m-%d %H:%M:%S.%L"
+  time_key Timestamp
+  keep_time_key true
+  </parse>
+</filter>

ansible/roles/common/templates/conf/input/00-global.conf.j2

+<source>
+  @type tail
+  path /var/log/kolla/*/*.log
+  exclude_path ["/var/log/kolla/rabbitmq/*.log", "/var/log/kolla/mariadb/mariadb.log", "/var/log/kolla/haproxy/*.log", "/var/log/kolla/swift/*.log"]
+  pos_file /var/run/td-agent/kolla.pos
+  tag kolla.*
+  format /^(?<message>.*)$/
+</source>

ansible/roles/common/templates/conf/input/01-syslog.conf.j2

+<source>
+  @type syslog
+  port {{ fluentd_syslog_port }}
+  bind {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
+  tag syslog
+  format /^(?<Payload>.*)$/
+</source>

ansible/roles/common/templates/conf/input/02-mariadb.conf.j2

+<source>
+  @type tail
+  path /var/log/kolla/mariadb/mariadb.log
+  pos_file /var/run/td-agent/mariadb.pos
+  tag mariadb.*
+  format multiline
+  format_firstline /^\d{6}/
+  format1 /^(?<time>\d{6} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<severity_label>\S+)\] (?<Payload>.*)/
+  time_format %y%m%d %k:%M:%S
+</source>

ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2

+<source>
+  @type tail
+  path /var/log/kolla/rabbitmq/rabbit.log
+  pos_file /var/run/td-agent/rabbit.pos
+  tag rabbit.*
+  format multiline
+  format_firstline /^=/
+  format1 /^.*\n/
+  format2 /^(?<Payload>.*)\n/
+</source>

ansible/roles/common/templates/conf/output/00-local.conf.j2

+<match syslog.local0.**>
+  @type copy
+  <store>
+    @type file
+    path /var/log/kolla/swift/swift_latest.*.log
+    symlink_path /var/log/kolla/swift/swift_latest.log
+    utc
+    append true
+    compress gzip
+  </store>
+  <store>
+       type elasticsearch
+       host {{ kolla_external_vip_address }}
+       port {{ elasticsearch_port }}
+       logstash_format true
+       logstash_prefix flog
+       flush_interval 15s
+  </store>
+</match>
+
+<match syslog.local1.**>
+  @type copy
+  <store>
+    @type file
+    path /var/log/kolla/haproxy/haproxy_latest.*.log
+    symlink_path /var/log/kolla/haproxy/haproxy_latest.log
+    output_tag false
+    output_time false
+    utc
+    append true
+    compress gzip
+  </store>
+  <store>
+       type elasticsearch
+       host {{ kolla_external_vip_address }}
+       port {{ elasticsearch_port }}
+       logstash_format true
+       logstash_prefix flog
+       flush_interval 15s
+  </store>
+</match>

ansible/roles/common/templates/conf/output/01-es.conf.j2

+<match *.**>
+    type copy
+    <store>
+       type elasticsearch
+       host {{ kolla_external_vip_address }} 
+       port {{ elasticsearch_port }}
+       logstash_format true
+       logstash_prefix flog
+       flush_interval 15s
+    </store>
+</match>

ansible/roles/common/templates/fluentd.json.j2

+{
+    "command": "/usr/sbin/td-agent",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/td-agent.conf",
+            "dest": "/etc/td-agent/td-agent.conf",
+            "owner": "td-agent",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/input/00-global.conf",
+            "dest": "/etc/td-agent/input/00-global.conf",
+            "owner": "td-agent",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/input/02-mariadb.conf",
+            "dest": "/etc/td-agent/input/02-mariadb.conf",
+            "owner": "td-agent",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/input/03-rabbitmq.conf",
+            "dest": "/etc/td-agent/input/03-rabbitmq.conf",
+            "owner": "td-agent",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/input/01-syslog.conf",
+            "dest": "/etc/td-agent/input/01-syslog.conf",
+            "owner": "td-agent",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/filter/00-record_transformer.conf",
+            "dest": "/etc/td-agent/filter/00-record_transformer.conf",
+            "owner": "td-agent",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/filter/01-rewrite.conf",
+            "dest": "/etc/td-agent/filter/01-rewrite.conf",
+            "owner": "td-agent",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/format/apache_access.conf",
+            "dest": "/etc/td-agent/format/apache_access.conf",
+            "owner": "td-agent",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/format/wsgi_access.conf",
+            "dest": "/etc/td-agent/format/wsgi_access.conf",
+            "owner": "td-agent",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/format/wsgi_python.conf",
+            "dest": "/etc/td-agent/format/wsgi_python.conf",
+            "owner": "td-agent",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/output/01-es.conf",
+            "dest": "/etc/td-agent/output/01-es.conf",
+            "owner": "td-agent",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/output/00-local.conf",
+            "dest": "/etc/td-agent/output/00-local.conf",
+            "owner": "td-agent",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+       {
+            "path": "/var/log/kolla",
+            "owner": "td-agent:td-agent",
+            "recurse": true
+       }
+    ]
+}