Support deploying Monasca Log Metrics

The log metrics service generates metrics from log messages
which allows further analysis and alerting to be performed
on them. Basic configuration is provided so that metrics
are generated for high level warning logs such as error, or
warning.

Change-Id: I45cc17817c716296451f620f304c0b1108162a56
Partially-Implements: blueprint monasca-roles

ansible/inventory/all-in-one

@@ -455,6 +455,9 @@ monasca
 [monasca-log-persister:children]
 monasca
 
+[monasca-log-metrics:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/inventory/multinode

@@ -464,6 +464,9 @@ monasca
 [monasca-log-persister:children]
 monasca
 
+[monasca-log-metrics:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/roles/monasca/defaults/main.yml

@@ -40,6 +40,16 @@ monasca_services:
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla"
     dimensions: "{{ monasca_log_persister_dimensions }}"
+  monasca-log-metrics:
+    container_name: monasca_log_metrics
+    group: monasca-log-metrics
+    enabled: true
+    image: "{{ monasca_logstash_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/monasca-log-metrics/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
+    dimensions: "{{ monasca_log_metrics_dimensions }}"
 
 ####################
 # Databases
@@ -94,6 +104,7 @@ monasca_api_dimensions: "{{ default_container_dimensions }}"
 monasca_log_api_dimensions: "{{ default_container_dimensions }}"
 monasca_log_transformer_dimensions: "{{ default_container_dimensions }}"
 monasca_log_persister_dimensions: "{{ default_container_dimensions }}"
+monasca_log_metrics_dimensions: "{{ default_container_dimensions }}"
 
 
 ####################

ansible/roles/monasca/handlers/main.yml

@@ -87,3 +87,24 @@
       or monasca_log_persister_confs.changed | bool
       or monasca_log_persister_elasticsearch_template.changed | bool
       or monasca_log_persister_container.changed | bool
+
+- name: Restart monasca-log-metrics container
+  vars:
+    service_name: "monasca-log-metrics"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_log_metrics_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_log_metrics_confs.changed | bool
+      or monasca_log_metrics_container.changed | bool

ansible/roles/monasca/tasks/config.yml

@@ -156,6 +156,24 @@
   notify:
     - Restart monasca-log-persister container
 
+- name: Copying over monasca-log-metrics config
+  vars:
+    service: "{{ monasca_services['monasca-log-metrics'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-log-metrics/log-metrics.conf"
+    mode: "0660"
+  become: true
+  register: monasca_log_metrics_confs
+  with_first_found:
+    - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-metrics.conf"
+    - "{{ node_custom_config }}/monasca/log-metrics.conf"
+    - "{{ role_path }}/templates/monasca-log-metrics/log-metrics.conf.j2"
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-log-metrics container
 
 - name: Check monasca containers
   become: true

ansible/roles/monasca/tasks/deploy.yml

@@ -7,7 +7,8 @@
   when: inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api'] or
         inventory_hostname in groups['monasca-log-transformer'] or
-        inventory_hostname in groups['monasca-log-persister']
+        inventory_hostname in groups['monasca-log-persister'] or
+        inventory_hostname in groups['monasca-log-metrics']
 
 - include_tasks: bootstrap.yml
   when: inventory_hostname in groups['monasca-api']
@@ -19,4 +20,5 @@
   when: inventory_hostname in groups['monasca-api'] or
         inventory_hostname in groups['monasca-log-api'] or
         inventory_hostname in groups['monasca-log-transformer'] or
-        inventory_hostname in groups['monasca-log-persister']
+        inventory_hostname in groups['monasca-log-persister'] or
+        inventory_hostname in groups['monasca-log-metrics']

ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2

+# This config file is used to generate Monasca metrics from logs ingested
+# by the Monasca Log API. Alarms and notifications can then be set to alert
+# users when a particular log message is ingested. This example config file
+# generates metrics for all logs which have a log level which isn't
+# debug, trace or info level. A user may want to apply additional logic to
+# generate special metrics when particular log messages are ingested. For
+# example, if HAProxy fails over, file system corruption is detected or
+# other scenarios of interest.
+
+input {
+    kafka {
+        zk_connect => "{{ monasca_zookeeper_servers }}"
+        topic_id => "{{ monasca_transformed_logs_topic }}"
+        group_id => "log_metrics"
+        consumer_id => "log_metrics_{{ ansible_hostname }}"
+        consumer_threads => "{{ monasca_log_pipeline_threads }}"
+    }
+}
+
+filter {
+    # Drop everything we don't want to create metrics for.
+    if ![log][dimensions][log_level] or [log][dimensions][log_level] in [ "debug", "trace", "info" ] {
+        drop {
+        }
+    }
+
+    # Generate a metric name based on the program and log level
+    mutate {
+        add_field => { "[metric][name]" => "log.%{[log][dimensions][programname]}.%{[log][dimensions][log_level]}" }
+    }
+
+    # Form the metric structure.
+    mutate {
+        add_field => { "[metric][value]" => 1 }
+        rename => { "[log][dimensions]" => "[metric][dimensions]" }
+        rename => { "[metric][dimensions][Hostname]" => "[metric][dimensions][hostname]" }
+        rename => { "[metric][dimensions][programname]" => "[metric][dimensions][service]" }
+    }
+    mutate {
+         convert => { "[metric][value]" => "float" }
+    }
+
+    # Convert the timestamp of the event to milliseconds since epoch.
+    ruby {
+        code => "event['metric']['timestamp'] = event['@timestamp'].to_i * 1000"
+    }
+
+    # Clean up any fields which aren't required from the new metric to save space
+    mutate {
+        remove_field => ["[metric][dimensions][log_level]",
+                         "[metric][dimensions][domain_id]",
+                         "[metric][dimensions][user_id]",
+                         "[metric][dimensions][tenant_id]",
+                         "[metric][dimensions][project_domain]",
+                         "[metric][dimensions][tag]",
+                         "[metric][dimensions][Logger]",
+                         "[metric][dimensions][Pid]",
+                         "[metric][dimensions][user_domain]",
+                         "[metric][dimensions][request_id]",
+                         "[metric][dimensions][python_module]",
+                         "[metric][meta]",
+                         "creation_time",
+                         "log",
+                         "@version",
+                         "@timestamp"]
+    }
+}
+
+output {
+    kafka {
+        bootstrap_servers => "{{ monasca_kafka_servers }}"
+        topic_id => "{{ monasca_metrics_topic }}"
+        client_id => "log_metrics_{{ ansible_hostname }}"
+        workers => "{{ monasca_log_pipeline_threads }}"
+    }
+}

ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2

+{
+    "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-metrics.log -f /etc/logstash/conf.d/log-metrics.conf",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/log-metrics.conf",
+            "dest": "/etc/logstash/conf.d/log-metrics.conf",
+            "owner": "logstash",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/logstash",
+            "owner": "logstash:kolla",
+            "recurse": true
+        }
+    ]
+}

releasenotes/notes/add-monasca-log-metrics-370846df015ff96a.yaml

+---
+features:
+  - |
+    Add the Monasca Log Metrics service. This service is responsible for
+    generating metrics from log files.