diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one
index 4596fa13f64e759a01cdc087a6c76b57e92daf43..1bb26e922661f31037d51c679453ace09baf9b46 100644
--- a/ansible/inventory/all-in-one
+++ b/ansible/inventory/all-in-one
@@ -454,6 +454,9 @@ monasca
[monasca-log-persister:children]
monasca
+[monasca-log-metrics:children]
+monasca
+
# Ironic
[ironic-api:children]
ironic
diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode
index 80a39b601b84ada9f7bbefdc3958773b73df7355..826f6ea6bec00388965a7419f16b80b2a6059f4b 100644
--- a/ansible/inventory/multinode
+++ b/ansible/inventory/multinode
@@ -463,6 +463,9 @@ monasca
[monasca-log-persister:children]
monasca
+[monasca-log-metrics:children]
+monasca
+
# Ironic
[ironic-api:children]
ironic
diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml
index 55c2f125dc2d6da6d7dcad4e2b14f822345692b7..309d545dd36c7fd45956295e9d365615d42af93e 100644
--- a/ansible/roles/monasca/defaults/main.yml
+++ b/ansible/roles/monasca/defaults/main.yml
@@ -40,6 +40,16 @@ monasca_services:
- "/etc/localtime:/etc/localtime:ro"
- "kolla_logs:/var/log/kolla"
dimensions: "{{ monasca_log_persister_dimensions }}"
+ monasca-log-metrics:
+ container_name: monasca_log_metrics
+ group: monasca-log-metrics
+ enabled: true
+ image: "{{ monasca_logstash_image_full }}"
+ volumes:
+ - "{{ node_config_directory }}/monasca-log-metrics/:{{ container_config_directory }}/:ro"
+ - "/etc/localtime:/etc/localtime:ro"
+ - "kolla_logs:/var/log/kolla"
+ dimensions: "{{ monasca_log_metrics_dimensions }}"
####################
# Databases
@@ -94,6 +104,7 @@ monasca_api_dimensions: "{{ default_container_dimensions }}"
monasca_log_api_dimensions: "{{ default_container_dimensions }}"
monasca_log_transformer_dimensions: "{{ default_container_dimensions }}"
monasca_log_persister_dimensions: "{{ default_container_dimensions }}"
+monasca_log_metrics_dimensions: "{{ default_container_dimensions }}"
####################
diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml
index d20d442741af210741ce8a728745460f73e4e3c2..e47955b1e0ca6a62f4adc9c3eb1bb64a9dadb60d 100644
--- a/ansible/roles/monasca/handlers/main.yml
+++ b/ansible/roles/monasca/handlers/main.yml
@@ -87,3 +87,24 @@
or monasca_log_persister_confs.changed | bool
or monasca_log_persister_elasticsearch_template.changed | bool
or monasca_log_persister_container.changed | bool
+
+- name: Restart monasca-log-metrics container
+ vars:
+ service_name: "monasca-log-metrics"
+ service: "{{ monasca_services[service_name] }}"
+ config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ monasca_log_metrics_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ kolla_docker:
+ action: "recreate_or_restart_container"
+ common_options: "{{ docker_common_options }}"
+ name: "{{ service.container_name }}"
+ image: "{{ service.image }}"
+ volumes: "{{ service.volumes }}"
+ dimensions: "{{ service.dimensions }}"
+ when:
+ - kolla_action != "config"
+ - inventory_hostname in groups[service.group]
+ - service.enabled | bool
+ - config_json.changed | bool
+ or monasca_log_metrics_confs.changed | bool
+ or monasca_log_metrics_container.changed | bool
diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml
index 454452270eac60a9ef60f35295062cffb8d34c87..4c07bd31b7291235e2f0795ec4da05df54dbda14 100644
--- a/ansible/roles/monasca/tasks/config.yml
+++ b/ansible/roles/monasca/tasks/config.yml
@@ -156,6 +156,24 @@
notify:
- Restart monasca-log-persister container
+- name: Copying over monasca-log-metrics config
+ vars:
+ service: "{{ monasca_services['monasca-log-metrics'] }}"
+ template:
+ src: "{{ item }}"
+ dest: "{{ node_config_directory }}/monasca-log-metrics/log-metrics.conf"
+ mode: "0660"
+ become: true
+ register: monasca_log_metrics_confs
+ with_first_found:
+ - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-metrics.conf"
+ - "{{ node_custom_config }}/monasca/log-metrics.conf"
+ - "{{ role_path }}/templates/monasca-log-metrics/log-metrics.conf.j2"
+ when:
+ - inventory_hostname in groups[service['group']]
+ - service.enabled | bool
+ notify:
+ - Restart monasca-log-metrics container
- name: Check monasca containers
become: true
diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml
index 15dbbbdecc501fc9bdb0480bff90a29155635179..aa233dc2c56eebe17a91c9312e4fee4d2c84970c 100644
--- a/ansible/roles/monasca/tasks/deploy.yml
+++ b/ansible/roles/monasca/tasks/deploy.yml
@@ -7,7 +7,8 @@
when: inventory_hostname in groups['monasca-api'] or
inventory_hostname in groups['monasca-log-api'] or
inventory_hostname in groups['monasca-log-transformer'] or
- inventory_hostname in groups['monasca-log-persister']
+ inventory_hostname in groups['monasca-log-persister'] or
+ inventory_hostname in groups['monasca-log-metrics']
- include_tasks: bootstrap.yml
when: inventory_hostname in groups['monasca-api']
@@ -19,4 +20,5 @@
when: inventory_hostname in groups['monasca-api'] or
inventory_hostname in groups['monasca-log-api'] or
inventory_hostname in groups['monasca-log-transformer'] or
- inventory_hostname in groups['monasca-log-persister']
+ inventory_hostname in groups['monasca-log-persister'] or
+ inventory_hostname in groups['monasca-log-metrics']
diff --git a/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2
new file mode 100644
index 0000000000000000000000000000000000000000..805164541e1529b5a9b8457925a3e3d2b419bf73
--- /dev/null
+++ b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2
@@ -0,0 +1,76 @@
+# This config file is used to generate Monasca metrics from logs ingested
+# by the Monasca Log API. Alarms and notifications can then be set to alert
+# users when a particular log message is ingested. This example config file
+# generates metrics for all logs which have a log level which isn't
+# debug, trace or info level. A user may want to apply additional logic to
+# generate special metrics when particular log messages are ingested. For
+# example, if HAProxy fails over, file system corruption is detected or
+# other scenarios of interest.
+
+input {
+ kafka {
+ zk_connect => "{{ monasca_zookeeper_servers }}"
+ topic_id => "{{ monasca_transformed_logs_topic }}"
+ group_id => "log_metrics"
+ consumer_id => "log_metrics_{{ ansible_hostname }}"
+ consumer_threads => "{{ monasca_log_pipeline_threads }}"
+ }
+}
+
+filter {
+ # Drop everything we don't want to create metrics for.
+ if ![log][dimensions][log_level] or [log][dimensions][log_level] in [ "debug", "trace", "info" ] {
+ drop {
+ }
+ }
+
+ # Generate a metric name based on the program and log level
+ mutate {
+ add_field => { "[metric][name]" => "log.%{[log][dimensions][programname]}.%{[log][dimensions][log_level]}" }
+ }
+
+ # Form the metric structure.
+ mutate {
+ add_field => { "[metric][value]" => 1 }
+ rename => { "[log][dimensions]" => "[metric][dimensions]" }
+ rename => { "[metric][dimensions][Hostname]" => "[metric][dimensions][hostname]" }
+ rename => { "[metric][dimensions][programname]" => "[metric][dimensions][service]" }
+ }
+ mutate {
+ convert => { "[metric][value]" => "float" }
+ }
+
+ # Convert the timestamp of the event to milliseconds since epoch.
+ ruby {
+ code => "event['metric']['timestamp'] = event['@timestamp'].to_i * 1000"
+ }
+
+ # Clean up any fields which aren't required from the new metric to save space
+ mutate {
+ remove_field => ["[metric][dimensions][log_level]",
+ "[metric][dimensions][domain_id]",
+ "[metric][dimensions][user_id]",
+ "[metric][dimensions][tenant_id]",
+ "[metric][dimensions][project_domain]",
+ "[metric][dimensions][tag]",
+ "[metric][dimensions][Logger]",
+ "[metric][dimensions][Pid]",
+ "[metric][dimensions][user_domain]",
+ "[metric][dimensions][request_id]",
+ "[metric][dimensions][python_module]",
+ "[metric][meta]",
+ "creation_time",
+ "log",
+ "@version",
+ "@timestamp"]
+ }
+}
+
+output {
+ kafka {
+ bootstrap_servers => "{{ monasca_kafka_servers }}"
+ topic_id => "{{ monasca_metrics_topic }}"
+ client_id => "log_metrics_{{ ansible_hostname }}"
+ workers => "{{ monasca_log_pipeline_threads }}"
+ }
+}
diff --git a/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 b/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2
new file mode 100644
index 0000000000000000000000000000000000000000..287ffcd9751ef07cdb1ba76a94d6f1574d04bac9
--- /dev/null
+++ b/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2
@@ -0,0 +1,18 @@
+{
+ "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-metrics.log -f /etc/logstash/conf.d/log-metrics.conf",
+ "config_files": [
+ {
+ "source": "{{ container_config_directory }}/log-metrics.conf",
+ "dest": "/etc/logstash/conf.d/log-metrics.conf",
+ "owner": "logstash",
+ "perm": "0600"
+ }
+ ],
+ "permissions": [
+ {
+ "path": "/var/log/kolla/logstash",
+ "owner": "logstash:kolla",
+ "recurse": true
+ }
+ ]
+}
diff --git a/releasenotes/notes/add-monasca-log-metrics-370846df015ff96a.yaml b/releasenotes/notes/add-monasca-log-metrics-370846df015ff96a.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8abda7ab3433046b89db31b3dd4681735ca4241f
--- /dev/null
+++ b/releasenotes/notes/add-monasca-log-metrics-370846df015ff96a.yaml
@@ -0,0 +1,5 @@
+---
+features:
+ - |
+ Add the Monasca Log Metrics service. This service is responsible for
+ generating metrics from log files.