add grafana log in heka

Change-Id: I38b57cc43411b703dbb0639a600aaef58e01bc38
Closes-Bug: #1631818

ansible/roles/common/tasks/config.yml

@@ -30,6 +30,7 @@
     - { name: "elasticsearch", enabled: "{{ enable_central_logging }}" }
     - { name: "global", enabled: "yes" }
     - { name: "gnocchi", enabled: "{{ enable_gnocchi }}" }
+    - { name: "grafana", enabled: "{{ enable_grafana }}" }
     - { name: "haproxy", enabled: "{{ enable_haproxy }}" }
     - { name: "horizon", enabled: "{{ enable_horizon }}" }
     - { name: "keepalived", enabled: "{{ enable_haproxy }}" }
@@ -77,6 +78,7 @@
     - { name: "glance", enabled: "{{ enable_glance }}" }
     - { name: "global", enabled: "yes" }
     - { name: "gnocchi", enabled: "{{ enable_gnocchi }}" }
+    - { name: "grafana", enabled: "{{ enable_grafana }}" }
     - { name: "haproxy", enabled: "{{ enable_haproxy }}" }
     - { name: "heat", enabled: "{{ enable_heat }}" }
     - { name: "keepalived", enabled: "{{ enable_haproxy }}" }

ansible/roles/common/templates/cron-logrotate-grafana.conf.j2

+"/var/log/kolla/grafana/*.log"
+{
+}

ansible/roles/common/templates/cron.json.j2

@@ -7,6 +7,7 @@
     ( 'cloudkitty', enable_cloudkitty ),
     ( 'glance', enable_glance ),
     ( 'gnocchi', enable_gnocchi ),
+    ( 'grafana', enable_grafana ),
     ( 'haproxy', enable_haproxy ),
     ( 'heat', enable_heat ),
     ( 'keepalived', enable_haproxy ),

ansible/roles/common/templates/heka-grafana.toml.j2

+[grafana_log_decoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/os_grafana_log.lua"
+
+[grafana_log_splitter]
+type = "RegexSplitter"
+delimiter = '\n\n(=[^=]+====)'
+delimiter_eol = false
+deliver_incomplete_final = true
+
+[grafana_logstreamer_input]
+type = "LogstreamerInput"
+decoder = "grafana_log_decoder"
+splitter = "grafana_log_splitter"
+log_directory = "/var/log/kolla"
+file_match = 'grafana/(?P<Service>grafana.*)\.log\.?(?P<Seq>\d*)$'
+priority = ["^Seq"]
+differentiator = ["Service"]

ansible/roles/common/templates/heka.json.j2

@@ -29,6 +29,13 @@
             "perm": "0600",
             "optional": {{ (not enable_gnocchi | bool) | string | lower }}
         },
+        {
+            "source": "{{ container_config_directory }}/heka-grafana.toml",
+            "dest": "/etc/heka/heka-grafana.toml",
+            "owner": "heka",
+            "perm": "0600",
+            "optional": {{ (not enable_grafana | bool) | string | lower }}
+        },
         {
             "source": "{{ container_config_directory }}/heka-barbican.toml",
             "dest": "/etc/heka/heka-barbican.toml",

docker/heka/plugins/decoders/os_grafana_log.lua

+-- Copyright 2015-2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local dt     = require "date_time"
+local l      = require 'lpeg'
+l.locale(l)
+
+local patt   = require 'os_patterns'
+local utils  = require 'os_utils'
+
+local msg = {
+    Timestamp   = nil,
+    Type        = 'log',
+    Hostname    = nil,
+    Payload     = nil,
+    Pid         = nil,
+    Fields      = nil,
+    Severity    = nil,
+}
+
+--   Grafana message logs are formatted like this:
+--   =ERROR REPORT==== 2-Jan-2015::09:17:22 ===
+--   Blabla
+--   Blabla
+--
+local message   = l.Cg(patt.Message / utils.chomp, "Message")
+-- The token before 'REPORT' isn't standardized so it can be a valid severity
+-- level as 'INFO' or 'ERROR' but also 'CRASH' or 'SUPERVISOR'.
+local severity  = l.Cg(l.R"AZ"^1, "SeverityLabel")
+local day = l.R"13" * l.R"09" + l.R"19"
+local datetime = l.Cg(day, "day") * patt.dash * dt.date_mabbr * patt.dash * dt.date_fullyear *
+                 "::" * dt.rfc3339_partial_time
+local timestamp = l.Cg(l.Ct(datetime)/ dt.time_to_ns, "Timestamp")
+
+local grammar = l.Ct("=" * severity * " REPORT==== " * timestamp * " ===" * l.P'\n' * message)
+
+function process_message ()
+    local log = read_message("Payload")
+
+    local m = grammar:match(log)
+    if not m then
+        return -1
+    end
+
+    msg.Timestamp = m.Timestamp
+    msg.Payload = m.Message
+    msg.Logger = read_message("Logger")
+
+    if utils.label_to_severity_map[m.SeverityLabel] then
+        msg.Severity = utils.label_to_severity_map[m.SeverityLabel]
+    elseif m.SeverityLabel == 'CRASH' then
+        msg.Severity = 2 -- CRITICAL
+    else
+        msg.Severity = 5 -- NOTICE
+    end
+
+    msg.Fields = {}
+    msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity]
+    msg.Fields.programname = 'grafana'
+
+    return utils.safe_inject_message(msg)
+end

releasenotes/notes/add-grafana-in-heka-6397498442c00670.yaml

+---
+features:
+  - add grafana log in heka