Skip to content
Snippets Groups Projects
Commit 4180bee0 authored by Michal Nasiadka's avatar Michal Nasiadka
Browse files

Use fluentd image labels

In order to orchestrate smooth transition to fluentd 0.14.x
aka 1.0 stable branch aka td-agent 3
from td-agent repository - use image labels (fluentd_version
and fluentd_binary).

Depends-On: https://review.opendev.org/676411
Change-Id: Iab8518c34ef876056c6abcdb5f2e9fc9f1f7dbdd
parent b93e33e7
No related branches found
No related tags found
No related merge requests found
Showing
with 84 additions and 43 deletions
......@@ -51,6 +51,7 @@ options:
- compare_container
- compare_image
- create_volume
- ensure_image
- get_container_env
- get_container_state
- pull_image
......@@ -886,6 +887,10 @@ class DockerWorker(object):
)
raise
def ensure_image(self):
if not self.check_image():
self.pull_image()
def generate_module():
# NOTE(jeffrey4l): add empty string '' to choices let us use
......@@ -893,13 +898,20 @@ def generate_module():
argument_spec = dict(
common_options=dict(required=False, type='dict', default=dict()),
action=dict(required=True, type='str',
choices=['compare_container', 'compare_image',
'create_volume', 'get_container_env',
'get_container_state', 'pull_image',
choices=['compare_container',
'compare_image',
'create_volume',
'ensure_image',
'get_container_env',
'get_container_state',
'pull_image',
'recreate_or_restart_container',
'remove_container', 'remove_image',
'remove_volume', 'restart_container',
'start_container', 'stop_container',
'remove_container',
'remove_image',
'remove_volume',
'restart_container',
'start_container',
'stop_container',
'stop_and_remove_container']),
api_version=dict(required=False, type='str', default='auto'),
auth_email=dict(required=False, type='str'),
......@@ -947,6 +959,7 @@ def generate_module():
['action', 'compare_container', ['name']],
['action', 'compare_image', ['name']],
['action', 'create_volume', ['name']],
['action', 'ensure_image', ['image']],
['action', 'get_container_env', ['name']],
['action', 'get_container_state', ['name']],
['action', 'recreate_or_restart_container', ['name']],
......
......@@ -26,6 +26,32 @@
- "fluentd/filter"
when: enable_fluentd | bool
- name: Ensure fluentd image is present for label check
vars:
service_name: "fluentd"
service: "{{ common_services[service_name] }}"
become: true
kolla_docker:
action: "ensure_image"
common_options: "{{ docker_common_options }}"
image: "{{ service.image }}"
when: enable_fluentd | bool
- name: Fetch fluentd image labels
vars:
service_name: "fluentd"
service: "{{ common_services[service_name] }}"
become: true
docker_image_facts:
name: "{{ service.image }}"
register: fluentd_labels
when: enable_fluentd | bool
- name: Set fluentd facts
set_fact:
fluentd_binary: "{% if fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary is not defined %}{% if kolla_base_distro in 'ubuntu' and ansible_architecture == 'x86_64' %}td-agent{% else %}fluentd{% endif %}{% else %}{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary }}{% endif %}"
when: enable_fluentd | bool
- name: Copying over config.json files for services
template:
src: "{{ item.key }}.json.j2"
......@@ -199,6 +225,7 @@
- name: Copying over fluentd filter config files
vars:
customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | map('basename') | list }}"
fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
template:
src: "conf/filter/{{ item.src }}.conf.j2"
dest: "{{ node_config_directory }}/fluentd/filter/{{ item.dest }}.conf"
......@@ -207,7 +234,7 @@
with_items:
- src: 00-record_transformer
dest: 00-record_transformer
- src: "{{ '01-rewrite-0.14' if kolla_base_distro in ['debian', 'ubuntu'] else '01-rewrite-0.12' }}"
- src: "{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}"
dest: 01-rewrite
when:
- enable_fluentd | bool
......
#jinja2: trim_blocks: False
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
{% set services = [
( 'aodh', enable_aodh ),
( 'barbican', enable_barbican ),
......@@ -51,7 +50,7 @@
"/var/log/kolla/neutron/dnsmasq.log",
"/var/log/kolla/*/*-access.log",
"/var/log/kolla/*/*-error.log"]
pos_file /var/run/{{ fluentd_dir }}/kolla-openstack.pos
pos_file /var/run/{{ fluentd_binary }}/kolla-openstack.pos
tag kolla.*
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
......
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/mariadb/mariadb.log
pos_file /var/run/{{ fluentd_dir }}/mariadb.pos
pos_file /var/run/{{ fluentd_binary }}/mariadb.pos
tag infra.mariadb
format multiline
format_firstline /^\d{6}/
......
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/rabbitmq/rabbit.log
pos_file /var/run/{{ fluentd_dir }}/rabbit.pos
pos_file /var/run/{{ fluentd_binary }}/rabbit.pos
tag infra.rabbit
format multiline
format_firstline /^\s*$/
......
# Note (blallau): to manage Apache and WSGI log files
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log
pos_file /var/run/{{ fluentd_dir }}/kolla-openstack-wsgi.pos
pos_file /var/run/{{ fluentd_binary }}/kolla-openstack-wsgi.pos
tag kolla.*
format /^(?<Payload>.*)$/
enable_watch_timer false
......
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/libvirt/libvirtd.log
pos_file /var/run/{{ fluentd_dir }}/libvirt.pos
pos_file /var/run/{{ fluentd_binary }}/libvirt.pos
tag infra.libvirt
format /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?<Pid>\d+): (?<log_level>\S+) : (?<Payload>.*)?$/
time_key Timestamp
......
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/zookeeper/zookeeper.log
pos_file /var/run/{{ fluentd_dir }}/zookeeper.pos
pos_file /var/run/{{ fluentd_binary }}/zookeeper.pos
tag infra.*
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/
......
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/kafka/controller.log, /var/log/kolla/kafka/server.log, /var/log/kolla/kafka/state-change.log
pos_file /var/run/{{ fluentd_dir }}/kafka.pos
pos_file /var/run/{{ fluentd_binary }}/kafka.pos
tag infra.*
format multiline
format_firstline /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] \S+ .*$/
......
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/opendaylight/karaf.log
pos_file /var/run/{{ fluentd_dir }}/opendaylight.pos
pos_file /var/run/{{ fluentd_binary }}/opendaylight.pos
tag infra.opendaylight
format multiline
format_firstline /\d{4}-\d{2}-\d{2}/
......
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/monasca/agent*.log
pos_file /var/run/{{ fluentd_dir }}/monasca-agent.pos
pos_file /var/run/{{ fluentd_binary }}/monasca-agent.pos
tag kolla.*
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/
......@@ -13,7 +12,7 @@
<source>
@type tail
path /var/log/kolla/monasca/grafana.log
pos_file /var/run/{{ fluentd_dir }}/monasca-grafana.pos
pos_file /var/run/{{ fluentd_binary }}/monasca-grafana.pos
tag infra.*
format multiline
format_firstline /^t=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{4} lvl=\S+ msg=.*$/
......
{% if kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64' %}
{% set fluentd_cmd = '/usr/sbin/td-agent' %}
{% set fluentd_dir = '/etc/td-agent' %}
{% set fluentd_conf = 'td-agent.conf' %}
{% set fluentd_user = 'td-agent' %}
{% else %}
{% set fluentd_user = fluentd_binary %}
{% set fluentd_dir = '/etc/' ~ fluentd_binary %}
{% if fluentd_binary == 'fluentd' %}
{% set fluentd_conf = 'fluent.conf' %}
{% if kolla_base_distro in ['ubuntu', 'debian'] %}
{% set fluentd_cmd = '/usr/local/bin/fluentd -c /etc/fluentd/fluent.conf' %}
{% set fluentd_cmd = '/usr/local/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %}
{% else %}
{% set fluentd_cmd = '/usr/bin/fluentd -c /etc/fluentd/fluent.conf' %}
{% set fluentd_cmd = '/usr/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %}
{% endif %}
{% set fluentd_dir = '/etc/fluentd' %}
{% set fluentd_conf = 'fluent.conf' %}
{% set fluentd_user = 'fluentd' %}
{% elif fluentd_binary == 'td-agent' %}
{% set fluentd_conf = fluentd_binary ~ '.conf' %}
{% set fluentd_cmd = '/usr/sbin/td-agent' %}
{% endif %}
{
"command": "{{ fluentd_cmd }}",
"config_files": [
......
......@@ -44,12 +44,21 @@ class ModuleArgsTest(base.BaseTestCase):
common_options=dict(required=False, type='dict', default=dict()),
action=dict(
required=True, type='str',
choices=['compare_container', 'compare_image', 'create_volume',
'get_container_env', 'get_container_state',
'pull_image', 'recreate_or_restart_container',
'remove_container', 'remove_image', 'remove_volume',
'restart_container', 'start_container',
'stop_container', 'stop_and_remove_container']),
choices=['compare_container',
'compare_image',
'create_volume',
'ensure_image',
'get_container_env',
'get_container_state',
'pull_image',
'recreate_or_restart_container',
'remove_container',
'remove_image',
'remove_volume',
'restart_container',
'start_container',
'stop_container',
'stop_and_remove_container']),
api_version=dict(required=False, type='str', default='auto'),
auth_email=dict(required=False, type='str'),
auth_password=dict(required=False, type='str', no_log=True),
......@@ -96,6 +105,7 @@ class ModuleArgsTest(base.BaseTestCase):
['action', 'compare_container', ['name']],
['action', 'compare_image', ['name']],
['action', 'create_volume', ['name']],
['action', 'ensure_image', ['image']],
['action', 'get_container_env', ['name']],
['action', 'get_container_state', ['name']],
['action', 'recreate_or_restart_container', ['name']],
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment