Skip to content
Snippets Groups Projects
Commit 4180bee0 authored by Michal Nasiadka's avatar Michal Nasiadka
Browse files

Use fluentd image labels 

In order to orchestrate smooth transition to fluentd 0.14.x
aka 1.0 stable branch aka td-agent 3
from td-agent repository - use image labels (fluentd_version
and fluentd_binary).

Depends-On: https://review.opendev.org/676411
Change-Id: Iab8518c34ef876056c6abcdb5f2e9fc9f1f7dbdd
parent b93e33e7
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 84 additions and 43 deletions
ansible/library/kolla_docker.py
+ 19
− 6
View file @ 4180bee0
......@@ -51,6 +51,7 @@ options:
- compare_container
- compare_image
- create_volume
- ensure_image
- get_container_env
- get_container_state
- pull_image
......@@ -886,6 +887,10 @@ class DockerWorker(object):
)
raise
def ensure_image(self):
if not self.check_image():
self.pull_image()
def generate_module():
# NOTE(jeffrey4l): add empty string '' to choices let us use
......@@ -893,13 +898,20 @@ def generate_module():
argument_spec = dict(
common_options=dict(required=False, type='dict', default=dict()),
action=dict(required=True, type='str',
choices=['compare_container', 'compare_image',
'create_volume', 'get_container_env',
'get_container_state', 'pull_image',
choices=['compare_container',
'compare_image',
'create_volume',
'ensure_image',
'get_container_env',
'get_container_state',
'pull_image',
'recreate_or_restart_container',
'remove_container', 'remove_image',
'remove_volume', 'restart_container',
'start_container', 'stop_container',
'remove_container',
'remove_image',
'remove_volume',
'restart_container',
'start_container',
'stop_container',
'stop_and_remove_container']),
api_version=dict(required=False, type='str', default='auto'),
auth_email=dict(required=False, type='str'),
......@@ -947,6 +959,7 @@ def generate_module():
['action', 'compare_container', ['name']],
['action', 'compare_image', ['name']],
['action', 'create_volume', ['name']],
['action', 'ensure_image', ['image']],
['action', 'get_container_env', ['name']],
['action', 'get_container_state', ['name']],
['action', 'recreate_or_restart_container', ['name']],
......
ansible/roles/common/tasks/config.yml
+ 28
− 1
View file @ 4180bee0
......@@ -26,6 +26,32 @@
- "fluentd/filter"
when: enable_fluentd | bool
- name: Ensure fluentd image is present for label check
vars:
service_name: "fluentd"
service: "{{ common_services[service_name] }}"
become: true
kolla_docker:
action: "ensure_image"
common_options: "{{ docker_common_options }}"
image: "{{ service.image }}"
when: enable_fluentd | bool
- name: Fetch fluentd image labels
vars:
service_name: "fluentd"
service: "{{ common_services[service_name] }}"
become: true
docker_image_facts:
name: "{{ service.image }}"
register: fluentd_labels
when: enable_fluentd | bool
- name: Set fluentd facts
set_fact:
fluentd_binary: "{% if fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary is not defined %}{% if kolla_base_distro in 'ubuntu' and ansible_architecture == 'x86_64' %}td-agent{% else %}fluentd{% endif %}{% else %}{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary }}{% endif %}"
when: enable_fluentd | bool
- name: Copying over config.json files for services
template:
src: "{{ item.key }}.json.j2"
......@@ -199,6 +225,7 @@
- name: Copying over fluentd filter config files
vars:
customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | map('basename') | list }}"
fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
template:
src: "conf/filter/{{ item.src }}.conf.j2"
dest: "{{ node_config_directory }}/fluentd/filter/{{ item.dest }}.conf"
......@@ -207,7 +234,7 @@
with_items:
- src: 00-record_transformer
dest: 00-record_transformer
- src: "{{ '01-rewrite-0.14' if kolla_base_distro in ['debian', 'ubuntu'] else '01-rewrite-0.12' }}"
- src: "{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}"
dest: 01-rewrite
when:
- enable_fluentd | bool
......
ansible/roles/common/templates/conf/input/00-global.conf.j2
+ 1
− 2
View file @ 4180bee0
#jinja2: trim_blocks: False
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
{% set services = [
( 'aodh', enable_aodh ),
( 'barbican', enable_barbican ),
......@@ -51,7 +50,7 @@
"/var/log/kolla/neutron/dnsmasq.log",
"/var/log/kolla/*/*-access.log",
"/var/log/kolla/*/*-error.log"]
pos_file /var/run/{{ fluentd_dir }}/kolla-openstack.pos
pos_file /var/run/{{ fluentd_binary }}/kolla-openstack.pos
tag kolla.*
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
......
ansible/roles/common/templates/conf/input/02-mariadb.conf.j2
+ 1
− 2
View file @ 4180bee0
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/mariadb/mariadb.log
pos_file /var/run/{{ fluentd_dir }}/mariadb.pos
pos_file /var/run/{{ fluentd_binary }}/mariadb.pos
tag infra.mariadb
format multiline
format_firstline /^\d{6}/
......
ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2
+ 1
− 2
View file @ 4180bee0
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/rabbitmq/rabbit.log
pos_file /var/run/{{ fluentd_dir }}/rabbit.pos
pos_file /var/run/{{ fluentd_binary }}/rabbit.pos
tag infra.rabbit
format multiline
format_firstline /^\s*$/
......
ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2
+ 1
− 2
View file @ 4180bee0
# Note (blallau): to manage Apache and WSGI log files
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log
pos_file /var/run/{{ fluentd_dir }}/kolla-openstack-wsgi.pos
pos_file /var/run/{{ fluentd_binary }}/kolla-openstack-wsgi.pos
tag kolla.*
format /^(?<Payload>.*)$/
enable_watch_timer false
......
ansible/roles/common/templates/conf/input/05-libvirt.conf.j2
+ 1
− 2
View file @ 4180bee0
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/libvirt/libvirtd.log
pos_file /var/run/{{ fluentd_dir }}/libvirt.pos
pos_file /var/run/{{ fluentd_binary }}/libvirt.pos
tag infra.libvirt
format /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?<Pid>\d+): (?<log_level>\S+) : (?<Payload>.*)?$/
time_key Timestamp
......
ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2
+ 1
− 2
View file @ 4180bee0
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/zookeeper/zookeeper.log
pos_file /var/run/{{ fluentd_dir }}/zookeeper.pos
pos_file /var/run/{{ fluentd_binary }}/zookeeper.pos
tag infra.*
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/
......
ansible/roles/common/templates/conf/input/07-kafka.conf.j2
+ 1
− 2
View file @ 4180bee0
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/kafka/controller.log, /var/log/kolla/kafka/server.log, /var/log/kolla/kafka/state-change.log
pos_file /var/run/{{ fluentd_dir }}/kafka.pos
pos_file /var/run/{{ fluentd_binary }}/kafka.pos
tag infra.*
format multiline
format_firstline /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] \S+ .*$/
......
ansible/roles/common/templates/conf/input/08-opendaylight.conf.j2
+ 1
− 2
View file @ 4180bee0
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/opendaylight/karaf.log
pos_file /var/run/{{ fluentd_dir }}/opendaylight.pos
pos_file /var/run/{{ fluentd_binary }}/opendaylight.pos
tag infra.opendaylight
format multiline
format_firstline /\d{4}-\d{2}-\d{2}/
......
ansible/roles/common/templates/conf/input/09-monasca.conf.j2
+ 2
− 3
View file @ 4180bee0
{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/monasca/agent*.log
pos_file /var/run/{{ fluentd_dir }}/monasca-agent.pos
pos_file /var/run/{{ fluentd_binary }}/monasca-agent.pos
tag kolla.*
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/
......@@ -13,7 +12,7 @@
<source>
@type tail
path /var/log/kolla/monasca/grafana.log
pos_file /var/run/{{ fluentd_dir }}/monasca-grafana.pos
pos_file /var/run/{{ fluentd_binary }}/monasca-grafana.pos
tag infra.*
format multiline
format_firstline /^t=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{4} lvl=\S+ msg=.*$/
......
ansible/roles/common/templates/fluentd.json.j2
+ 11
− 11
View file @ 4180bee0
{% if kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64' %}
{% set fluentd_cmd = '/usr/sbin/td-agent' %}
{% set fluentd_dir = '/etc/td-agent' %}
{% set fluentd_conf = 'td-agent.conf' %}
{% set fluentd_user = 'td-agent' %}
{% else %}
{% set fluentd_user = fluentd_binary %}
{% set fluentd_dir = '/etc/' ~ fluentd_binary %}
{% if fluentd_binary == 'fluentd' %}
{% set fluentd_conf = 'fluent.conf' %}
{% if kolla_base_distro in ['ubuntu', 'debian'] %}
{% set fluentd_cmd = '/usr/local/bin/fluentd -c /etc/fluentd/fluent.conf' %}
{% set fluentd_cmd = '/usr/local/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %}
{% else %}
{% set fluentd_cmd = '/usr/bin/fluentd -c /etc/fluentd/fluent.conf' %}
{% set fluentd_cmd = '/usr/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %}
{% endif %}
{% set fluentd_dir = '/etc/fluentd' %}
{% set fluentd_conf = 'fluent.conf' %}
{% set fluentd_user = 'fluentd' %}
{% elif fluentd_binary == 'td-agent' %}
{% set fluentd_conf = fluentd_binary ~ '.conf' %}
{% set fluentd_cmd = '/usr/sbin/td-agent' %}
{% endif %}
{
"command": "{{ fluentd_cmd }}",
"config_files": [
......
tests/test_kolla_docker.py
+ 16
− 6
View file @ 4180bee0
......@@ -44,12 +44,21 @@ class ModuleArgsTest(base.BaseTestCase):
common_options=dict(required=False, type='dict', default=dict()),
action=dict(
required=True, type='str',
choices=['compare_container', 'compare_image', 'create_volume',
'get_container_env', 'get_container_state',
'pull_image', 'recreate_or_restart_container',
'remove_container', 'remove_image', 'remove_volume',
'restart_container', 'start_container',
'stop_container', 'stop_and_remove_container']),
choices=['compare_container',
'compare_image',
'create_volume',
'ensure_image',
'get_container_env',
'get_container_state',
'pull_image',
'recreate_or_restart_container',
'remove_container',
'remove_image',
'remove_volume',
'restart_container',
'start_container',
'stop_container',
'stop_and_remove_container']),
api_version=dict(required=False, type='str', default='auto'),
auth_email=dict(required=False, type='str'),
auth_password=dict(required=False, type='str', no_log=True),
......@@ -96,6 +105,7 @@ class ModuleArgsTest(base.BaseTestCase):
['action', 'compare_container', ['name']],
['action', 'compare_image', ['name']],
['action', 'create_volume', ['name']],
['action', 'ensure_image', ['image']],
['action', 'get_container_env', ['name']],
['action', 'get_container_state', ['name']],
['action', 'recreate_or_restart_container', ['name']],
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment