Merge "Add gnocchi ansible role"

ansible/group_vars/all.yml

@@ -122,6 +122,8 @@ congress_api_port: "1789"
 
 iscsi_port: "3260"
 
+gnocchi_api_port: "8041"
+
 mariadb_port: "{{ database_port }}"
 mariadb_wsrep_port: "4567"
 mariadb_ist_port: "4568"
@@ -230,6 +232,7 @@ enable_ceph_rgw: "no"
 enable_cinder: "no"
 enable_cinder_backend_lvm: "no"
 enable_congress: "no"
+enable_gnocchi: "no"
 enable_heat: "yes"
 enable_horizon: "yes"
 enable_ironic: "no"

ansible/inventory/all-in-one

@@ -84,6 +84,9 @@ control
 [manila:children]
 control
 
+[gnocchi:children]
+control
+
 [ceilometer:children]
 control
 
@@ -256,6 +259,16 @@ aodh
 [aodh-notifier:children]
 aodh
 
+# Gnocchi
+[gnocchi-api:children]
+gnocchi
+
+[gnocchi-statsd:children]
+gnocchi
+
+[gnocchi-metricd:children]
+gnocchi
+
 # Ceilometer
 [ceilometer-api:children]
 ceilometer

ansible/inventory/multinode

@@ -110,6 +110,9 @@ control
 [congress:children]
 control
 
+[gnocchi:children]
+control
+
 # Tempest
 [tempest:children]
 control
@@ -299,6 +302,16 @@ congress
 [congress-policy-engine:children]
 congress
 
+# Gnocchi
+[gnocchi-api:children]
+gnocchi
+
+[gnocchi-statsd:children]
+gnocchi
+
+[gnocchi-metricd:children]
+gnocchi
+
 # Multipathd
 [multipathd:children]
 compute

ansible/roles/common/tasks/config.yml

@@ -28,6 +28,7 @@
     - { name: "aodh", enabled: "{{ enable_aodh }}" }
     - { name: "elasticsearch", enabled: "{{ enable_central_logging }}" }
     - { name: "global", enabled: "yes" }
+    - { name: "gnocchi", enabled: "{{ enable_gnocchi }}" }
     - { name: "haproxy", enabled: "{{ enable_haproxy }}" }
     - { name: "horizon", enabled: "{{ enable_horizon }}" }
     - { name: "keepalived", enabled: "{{ enable_haproxy }}" }
@@ -71,6 +72,7 @@
     - "cinder"
     - "glance"
     - "global"
+    - "gnocchi"
     - "haproxy"
     - "heat"
     - "keepalived"

ansible/roles/common/templates/cron-logrotate-gnocchi.conf.j2

+"/var/log/kolla/gnocchi/*.log"
+{
+}

ansible/roles/common/templates/cron.json.j2

 {% set cron_cmd = 'cron -f' if kolla_base_distro in ['ubuntu', 'debian'] else 'crond -s -n' %}
-{% set services = ["ansible", "aodh", "cinder", "glance", "haproxy", "heat", "keepalived", "keystone", "magnum", "manila", "mariadb", "mistral", "murano", "neutron", "nova", "rabbitmq", "swift"] %}
+{% set services = ["ansible", "aodh", "cinder", "glance", "gnocchi", "haproxy", "heat", "keepalived", "keystone", "magnum", "manila", "mariadb", "mistral", "murano", "neutron", "nova", "rabbitmq", "swift"] %}
 {
     "command": "{{ cron_cmd }}",
     "config_files": [

ansible/roles/common/templates/heka-gnocchi.toml.j2

+[gnocchi_apache_log_decoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/os_gnocchi_apache_log.lua"
+    [gnocchi_apache_log_decoder.config]
+    apache_log_pattern = '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"'
+
+[gnocchi_apache_logstreamer_input]
+type = "LogstreamerInput"
+decoder = "gnocchi_apache_log_decoder"
+log_directory = "/var/log/kolla"
+file_match = 'gnocchi/gnocchi-apache-(?P<Service>.+)-access\.log\.?(?P<Seq>\d*)$'
+priority = ["^Seq"]
+differentiator = ["gnocchi-apache-", "Service"]

ansible/roles/gnocchi/defaults/main.yml

+---
+project_name: "gnocchi"
+
+
+####################
+# Database
+####################
+gnocchi_database_name: "gnocchi"
+gnocchi_database_user: "gnocchi"
+gnocchi_database_address: "{{ kolla_internal_fqdn }}:{{ database_port }}"
+
+
+####################
+# Docker
+####################
+gnocchi_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-gnocchi-api"
+gnocchi_api_tag: "{{ openstack_release }}"
+gnocchi_api_image_full: "{{ gnocchi_api_image }}:{{ gnocchi_api_tag }}"
+
+gnocchi_statsd_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-gnocchi-statsd"
+gnocchi_statsd_tag: "{{ openstack_release }}"
+gnocchi_statsd_image_full: "{{ gnocchi_statsd_image }}:{{ gnocchi_statsd_tag }}"
+
+gnocchi_metricd_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-gnocchi-metricd"
+gnocchi_metricd_tag: "{{ openstack_release }}"
+gnocchi_metricd_image_full: "{{ gnocchi_metricd_image }}:{{ gnocchi_metricd_tag }}"
+
+####################
+# OpenStack
+####################
+gnocchi_admin_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ gnocchi_api_port }}"
+gnocchi_internal_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ gnocchi_api_port }}"
+gnocchi_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn }}:{{ gnocchi_api_port }}"
+
+gnocchi_logging_debug: "{{ openstack_logging_debug }}"
+
+gnocchi_keystone_user: "gnocchi"
+
+openstack_gnocchi_auth: "{'auth_url':'{{ openstack_auth.auth_url }}','username':'{{ openstack_auth.username }}','password':'{{ openstack_auth.password }}','project_name':'{{ openstack_auth.project_name }}','domain_name':'default'}"

ansible/roles/gnocchi/meta/main.yml

+---
+dependencies:
+  - { role: common }

ansible/roles/gnocchi/tasks/bootstrap.yml

+---
+- name: Creating gnocchi database
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m mysql_db
+    -a "login_host='{{ database_address }}'
+        login_port='{{ database_port }}'
+        login_user='{{ database_user }}'
+        login_password='{{ database_password }}'
+        name='{{ gnocchi_database_name }}'"
+  register: database
+  changed_when: "{{ database.stdout.find('localhost | SUCCESS => ') != -1 and
+                    (database.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  failed_when: database.stdout.split()[2] != 'SUCCESS'
+  run_once: True
+  delegate_to: "{{ groups['gnocchi-api'][0] }}"
+
+- name: Reading json from variable
+  set_fact:
+    database_created: "{{ (database.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+
+- name: Creating gnocchi database user and setting permissions
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m mysql_user
+    -a "login_host='{{ database_address }}'
+        login_port='{{ database_port }}'
+        login_user='{{ database_user }}'
+        login_password='{{ database_password }}'
+        name='{{ gnocchi_database_name }}'
+        password='{{ gnocchi_database_password }}'
+        host='%'
+        priv='{{ gnocchi_database_name }}.*:ALL'
+        append_privs='yes'"
+  register: database_user_create
+  changed_when: "{{ database_user_create.stdout.find('localhost | SUCCESS => ') != -1 and
+                    (database_user_create.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  failed_when: database_user_create.stdout.split()[2] != 'SUCCESS'
+  run_once: True
+  delegate_to: "{{ groups['gnocchi-api'][0] }}"
+
+- include: bootstrap_service.yml
+  when: database_created

ansible/roles/gnocchi/tasks/bootstrap_service.yml

+---
+- name: Running gnocchi bootstrap container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ gnocchi_api_image_full }}"
+    labels:
+      BOOTSTRAP:
+    name: "bootstrap_gnocchi"
+    restart_policy: "never"
+    volumes:
+      - "{{ node_config_directory }}/gnocchi-api/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "gnocchi:/var/lib/gnocchi/"
+      - "kolla_logs:/var/log/kolla/"
+  run_once: True
+  delegate_to: "{{ groups['gnocchi-api'][0] }}"

ansible/roles/gnocchi/tasks/config.yml

+---
+- name: Ensuring config directories exist
+  file:
+    path: "{{ node_config_directory }}/{{ item }}"
+    state: "directory"
+    recurse: yes
+  with_items:
+    - "gnocchi-api"
+    - "gnocchi-statsd"
+    - "gnocchi-metricd"
+
+- name: Copying over config.json files for services
+  template:
+    src: "{{ item }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ item }}/config.json"
+  with_items:
+    - "gnocchi-api"
+    - "gnocchi-metricd"
+    - "gnocchi-statsd"
+
+- name: Copying over api-paste.ini files for services
+  template:
+    src: "gnocchi-api-paste.ini.j2"
+    dest: "{{ node_config_directory }}/{{ item }}/gnocchi-api-paste.ini"
+  with_items:
+    -  "gnocchi-api"
+
+- name: Copying over gnocchi.conf
+  merge_configs:
+    vars:
+      service_name: "{{ item }}"
+    sources:
+      - "{{ role_path }}/templates/gnocchi.conf.j2"
+      - "{{ node_custom_config }}/global.conf"
+      - "{{ node_custom_config }}/database.conf"
+      - "{{ node_custom_config }}/messaging.conf"
+      - "{{ node_custom_config }}/gnocchi.conf"
+      - "{{ node_custom_config }}/gnocchi/{{ item }}.conf"
+      - "{{ node_custom_config }}/gnocchi/{{ inventory_hostname }}/gnocchi.conf"
+    dest: "{{ node_config_directory }}/{{ item }}/gnocchi.conf"
+  with_items:
+    - "gnocchi-api"
+    - "gnocchi-statsd"
+    - "gnocchi-metricd"
+
+- name: Copying over wsgi-gnocchi.conf
+  template:
+    src: "wsgi-gnocchi.conf.j2"
+    dest: "{{ node_config_directory }}/{{ item }}/wsgi-gnocchi.conf"
+  with_items:
+    -  "gnocchi-api"

ansible/roles/gnocchi/tasks/deploy.yml

+---
+- include: register.yml
+  when: inventory_hostname in groups['gnocchi-api']
+
+- include: config.yml
+  when: inventory_hostname in groups['gnocchi-statsd'] or
+        inventory_hostname in groups['gnocchi-api'] or
+        inventory_hostname in groups['gnocchi-metricd']
+
+- include: bootstrap.yml
+  when: inventory_hostname in groups['gnocchi-api']
+
+- include: start.yml
+  when: inventory_hostname in groups['gnocchi-statsd'] or
+        inventory_hostname in groups['gnocchi-api'] or
+        inventory_hostname in groups['gnocchi-metricd']

ansible/roles/gnocchi/tasks/do_reconfigure.yml

+---
+- name: Ensuring the containers up
+  kolla_docker:
+    name: "{{ item.name }}"
+    action: "get_container_state"
+  register: container_state
+  failed_when: container_state.Running == false
+  when: inventory_hostname in groups[item.group]
+  with_items:
+    - { name: gnocchi_api, group: gnocchi-api }
+    - { name: gnocchi_metricd, group: gnocchi-metricd }
+    - { name: gnocchi_statsd, group: gnocchi-statsd }
+
+- include: config.yml
+
+- name: Check the configs
+  command: docker exec {{ item.name }} /usr/local/bin/kolla_set_configs --check
+  changed_when: false
+  failed_when: false
+  register: check_results
+  when: inventory_hostname in groups[item.group]
+  with_items:
+    - { name: gnocchi_api, group: gnocchi-api }
+    - { name: gnocchi_metricd, group: gnocchi-metricd }
+    - { name: gnocchi_statsd, group: gnocchi-statsd }
+
+# NOTE(jeffrey4l): when config_strategy == 'COPY_ALWAYS'
+# and container env['KOLLA_CONFIG_STRATEGY'] == 'COPY_ONCE',
+# just remove the container and start again
+- name: Containers config strategy
+  kolla_docker:
+    name: "{{ item.name }}"
+    action: "get_container_env"
+  register: container_envs
+  when: inventory_hostname in groups[item.group]
+  with_items:
+    - { name: gnocchi_api, group: gnocchi-api }
+    - { name: gnocchi_metricd, group: gnocchi-metricd }
+    - { name: gnocchi_statsd, group: gnocchi-statsd }
+
+- name: Remove the containers
+  kolla_docker:
+    name: "{{ item[0]['name'] }}"
+    action: "remove_container"
+  register: remove_containers
+  when:
+    - inventory_hostname in groups[item[0]['group']]
+    - config_strategy == "COPY_ONCE" or item[1]['KOLLA_CONFIG_STRATEGY'] == 'COPY_ONCE'
+    - item[2]['rc'] == 1
+  with_together:
+    - [{ name: gnocchi_api, group: gnocchi-api },
+       { name: gnocchi_metricd, group: gnocchi-metricd },
+       { name: gnocchi_statsd, group: gnocchi-statsd }]
+    - "{{ container_envs.results }}"
+    - "{{ check_results.results }}"
+
+- include: start.yml
+  when: remove_containers.changed
+
+- name: Restart containers
+  kolla_docker:
+    name: "{{ item[0]['name'] }}"
+    action: "restart_container"
+  when:
+    - inventory_hostname in groups[item[0]['group']]
+    - config_strategy == 'COPY_ALWAYS'
+    - item[1]['KOLLA_CONFIG_STRATEGY'] != 'COPY_ONCE'
+    - item[2]['rc'] == 1
+  with_together:
+    - [{ name: gnocchi_api, group: gnocchi-api },
+       { name: gnocchi_metricd, group: gnocchi-metricd },
+       { name: gnocchi_statsd, group: gnocchi-statsd }]
+    - "{{ container_envs.results }}"
+    - "{{ check_results.results }}"

ansible/roles/gnocchi/tasks/main.yml

+---
+- include: "{{ action }}.yml"

ansible/roles/gnocchi/tasks/pull.yml

+---
+- name: Pulling gnocchi-api image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ gnocchi_api_image_full }}"
+  when: inventory_hostname in groups['gnocchi-api']
+
+- name: Pulling gnocchi-metricd image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ gnocchi_metricd_image_full }}"
+  when: inventory_hostname in groups['gnocchi-metricd']
+
+- name: Pulling gnocchi-statsd image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ gnocchi_statsd_image_full }}"
+  when: inventory_hostname in groups['gnocchi-statsd']

ansible/roles/gnocchi/tasks/reconfigure.yml

+---
+- include: do_reconfigure.yml
+  serial: "30%"
+  when: inventory_hostname in groups['gnocchi-api']
+        or inventory_hostname in groups['gnocchi-metricd']
+        or inventory_hostname in groups['gnocchi-statsd']

ansible/roles/gnocchi/tasks/register.yml

+---
+- name: Creating the gnocchi service and endpoint
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m kolla_keystone_service
+    -a "service_name=gnocchi
+        service_type=metric
+        description='OpenStack Metric Service'
+        endpoint_region={{ openstack_region_name }}
+        url='{{ item.url }}'
+        interface='{{ item.interface }}'
+        region_name={{ openstack_region_name }}
+        auth={{ '{{ openstack_gnocchi_auth }}' }}"
+    -e "{'openstack_gnocchi_auth':{{ openstack_gnocchi_auth }}}"
+  register: gnocchi_endpoint
+  changed_when: "{{ gnocchi_endpoint.stdout.find('localhost | SUCCESS => ') != -1 and (gnocchi_endpoint.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: gnocchi_endpoint.stdout.split()[2] == 'SUCCESS'
+  retries: 10
+  delay: 5
+  run_once: True
+  with_items:
+    - {'interface': 'admin', 'url': '{{ gnocchi_admin_endpoint }}'}
+    - {'interface': 'internal', 'url': '{{ gnocchi_internal_endpoint }}'}
+    - {'interface': 'public', 'url': '{{ gnocchi_public_endpoint }}'}
+
+- name: Creating the gnocchi project, user, and role
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m kolla_keystone_user
+    -a "project=service
+        user=gnocchi
+        password={{ gnocchi_keystone_password }}
+        role=admin
+        region_name={{ openstack_region_name }}
+        auth={{ '{{ openstack_gnocchi_auth }}' }}"
+    -e "{'openstack_gnocchi_auth':{{ openstack_gnocchi_auth }}}"
+  register: gnocchi_user
+  changed_when: "{{ gnocchi_user.stdout.find('localhost | SUCCESS => ') != -1 and (gnocchi_user.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: gnocchi_user.stdout.split()[2] == 'SUCCESS'
+  retries: 10
+  delay: 5
+  run_once: True

ansible/roles/gnocchi/tasks/start.yml

+---
+- name: Starting gnocchi-api container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ gnocchi_api_image_full }}"
+    name: "gnocchi_api"
+    volumes:
+      - "{{ node_config_directory }}/gnocchi-api/:{{ container_config_directory }}/:ro"
+      - "gnocchi:/var/lib/gnocchi/"
+      - "kolla_logs:/var/log/kolla/"
+  when: inventory_hostname in groups['gnocchi-api']
+
+- name: Starting gnocchi-metricd container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ gnocchi_metricd_image_full }}"
+    name: "gnocchi_metricd"
+    volumes:
+      - "{{ node_config_directory }}/gnocchi-metricd/:{{ container_config_directory }}/:ro"
+      - "kolla_logs:/var/log/kolla/"
+  when: inventory_hostname in groups['gnocchi-metricd']
+
+- name: Starting gnocchi-statsd container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ gnocchi_statsd_image_full }}"
+    name: "gnocchi_statsd"
+    volumes:
+      - "{{ node_config_directory }}/gnocchi-statsd/:{{ container_config_directory }}/:ro"
+      - "kolla_logs:/var/log/kolla/"
+  when: inventory_hostname in groups['gnocchi-statsd']

ansible/roles/gnocchi/templates/gnocchi-api-paste.ini.j2

+# Use gnocchi+noauth in the pipeline if you don't want keystone authentication
+[pipeline:main]
+pipeline = gnocchi+auth
+
+[composite:gnocchi+noauth]
+use = egg:Paste#urlmap
+/ = gnocchiversions
+/v1 = gnocchiv1
+
+[composite:gnocchi+auth]
+use = egg:Paste#urlmap
+/ = gnocchiversions
+/v1 = gnocchiv1+auth
+
+[pipeline:gnocchiv1+auth]
+pipeline = healthcheck keystone_authtoken gnocchiv1
+
+[app:gnocchiversions]
+paste.app_factory = gnocchi.rest.app:app_factory
+root = gnocchi.rest.VersionsController
+
+[app:gnocchiv1]
+paste.app_factory = gnocchi.rest.app:app_factory
+root = gnocchi.rest.V1Controller
+
+[filter:healthcheck]
+paste.filter_factory = oslo_middleware:Healthcheck.factory
+path = /status
+backends = disable_by_file
+
+
+[filter:keystone_authtoken]
+paste.filter_factory = keystonemiddleware.auth_token:filter_factory
+oslo_config_project = gnocchi
+
+[filter:cors]
+paste.filter_factory = oslo_middleware.cors:filter_factory
+oslo_config_project = gnocchi