ADD venus for kolla-ansible

This project [1] can provide a one-stop solution to log collection,
cleaning, indexing, analysis, alarm, visualization, report generation
and other needs, which involves helping operator or maintainer to
quickly solve retrieve problems, grasp the operational health of the
platform, and improve the level of platform management.

[1] https://wiki.openstack.org/wiki/Venus

Change-Id: If3562bbed6181002b76831bab54f863041c5a885

ansible/group_vars/all.yml

@@ -517,6 +517,8 @@ tacker_server_port: "9890"
 
 trove_api_port: "8779"
 
+venus_api_port: "10010"
+
 watcher_api_port: "9322"
 
 zookeeper_client_port: "2181"
@@ -710,6 +712,7 @@ enable_tacker: "no"
 enable_telegraf: "no"
 enable_trove: "no"
 enable_trove_singletenant: "no"
+enable_venus: "no"
 enable_vitrage: "no"
 enable_watcher: "no"
 enable_zookeeper: "{{ enable_kafka | bool or enable_storm | bool }}"

ansible/inventory/all-in-one

@@ -735,3 +735,12 @@ ovn-database
 
 [ovn-sb-db:children]
 ovn-database
+
+[venus:children]
+monitoring
+
+[venus-api:children]
+venus
+
+[venus-manager:children]
+venus

ansible/inventory/multinode

@@ -753,3 +753,12 @@ ovn-database
 
 [ovn-sb-db:children]
 ovn-database
+
+[venus:children]
+monitoring
+
+[venus-api:children]
+venus
+
+[venus-manager:children]
+venus

ansible/roles/common/tasks/config.yml

@@ -201,6 +201,7 @@
       - { name: "swift", enabled: "{{ enable_swift | bool }}" }
       - { name: "tacker", enabled: "{{ enable_tacker | bool }}" }
       - { name: "trove", enabled: "{{ enable_trove | bool }}" }
+      - { name: "venus", enabled: "{{ enable_venus | bool }}" }
       - { name: "vitrage", enabled: "{{ enable_vitrage | bool }}" }
       - { name: "watcher", enabled: "{{ enable_watcher | bool }}" }
       - { name: "zookeeper", enabled: "{{ enable_zookeeper | bool }}" }

ansible/roles/common/templates/conf/filter/01-rewrite.conf.j2

@@ -166,6 +166,11 @@
     pattern ^(masakari-engine|masakari-api)$
     tag openstack_python
   </rule>
+  <rule>
+    key     programname
+    pattern ^(venus-api|venus-manager)$
+    tag openstack_python
+  </rule>
   <rule>
     key     programname
     pattern .+

ansible/roles/common/templates/cron-logrotate-venus.conf.j2

+"/var/log/kolla/venus/*.log"
+{
+}

ansible/roles/venus/defaults/main.yml

+---
+venus_services:
+  venus-api:
+    container_name: venus_api
+    group: venus-api
+    enabled: true
+    image: "{{ venus_api_image_full }}"
+    volumes: "{{ venus_api_default_volumes + venus_api_extra_volumes }}"
+    dimensions: "{{ venus_api_dimensions }}"
+    healthcheck: "{{ venus_api_healthcheck }}"
+    haproxy:
+      venus_api:
+        enabled: "{{ enable_venus }}"
+        mode: "http"
+        external: false
+        port: "{{ venus_api_port }}"
+      venus_api_external:
+        enabled: "{{ enable_venus }}"
+        mode: "http"
+        external: true
+        port: "{{ venus_api_port }}"
+  venus-manager:
+    container_name: venus_manager
+    group: venus-manager
+    enabled: true
+    image: "{{ venus_manager_image_full }}"
+    volumes: "{{ venus_manager_default_volumes + venus_manager_extra_volumes }}"
+    dimensions: "{{ venus_manager_dimensions }}"
+
+####################
+# Database
+####################
+venus_database_name: "venus"
+venus_database_user: "{% if use_preconfigured_databases | bool and use_common_mariadb_user | bool %}{{ database_user }}{% else %}venus{% endif %}"
+venus_database_address: "{{ database_address | put_address_in_context('url') }}:{{ database_port }}"
+
+
+####################
+# Docker
+####################
+venus_install_type: "{{ kolla_install_type }}"
+venus_tag: "{{ openstack_tag }}"
+
+venus_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ venus_install_type }}-venus-api"
+venus_api_tag: "{{ venus_tag }}"
+venus_api_image_full: "{{ venus_api_image }}:{{ venus_api_tag }}"
+
+venus_manager_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ venus_install_type }}-venus-manager"
+venus_manager_tag: "{{ venus_tag }}"
+venus_manager_image_full: "{{ venus_manager_image }}:{{ venus_manager_tag }}"
+
+venus_api_dimensions: "{{ default_container_dimensions }}"
+venus_manager_dimensions: "{{ default_container_dimensions }}"
+
+venus_api_enable_healthchecks: "{{ enable_container_healthchecks }}"
+venus_api_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
+venus_api_healthcheck_retries: "{{ default_container_healthcheck_retries }}"
+venus_api_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}"
+venus_api_healthcheck_test: ["CMD-SHELL", "healthcheck_curl http://{{ api_interface_address |  put_address_in_context('url') }}:{{ venus_api_port }}"]
+venus_api_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}"
+venus_api_healthcheck:
+  interval: "{{ venus_api_healthcheck_interval }}"
+  retries: "{{ venus_api_healthcheck_retries }}"
+  start_period: "{{ venus_api_healthcheck_start_period }}"
+  test: "{% if venus_api_enable_healthchecks | bool %}{{ venus_api_healthcheck_test }}{% else %}NONE{% endif %}"
+  timeout: "{{ venus_api_healthcheck_timeout }}"
+
+venus_api_default_volumes:
+  - "{{ node_config_directory }}/venus-api/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
+  - "kolla_logs:/var/log/kolla/"
+  - "{{ kolla_dev_repos_directory ~ '/venus/venus:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/venus' if venus_dev_mode | bool else '' }}"
+  - "venus:/var/lib/venus/"
+venus_manager_default_volumes:
+  - "{{ node_config_directory }}/venus-manager/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
+  - "kolla_logs:/var/log/kolla/"
+  - "{{ kolla_dev_repos_directory ~ '/venus/venus:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/venus' if venus_dev_mode | bool else '' }}"
+  - "venus:/var/lib/venus/"
+
+venus_extra_volumes: "{{ default_extra_volumes }}"
+venus_api_extra_volumes: "{{ venus_extra_volumes }}"
+venus_manager_extra_volumes: "{{ venus_extra_volumes }}"
+
+####################
+# OpenStack
+####################
+venus_internal_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ venus_api_port }}/v1.0/%(tenant_id)s"
+venus_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn | put_address_in_context('url') }}:{{ venus_api_port }}/v1.0/%(tenant_id)s"
+
+venus_logging_debug: "{{ openstack_logging_debug }}"
+
+venus_keystone_user: "venus"
+
+openstack_venus_auth: "{{ openstack_auth }}"
+
+
+####################
+# Kolla
+####################
+venus_git_repository: "https://opendev.org/inspur/venus"
+venus_dev_repos_pull: "{{ kolla_dev_repos_pull }}"
+venus_dev_mode: "{{ kolla_dev_mode }}"
+venus_source_version: "{{ kolla_source_version }}"
+
+####################
+# logging
+####################
+openstack_logging_default_format_string: "%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [- req-None - - - - -] %(instance)s%(message)s"
+openstack_logging_context_format_string: "%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s"
+
+####################
+# Notifications
+####################
+venus_notification_topics:
+  - name: notifications
+    enabled: "{{ enable_ceilometer | bool }}"
+
+venus_enabled_notification_topics: "{{ venus_notification_topics | selectattr('enabled', 'equalto', true) | list }}"
+
+####################
+# Keystone
+####################
+venus_ks_services:
+  - name: "venus"
+    type: "LMS"
+    description: "Log Manager Service"
+    endpoints:
+      - {'interface': 'internal', 'url': '{{ venus_internal_endpoint }}'}
+      - {'interface': 'public', 'url': '{{ venus_public_endpoint }}'}
+
+venus_ks_users:
+  - project: "service"
+    user: "{{ venus_keystone_user }}"
+    password: "{{ venus_keystone_password }}"
+    role: "admin"

ansible/roles/venus/handlers/main.yml

+---
+- name: Restart venus-api container
+  vars:
+    service_name: "venus-api"
+    service: "{{ venus_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
+    healthcheck: "{{ service.healthcheck | default(omit) }}"
+  when:
+    - kolla_action != "config"
+
+- name: Restart venus-manager container
+  vars:
+    service_name: "venus-manager"
+    service: "{{ venus_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
+    healthcheck: "{{ service.healthcheck | default(omit) }}"
+  when:
+    - kolla_action != "config"

ansible/roles/venus/tasks/bootstrap.yml

+---
+- name: Creating venus database
+  become: true
+  kolla_toolbox:
+    module_name: mysql_db
+    module_args:
+      login_host: "{{ database_address }}"
+      login_port: "{{ database_port }}"
+      login_user: "{{ database_user }}"
+      login_password: "{{ database_password }}"
+      name: "{{ venus_database_name }}"
+  run_once: True
+  delegate_to: "{{ groups['venus-api'][0] }}"
+  when:
+    - not use_preconfigured_databases | bool
+
+- name: Creating venus database user and setting permissions
+  become: true
+  kolla_toolbox:
+    module_name: mysql_user
+    module_args:
+      login_host: "{{ database_address }}"
+      login_port: "{{ database_port }}"
+      login_user: "{{ database_user }}"
+      login_password: "{{ database_password }}"
+      name: "{{ venus_database_user }}"
+      password: "{{ venus_database_password }}"
+      host: "%"
+      priv: "{{ venus_database_name }}.*:ALL"
+      append_privs: "yes"
+  run_once: True
+  delegate_to: "{{ groups['venus-api'][0] }}"
+  when:
+    - not use_preconfigured_databases | bool

ansible/roles/venus/tasks/check-containers.yml

+---
+- name: Check venus containers
+  become: true
+  kolla_docker:
+    action: "compare_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ item.value.container_name }}"
+    image: "{{ item.value.image }}"
+    volumes: "{{ item.value.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ item.value.dimensions }}"
+    healthcheck: "{{ item.value.healthcheck | default(omit) }}"
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ venus_services }}"
+  notify:
+    - "Restart {{ item.key }} container"

ansible/roles/venus/tasks/check.yml

+---

ansible/roles/venus/tasks/clone.yml

+---
+- name: Cloning venus source repository for development
+  become: true
+  git:
+    repo: "{{ venus_git_repository }}"
+    dest: "{{ kolla_dev_repos_directory }}/{{ project_name }}"
+    update: "{{ venus_dev_repos_pull }}"
+    version: "{{ venus_source_version }}"

ansible/roles/venus/tasks/config.yml

+---
+- name: Ensuring config directories exist
+  file:
+    path: "{{ node_config_directory }}/{{ item.key }}"
+    state: "directory"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0770"
+  become: true
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ venus_services }}"
+
+- name: Check if policies shall be overwritten
+  stat:
+    path: "{{ item }}"
+  run_once: True
+  delegate_to: localhost
+  register: venus_policy
+  with_first_found:
+    - files: "{{ supported_policy_format_list }}"
+      paths:
+        - "{{ node_custom_config }}/venus/"
+      skip: true
+
+- name: Set venus policy file
+  set_fact:
+    venus_policy_file: "{{ venus_policy.results.0.stat.path | basename }}"
+    venus_policy_file_path: "{{ venus_policy.results.0.stat.path }}"
+  when:
+    - venus_policy.results
+
+- include_tasks: copy-certs.yml
+  when:
+    - kolla_copy_ca_into_containers | bool
+
+- name: Copying over config.json files for services
+  template:
+    src: "{{ item.key }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
+    mode: "0660"
+  become: true
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ venus_services }}"
+  notify:
+    - "Restart {{ item.key }} container"
+
+- name: Copying over venus.conf
+  vars:
+    service_name: "{{ item.key }}"
+  merge_configs:
+    sources:
+      - "{{ role_path }}/templates/venus.conf.j2"
+      - "{{ node_custom_config }}/global.conf"
+      - "{{ node_custom_config }}/venus.conf"
+      - "{{ node_custom_config }}/venus/{{ item.key }}.conf"
+      - "{{ node_custom_config }}/venus/{{ inventory_hostname }}/venus.conf"
+    dest: "{{ node_config_directory }}/{{ item.key }}/venus.conf"
+    mode: "0660"
+  become: true
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ venus_services }}"
+  notify:
+    - "Restart {{ item.key }} container"
+
+- name: Copying over existing policy file
+  template:
+    src: "{{ venus_policy_file_path }}"
+    dest: "{{ node_config_directory }}/{{ item.key }}/{{ venus_policy_file }}"
+    mode: "0660"
+  when:
+    - venus_policy_file is defined
+    - inventory_hostname in groups[item.value.group]
+  with_dict: "{{ venus_services }}"
+  notify:
+    - Restart {{ item.key }} container

ansible/roles/venus/tasks/copy-certs.yml

+---
+- name: "Copy certificates and keys for {{ project_name }}"
+  import_role:
+    role: service-cert-copy
+  vars:
+    project_services: "{{ venus_services }}"

ansible/roles/venus/tasks/deploy-containers.yml

+---
+- import_tasks: check-containers.yml

ansible/roles/venus/tasks/deploy.yml

+---
+- import_tasks: register.yml
+
+- import_tasks: config.yml
+
+- import_tasks: check-containers.yml
+
+- include_tasks: clone.yml
+  when: venus_dev_mode | bool
+
+- import_tasks: bootstrap.yml
+
+- name: Flush handlers
+  meta: flush_handlers

ansible/roles/venus/tasks/loadbalancer.yml

+---
+- name: "Configure haproxy for {{ project_name }}"
+  import_role:
+    name: haproxy-config
+  vars:
+    project_services: "{{ venus_services }}"
+  tags: always

ansible/roles/venus/tasks/main.yml

+---
+- include_tasks: "{{ kolla_action }}.yml"

ansible/roles/venus/tasks/precheck.yml

+---
+- import_role:
+    name: service-precheck
+  vars:
+    service_precheck_services: "{{ venus_services }}"
+    service_name: "{{ project_name }}"
+
+- name: Get container facts
+  become: true
+  kolla_container_facts:
+    name:
+      - venus_api
+  register: container_facts
+
+- name: Checking free port for Venus API
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ venus_api_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - container_facts['venus_api'] is not defined
+    - inventory_hostname in groups['venus-api']

ansible/roles/venus/tasks/pull.yml

+---
+- name: Pulling venus images
+  become: true
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ item.value.image }}"
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+  with_dict: "{{ venus_services }}"