Support exposing prometheus_server externally

This avoids the need to use a proxy, or some other means, to connect to
Prometheus. This is disabled by default and can be enabled by setting
enable_prometheus_server_external to true.

Change-Id: Ia0af044ff436c2a204b357750a16ff49fcdfec45

ansible/group_vars/all.yml

@@ -570,7 +570,10 @@ placement_api_port: "8780"
 placement_api_listen_port: "{{ placement_api_port }}"
 placement_api_public_port: "{{ haproxy_single_external_frontend_public_port if haproxy_single_external_frontend | bool else placement_api_port }}"
 
+prometheus_external_fqdn: "{{ kolla_external_fqdn }}"
+prometheus_internal_fqdn: "{{ kolla_internal_fqdn }}"
 prometheus_port: "9091"
+prometheus_public_port: "{{ haproxy_single_external_frontend_public_port if haproxy_single_external_frontend | bool else prometheus_port }}"
 prometheus_node_exporter_port: "9100"
 prometheus_mysqld_exporter_port: "9104"
 prometheus_haproxy_exporter_port: "9101"
@@ -1281,6 +1284,7 @@ enable_prometheus_etcd_integration: "{{ enable_prometheus | bool and enable_etcd
 enable_prometheus_msteams: "no"
 
 prometheus_alertmanager_user: "admin"
+prometheus_grafana_user: "grafana"
 prometheus_scrape_interval: "60s"
 prometheus_openstack_exporter_interval: "{{ prometheus_scrape_interval }}"
 prometheus_openstack_exporter_timeout: "45s"
@@ -1292,6 +1296,9 @@ prometheus_openstack_exporter_compute_api_version: "latest"
 prometheus_libvirt_exporter_interval: "60s"
 prometheus_msteams_webhook_url:
 
+prometheus_public_endpoint: "{{ prometheus_external_fqdn | kolla_url(public_protocol, prometheus_public_port) }}"
+prometheus_internal_endpoint: "{{ prometheus_internal_fqdn | kolla_url(internal_protocol, prometheus_port) }}"
+
 ############
 # Vitrage
 ############

ansible/roles/grafana/templates/prometheus.yaml.j2

@@ -4,6 +4,9 @@ datasources:
 - name: Prometheus
   type: prometheus
   access: proxy
+  basicAuth: true
+  basicAuthPassword: "{{ prometheus_grafana_password }}"
+  basicAuthUser: "{{ prometheus_grafana_user }}"
   orgId: 1
   url: {{ grafana_prometheus_url }}
   version: 1

ansible/roles/prometheus/defaults/main.yml

@@ -14,6 +14,12 @@ prometheus_services:
         external: false
         port: "{{ prometheus_port }}"
         active_passive: "{{ prometheus_active_passive | bool }}"
+      prometheus_server_external:
+        enabled: "{{ enable_prometheus_server_external | bool }}"
+        mode: "http"
+        external: true
+        port: "{{ prometheus_public_port }}"
+        active_passive: "{{ prometheus_active_passive | bool }}"
   prometheus-node-exporter:
     container_name: prometheus_node_exporter
     group: prometheus-node-exporter
@@ -132,6 +138,26 @@ prometheus_services:
 prometheus_external_labels:
 #  <labelname>: <labelvalue>
 
+####################
+# Server
+####################
+enable_prometheus_server_external: false
+
+####################
+# Basic Auth
+####################
+prometheus_basic_auth_users: "{{ prometheus_basic_auth_users_default + prometheus_basic_auth_users_extra }}"
+
+prometheus_basic_auth_users_default:
+  - username: admin
+    password: "{{ prometheus_password }}"
+    enabled: true
+  - username: "{{ prometheus_grafana_user }}"
+    password: "{{ prometheus_grafana_password }}"
+    enabled: "{{ enable_grafana }}"
+
+prometheus_basic_auth_users_extra: []
+
 ####################
 # Database
 ####################
@@ -315,6 +341,12 @@ prometheus_openstack_exporter_disabled_object: "{{ '--disable-service.object-sto
 prometheus_openstack_exporter_disabled_lb: "{{ '--disable-service.load-balancer --disable-metric=neutron-loadbalancers --disable-metric=neutron-loadbalancers_not_active' if not enable_octavia | bool else '' }}"
 prometheus_openstack_exporter_disabled_items: "{{ [prometheus_openstack_exporter_disabled_volume, prometheus_openstack_exporter_disabled_dns, prometheus_openstack_exporter_disabled_object, prometheus_openstack_exporter_disabled_lb | trim] | join(' ') | trim }}"
 
+prometheus_server_command: >-
+  /opt/prometheus/prometheus --web.config.file=/etc/prometheus/web.yml --config.file /etc/prometheus/prometheus.yml
+  --web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_port }}
+  --web.external-url={{ prometheus_public_endpoint if enable_prometheus_server_external else prometheus_internal_endpoint }}
+  --storage.tsdb.path /var/lib/prometheus{% if prometheus_cmdline_extras %} {{ prometheus_cmdline_extras }}{% endif %}
+
 prometheus_blackbox_exporter_cmdline_extras: ""
 prometheus_cadvisor_cmdline_extras: "--docker_only --store_container_labels=false --disable_metrics=percpu,referenced_memory,cpu_topology,resctrl,udp,advtcp,sched,hugetlb,memory_numa,tcp,process"
 prometheus_elasticsearch_exporter_cmdline_extras: ""

ansible/roles/prometheus/tasks/config.yml

@@ -97,6 +97,24 @@
   notify:
     - Restart prometheus-server container
 
+- name: Copying over prometheus web config file
+  become: true
+  vars:
+    service: "{{ prometheus_services['prometheus-server'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/prometheus-server/web.yml"
+    mode: "0600"
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+  with_first_found:
+    - "{{ node_custom_config }}/prometheus/{{ inventory_hostname }}/web.yml"
+    - "{{ node_custom_config }}/prometheus/web.yml"
+    - "{{ role_path }}/templates/prometheus-web.yml.j2"
+  notify:
+    - Restart prometheus-server container
+
 - name: Copying over prometheus alertmanager config file
   become: true
   vars:

ansible/roles/prometheus/tasks/precheck.yml

@@ -25,6 +25,28 @@
   check_mode: false
   register: container_facts
 
+- name: Check that prometheus_bcrypt_salt is correctly set
+  assert:
+    that:
+      - prometheus_bcrypt_salt is defined
+      - prometheus_bcrypt_salt is string
+      - prometheus_bcrypt_salt | length == 22
+
+- name: Check that prometheus_password is correctly set
+  assert:
+    that:
+      - prometheus_password is defined
+      - prometheus_password is string
+      - prometheus_password | length > 0
+
+- name: Check that prometheus_grafana_password is correctly set
+  assert:
+    that:
+      - prometheus_grafana_password is defined
+      - prometheus_grafana_password is string
+      - prometheus_grafana_password | length > 0
+  when: enable_grafana | bool
+
 - name: Checking free port for Prometheus server
   wait_for:
     host: "{{ 'api' | kolla_address }}"

ansible/roles/prometheus/templates/prometheus-server.json.j2

 {
-    "command": "/opt/prometheus/prometheus --config.file /etc/prometheus/prometheus.yml --web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_port }} --web.external-url={{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ prometheus_port }} --storage.tsdb.path /var/lib/prometheus{% if prometheus_cmdline_extras %} {{ prometheus_cmdline_extras }}{% endif %}",
+    "command": "{{ prometheus_server_command }}",
     "config_files": [
         {
             "source": "{{ container_config_directory }}/prometheus.yml",
@@ -7,6 +7,12 @@
             "owner": "prometheus",
             "perm": "0600"
         },
+        {
+            "source": "{{ container_config_directory }}/web.yml",
+            "dest": "/etc/prometheus/web.yml",
+            "owner": "prometheus",
+            "perm": "0600"
+        },
         {
             "source": "{{ container_config_directory }}/extras/*",
             "dest": "/etc/prometheus/extras/",

ansible/roles/prometheus/templates/prometheus-web.yml.j2

+basic_auth_users:
+{% for user in prometheus_basic_auth_users | selectattr('enabled') | list %}
+    {{ user.username }}: {{ user.password | password_hash('bcrypt', salt=prometheus_bcrypt_salt) }}
+{% endfor %}

doc/source/reference/logging-and-monitoring/prometheus-guide.rst

@@ -34,6 +34,26 @@ In order to remove leftover volume containing Prometheus 1.x data, execute:
 
 on all hosts wherever Prometheus was previously deployed.
 
+Basic Auth
+~~~~~~~~~~
+
+Prometheus is protected with basic HTTP authentication. Kolla-ansible will
+create the following users: ``admin`` and ``grafana`` (if grafana is
+enabled). The grafana username can be overidden using the variable
+``prometheus_grafana_user``. The passwords are defined by the
+``prometheus_password`` and ``prometheus_grafana_password`` variables in
+``passwords.yml``. The list of basic auth users can be extended using the
+``prometheus_basic_auth_users_extra`` variable:
+
+.. code-block:: yaml
+
+   prometheus_basic_auth_users_extra:
+      - username: user
+        password: hello
+        enabled: true
+
+or completely overriden with the ``prometheus_basic_auth_users`` variable.
+
 Extending the default command line options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

etc/kolla/passwords.yml

@@ -247,6 +247,9 @@ redis_master_password:
 ####################
 prometheus_mysql_exporter_database_password:
 prometheus_alertmanager_password:
+prometheus_password:
+prometheus_grafana_password:
+prometheus_bcrypt_salt:
 
 ###############################
 # OpenStack identity federation

kolla_ansible/cmd/genpwd.py

@@ -20,6 +20,7 @@ import stat
 import string
 import sys
 
+from ansible.utils.encrypt import random_salt
 from cryptography import fernet
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives.asymmetric import rsa
@@ -56,7 +57,7 @@ def generate_RSA(bits=4096):
 
 
 def genpwd(passwords_file, length, uuid_keys, ssh_keys, blank_keys,
-           fernet_keys, hmac_md5_keys):
+           fernet_keys, hmac_md5_keys, bcrypt_keys):
     try:
         with open(passwords_file, 'r') as f:
             passwords = yaml.safe_load(f.read())
@@ -98,6 +99,11 @@ def genpwd(passwords_file, length, uuid_keys, ssh_keys, blank_keys,
                     .hexdigest())
             elif k in fernet_keys:
                 passwords[k] = fernet.Fernet.generate_key().decode()
+            elif k in bcrypt_keys:
+                # NOTE(wszusmki) To be compatible with the ansible
+                # password_hash filter, we use the utility function from the
+                # ansible library.
+                passwords[k] = random_salt(22)
             else:
                 passwords[k] = ''.join([
                     random.SystemRandom().choice(
@@ -151,11 +157,14 @@ def main():
     # Fernet keys
     fernet_keys = ['barbican_crypto_key']
 
+    # bcrypt salts
+    bcrypt_keys = ['prometheus_bcrypt_salt']
+
     # length of password
     length = 40
 
     genpwd(passwords_file, length, uuid_keys, ssh_keys, blank_keys,
-           fernet_keys, hmac_md5_keys)
+           fernet_keys, hmac_md5_keys, bcrypt_keys)
 
 
 if __name__ == '__main__':

releasenotes/notes/expose-prometheus-on-external-api-78d5fff60f6e75a5.yaml

+---
+features:
+  - |
+    Adds support for exposing Prometheus server on the external interface.
+    This is disabled by default and can be enabled by setting
+    ``enable_prometheus_server_external`` to ``true``. Basic auth is used to
+    protect the endpoint.
+  - |
+    Adds ``prometheus_external_fqdn`` and ``prometheus_internal_fqdn`` to
+    customise prometheus FQDNs.
+upgrade:
+  - |
+    Prometheus now uses basic auth. The password is under the key
+    ``prometheus_password`` in the Kolla passwords file. The username is
+    ``admin``. The default set of users can be changed using the variable:
+    ``prometheus_basic_auth_users``.

requirements.txt

@@ -18,3 +18,7 @@ jmespath>=0.9.3 # MIT
 
 # Hashicorp Vault
 hvac>=0.10.1
+
+# Password hashing
+bcrypt>=3.0.0 # Apache-2.0
+passlib[bcrypt]>=1.0.0 # BSD

tests/j2lint.py

@@ -23,6 +23,7 @@ Simple j2 linter, useful for checking jinja2 template syntax
 Adapted for OpenStack Kolla/Kolla-Ansible purposes
 """
 
+from ansible.plugins.filter.core import get_encrypted_password
 from ansible.plugins.filter.core import to_json
 from functools import reduce
 from jinja2 import BaseLoader
@@ -51,6 +52,9 @@ def check(template, out, err, env=Environment(loader=AbsolutePathLoader(),
         env.filters['bool'] = bool
         env.filters['hash'] = hash
         env.filters['to_json'] = to_json
+        # NOTE(wszumski): password_hash is mapped to the function:
+        # get_encrypted_password in ansible.filters.core.
+        env.filters['password_hash'] = get_encrypted_password
         env.filters['kolla_address'] = kolla_address
         env.filters['put_address_in_context'] = put_address_in_context
         env.get_template(template)

tests/run-hashi-vault.yml

@@ -36,10 +36,12 @@
 
     - name: install kolla-ansible and dependencies
       pip:
-        name:
-          - "{{ kolla_ansible_src_dir }}"
         executable: "pip3"
         extra_args: "-c {{ upper_constraints_file }} --user"
+        name:
+          - "{{ kolla_ansible_src_dir }}"
+          - "ansible-core{{ ansible_core_version_constraint }}"
+          - "ansible{{ ansible_version_constraint }}"
 
     - name: copy passwords.yml file
       copy:

tests/test-prometheus-opensearch.sh

@@ -79,11 +79,14 @@ function check_prometheus {
     # Query prometheus graph, and check that the returned page looks like a
     # prometheus page.
     PROMETHEUS_URL=${OS_AUTH_URL%:*}:9091/graph
+    prometheus_password=$(awk '$1 == "prometheus_password:" { print $2 }' /etc/kolla/passwords.yml)
     output_path=$1
     args=(
         --include
         --location
         --fail
+        --user
+        admin:$prometheus_password
     )
     if [[ "$TLS_ENABLED" = "True" ]]; then
         args+=(--cacert $OS_CACERT)

zuul.d/base.yaml

@@ -272,6 +272,7 @@
 
 - job:
     name: kolla-ansible-hashi-vault-base
+    parent: kolla-ansible-variables
     run: tests/run-hashi-vault.yml
     required-projects:
       - openstack/kolla-ansible