Update rabbitmq to use new conf & clustering

Depends-On: I75e00312b36e1678b90a42cf58d24652323eff27
Change-Id: Ia716fabffca41eff816e59bbf9f4cab79ee8b72f

ansible/roles/rabbitmq/defaults/main.yml

@@ -39,3 +39,4 @@ rabbitmq_dimensions: "{{ default_container_dimensions }}"
 rabbitmq_user: "openstack"
 rabbitmq_cluster_name: "openstack"
 rabbitmq_hostname: "{{ ansible_hostname }}"
+rabbitmq_pid_file: "/var/lib/rabbitmq/mnesia/rabbitmq.pid"

ansible/roles/rabbitmq/handlers/main.yml

 ---
-- name: Restart rabbitmq container
+- name: Restart rabbitmq container (first node)
   vars:
     service_name: "rabbitmq"
     service: "{{ rabbitmq_services[service_name] }}"
@@ -16,7 +16,40 @@
     dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
-    - inventory_hostname in groups[service.group]
+    - inventory_hostname == groups[service.group]|first
+    - service.enabled | bool
+    - config_json.changed | bool
+      or rabbitmq_confs.changed | bool
+      or rabbitmq_container.changed | bool
+  notify:
+    - Waiting for rabbitmq to start on first node
+
+- name: Waiting for rabbitmq to start on first node
+  vars:
+    service_name: "rabbitmq"
+    service: "{{ rabbitmq_services[service_name] }}"
+  shell: "docker exec {{ service.container_name }} rabbitmqctl wait {{ rabbitmq_pid_file }}"
+  when:
+    - inventory_hostname == groups[service.group]|first
+
+- name: Restart rabbitmq container (rest of nodes)
+  vars:
+    service_name: "rabbitmq"
+    service: "{{ rabbitmq_services[service_name] }}"
+    config_json: "{{ rabbitmq_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    rabbitmq_container: "{{ check_rabbitmq_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    environment: "{{ service.environment }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname != groups[service.group]|first
     - service.enabled | bool
     - config_json.changed | bool
       or rabbitmq_confs.changed | bool

ansible/roles/rabbitmq/tasks/config.yml

@@ -24,7 +24,8 @@
     - item.value.enabled | bool
   with_dict: "{{ rabbitmq_services }}"
   notify:
-    - Restart rabbitmq container
+    - Restart rabbitmq container (first node)
+    - Restart rabbitmq container (rest of nodes)
 
 - name: Copying over rabbitmq configs
   vars:
@@ -40,10 +41,11 @@
     - service.enabled | bool
   with_items:
     - "rabbitmq-env.conf"
-    - "rabbitmq.config"
+    - "rabbitmq.conf"
     - "definitions.json"
   notify:
-    - Restart rabbitmq container
+    - Restart rabbitmq container (first node)
+    - Restart rabbitmq container (rest of nodes)
 
 - name: Check rabbitmq containers
   become: true
@@ -62,4 +64,5 @@
     - item.value.enabled | bool
   with_dict: "{{ rabbitmq_services }}"
   notify:
-    - Restart rabbitmq container
+    - Restart rabbitmq container (first node)
+    - Restart rabbitmq container (rest of nodes)

ansible/roles/rabbitmq/tasks/upgrade.yml

 ---
+# NOTE(pbourke): These tasks perform a 'full stop upgrade', which is necessary when moving between
+# major releases. In future kolla-ansible releases we may be able to change this to a rolling
+# restart. For info on this process see https://www.rabbitmq.com/upgrade.html
 - name: Checking if rabbitmq container needs upgrading
   vars:
     service_name: "rabbitmq"
@@ -15,14 +18,25 @@
 
 - include_tasks: config.yml
 
-- name: Flush handlers
-  meta: flush_handlers
+- name: Stopping all rabbitmq instances but the first node
+  become: true
+  kolla_docker:
+    action: "stop_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ project_name }}"
   when:
-    - rabbitmq_hostname == (gospel_node.stdout | from_json).hostname
+    - inventory_hostname != groups[role_rabbitmq_groups]|first
     - rabbitmq_differs['result']
 
-- name: Flush handlers
-  meta: flush_handlers
+- name: Stopping rabbitmq on the first node
+  become: true
+  kolla_docker:
+    action: "stop_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ project_name }}"
   when:
-    - rabbitmq_hostname != (gospel_node.stdout | from_json).hostname
+    - inventory_hostname == groups[role_rabbitmq_groups]|first
     - rabbitmq_differs['result']
+
+- name: Flush handlers
+  meta: flush_handlers

ansible/roles/rabbitmq/templates/definitions.json.j2

@@ -5,7 +5,7 @@
     {% endif %}
   ],
   "users": [
-    {"name": "{{ role_rabbitmq_user }}", "password": "password", "tags": "administrator"}{% if role_rabbitmq_monitoring_user is defined and role_rabbitmq_monitoring_user %},
+    {"name": "{{ role_rabbitmq_user }}", "password": "{{ role_rabbitmq_password }}", "tags": "administrator"}{% if role_rabbitmq_monitoring_user is defined and role_rabbitmq_monitoring_user %},
     {"name": "{{ role_rabbitmq_monitoring_user }}", "password": "{{ role_rabbitmq_monitoring_password }}", "tags": "monitoring"}{% endif %}{% if project_name == 'outward_rabbitmq' %},
     {"name": "{{ murano_agent_rabbitmq_user }}", "password": "{{ murano_agent_rabbitmq_password }}", "tags": "management"}
     {% endif %}

ansible/roles/rabbitmq/templates/rabbitmq-env.conf.j2

 RABBITMQ_NODENAME=rabbit@{{ ansible_hostname }}
 RABBITMQ_LOG_BASE=/var/log/kolla/{{ project_name }}
+RABBITMQ_DIST_PORT={{ role_rabbitmq_cluster_port }}
+RABBITMQ_PID_FILE={{ rabbitmq_pid_file }}
 
-# TODO(sdake, vhosakot)
-# erlang by default binds to wildcard (all interfaces) and can potentially
-# interfere with the neutron external or tenant networks. We should in theory
-# bind epmd to the host's IPv4 address to address the issue however this also
-# has issues and can crash erlang when it is compiled with IPv6 support.
-# See bugs:
-# https://bugs.launchpad.net/ubuntu/+source/erlang/+bug/1374109
-# https://bugs.launchpad.net/kolla/+bug/1562701
-# https://bugzilla.redhat.com/show_bug.cgi?id=1324922
-#export ERL_EPMD_ADDRESS={{ api_interface_address }}
+export ERL_EPMD_ADDRESS={{ api_interface_address }}
 export ERL_EPMD_PORT={{ role_rabbitmq_epmd_port }}

ansible/roles/rabbitmq/templates/rabbitmq.conf.j2

+listeners.tcp.1 = {{ api_interface_address }}:{{ role_rabbitmq_port }}
+{% if rabbitmq_hipe_compile|bool %}
+hipe_compile = true
+{% endif %}
+cluster_partition_handling = autoheal
+
+management.listener.ip = {{ api_interface_address }}
+management.listener.port = {{ role_rabbitmq_management_port }}
+management.load_definitions = /etc/rabbitmq/definitions.json
+
+cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config
+{% for host in groups[role_rabbitmq_groups] %}
+cluster_formation.classic_config.nodes.{{ loop.index0 }} = rabbit@{{ hostvars[host]['ansible_hostname'] }}
+{% endfor %}

ansible/roles/rabbitmq/templates/rabbitmq.config.j2

-[
-  {kernel, [
-    {inet_dist_use_interface, {% raw %}{{% endraw %}{{ api_interface_address | regex_replace('\.', ',') }}}},
-    {inet_dist_listen_min, {{ role_rabbitmq_cluster_port }}},
-    {inet_dist_listen_max, {{ role_rabbitmq_cluster_port }}}
-  ]},
-  {rabbit, [
-{% if rabbitmq_hipe_compile|bool %}
-    {hipe_compile, true},
-{% endif %}
-    {tcp_listeners, [
-      {"{{ api_interface_address }}", {{ role_rabbitmq_port }}}
-    ]},
-    {cluster_partition_handling, autoheal}
-  ]},
-  {rabbitmq_management, [
-    {listener, [
-      {ip, "{{ api_interface_address }}"},
-      {port, {{ role_rabbitmq_management_port }}}
-    ]},
-    {load_definitions, "/etc/rabbitmq/definitions.json"}
-  ]}
-].
-% EOF

ansible/roles/rabbitmq/templates/rabbitmq.json.j2

@@ -8,8 +8,8 @@
             "perm": "0600"
         },
         {
-            "source": "{{ container_config_directory }}/rabbitmq.config",
-            "dest": "/etc/rabbitmq/rabbitmq.config",
+            "source": "{{ container_config_directory }}/rabbitmq.conf",
+            "dest": "/etc/rabbitmq/rabbitmq.conf",
             "owner": "rabbitmq",
             "perm": "0600"
         },

releasenotes/notes/update-rabbitmq-5db770469b9bae18.yaml

+---
+upgrade:
+  - |
+    Rabbitmq has been updated to 3.7.x. This comes with a new config format
+    which is now called rabbitmq.conf rather than rabbitmq.config.

tests/templates/globals-default.j2

@@ -45,6 +45,3 @@ ceph_pool_pgp_num: 8
 {% endif %}
 
 keystone_token_provider: "fernet"
-
-# TODO(pbourke): remove once https://review.openstack.org/#/c/584427/ is merged
-rpc_transport_url: "rabbit://openstack:password@{{ api_interface_address }}:5672"

tools/setup_gate.sh

@@ -149,8 +149,7 @@ function test_openstack {
     tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks1
     # TODO(jeffrey4l): add pull action when we have a local registry
     # service in CI
-    # TODO(pbourke): remove '-e rabbitmq_password=password' once https://review.openstack.org/#/c/584427/ is merged
-    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy -e rabbitmq_password=password > /tmp/logs/ansible/deploy
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy
     tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy
     tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-deploy