From 80a32c3c74cad4d46671faff067b274ffed74bba Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 15 Dec 2021 16:07:50 +0000
Subject: [PATCH] cinder: restart services after upgrade
This patch is roughly an adaptation of
Ia6fc9011ee6f5461f40a1307b72709d769814a79 for cinder.
During an upgrade, cinder pins the version of RPC calls to the minimum
seen across all services. This ensures that old services do not receive
data they cannot handle. After the upgrade is complete, all cinder
services are supposed to be reloaded to cause them to check again the
RPC versions of services and use the new latest version which should now
be supported by all running services.
There is a second issue in that it takes some time for the upgraded
services to update the cinder services database table with their new
version. We need to wait until all cinder services have done this
before the restart is performed, otherwise the RPC version cap will
remain in place. There is currently no interface in cinder available for
checking these versions, so as a workaround we use a configurable
delay with a default duration of 30 seconds, as we do for nova.
This change restarts all cinder services after an upgrade, after a 30
second delay.
Closes-Bug: #1954932
Related-Bug: #1833069
Change-Id: I9164dc589386d2c2d4daf1bf84061b806ba9988d
---
ansible/roles/cinder/defaults/main.yml | 9 +++++++++
ansible/roles/cinder/handlers/main.yml | 20 +++++++++++++++++++
ansible/roles/cinder/tasks/reload.yml | 10 ++++++++++
ansible/roles/cinder/tasks/upgrade.yml | 2 ++
.../unpin-cinder-rpcs-8eb7e0858a91b9b8.yaml | 6 ++++++
5 files changed, 47 insertions(+)
create mode 100644 ansible/roles/cinder/tasks/reload.yml
create mode 100644 releasenotes/notes/unpin-cinder-rpcs-8eb7e0858a91b9b8.yaml
diff --git a/ansible/roles/cinder/defaults/main.yml b/ansible/roles/cinder/defaults/main.yml
index c750672876..62dac7c7cc 100644
--- a/ansible/roles/cinder/defaults/main.yml
+++ b/ansible/roles/cinder/defaults/main.yml
@@ -203,6 +203,15 @@ cinder_logging_debug: "{{ openstack_logging_debug }}"
openstack_cinder_auth: "{{ openstack_auth }}"
+# After upgrading cinder, services will have an RPC version cap in place. We
+# need to restart all services in order to allow them to use the latest RPC
+# version. Ideally, there would be a way to check whether all cinder services
+# are using the latest version, but currently there is not. Instead, wait a
+# short time for all cinder services to update the version of their service in
+# the database. This seems to take around 10 seconds, but the default is 30 to
+# allow room for slowness.
+cinder_rpc_version_startup_delay: 30
+
####################
# Cinder
diff --git a/ansible/roles/cinder/handlers/main.yml b/ansible/roles/cinder/handlers/main.yml
index ecd3b8f069..59a08ca8d0 100644
--- a/ansible/roles/cinder/handlers/main.yml
+++ b/ansible/roles/cinder/handlers/main.yml
@@ -66,3 +66,23 @@
healthcheck: "{{ service.healthcheck | default(omit) }}"
when:
- kolla_action != "config"
+
+# NOTE(mgoddard): After upgrading cinder, services will have an RPC version cap
+# in place. We need to restart all services in order to allow them to use the
+# latest RPC version. Ideally, there would be a way to check whether all cinder
+# services are using the latest version, but currently there is not. Instead,
+# wait a short time for all cinder services to update the version of their
+# service in the database. This seems to take around 10 seconds, but the
+# default is 30 to allow room for slowness.
+
+- name: Wait for cinder services to update service versions
+ pause:
+ seconds: "{{ cinder_rpc_version_startup_delay }}"
+ run_once: true
+ when:
+ - kolla_action == 'upgrade'
+ listen:
+ - Restart cinder-api container
+ - Restart cinder-scheduler container
+ - Restart cinder-volume container
+ - Restart cinder-backup container
diff --git a/ansible/roles/cinder/tasks/reload.yml b/ansible/roles/cinder/tasks/reload.yml
new file mode 100644
index 0000000000..a8d6dd3289
--- /dev/null
+++ b/ansible/roles/cinder/tasks/reload.yml
@@ -0,0 +1,10 @@
+---
+- name: Reload cinder services to remove RPC version pin
+ vars:
+ service: "{{ item.value }}"
+ become: true
+ kolla_docker:
+ action: "restart_container"
+ common_options: "{{ docker_common_options }}"
+ name: "{{ service.container_name }}"
+ with_dict: "{{ cinder_services | select_services_enabled_and_mapped_to_host }}"
diff --git a/ansible/roles/cinder/tasks/upgrade.yml b/ansible/roles/cinder/tasks/upgrade.yml
index 17bbdae0cf..a30a1c726b 100644
--- a/ansible/roles/cinder/tasks/upgrade.yml
+++ b/ansible/roles/cinder/tasks/upgrade.yml
@@ -13,6 +13,8 @@
- name: Flush handlers
meta: flush_handlers
+- import_tasks: reload.yml
+
- name: Running Cinder online schema migration
vars:
cinder_api: "{{ cinder_services['cinder-api'] }}"
diff --git a/releasenotes/notes/unpin-cinder-rpcs-8eb7e0858a91b9b8.yaml b/releasenotes/notes/unpin-cinder-rpcs-8eb7e0858a91b9b8.yaml
new file mode 100644
index 0000000000..acb8f66bf1
--- /dev/null
+++ b/releasenotes/notes/unpin-cinder-rpcs-8eb7e0858a91b9b8.yaml
@@ -0,0 +1,6 @@
+---
+fixes:
+ - |
+ Fixes an issue with Cinder upgrade where Cinder services would remain
+ pinned to the previous release's RPC & object versions. `LP#1954932
+ <https://bugs.launchpad.net/kolla-ansible/+bug/1954932>`__
--
GitLab