From e34fbb174c7042f61fc7742363ee7e81569aaba5 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Fri, 20 Jan 2023 13:32:11 +0000
Subject: [PATCH] Use the upgraded image to run Nova upgrade checks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
When upgrading Nova, we sometimes hit an error where an old hypervisor
that hasn’t been upgraded recently (for example due to broken hardware)
is preventing Nova API from starting properly. This can be detected
using the tool ``nova-status upgrade check`` to make sure that there are
no ``nova-compute`` that are older than N-1 releases. This is already
used in the Kolla Ansible upgrade task for Nova. However, this task uses
the current ``nova-api`` container, so computes which will be too old
after the upgrade are not caught.
This patch changes Kolla Ansible so that the upgraded ``nova-api`` image
is used to run the upgrade checks, allowing computes that will be too
old to be detected before the upgrades are performed.
Depends-On: https://review.opendev.org/c/openstack/kolla/+/878744
Closes-Bug: #1957080
Co-Authored-By: Pierre Riteau <pierre@stackhpc.com>
Change-Id: I3a899411001834a0c88e37f45a756247ee11563d
---
ansible/roles/nova/tasks/upgrade.yml | 21 ++++++++++++++++---
...-with-upgraded-image-dca9c515bcd89ec8.yaml | 10 +++++++++
2 files changed, 28 insertions(+), 3 deletions(-)
create mode 100644 releasenotes/notes/perform-nova-upgrade-checks-with-upgraded-image-dca9c515bcd89ec8.yaml
diff --git a/ansible/roles/nova/tasks/upgrade.yml b/ansible/roles/nova/tasks/upgrade.yml
index d6f3f70c42..07f91379af 100644
--- a/ansible/roles/nova/tasks/upgrade.yml
+++ b/ansible/roles/nova/tasks/upgrade.yml
@@ -1,9 +1,24 @@
---
-- name: Check nova upgrade status
+- name: Run Nova upgrade checks
become: true
- command: "{{ kolla_container_engine }} exec -t nova_api nova-status upgrade check"
+ vars:
+ nova_api: "{{ nova_services['nova-api'] }}"
+ kolla_docker:
+ action: "start_container"
+ common_options: "{{ docker_common_options }}"
+ detach: False
+ environment:
+ KOLLA_UPGRADE_CHECK:
+ KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+ image: "{{ nova_api.image }}"
+ labels:
+ UPGRADE:
+ name: "nova_upgrade_checks"
+ restart_policy: no
+ volumes: "{{ nova_api_default_volumes + nova_api_extra_volumes }}"
+ run_once: True
register: nova_upgrade_check_stdout
- when: inventory_hostname == groups['nova-api'][0]
+ delegate_to: "{{ groups['nova-api'][0] }}"
failed_when: false
- name: Upgrade status check result
diff --git a/releasenotes/notes/perform-nova-upgrade-checks-with-upgraded-image-dca9c515bcd89ec8.yaml b/releasenotes/notes/perform-nova-upgrade-checks-with-upgraded-image-dca9c515bcd89ec8.yaml
new file mode 100644
index 0000000000..746bd9988d
--- /dev/null
+++ b/releasenotes/notes/perform-nova-upgrade-checks-with-upgraded-image-dca9c515bcd89ec8.yaml
@@ -0,0 +1,10 @@
+---
+fixes:
+ - |
+ When upgrading Nova to a new release, we use the tool ``nova-status upgrade
+ check`` to make sure that there are no ``nova-compute`` that are older than
+ N-1 releases. This was performed using the current ``nova-api`` container,
+ so computes which will be too old after the upgrade were not caught. Now
+ the upgraded ``nova-api`` container image is used, so older computes are
+ identified correctly.
+ `LP#1957080 <https://launchpad.net/bugs/1957080>`__
--
GitLab