Merge "Apply Nova rolling upgrade logic"

ansible/roles/nova/defaults/main.yml

@@ -290,6 +290,8 @@ openstack_nova_auth: "{{ openstack_auth }}"
 openstack_placement_auth: "{{ openstack_auth }}"
 
 nova_compute_host_rp_filter_mode: 0
+nova_enable_rolling_upgrade: "yes"
+nova_safety_upgrade: "no"
 
 nova_libvirt_port: "16509"
 nova_ssh_port: "8022"

ansible/roles/nova/handlers/main.yml

 ---
-- name: Restart nova-ssh container
+- name: Restart placement-api container
   vars:
-    service_name: "nova-ssh"
+    service_name: "placement-api"
     service: "{{ nova_services[service_name] }}"
     config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_ssh_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    placement_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
   become: true
   kolla_docker:
     action: "recreate_or_restart_container"
     common_options: "{{ docker_common_options }}"
     name: "{{ service.container_name }}"
     image: "{{ service.image }}"
-    pid_mode: "{{ service.pid_mode | default('') }}"
-    privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
     dimensions: "{{ service.dimensions }}"
   when:
@@ -20,52 +20,51 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
     - config_json.changed | bool
-      or nova_ssh_confs.changed | bool
-      or nova_ssh_container.changed | bool
+      or nova_conf.changed | bool
+      or policy_overwriting.changed | bool
+      or placement_api_wsgi_conf | changed
+      or placement_api_container.changed | bool
 
-- name: Restart nova-libvirt container
+- name: Restart nova-conductor container
   vars:
-    service_name: "nova-libvirt"
+    service_name: "nova-conductor"
     service: "{{ nova_services[service_name] }}"
     config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_libvirt_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_conductor_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
   become: true
   kolla_docker:
     action: "recreate_or_restart_container"
     common_options: "{{ docker_common_options }}"
     name: "{{ service.container_name }}"
     image: "{{ service.image }}"
-    pid_mode: "{{ service.pid_mode | default('') }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
     dimensions: "{{ service.dimensions }}"
-  register: restart_nova_libvirt
-  # NOTE(Jeffrey4l): retry 5 to remove nova_libvirt container because when
-  # guests running, nova_libvirt will raise error even though it is removed.
-  retries: 5
-  until: restart_nova_libvirt is success
   when:
     - kolla_action != "config"
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
     - config_json.changed | bool
-      or nova_libvirt_confs.changed | bool
-      or nova_libvirt_container.changed | bool
+      or nova_conf.changed | bool
+      or policy_overwriting.changed | bool
+      or nova_conductor_container.changed | bool
 
-- name: Restart placement-api container
+- name: Restart nova-ssh container
   vars:
-    service_name: "placement-api"
+    service_name: "nova-ssh"
     service: "{{ nova_services[service_name] }}"
     config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    placement_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_ssh_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
   become: true
   kolla_docker:
     action: "recreate_or_restart_container"
     common_options: "{{ docker_common_options }}"
     name: "{{ service.container_name }}"
     image: "{{ service.image }}"
+    pid_mode: "{{ service.pid_mode | default('') }}"
+    privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
     dimensions: "{{ service.dimensions }}"
   when:
@@ -73,36 +72,37 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
     - config_json.changed | bool
-      or nova_conf.changed | bool
-      or policy_overwriting.changed | bool
-      or placement_api_wsgi_conf | changed
-      or placement_api_container.changed | bool
+      or nova_ssh_confs.changed | bool
+      or nova_ssh_container.changed | bool
 
-- name: Restart nova-api container
+- name: Restart nova-libvirt container
   vars:
-    service_name: "nova-api"
+    service_name: "nova-libvirt"
     service: "{{ nova_services[service_name] }}"
     config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_libvirt_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
   become: true
   kolla_docker:
     action: "recreate_or_restart_container"
     common_options: "{{ docker_common_options }}"
     name: "{{ service.container_name }}"
     image: "{{ service.image }}"
+    pid_mode: "{{ service.pid_mode | default('') }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
     dimensions: "{{ service.dimensions }}"
+  register: restart_nova_libvirt
+  # NOTE(Jeffrey4l): retry 5 to remove nova_libvirt container because when
+  # guests running, nova_libvirt will raise error even though it is removed.
+  retries: 5
+  until: restart_nova_libvirt is success
   when:
     - kolla_action != "config"
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
     - config_json.changed | bool
-      or nova_conf.changed | bool
-      or policy_overwriting.changed | bool
-      or nova_api_container.changed | bool
+      or nova_libvirt_confs.changed | bool
+      or nova_libvirt_container.changed | bool
 
 - name: Restart nova-scheduler container
   vars:
@@ -130,14 +130,14 @@
       or policy_overwriting.changed | bool
       or nova_scheduler_container.changed | bool
 
-- name: Restart nova-conductor container
+- name: Restart nova-consoleauth container
   vars:
-    service_name: "nova-conductor"
+    service_name: "nova-consoleauth"
     service: "{{ nova_services[service_name] }}"
     config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
     nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
     policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_conductor_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_consoleauth_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
   become: true
   kolla_docker:
     action: "recreate_or_restart_container"
@@ -154,16 +154,16 @@
     - config_json.changed | bool
       or nova_conf.changed | bool
       or policy_overwriting.changed | bool
-      or nova_conductor_container.changed | bool
+      or nova_consoleauth_container.changed | bool
 
-- name: Restart nova-consoleauth container
+- name: Restart nova-novncproxy container
   vars:
-    service_name: "nova-consoleauth"
+    service_name: "nova-novncproxy"
     service: "{{ nova_services[service_name] }}"
     config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
     nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
     policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_consoleauth_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_novncproxy_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
   become: true
   kolla_docker:
     action: "recreate_or_restart_container"
@@ -180,16 +180,16 @@
     - config_json.changed | bool
       or nova_conf.changed | bool
       or policy_overwriting.changed | bool
-      or nova_consoleauth_container.changed | bool
+      or nova_novncproxy_container.changed | bool
 
-- name: Restart nova-novncproxy container
+- name: Restart nova-spicehtml5proxy container
   vars:
-    service_name: "nova-novncproxy"
+    service_name: "nova-spicehtml5proxy"
     service: "{{ nova_services[service_name] }}"
     config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
     nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
     policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_novncproxy_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_spicehtml5proxy_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
   become: true
   kolla_docker:
     action: "recreate_or_restart_container"
@@ -206,16 +206,16 @@
     - config_json.changed | bool
       or nova_conf.changed | bool
       or policy_overwriting.changed | bool
-      or nova_novncproxy_container.changed | bool
+      or nova_spicehtml5proxy_container.changed | bool
 
-- name: Restart nova-spicehtml5proxy container
+- name: Restart nova-serialproxy container
   vars:
-    service_name: "nova-spicehtml5proxy"
+    service_name: "nova-serialproxy"
     service: "{{ nova_services[service_name] }}"
     config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
     nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
     policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_spicehtml5proxy_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_serialproxy_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
   become: true
   kolla_docker:
     action: "recreate_or_restart_container"
@@ -232,16 +232,16 @@
     - config_json.changed | bool
       or nova_conf.changed | bool
       or policy_overwriting.changed | bool
-      or nova_spicehtml5proxy_container.changed | bool
+      or nova_serialproxy_container.changed | bool
 
-- name: Restart nova-serialproxy container
+- name: Restart nova-api container
   vars:
-    service_name: "nova-serialproxy"
+    service_name: "nova-api"
     service: "{{ nova_services[service_name] }}"
     config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
     nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
     policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    nova_serialproxy_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    nova_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
   become: true
   kolla_docker:
     action: "recreate_or_restart_container"
@@ -258,7 +258,7 @@
     - config_json.changed | bool
       or nova_conf.changed | bool
       or policy_overwriting.changed | bool
-      or nova_serialproxy_container.changed | bool
+      or nova_api_container.changed | bool
 
 - name: Restart nova-compute container
   vars:

ansible/roles/nova/tasks/bootstrap_service.yml

@@ -2,14 +2,18 @@
 - name: Running Nova bootstrap container
   vars:
     nova_api: "{{ nova_services['nova-api'] }}"
+    bootstrap_environment:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    upgrade_environment:
+      KOLLA_UPGRADE:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
   become: true
   kolla_docker:
     action: "start_container"
     common_options: "{{ docker_common_options }}"
     detach: False
-    environment:
-      KOLLA_BOOTSTRAP:
-      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    environment: "{{ upgrade_environment if nova_enable_rolling_upgrade|bool else bootstrap_environment }}"
     image: "{{ nova_api.image }}"
     labels:
       BOOTSTRAP:

ansible/roles/nova/tasks/legacy_upgrade.yml

+---
+- include_tasks: config.yml
+
+- include_tasks: bootstrap_service.yml
+
+- name: Checking if conductor container needs upgrading
+  become: true
+  kolla_docker:
+    action: "compare_image"
+    common_options: "{{ docker_common_options }}"
+    name: "nova_conductor"
+    image: "{{ nova_conductor_image_full }}"
+  when: inventory_hostname in groups['nova-conductor']
+  register: conductor_differs
+
+# Short downtime here, but from user perspective his call will just timeout or execute later
+- name: Stopping all nova_conductor containers
+  become: true
+  kolla_docker:
+    action: "stop_container"
+    common_options: "{{ docker_common_options }}"
+    name: "nova_conductor"
+  when:
+    - inventory_hostname in groups['nova-conductor']
+    - conductor_differs['result']
+
+- name: Flush handlers
+  meta: flush_handlers
+
+- include_tasks: reload.yml

ansible/roles/nova/tasks/rolling_upgrade.yml

+---
+# Create new set of configs on nodes
+- include_tasks: pull.yml
+
+- include_tasks: config.yml
+
+- include_tasks: bootstrap_service.yml
+
+- name: Stopping all nova services except nova-compute
+  become: true
+  kolla_docker:
+    action: "stop_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ item.value.container_name }}"
+  with_dict: "{{ nova_services }}"
+  when:
+    - "'nova-compute' not in item.key"
+    - inventory_hostname in groups[item.value.group]
+    - item.value.enabled | bool
+    - nova_safety_upgrade | bool
+
+# TODO(donghm): Flush_handlers to restart nova services
+# should be run in serial nodes to decrease downtime if
+# the previous task did not run. Update when the
+# Ansible strategy module for rolling upgrade is finished.
+
+- name: Flush handlers
+  meta: flush_handlers
+
+- include_tasks: reload.yml
+
+- name: Migrate Nova database
+  vars:
+    nova_api: "{{ nova_services['nova-api'] }}"
+  become: true
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_OSM:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ nova_api.image }}"
+    labels:
+      BOOTSTRAP:
+    name: "bootstrap_nova"
+    restart_policy: "never"
+    volumes: "{{ nova_api.volumes }}"
+  run_once: True
+  delegate_to: "{{ groups[nova_api.group][0] }}"

ansible/roles/nova/tasks/upgrade.yml

 ---
-# Create new set of configs on nodes
-- include_tasks: config.yml
-
-- include_tasks: bootstrap_service.yml
-
-- name: Checking if conductor container needs upgrading
+- name: Check nova upgrade status
   become: true
-  kolla_docker:
-    action: "compare_image"
-    common_options: "{{ docker_common_options }}"
-    name: "nova_conductor"
-    image: "{{ nova_conductor_image_full }}"
-  when: inventory_hostname in groups['nova-conductor']
-  register: conductor_differs
+  command: docker exec -t nova_api nova-status upgrade check
+  register: nova_upgrade_check_stdout
+  when: inventory_hostname == groups['nova-api'][0]
 
-# Short downtime here, but from user perspective his call will just timeout or execute later
-- name: Stopping all nova_conductor containers
-  become: true
-  kolla_docker:
-    action: "stop_container"
-    common_options: "{{ docker_common_options }}"
-    name: "nova_conductor"
-  when:
-    - inventory_hostname in groups['nova-conductor']
-    - conductor_differs['result']
+- name: Upgrade status check result
+  fail:
+    msg:
+      - "There was an upgrade status check warning or failure!"
+      - "See the detail at https://docs.openstack.org/nova/latest/cli/nova-status.html#nova-status-checks"
+  vars:
+    first_nova_api_host: "{{ groups['nova-api'][0] }}"
+  when: hostvars[first_nova_api_host]['nova_upgrade_check_stdout']['rc'] != 0
 
-- name: Flush handlers
-  meta: flush_handlers
+- include_tasks: legacy_upgrade.yml
+  when: not nova_enable_rolling_upgrade | bool
 
-- include_tasks: reload.yml
+- include_tasks: rolling_upgrade.yml
+  when: nova_enable_rolling_upgrade | bool

etc/kolla/globals.yml

@@ -401,6 +401,21 @@ glance_enable_rolling_upgrade: "no"
 # The number of fake driver per compute node
 #num_nova_fake_per_node: 5
 
+# Configure nova upgrade option, due to currently kolla support
+# two upgrade ways for nova: legacy_upgrade and rolling_upgrade
+# The variable "nova_enable_rolling_upgrade: yes" is meaning
+# rolling_upgrade were enabled and opposite
+#nova_enable_rolling_upgrade: "yes"
+
+# The flag "nova_safety_upgrade" need to be consider when
+# "nova_enable_rolling_upgrade" is enabled. The "nova_safety_upgrade"
+# controls whether the nova services are all stopped before rolling
+# upgrade to the new version, for the safety and availability.
+# If "nova_safety_upgrade" is "yes", that will stop all nova services (except
+# nova-compute) for no failed API operations before upgrade to the
+# new version. And opposite.
+#nova_safety_upgrade: "no"
+
 #################
 # Hyper-V options
 #################

releasenotes/notes/implement-nova-rolling-upgrade-f3b2d8382f725cb2.yaml

+---
+features:
+  - |
+    Implement Nova rolling upgrade logic