CI: retry smoke tests and instance creation

Sometimes services have problems connecting to the database
or rmq on upgrade jobs, and we're not retrying the test.

Depends-On: https://review.opendev.org/c/openstack/ansible-collection-kolla/+/882344

Change-Id: I9fca3d4336375b72d09633b51c8596e0c08abbf9

tests/test-core-openstack.sh

@@ -8,11 +8,52 @@ set -o pipefail
 export PYTHONUNBUFFERED=1
 
 function test_smoke {
-    openstack --debug compute service list
-    openstack --debug network agent list
-    openstack --debug orchestration service list
+    attempts=10
+
+    for i in $(seq 1 ${attempts}); do
+        if openstack --debug compute service list; then
+            break
+        elif [[ $i -eq ${attempts} ]]; then
+            echo "Failed to list compute services after ${attempts} attempts"
+        else
+            echo "Cannot list compute services, retrying"
+        fi
+        sleep 10
+    done
+
+    for i in $(seq 1 ${attempts}); do
+        if openstack --debug network agent list; then
+            break
+        elif [[ $i -eq ${attempts} ]]; then
+            echo "Failed to list network services after ${attempts} attempts"
+        else
+            echo "Cannot list network services, retrying"
+        fi
+        sleep 10
+    done
+
+    for i in $(seq 1 ${attempts}); do
+        if openstack --debug orchestration service list; then
+            break
+        elif [[ $i -eq ${attempts} ]]; then
+            echo "Failed to list orchestration services after ${attempts} attempts"
+        else
+            echo "Cannot list orchestration services, retrying"
+        fi
+        sleep 10
+    done
+
     if [[ $SCENARIO == "cephadm" ]] || [[ $SCENARIO == "zun" ]]; then
-        openstack --debug volume service list
+        for i in $(seq 1 ${attempts}); do
+            if openstack --debug volume service list; then
+                break
+            elif [[ $i -eq ${attempts} ]]; then
+                echo "Failed to list volume services after ${attempts} attempts"
+            else
+                echo "Cannot list volume services, retrying"
+            fi
+            sleep 10
+        done
     fi
 }
 
@@ -166,13 +207,25 @@ function create_instance {
         server_create_extra="${server_create_extra} --config-drive True"
     fi
 
-    openstack server create --wait --image cirros --flavor m1.tiny --key-name mykey --network demo-net ${server_create_extra} ${name}
-    # If the status is not ACTIVE, print info and exit 1
-    if [[ $(openstack server show ${name} -f value -c status) != "ACTIVE" ]]; then
-        echo "FAILED: Instance is not active"
-        openstack --debug server show ${name}
-        return 1
-    fi
+    attempts=10
+    for i in $(seq 1 ${attempts}); do
+        if openstack server create --wait --image cirros --flavor m1.tiny --key-name mykey --network demo-net ${server_create_extra} ${name}; then
+            # If the status is not ACTIVE, print info and exit 1
+            if [[ $(openstack server show ${name} -f value -c status) != "ACTIVE" ]]; then
+                echo "FAILED: Instance is not active"
+                openstack --debug server show ${name}
+                openstack server delete ${name}
+            else
+                break
+            fi
+        elif [[ $i -eq ${attempts} ]]; then
+            echo "Failed to create instance after ${attempts} attempts"
+        else
+            echo "Cannot create instance, retrying"
+            openstack server delete ${name}
+        fi
+        sleep 10
+    done
 }
 
 function delete_instance {