Use explicit version of community.docker

This works around a bug in the ansible packaged version of
community.docker when using requests>=2.32. It is also more flexible if
we can control the versions.

Also fixes an issue where kolla collections would not be installed if
the same role existed in the kayobe ansible collections path.

This relies on the support added in kolla-ansible to use ansible-core,
see: https://review.opendev.org/c/openstack/kolla-ansible/+/896406.

Closes-Bug: #2072980
Change-Id: Id9b19f10bc8ac38927914fb9782892b7daf1be82

ansible/roles/kolla-ansible/defaults/main.yml

@@ -23,13 +23,17 @@ kolla_ansible_venv_extra_requirements: []
 
 # Pip requirement specifier for the ansible package. NOTE: This limits the
 # version of ansible used by kolla-ansible to avoid new releases from breaking
-# tested code. Changes to this limit should be tested.
-kolla_ansible_venv_ansible: 'ansible>=8,<10.0'
+# tested code. Changes to this limit should be tested. It is possible to only
+# install ansible-core by setting kolla_ansible_venv_ansible to None.
+kolla_ansible_venv_ansible:
 kolla_ansible_venv_ansible_core: 'ansible-core>=2.15,<2.17'
 
 # Path to a requirements.yml file for Ansible collections.
 kolla_ansible_requirements_yml: "{{ kolla_ansible_venv }}/share/kolla-ansible/requirements.yml"
 
+# Path to a an additional requirements.yml file for Ansible collections when using ansible-core.
+kolla_ansible_core_requirements_yml: "{{ kolla_ansible_venv }}/share/kolla-ansible/requirements-core.yml"
+
 # Virtualenv directory where Kolla-ansible's ansible modules will execute
 # remotely on the target nodes. If None, no virtualenv will be used.
 kolla_ansible_target_venv:

ansible/roles/kolla-ansible/tasks/install.yml

@@ -108,6 +108,13 @@
 - name: Ensure Ansible collections are installed
   command:
     cmd: >-
-      ansible-galaxy collection install
+      ansible-galaxy collection install --force
       -r {{ kolla_ansible_requirements_yml }}
+      {% if not kolla_ansible_venv_ansible %}-r {{ kolla_ansible_core_requirements_yml }}{% endif %}
       -p {{ kolla_ansible_venv }}/share/kolla-ansible/ansible/collections/
+  environment:
+    # NOTE(wszumski): Ignore collections shipped with ansible, so that we can install
+    # newer versions.
+    ANSIBLE_COLLECTIONS_SCAN_SYS_PATH: "False"
+    # NOTE(wszumski): Don't use path configured for kayobe
+    ANSIBLE_COLLECTIONS_PATH:

kayobe/tests/unit/test_utils.py

@@ -48,11 +48,13 @@ class TestCase(unittest.TestCase):
         mock_read.return_value = {"collections": []}
         utils.galaxy_collection_install("/path/to/collection/file",
                                         "/path/to/collections")
+        env = {'ANSIBLE_COLLECTIONS_SCAN_SYS_PATH': 'False'} | os.environ
         mock_run.assert_called_once_with(["ansible-galaxy", "collection",
                                           "install", "--collections-path",
                                           "/path/to/collections",
                                           "--requirements-file",
-                                          "/path/to/collection/file"])
+                                          "/path/to/collection/file"],
+                                          env=env)
 
     @mock.patch.object(utils, "run_command")
     @mock.patch.object(utils, "read_yaml_file")
@@ -77,11 +79,13 @@ class TestCase(unittest.TestCase):
         mock_read.return_value = {"roles": []}
         utils.galaxy_collection_install("/path/to/collection/file",
                                         "/path/to/collections")
+        env = {'ANSIBLE_COLLECTIONS_SCAN_SYS_PATH': 'False'} | os.environ
         mock_run.assert_called_once_with(["ansible-galaxy", "collection",
                                           "install", "--collections-path",
                                           "/path/to/collections",
                                           "--requirements-file",
-                                          "/path/to/collection/file"])
+                                          "/path/to/collection/file"],
+                                          env=env)
 
     @mock.patch.object(utils, "run_command")
     @mock.patch.object(utils, "read_yaml_file")

kayobe/utils.py

@@ -118,10 +118,16 @@ def galaxy_collection_install(requirements_file, collections_path,
     cmd = ["ansible-galaxy", "collection", "install"]
     cmd += ["--collections-path", collections_path]
     cmd += ["--requirements-file", requirements_file]
+    env_defaults = {
+        # NOTE(wszumski): Allow overriding of ansible builtin collections in
+        # kayobe requirements.yml.
+        "ANSIBLE_COLLECTIONS_SCAN_SYS_PATH": "False",
+    }
+    env = env_defaults | os.environ
     if force:
         cmd += ["--force"]
     try:
-        run_command(cmd)
+        run_command(cmd, env=env)
     except subprocess.CalledProcessError as e:
         LOG.error("Failed to install Ansible collections from %s via Ansible "
                   "Galaxy: returncode %d", requirements_file, e.returncode)

releasenotes/notes/switch-to-using-newer-docker-collection-7594e079325bfa8d.yaml

+---
+fixes:
+  - |
+    Switches to using ``ansible-core`` based kolla-ansible install.  This is a
+    workaround for `LP#2072979
+    <https://bugs.launchpad.net/kayobe/+bug/2072979>`__, but also results in a
+    lighter weight install.
+  - |
+    Switches to using a newer version of the docker community collection to
+    workaround issues using the docker ansible modules with certain
+    combinations of python libraries. See `LP#2072979
+    <https://bugs.launchpad.net/kayobe/+bug/2072979>`__.

requirements.yml

@@ -3,6 +3,8 @@ collections:
   - name: https://opendev.org/openstack/ansible-collection-kolla
     type: git
     version: master
+  - name: community.docker
+    version: 3.11.0
   - name: dellemc.os10
     version: 1.1.1
   - name: openstack.cloud