Merge "Use environment variables for plugin paths"

ansible/inventory/hosts

+# Dummy inventory file to allow Ansible to consume this inventory directory.

doc/source/custom-ansible-playbooks.rst

@@ -49,27 +49,19 @@ playbooks in this repository makes a lot of sense, and kayobe has special
 support for this.
 
 It is recommended to store custom playbooks in
-``$KAYOBE_CONFIG_PATH/ansible/``.  Roles located in
-``$KAYOBE_CONFIG_PATH/ansible/roles/`` will be automatically available to
-playbooks in this directory.
-
-With this directory layout, the following commands could be used to create
-symlinks that allow access to Kayobe's filter plugins, group variables and test
-plugins:
-
-.. code-block:: console
-
-   cd ${KAYOBE_CONFIG_PATH}/ansible/
-   ln -s ../../../../kayobe/ansible/filter_plugins/ filter_plugins
-   ln -s ../../../../kayobe/ansible/test_plugins/ test_plugins
-
-These symlinks can even be committed to the kayobe-config Git repository.
-
-.. note::
-
-   These symlinks rely on having a kayobe source checkout at the same level as
-   the kayobe-config repository checkout, as described in
-   :ref:`installation-source`.
+``$KAYOBE_CONFIG_PATH/ansible/``.  It is also possible to use the following
+subdirectories, and since the Zed 13.0.0 release these will be available to all
+Kayobe playbook executions.
+
+* ``roles``
+* ``collections``
+* ``action_plugins``
+* ``filter_plugins``
+* ``test_plugins``
+
+Note that since the Zed 13.0.0 release, it is no longer necessary to create
+symlinks in order to use Kayobe's roles, collections or plugins. Existing
+symlinks may be removed.
 
 Ansible Galaxy
 --------------

kayobe/ansible.py

@@ -21,6 +21,8 @@ import subprocess
 import sys
 import tempfile
 
+import ansible.constants
+
 from kayobe import exception
 from kayobe import utils
 from kayobe import vault
@@ -231,6 +233,40 @@ def _get_environment(parsed_args):
     ansible_cfg_path = os.path.join(parsed_args.config_path, "ansible.cfg")
     if utils.is_readable_file(ansible_cfg_path)["result"]:
         env.setdefault("ANSIBLE_CONFIG", ansible_cfg_path)
+
+    # Update various role, collection and plugin paths to include the Kayobe
+    # roles, collections and plugins. This allows custom playbooks to use these
+    # resources.
+    roles_paths = [
+        os.path.join(parsed_args.config_path, "ansible", "roles"),
+        utils.get_data_files_path("ansible", "roles"),
+    ] + ansible.constants.DEFAULT_ROLES_PATH
+    env.setdefault("ANSIBLE_ROLES_PATH", ":".join(roles_paths))
+
+    collections_paths = [
+        os.path.join(parsed_args.config_path, "ansible", "collections"),
+        utils.get_data_files_path("ansible", "collections"),
+    ] + ansible.constants.COLLECTIONS_PATHS
+    env.setdefault("ANSIBLE_COLLECTIONS_PATH", ":".join(collections_paths))
+
+    action_plugins = [
+        os.path.join(parsed_args.config_path, "ansible", "action_plugins"),
+        utils.get_data_files_path("ansible", "action_plugins"),
+    ] + ansible.constants.DEFAULT_ACTION_PLUGIN_PATH
+    env.setdefault("ANSIBLE_ACTION_PLUGINS", ":".join(action_plugins))
+
+    filter_plugins = [
+        os.path.join(parsed_args.config_path, "ansible", "filter_plugins"),
+        utils.get_data_files_path("ansible", "filter_plugins"),
+    ] + ansible.constants.DEFAULT_FILTER_PLUGIN_PATH
+    env.setdefault("ANSIBLE_FILTER_PLUGINS", ":".join(filter_plugins))
+
+    test_plugins = [
+        os.path.join(parsed_args.config_path, "ansible", "test_plugins"),
+        utils.get_data_files_path("ansible", "test_plugins"),
+    ] + ansible.constants.DEFAULT_TEST_PLUGIN_PATH
+    env.setdefault("ANSIBLE_TEST_PLUGINS", ":".join(test_plugins))
+
     return env
 
 

kayobe/tests/unit/test_ansible.py

@@ -15,6 +15,7 @@
 import argparse
 import errno
 import os
+import os.path
 import shutil
 import subprocess
 import tempfile
@@ -52,7 +53,41 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe"}
+        home = os.path.expanduser("~")
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": ":".join([
+                "/etc/kayobe/ansible/roles",
+                utils.get_data_files_path("ansible", "roles"),
+                home + "/.ansible/roles",
+                "/usr/share/ansible/roles",
+                "/etc/ansible/roles",
+            ]),
+            "ANSIBLE_COLLECTIONS_PATH": ":".join([
+                "/etc/kayobe/ansible/collections",
+                utils.get_data_files_path("ansible", "collections"),
+                home + "/.ansible/collections",
+                "/usr/share/ansible/collections",
+            ]),
+            "ANSIBLE_ACTION_PLUGINS": ":".join([
+                "/etc/kayobe/ansible/action_plugins",
+                utils.get_data_files_path("ansible", "action_plugins"),
+                home + "/.ansible/plugins/action",
+                "/usr/share/ansible/plugins/action",
+            ]),
+            "ANSIBLE_FILTER_PLUGINS": ":".join([
+                "/etc/kayobe/ansible/filter_plugins",
+                utils.get_data_files_path("ansible", "filter_plugins"),
+                home + "/.ansible/plugins/filter",
+                "/usr/share/ansible/plugins/filter",
+            ]),
+            "ANSIBLE_TEST_PLUGINS": ":".join([
+                "/etc/kayobe/ansible/test_plugins",
+                utils.get_data_files_path("ansible", "test_plugins"),
+                home + "/.ansible/plugins/test",
+                "/usr/share/ansible/plugins/test",
+            ]),
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with(["/etc/kayobe"])
@@ -99,8 +134,42 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/path/to/config",
-                        "KAYOBE_ENVIRONMENT": "test-env"}
+        home = os.path.expanduser("~")
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/path/to/config",
+            "KAYOBE_ENVIRONMENT": "test-env",
+            "ANSIBLE_ROLES_PATH": ":".join([
+                "/path/to/config/ansible/roles",
+                utils.get_data_files_path("ansible", "roles"),
+                home + "/.ansible/roles",
+                "/usr/share/ansible/roles",
+                "/etc/ansible/roles",
+            ]),
+            "ANSIBLE_COLLECTIONS_PATH": ":".join([
+                "/path/to/config/ansible/collections",
+                utils.get_data_files_path("ansible", "collections"),
+                home + "/.ansible/collections",
+                "/usr/share/ansible/collections",
+            ]),
+            "ANSIBLE_ACTION_PLUGINS": ":".join([
+                "/path/to/config/ansible/action_plugins",
+                utils.get_data_files_path("ansible", "action_plugins"),
+                home + "/.ansible/plugins/action",
+                "/usr/share/ansible/plugins/action",
+            ]),
+            "ANSIBLE_FILTER_PLUGINS": ":".join([
+                "/path/to/config/ansible/filter_plugins",
+                utils.get_data_files_path("ansible", "filter_plugins"),
+                home + "/.ansible/plugins/filter",
+                "/usr/share/ansible/plugins/filter",
+            ]),
+            "ANSIBLE_TEST_PLUGINS": ":".join([
+                "/path/to/config/ansible/test_plugins",
+                utils.get_data_files_path("ansible", "test_plugins"),
+                home + "/.ansible/plugins/test",
+                "/usr/share/ansible/plugins/test",
+            ]),
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with(
@@ -153,9 +222,16 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/path/to/config",
-                        "KAYOBE_ENVIRONMENT": "test-env",
-                        "KAYOBE_VAULT_PASSWORD": "test-pass"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/path/to/config",
+            "KAYOBE_ENVIRONMENT": "test-env",
+            "KAYOBE_VAULT_PASSWORD": "test-pass",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         expected_calls = [
             mock.call(["which", "kayobe-vault-password-helper"],
                       check_output=True, universal_newlines=True),
@@ -189,7 +265,14 @@ class TestCase(unittest.TestCase):
             "--inventory", "/etc/kayobe/inventory",
             "playbook1.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_update.assert_called_once_with(mock.ANY, expected_env)
@@ -219,8 +302,15 @@ class TestCase(unittest.TestCase):
             "--inventory", "/etc/kayobe/inventory",
             "playbook1.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe",
-                        "KAYOBE_VAULT_PASSWORD": "test-pass"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "KAYOBE_VAULT_PASSWORD": "test-pass",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
 
@@ -279,7 +369,14 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with(["/etc/kayobe"])
@@ -309,7 +406,14 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with(["/etc/kayobe"])
@@ -339,7 +443,14 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with(["/etc/kayobe"])
@@ -365,7 +476,12 @@ class TestCase(unittest.TestCase):
         ]
         expected_env = {
             "ANSIBLE_CONFIG": "/etc/kayobe/ansible.cfg",
-            "KAYOBE_CONFIG_PATH": "/etc/kayobe"
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
         }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
@@ -394,7 +510,12 @@ class TestCase(unittest.TestCase):
         ]
         expected_env = {
             "ANSIBLE_CONFIG": "/path/to/ansible.cfg",
-            "KAYOBE_CONFIG_PATH": "/etc/kayobe"
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
         }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
@@ -689,7 +810,14 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         mock_run.assert_called_once_with(expected_cmd, check_output=False,
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with(["/etc/kayobe"])
@@ -722,8 +850,15 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe",
-                        "KAYOBE_ENVIRONMENT": "test-env"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "KAYOBE_ENVIRONMENT": "test-env",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         expected_calls = [
             mock.call("/etc/kayobe/inventory"),
             mock.call("/etc/kayobe/environments/test-env/inventory"),
@@ -762,8 +897,15 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe",
-                        "KAYOBE_ENVIRONMENT": "test-env"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "KAYOBE_ENVIRONMENT": "test-env",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         expected_calls = [
             mock.call("/etc/kayobe/inventory"),
             mock.call("/etc/kayobe/environments/test-env/inventory"),
@@ -802,8 +944,15 @@ class TestCase(unittest.TestCase):
             "playbook1.yml",
             "playbook2.yml",
         ]
-        expected_env = {"KAYOBE_CONFIG_PATH": "/etc/kayobe",
-                        "KAYOBE_ENVIRONMENT": "test-env"}
+        expected_env = {
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe",
+            "KAYOBE_ENVIRONMENT": "test-env",
+            "ANSIBLE_ROLES_PATH": mock.ANY,
+            "ANSIBLE_COLLECTIONS_PATH": mock.ANY,
+            "ANSIBLE_ACTION_PLUGINS": mock.ANY,
+            "ANSIBLE_FILTER_PLUGINS": mock.ANY,
+            "ANSIBLE_TEST_PLUGINS": mock.ANY,
+        }
         expected_calls = [
             mock.call("/etc/kayobe/inventory"),
             mock.call("/etc/kayobe/environments/test-env/inventory"),

releasenotes/notes/plugin-env-vars-e6265564b3441dfe.yaml

+---
+features:
+  - |
+    Roles, collections and plugins included with Kayobe configuration are now
+    accessible to all Kayobe playbook executions.
+upgrade:
+  - |
+    Changes the environment used during Kayobe playbook execution to include
+    Kayobe's collections, roles and plugins in the Ansible lookup paths.
+    This allows custom playbooks to use these items, without the requirement to
+    symlink into the Kayobe installation. Existing symlinks may be removed.