Rewrite kolla-ansible CLI to python

Moving the CLI to python allows for easier
maintenance and larger feature-set.

This patch introduces a few breaking changes!
The changes stem the nature of the cliff package.
- the order of parameters must be
  kolla-ansible <action> <arguments>
- mariadb_backup and mariadb_recovery now are
  mariadb-backup and mariadb-recovery

Closes-bug: #1589020
Signed-off-by: Roman Krček <roman.krcek@tietoevry.com>
Change-Id: I9749b320d4f5eeec601a055b597dfa7d8fb97ce2

doc/source/reference/deployment-and-bootstrapping/bifrost.rst

@@ -284,13 +284,15 @@ For development:
 
 .. code-block:: console
 
-   cd kolla-ansible
-   tools/kolla-ansible deploy-bifrost
+   pip install -e ./kolla-ansible
+   kolla-ansible deploy-bifrost
+
 
 For Production:
 
 .. code-block:: console
 
+   pip install -U ./kolla-ansible
    kolla-ansible deploy-bifrost
 
 Deploy Bifrost manually
@@ -376,12 +378,14 @@ For Development:
 
 .. code-block:: console
 
-   tools/kolla-ansible deploy-servers
+   pip install -e ./kolla-ansible
+   kolla-ansible deploy-servers
 
 For Production:
 
 .. code-block:: console
 
+   pip install -U ./kolla-ansible
    kolla-ansible deploy-servers
 
 Manually

kolla_ansible/ansible.py

+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import os
+import subprocess  # nosec
+import sys
+
+from kolla_ansible import utils
+from typing import List
+from typing import Tuple
+
+
+DEFAULT_CONFIG_PATH = "/etc/kolla"
+
+CONFIG_PATH_ENV = "KOLLA_CONFIG_PATH"
+
+LOG = logging.getLogger(__name__)
+
+
+def add_ansible_args(parser):
+    """Add arguments required for running Ansible playbooks to a parser."""
+    parser.add_argument(
+        "-b",
+        "--become",
+        action="store_true",
+        help="run operations with become (nopasswd implied)",
+    )
+    parser.add_argument(
+        "-C",
+        "--check",
+        action="store_true",
+        help="don't make any changes; instead, try to predict "
+        "some of the changes that may occur",
+    )
+    parser.add_argument(
+        "-D",
+        "--diff",
+        action="store_true",
+        help="when changing (small) files and templates, show "
+        "the differences in those files; works great "
+        "with --check",
+    )
+    parser.add_argument(
+        "-e",
+        "--extra-vars",
+        metavar="EXTRA_VARS",
+        action="append",
+        help="set additional variables as key=value or "
+        "YAML/JSON",
+    )
+    parser.add_argument(
+        "-i",
+        "--inventory",
+        metavar="INVENTORY",
+        action="append",
+        help="specify inventory host path ",
+    )
+    parser.add_argument(
+        "-l",
+        "--limit",
+        metavar="SUBSET",
+        help="further limit selected hosts to an additional "
+        "pattern",
+    )
+    parser.add_argument(
+        "--skip-tags",
+        metavar="TAGS",
+        help="only run plays and tasks whose tags do not "
+        "match these values",
+    )
+    parser.add_argument(
+        "-t",
+        "--tags",
+        metavar="TAGS",
+        help="only run plays and tasks tagged with these "
+        "values",
+    )
+    parser.add_argument(
+        "-lt",
+        "--list-tasks",
+        action="store_true",
+        help="only print names of tasks, don't run them, "
+        "note this has no affect on kolla-ansible.",
+    )
+    parser.add_argument(
+        "-p", "--playbook",
+        metavar="PLAYBOOKS",
+        action="append",
+        help="Specify custom playbooks for kolla ansible "
+        "to use"
+    ),
+    parser.add_argument(
+        "--vault-id",
+        metavar="VAULT_IDS",
+        action="append",
+        help="the vault identity to use. "
+             "This argument may be specified multiple times.",
+        default=[]
+    ),
+    parser.add_argument(
+        "--vault-password-file",
+        "--vault-pass-file",
+        metavar="VAULT_APSSWORD_FILES",
+        action="append",
+        help="vault password file",
+        default=[]
+    ),
+    parser.add_argument(
+        "-J",
+        "--ask-vault-password",
+        "--ask-vault-pass",
+        action="store_true",
+        help="ask for vault password"
+    )
+
+
+def add_kolla_ansible_args(parser):
+    """Add arguments required for running Kolla Ansible to a parser."""
+    default_config_path = os.getenv(CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH)
+    parser.add_argument(
+        "--configdir",
+        default=default_config_path,
+        dest="kolla_config_path",
+        help="path to Kolla configuration."
+        "(default=$%s or %s)" % (CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH),
+    )
+    parser.add_argument(
+        "--passwords",
+        dest="kolla_passwords",
+        help="Path to the kolla ansible passwords file"
+    )
+
+
+def _get_inventory_paths(parsed_args) -> List[str]:
+    """Return path to the Kolla Ansible inventory."""
+    if parsed_args.inventory:
+        return parsed_args.inventory
+
+    default_inventory = os.path.join(
+        os.path.abspath(parsed_args.kolla_config_path),
+        "ansible", "inventory", "all-in-one")
+    return [default_inventory]
+
+
+def _validate_args(parsed_args, playbooks: list) -> None:
+    """Validate Kolla Ansible arguments."""
+    result = utils.is_readable_dir(
+        os.path.abspath(parsed_args.kolla_config_path))
+    if not result["result"]:
+        LOG.error(
+            "Kolla Ansible configuration path %s is invalid: %s",
+            os.path.abspath(parsed_args.kolla_config_path),
+            result["message"],
+        )
+        sys.exit(1)
+
+    inventories = _get_inventory_paths(parsed_args)
+    for inventory in inventories:
+        result = utils.is_readable_dir(inventory)
+        if not result["result"]:
+            # NOTE(mgoddard): Previously the inventory was a file, now it is a
+            # directory to allow us to support inventory host_vars. Support
+            # both formats for now.
+            result_f = utils.is_readable_file(inventory)
+            if not result_f["result"]:
+                LOG.error(
+                    "Kolla inventory %s is invalid: %s",
+                    inventory, result["message"]
+                )
+                sys.exit(1)
+
+    for playbook in playbooks:
+        result = utils.is_readable_file(playbook)
+        if not result["result"]:
+            LOG.error(
+                "Kolla Ansible playbook %s is invalid: %s",
+                playbook, result["message"]
+            )
+            sys.exit(1)
+
+    if parsed_args.kolla_passwords:
+        passwd_file = parsed_args.kolla_passwords
+    else:
+        passwd_file = os.path.join(
+            os.path.abspath(parsed_args.kolla_config_path), "passwords.yml")
+    result = utils.is_readable_file(passwd_file)
+    if not result["result"]:
+        LOG.error("Kolla Ansible passwords file %s is invalid: %s",
+                  passwd_file, result["message"])
+
+    globals_file = os.path.join(os.path.abspath(
+        os.path.abspath(parsed_args.kolla_config_path)), "globals.yml")
+    result = utils.is_readable_file(globals_file)
+    if not result["result"]:
+        LOG.error("Kolla ansible globals file %s is invalid %s",
+                  globals_file, result["message"])
+
+
+def _get_vars_files(config_path: os.path) -> List[str]:
+    """Return a list of Kolla Ansible configuration variable files.
+
+    The globals.d directory in config path is searched to create the list of
+    variable files. The files will be sorted alphabetically by name for each
+    file, but ordering of file is kept to allow overrides.
+    """
+    vars_path = os.path.join(config_path, "globals.d")
+    result = utils.is_readable_dir(vars_path)
+    if not result["result"]:
+        return []
+
+    vars_files = []
+    for vars_file in os.listdir(vars_path):
+        abs_path = os.path.join(vars_path, vars_file)
+        if utils.is_readable_file(abs_path)["result"]:
+            root, ext = os.path.splitext(vars_file)
+            if ext in (".yml", ".yaml", ".json"):
+                vars_files.append(abs_path)
+
+    return sorted(vars_files)
+
+
+def build_args(parsed_args,
+               playbooks: list,
+               extra_vars: dict = {},
+               verbose_level: int = None) -> Tuple[str, List[str]]:
+    """Build arguments required for running Ansible playbooks."""
+    args = list()
+    if verbose_level:
+        args += ["-" + "v" * verbose_level]
+    if parsed_args.list_tasks:
+        args += ["--list-tasks"]
+    inventories = _get_inventory_paths(parsed_args)
+    for inventory in inventories:
+        args += ["--inventory", inventory]
+    args += ["-e", "@%s" % os.path.join(
+        os.path.abspath(parsed_args.kolla_config_path),
+        "globals.yml")]
+    args += ["-e", "@%s" % os.path.join(
+        os.path.abspath(parsed_args.kolla_config_path),
+        "passwords.yml")]
+    for vault_id in parsed_args.vault_id:
+        args += ["--vault-id", vault_id]
+    for vault_pass_file in parsed_args.vault_password_file:
+        args += ["--vault-password-file", vault_pass_file]
+    if parsed_args.ask_vault_password:
+        args += "--ask-vault-password"
+    vars_files = _get_vars_files(
+        os.path.abspath(parsed_args.kolla_config_path))
+    for vars_file in vars_files:
+        args += ["-e", "@%s" % vars_file]
+    if parsed_args.extra_vars:
+        for extra_var in parsed_args.extra_vars:
+            args += ["-e", extra_var]
+    if extra_vars:
+        for extra_var_name, extra_var_value in extra_vars.items():
+            args += ["-e", "%s=%s" % (extra_var_name, extra_var_value)]
+    args += ["-e", "CONFIG_DIR=%s" %
+             os.path.abspath(parsed_args.kolla_config_path)]
+    if parsed_args.become:
+        args += ["--become"]
+    if parsed_args.check:
+        args += ["--check"]
+    if parsed_args.diff:
+        args += ["--diff"]
+    if parsed_args.limit:
+        args += ["--limit", parsed_args.limit]
+    if parsed_args.skip_tags:
+        args += ["--skip-tags", parsed_args.skip_tags]
+    if parsed_args.tags:
+        args += ["--tags", parsed_args.tags]
+    args += [" ".join(playbooks)]
+    return ("ansible-playbook", args)
+
+
+def run_playbooks(parsed_args, playbooks: list, extra_vars: dict = {},
+                  quiet: bool = False, verbose_level: int = 0) -> None:
+    """Run a Kolla Ansible playbook."""
+    LOG.debug("Parsed arguments: %s" % parsed_args)
+    _validate_args(parsed_args, playbooks)
+    (executable, args) = build_args(
+        parsed_args,
+        playbooks,
+        extra_vars=extra_vars,
+        verbose_level=verbose_level,
+    )
+
+    try:
+        utils.run_command(executable, args, quiet=quiet)
+    except subprocess.CalledProcessError as e:
+        LOG.error(
+            "Kolla Ansible playbook(s) %s exited %d", ", ".join(
+                playbooks), e.returncode
+        )
+        sys.exit(e.returncode)
+
+
+def install_galaxy_collections(force: bool = True) -> None:
+    """Install Ansible Galaxy collection dependencies.
+
+    Installs collection dependencies specified in kolla-ansible,
+    and if present, in kolla-ansibnle configuration.
+
+    :param force: Whether to force reinstallation of roles.
+    """
+    requirements = utils.get_data_files_path("requirements.yml")
+    requirements_core = utils.get_data_files_path("requirements-core.yml")
+    utils.galaxy_collection_install(requirements, force=force)
+    utils.galaxy_collection_install(requirements_core, force=force)

kolla_ansible/cli/commands.py

+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from cliff.command import Command
+
+from kolla_ansible import ansible
+from kolla_ansible import utils
+
+# Serial is not recommended and disabled by default.
+# Users can enable it by configuring the variable.
+ANSIBLE_SERIAL = 0
+
+
+def _get_playbook_path(playbook):
+    """Return the absolute path of Kolla Ansible playbook"""
+    return utils.get_data_files_path("ansible", "%s.yml" % playbook)
+
+
+def _choose_playbooks(parsed_args, kolla_playbook="site"):
+    """Return user defined playbook if set, otherwise return Kolla playbook"""
+    if parsed_args.playbook:
+        playbooks = parsed_args.playbook
+    else:
+        playbooks = [_get_playbook_path(kolla_playbook)]
+    return playbooks
+
+
+class KollaAnsibleMixin:
+    """Mixin class for commands running Kolla Ansible."""
+
+    def get_parser(self, prog_name):
+        parser = super(KollaAnsibleMixin, self).get_parser(prog_name)
+        ansible_group = parser.add_argument_group("Ansible arguments")
+        ka_group = parser.add_argument_group("Kolla Ansible arguments")
+        self.add_ansible_args(ansible_group)
+        self.add_kolla_ansible_args(ka_group)
+        return parser
+
+    def add_kolla_ansible_args(self, group):
+        ansible.add_kolla_ansible_args(group)
+
+    def add_ansible_args(self, group):
+        ansible.add_ansible_args(group)
+
+    def _get_verbosity_args(self):
+        """Add quietness and verbosity level arguments."""
+        # Cliff's default verbosity level is 1, 0 means quiet.
+        verbosity_args = {}
+        if self.app.options.verbose_level:
+            ansible_verbose_level = self.app.options.verbose_level - 1
+            verbosity_args["verbose_level"] = ansible_verbose_level
+        else:
+            verbosity_args["quiet"] = True
+        return verbosity_args
+
+    def run_playbooks(self, parsed_args, *args, **kwargs):
+        kwargs.update(self._get_verbosity_args())
+        return ansible.run_playbooks(parsed_args, *args, **kwargs)
+
+
+class GatherFacts(KollaAnsibleMixin, Command):
+    """Gather Ansible facts on hosts"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Gathering Ansible facts")
+
+        playbooks = _choose_playbooks(parsed_args, "gather-facts")
+
+        self.run_playbooks(parsed_args, playbooks)
+
+
+class InstallDeps(KollaAnsibleMixin, Command):
+    """Install Ansible Galaxy dependencies"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Installing Ansible Galaxy dependencies")
+        ansible.install_galaxy_collections()
+
+
+class Prechecks(KollaAnsibleMixin, Command):
+    """Do pre-deployment checks for hosts"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Pre-deployment checking")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "precheck"
+
+        playbooks = _choose_playbooks(parsed_args,)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class GenConfig(KollaAnsibleMixin, Command):
+    """Generate configuration files for services. No container changes!"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info(
+            "Generate configuration files for enabled OpenStack services")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "config"
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Reconfigure(KollaAnsibleMixin, Command):
+    """Reconfigure enabled OpenStack service"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Reconfigure OpenStack service")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "reconfigure"
+        extra_vars["kolla_serial"] = ANSIBLE_SERIAL
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class ValidateConfig(KollaAnsibleMixin, Command):
+    """Validate configuration files for enabled OpenStack services"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Validate configuration files for enabled "
+                          "OpenStack services")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "config_validate"
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class BootstrapServers(KollaAnsibleMixin, Command):
+    """Bootstrap servers with Kolla Ansible deploy dependencies"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Bootstrapping servers")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "bootstrap-servers"
+
+        playbooks = _choose_playbooks(parsed_args, "kolla-host")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Pull(KollaAnsibleMixin, Command):
+    """Pull all images for containers. Only pulls, no container changes."""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Pulling Docker images")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "pull"
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Certificates(KollaAnsibleMixin, Command):
+    """Generate self-signed certificate for TLS *For Development Only*"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Generate TLS Certificates")
+
+        playbooks = _choose_playbooks(parsed_args, "certificates")
+
+        self.run_playbooks(parsed_args, playbooks)
+
+
+class OctaviaCertificates(KollaAnsibleMixin, Command):
+    """Generate certificates for octavia deployment"""
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        group = parser.add_argument_group("Octavia certificates action")
+        group.add_argument(
+            "--check-expiry",
+            type=int,
+            help="Check if the certificates will expire "
+                 "within given number of days",
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        extra_vars = {}
+
+        if hasattr(parsed_args, "check_expiry"):
+            self.app.LOG.info("Checking if certificates expire "
+                              "within given number of days.")
+            extra_vars["octavia_certs_check_expiry"] = "yes"
+            extra_vars["octavia_certs_expiry_limit"] = parsed_args.check_expiry
+        else:
+            self.app.LOG.info("Generate octavia Certificates")
+
+        playbooks = _choose_playbooks(parsed_args, "octavia-certificates")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Deploy(KollaAnsibleMixin, Command):
+    """Generate config, bootstrap and start all Kolla Ansible containers"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Deploying Playbooks")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "deploy"
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class DeployContainers(KollaAnsibleMixin, Command):
+    """Only deploy and start containers (no config updates or bootstrapping)"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Deploying Containers")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "deploy-containers"
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Postdeploy(KollaAnsibleMixin, Command):
+    """Do post deploy on deploy node"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Post-Deploying Playbooks")
+
+        playbooks = _choose_playbooks(parsed_args, "post-deploy")
+
+        self.run_playbooks(parsed_args, playbooks)
+
+
+class Upgrade(KollaAnsibleMixin, Command):
+    """Upgrades existing OpenStack Environment"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Upgrading OpenStack Environment")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "upgrade"
+        extra_vars["kolla_serial"] = ANSIBLE_SERIAL
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Stop(KollaAnsibleMixin, Command):
+    """Stop Kolla Ansible containers"""
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        group = parser.add_argument_group("Stop action")
+        group.add_argument(
+            "--yes-i-really-really-mean-it",
+            action="store_true",
+            required=True,
+            help="WARNING! This action will remove the Openstack deployment!",
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Stop Kolla containers")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "stop"
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Destroy(KollaAnsibleMixin, Command):
+    """Destroy Kolla Ansible containers, volumes and host configuration!"""
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        group = parser.add_argument_group("Destroy action")
+        group.add_argument(
+            "--yes-i-really-really-mean-it",
+            action="store_true",
+            required=True,
+            help="WARNING! This action will remove the Openstack deployment!",
+        )
+        group.add_argument(
+            "--include-dev",
+            action="store_true",
+            help="Remove devevelopment environment",
+        )
+        group.add_argument(
+            "--include-images",
+            action="store_true",
+            help="Remove leftover container images",
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        self.app.LOG.warning("WARNING: This will PERMANENTLY DESTROY "
+                             "all deployed kolla containers, volumes "
+                             "and host configuration. There is no way "
+                             "to recover from this action!")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "destroy"
+        extra_vars["destroy_include_dev"] = (
+            "yes" if parsed_args.include_dev else "no"
+        )
+        extra_vars["destroy_include_images"] = (
+            "yes" if parsed_args.include_images else "no"
+        )
+
+        playbooks = _choose_playbooks(parsed_args, "destroy")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class PruneImages(KollaAnsibleMixin, Command):
+    """Prune orphaned Kolla Ansible docker images"""
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        group = parser.add_argument_group("Prune images action")
+        group.add_argument(
+            "--yes-i-really-really-mean-it",
+            action="store_true",
+            required=True,
+            help="WARNING! This action will remove all orphaned images!",
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Prune orphaned Kolla images")
+
+        playbooks = _choose_playbooks(parsed_args, "prune-images")
+
+        self.run_playbooks(parsed_args, playbooks)
+
+
+class BifrostDeploy(KollaAnsibleMixin, Command):
+    """Deploy and start bifrost container"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Deploying Bifrost")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "deploy"
+
+        playbooks = _choose_playbooks(parsed_args, "bifrost")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class BifrostDeployServers(KollaAnsibleMixin, Command):
+    """Enroll and deploy servers with bifrost"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Deploying servers with bifrost")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "deploy-servers"
+
+        playbooks = _choose_playbooks(parsed_args, "bifrost")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class BifrostUpgrade(KollaAnsibleMixin, Command):
+    """Upgrades an existing bifrost container"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Upgrading Bifrost")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "upgrade"
+
+        playbooks = _choose_playbooks(parsed_args, "bifrost")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class RabbitMQResetState(KollaAnsibleMixin, Command):
+    """Force reset the state of RabbitMQ"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Force reset the state of RabbitMQ")
+
+        playbooks = _choose_playbooks(parsed_args, "rabbitmq-reset-state")
+
+        self.run_playbooks(parsed_args, playbooks)
+
+
+class MariaDBBackup(KollaAnsibleMixin, Command):
+    """Take a backup of MariaDB databases. See help for options."""
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        group = parser.add_argument_group("MariaDB backup type")
+        group.add_argument(
+            "--full",
+            action="store_const",
+            const="full",
+            dest="mariadb_backup_type",
+            default="full"
+        )
+        group.add_argument(
+            "--incremental",
+            action="store_const",
+            const="incremental",
+            dest="mariadb_backup_type"
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Backup MariaDB databases")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "backup"
+        extra_vars["mariadb_backup_type"] = parsed_args.mariadb_backup_type
+
+        playbooks = _choose_playbooks(parsed_args, "mariadb_backup")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class MariaDBRecovery(KollaAnsibleMixin, Command):
+    """Recover a completely stopped MariaDB cluster"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Attempting to restart MariaDB cluster")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "deploy"
+
+        playbooks = _choose_playbooks(parsed_args, "mariadb_recovery")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class NovaLibvirtCleanup(KollaAnsibleMixin, Command):
+    """Clean up disabled nova_libvirt containers"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Cleanup disabled nova_libvirt containers")
+
+        playbooks = _choose_playbooks(parsed_args, "nova-libvirt-cleanup")
+
+        self.run_playbooks(parsed_args, playbooks)

kolla_ansible/cmd/kolla_ansible.py

+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sys
+
+from cliff.app import App
+from cliff.commandmanager import CommandManager
+
+from kolla_ansible import version
+
+
+class KollaAnsibleApp(App):
+
+    def __init__(self):
+        release_version = version.version_info.release_string()
+        super().__init__(
+            description="Kolla Ansible Command Line Interface (CLI)",
+            version=release_version,
+            command_manager=CommandManager("kolla_ansible.cli"),
+            deferred_help=True,
+        )
+
+    def initialize_app(self, argv):
+        self.LOG.debug("initialize_app")
+
+    def prepare_to_run_command(self, cmd):
+        self.LOG.debug("prepare_to_run_command %s", cmd.__class__.__name__)
+
+    def clean_up(self, cmd, result, err):
+        self.LOG.debug("clean_up %s", cmd.__class__.__name__)
+        if err:
+            self.LOG.debug("got an error: %s", err)
+
+
+def main(argv=sys.argv[1:]):
+    myapp = KollaAnsibleApp()
+    return myapp.run(argv)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

kolla_ansible/utils.py

+# Copyright (c) 2017 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import glob
+import json
+import logging
+import os
+import subprocess  # nosec
+import sys
+import yaml
+
+from importlib.metadata import Distribution
+
+LOG = logging.getLogger(__name__)
+
+
+def get_data_files_path(*relative_path) -> os.path:
+    """Given a relative path to a data file, return the absolute path"""
+    # Detect editable pip install / python setup.py develop and use a path
+    # relative to the source directory
+    return os.path.join(_get_base_path(), *relative_path)
+
+
+def _detect_install_prefix(path: os.path) -> str:
+    script_path = os.path.realpath(path)
+    script_path = os.path.normpath(script_path)
+    components = script_path.split(os.sep)
+    # use heuristic: anything before the last 'lib' in path is the prefix
+    if 'lib' not in components:
+        return None
+    last_lib = len(components) - 1 - components[::-1].index('lib')
+    prefix = components[:last_lib]
+    prefix_path = os.sep.join(prefix)
+    return prefix_path
+
+
+def _get_direct_url(dist: Distribution) -> str:
+    direct_url = os.path.join(dist._path, 'direct_url.json')
+    if os.path.isfile(direct_url):
+        with open(direct_url, 'r') as f:
+            direct_url_content = json.loads(f.readline().strip())
+            url = direct_url_content['url']
+            prefix = 'file://'
+            if url.startswith(prefix):
+                return url[len(prefix):]
+
+    return None
+
+
+def _get_base_path() -> os.path:
+    """Return location where kolla-ansible package is installed."""
+    override = os.environ.get("KOLLA_ANSIBLE_DATA_FILES_PATH")
+    if override:
+        return os.path.join(override)
+
+    kolla_ansible_dist = list(Distribution.discover(name="kolla_ansible"))
+    if kolla_ansible_dist:
+        direct_url = _get_direct_url(kolla_ansible_dist[0])
+        if direct_url:
+            return direct_url
+
+    egg_glob = os.path.join(
+        sys.prefix, 'lib*', 'python*', '*-packages', 'kolla-ansible.egg-link'
+    )
+    egg_link = glob.glob(egg_glob)
+    if egg_link:
+        with open(egg_link[0], "r") as f:
+            realpath = f.readline().strip()
+        return os.path.join(realpath)
+
+    prefix = _detect_install_prefix(__file__)
+    if prefix:
+        return os.path.join(prefix, "share", "kolla-ansible")
+
+    # Assume uninstalled
+    return os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
+
+
+def galaxy_collection_install(requirements_file: str,
+                              collections_path: str = None,
+                              force: bool = False) -> None:
+    """Install ansible collections needed by kolla-ansible roles."""
+    requirements = read_yaml_file(requirements_file)
+    if not isinstance(requirements, dict):
+        # Handle legacy role list format, which causes the command to fail.
+        return
+    args = ["collection", "install"]
+    if collections_path:
+        args += ["--collections-path", collections_path]
+    args += ["--requirements-file", requirements_file]
+    if force:
+        args += ["--force"]
+    try:
+        run_command("ansible-galaxy", args)
+    except subprocess.CalledProcessError as e:
+        LOG.error("Failed to install Ansible collections from %s via Ansible "
+                  "Galaxy: returncode %d", requirements_file, e.returncode)
+        sys.exit(e.returncode)
+
+
+def read_file(path: os.path, mode: str = "r") -> str | bytes:
+    """Read the content of a file."""
+    with open(path, mode) as f:
+        return f.read()
+
+
+def read_yaml_file(path: os.path):
+    """Read and decode a YAML file."""
+    try:
+        content = read_file(path)
+    except IOError as e:
+        print("Failed to open YAML file %s: %s" %
+              (path, repr(e)))
+        sys.exit(1)
+    try:
+        return yaml.safe_load(content)
+    except yaml.YAMLError as e:
+        print("Failed to decode YAML file %s: %s" %
+              (path, repr(e)))
+        sys.exit(1)
+
+
+def is_readable_dir(path: os.path) -> bool:
+    """Check whether a path references a readable directory."""
+    if not os.path.exists(path):
+        return {"result": False, "message": "Path does not exist"}
+    if not os.path.isdir(path):
+        return {"result": False, "message": "Path is not a directory"}
+    if not os.access(path, os.R_OK):
+        return {"result": False, "message": "Directory is not readable"}
+    return {"result": True}
+
+
+def is_readable_file(path: os.path) -> bool:
+    """Check whether a path references a readable file."""
+    if not os.path.exists(path):
+        return {"result": False, "message": "Path does not exist"}
+    if not os.path.isfile(path):
+        return {"result": False, "message": "Path is not a file"}
+    if not os.access(path, os.R_OK):
+        return {"result": False, "message": "File is not readable"}
+    return {"result": True}
+
+
+def run_command(executable: str,
+                args: list,
+                quiet: bool = False,
+                **kwargs) -> None:
+    """Run a command, checking the output.
+
+    :param quiet: Redirect output to /dev/null
+    """
+    full_cmd = [executable] + args
+    cmd_string = " ".join(full_cmd)
+    LOG.debug("Running command: %s", cmd_string)
+
+    if quiet:
+        kwargs["stdout"] = subprocess.DEVNULL
+        kwargs["stderr"] = subprocess.DEVNULL
+        subprocess.run(full_cmd, shell=False, **kwargs)  # nosec
+    else:
+        subprocess.run(full_cmd, shell=False, **kwargs)  # nosec

releasenotes/notes/python-cli-3e568065b8706e73.yaml

+---
+upgrade:
+  - |
+    Rewrite kolla-ansible CLI to python
+
+    Moving the CLI to python allows for easier
+    maintenance and larger feature set.
+    The CLI was built using the cliff package
+    that is used in openstack-cli and kayobe-cli.
+
+    This patch introduces a few breaking changes.
+    The changes stem the nature of the cliff package.
+    1. the order of parameters must be
+    kolla-ansible <action> <arguments>
+    2. mariadb_backup and mariadb_recovery now are
+    mariadb-backup and mariadb-recovery

requirements.txt

@@ -23,7 +23,5 @@ oslo.utils>=3.33.0 # Apache-2.0
 # Password hashing
 passlib[bcrypt]>=1.0.0 # BSD
 
-pbr!=2.1.0,>=2.0.0 # Apache-2.0
-
-# YAML parsing
-PyYAML>=3.12 # MIT
+# CLI
+cliff>=4.7.0 # Apache-2.0

setup.cfg

@@ -38,12 +38,37 @@ data_files =
     share/kolla-ansible = requirements.yml
     share/kolla-ansible = requirements-core.yml
 
-scripts =
-    tools/kolla-ansible
-
 [entry_points]
 console_scripts =
     kolla-genpwd = kolla_ansible.cmd.genpwd:main
     kolla-mergepwd = kolla_ansible.cmd.mergepwd:main
     kolla-writepwd = kolla_ansible.cmd.writepwd:main
     kolla-readpwd = kolla_ansible.cmd.readpwd:main
+    kolla-ansible = kolla_ansible.cmd.kolla_ansible:main
+
+kolla-ansible.cli =
+    gather-facts = kolla_ansible.cli.commands:GatherFacts
+    install-deps = kolla_ansible.cli.commands:InstallDeps
+    prechecks = kolla_ansible.cli.commands:Prechecks
+    genconfig = kolla_ansible.cli.commands:GenConfig
+    reconfigure = kolla_ansible.cli.commands:Reconfigure
+    validate-config = kolla_ansible.cli.commands:ValidateConfig
+    bootstrap-servers = kolla_ansible.cli.commands:BootstrapServers
+    pull = kolla_ansible.cli.commands:Pull
+    certificates = kolla_ansible.cli.commands:Certificates
+    octavia-certificates = kolla_ansible.cli.commands:OctaviaCertificates
+    deploy = kolla_ansible.cli.commands:Deploy
+    deploy-containers = kolla_ansible.cli.commands:DeployContainers
+    post-deploy = kolla_ansible.cli.commands:Postdeploy
+    upgrade = kolla_ansible.cli.commands:Upgrade
+    stop = kolla_ansible.cli.commands:Stop
+    destroy = kolla_ansible.cli.commands:Destroy
+    prune-images = kolla_ansible.cli.commands:PruneImages
+    deploy-bifrost = kolla_ansible.cli.commands:BifrostDeploy
+    deploy-servers = kolla_ansible.cli.commands:BifrostDeployServers
+    upgrade-bifrost = kolla_ansible.cli.commands:BifrostUpgrade
+    rabbitmq-reset-state = kolla_ansible.cli.commands:RabbitMQResetState
+    mariadb-backup = kolla_ansible.cli.commands:MariaDBBackup
+    mariadb-recovery = kolla_ansible.cli.commands:MariaDBRecovery
+    nova-libvirt-cleanup = kolla_ansible.cli.commands:NovaLibvirtCleanup
+

tests/deploy-bifrost.sh

@@ -16,7 +16,7 @@ function deploy_bifrost {
     # Deploy the bifrost container.
     # TODO(mgoddard): add pull action when we have a local registry service in
     # CI.
-    kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost &> /tmp/logs/ansible/deploy-bifrost
+    kolla-ansible deploy-bifrost -i ${RAW_INVENTORY} -vvv  &> /tmp/logs/ansible/deploy-bifrost
 }
 
 

tests/deploy.sh

@@ -41,7 +41,7 @@ function certificates {
 
     # generate self-signed certificates for the optional internal TLS tests
     if [[ "$TLS_ENABLED" = "True" ]]; then
-        kolla-ansible -i ${RAW_INVENTORY} -vvv certificates > /tmp/logs/ansible/certificates
+        kolla-ansible certificates -i ${RAW_INVENTORY} -vvv > /tmp/logs/ansible/certificates
     fi
     if [[ "$LE_ENABLED" = "True" ]]; then
         init_pebble
@@ -64,13 +64,13 @@ function deploy {
     certificates
 
     # Actually do the deployment
-    kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/deploy-prechecks
-    kolla-ansible -i ${RAW_INVENTORY} -vvv pull &> /tmp/logs/ansible/pull
-    kolla-ansible -i ${RAW_INVENTORY} -vvv deploy &> /tmp/logs/ansible/deploy
-    kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy &> /tmp/logs/ansible/post-deploy
+    kolla-ansible prechecks -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/deploy-prechecks
+    kolla-ansible pull -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/pull
+    kolla-ansible deploy -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/deploy
+    kolla-ansible post-deploy -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/post-deploy
 
     if [[ $HAS_UPGRADE == 'no' ]]; then
-        kolla-ansible -i ${RAW_INVENTORY} -vvv validate-config &> /tmp/logs/ansible/validate-config
+        kolla-ansible validate-config -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/validate-config
     fi
 }
 

tests/reconfigure.sh

@@ -18,8 +18,8 @@ function reconfigure {
     if [[ $SCENARIO == "ovn" ]]; then
         sudo ${CONTAINER_ENGINE} rm -f ovn_nb_db ovn_sb_db && sudo ${CONTAINER_ENGINE} volume rm ovn_nb_db ovn_sb_db
     fi
-    kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/reconfigure-prechecks
-    kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure &> /tmp/logs/ansible/reconfigure
+    kolla-ansible prechecks -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/reconfigure-prechecks
+    kolla-ansible reconfigure -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/reconfigure
 }
 
 

tests/setup_gate.sh

@@ -116,7 +116,7 @@ EOF
 RAW_INVENTORY=/etc/kolla/inventory
 
 source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
-kolla-ansible -i ${RAW_INVENTORY} -vvv bootstrap-servers &> /tmp/logs/ansible/bootstrap-servers
+kolla-ansible bootstrap-servers -i ${RAW_INVENTORY} -vvv  &> /tmp/logs/ansible/bootstrap-servers
 deactivate
 
 prepare_images

tests/test-magnum.sh

@@ -125,7 +125,7 @@ EOF
     deactivate
     source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
     echo 'designate_enable_notifications_sink: "yes"' >> /etc/kolla/globals.yml
-    kolla-ansible -i ${RAW_INVENTORY} --tags designate,nova,nova-cell,neutron -vvv reconfigure &> /tmp/logs/ansible/reconfigure-designate
+    kolla-ansible reconfigure -i ${RAW_INVENTORY} --tags designate,nova,nova-cell,neutron -vvv &> /tmp/logs/ansible/reconfigure-designate
     deactivate
     source ~/openstackclient-venv/bin/activate
 

tests/test-mariadb.sh

@@ -11,7 +11,7 @@ export PYTHONUNBUFFERED=1
 
 function mariadb_stop {
     echo "Stopping the database cluster"
-    kolla-ansible -i ${RAW_INVENTORY} -vvv stop --yes-i-really-really-mean-it --tags mariadb --skip-tags common
+    kolla-ansible stop -i ${RAW_INVENTORY} -vvv --yes-i-really-really-mean-it --tags mariadb --skip-tags common
     if [[ $(sudo ${container_engine} ps -q | grep mariadb | wc -l) -ne 0 ]]; then
         echo "Failed to stop MariaDB cluster"
         return 1
@@ -21,7 +21,7 @@ function mariadb_stop {
 function mariadb_recovery {
     # Recover the database cluster.
     echo "Recovering the database cluster"
-    kolla-ansible -i ${RAW_INVENTORY} -vvv mariadb_recovery --tags mariadb --skip-tags common
+    kolla-ansible mariadb-recovery -i ${RAW_INVENTORY} -vvv --tags mariadb --skip-tags common
 }
 
 function test_recovery {
@@ -32,7 +32,7 @@ function test_recovery {
 
 function test_backup {
     echo "Performing full backup"
-    kolla-ansible -i ${RAW_INVENTORY} -vvv mariadb_backup --full
+    kolla-ansible mariadb-backup -i ${RAW_INVENTORY} -vvv --full
     # Sleep for 30 seconds, not because it's absolutely necessary.
     # The full backup is already completed at this point, as the
     # ansible job is waiting for the completion of the backup script
@@ -42,7 +42,7 @@ function test_backup {
     # data gets written within those 30 seconds.
     echo "Sleeping for 30 seconds"
     sleep 30
-    kolla-ansible -i ${RAW_INVENTORY} -vvv mariadb_backup --incremental
+    kolla-ansible mariadb-backup -i ${RAW_INVENTORY} -vvv --incremental
 }
 
 function test_backup_with_retries {

tests/upgrade-bifrost.sh

@@ -17,7 +17,7 @@ function upgrade_bifrost {
     # CI.
     # TODO(mgoddard): make some configuration file changes and trigger a real
     # upgrade.
-    kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost &> /tmp/logs/ansible/upgrade-bifrost
+    kolla-ansible deploy-bifrost -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade-bifrost
 }
 
 

tests/upgrade.sh

@@ -12,14 +12,14 @@ function upgrade {
 
     source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
 
-    kolla-ansible -i ${RAW_INVENTORY} -vvv certificates &> /tmp/logs/ansible/certificates
-    kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/upgrade-prechecks
-    kolla-ansible -i ${RAW_INVENTORY} -vvv pull &> /tmp/logs/ansible/pull-upgrade
-    kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade &> /tmp/logs/ansible/upgrade
+    kolla-ansible certificates -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/certificates
+    kolla-ansible prechecks -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade-prechecks
+    kolla-ansible pull -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/pull-upgrade
+    kolla-ansible upgrade -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade
 
-    kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy &> /tmp/logs/ansible/upgrade-post-deploy
+    kolla-ansible post-deploy -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade-post-deploy
 
-    kolla-ansible -i ${RAW_INVENTORY} -vvv validate-config &> /tmp/logs/ansible/validate-config
+    kolla-ansible validate-config -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/validate-config
 }