From c31688be3688d2a4fe24b02aaad7e20ce0a0c516 Mon Sep 17 00:00:00 2001
From: Jeffrey Zhang <jeffrey.zhang@99cloud.net>
Date: Mon, 6 Jun 2016 11:37:45 +0800
Subject: [PATCH] Make build.py importable lib
First step to break it down is to make it
importable lib rather than self-contained script.
Change-Id: Ibbc924c2849cc79006046f62f31a1909ce959066
Partially-implements: bp build-refactor
---
kolla/cmd/build.py | 912 +------------------------------------
kolla/image/__init__.py | 0
kolla/image/build.py | 924 ++++++++++++++++++++++++++++++++++++++
kolla/tests/base.py | 2 +-
kolla/tests/test_build.py | 11 +-
tests/test_build.py | 2 +-
6 files changed, 934 insertions(+), 917 deletions(-)
create mode 100644 kolla/image/__init__.py
create mode 100755 kolla/image/build.py
diff --git a/kolla/cmd/build.py b/kolla/cmd/build.py
index a84a49f6b..97dbafc67 100755
--- a/kolla/cmd/build.py
+++ b/kolla/cmd/build.py
@@ -12,919 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import print_function
-
-import datetime
-import errno
-import graphviz
-import json
-import logging
-import os
-import pprint
-import re
-import requests
-import shutil
-import signal
-import sys
-import tarfile
-import tempfile
-import threading
-import time
-
-import docker
-import git
-import jinja2
-from oslo_config import cfg
-from requests import exceptions as requests_exc
-import six
-
-PROJECT_ROOT = os.path.abspath(os.path.join(
- os.path.dirname(os.path.realpath(__file__)), '../..'))
-
-# NOTE(SamYaple): Update the search patch to prefer PROJECT_ROOT as the source
-# of packages to import if we are using local tools/build.py
-# instead of pip installed kolla-build tool
-if PROJECT_ROOT not in sys.path:
- sys.path.insert(0, PROJECT_ROOT)
-
-from kolla.common import config as common_config
-from kolla.common import task
-from kolla import version
-
-logging.basicConfig()
-LOG = logging.getLogger(__name__)
-LOG.setLevel(logging.INFO)
-
-
-def handle_ctrlc(single, frame):
- kollaobj = frame.f_locals['kolla']
- kollaobj.cleanup()
- sys.exit(1)
-
-signal.signal(signal.SIGINT, handle_ctrlc)
-
-
-class KollaDirNotFoundException(Exception):
- pass
-
-
-class KollaUnknownBuildTypeException(Exception):
- pass
-
-
-class KollaMismatchBaseTypeException(Exception):
- pass
-
-
-class KollaRpmSetupUnknownConfig(Exception):
- pass
-
-
-# Image status constants.
-#
-# TODO(harlowja): use enum lib in the future??
-STATUS_CONNECTION_ERROR = 'connection_error'
-STATUS_PUSH_ERROR = 'push_error'
-STATUS_ERROR = 'error'
-STATUS_PARENT_ERROR = 'parent_error'
-STATUS_BUILT = 'built'
-STATUS_BUILDING = 'building'
-STATUS_UNMATCHED = 'unmatched'
-STATUS_MATCHED = 'matched'
-STATUS_UNPROCESSED = 'unprocessed'
-
-# All error status constants.
-STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR,
- STATUS_ERROR, STATUS_PARENT_ERROR)
-
-
-class Recorder(object):
- """Recorder/buffer of (unicode) log lines for eventual display."""
-
- def __init__(self):
- self._lines = []
-
- def write(self, text=""):
- if isinstance(text, six.text_type):
- self._lines.append(text)
- elif isinstance(text, six.binary_type):
- self._lines.append(text.decode('utf8'))
- elif isinstance(text, Recorder):
- self._lines.extend(text._lines)
- else:
- self.write(text=str(text))
-
- def clear(self):
- self._lines = []
-
- def __iter__(self):
- for line in self._lines:
- yield line
-
- def __str__(self):
- return u"\n".join(self._lines)
-
-
-def docker_client():
- try:
- docker_kwargs = docker.utils.kwargs_from_env()
- return docker.Client(version='auto', **docker_kwargs)
- except docker.errors.DockerException:
- LOG.exception('Can not communicate with docker service.'
- 'Please check docker service is running without errors')
- sys.exit(1)
-
-
-class Image(object):
- def __init__(self, name, canonical_name, path, parent_name='',
- status=STATUS_UNPROCESSED, parent=None, source=None):
- self.name = name
- self.canonical_name = canonical_name
- self.path = path
- self.status = status
- self.parent = parent
- self.source = source
- self.parent_name = parent_name
- self.logs = Recorder()
- self.push_logs = Recorder()
- self.children = []
- self.plugins = []
-
- def __repr__(self):
- return ("Image(%s, %s, %s, parent_name=%s,"
- " status=%s, parent=%s, source=%s)") % (
- self.name, self.canonical_name, self.path,
- self.parent_name, self.status, self.parent, self.source)
-
-
-class PushIntoQueueTask(task.Task):
- """Task that pushes some other task into a queue."""
-
- def __init__(self, push_task, push_queue):
- super(PushIntoQueueTask, self).__init__()
- self.push_task = push_task
- self.push_queue = push_queue
-
- @property
- def name(self):
- return 'PushIntoQueueTask(%s=>%s)' % (self.push_task.name,
- self.push_queue)
-
- def run(self):
- self.push_queue.put(self.push_task)
- self.success = True
-
-
-class PushTask(task.Task):
- """Task that pushes a image to a docker repository."""
-
- def __init__(self, conf, image):
- super(PushTask, self).__init__()
- self.dc = docker_client()
- self.conf = conf
- self.image = image
-
- @property
- def name(self):
- return 'PushTask(%s)' % self.image.name
-
- def run(self):
- image = self.image
- try:
- LOG.debug('%s:Try to push the image', image.name)
- self.push_image(image)
- except requests_exc.ConnectionError:
- LOG.exception('%s:Make sure Docker is running and that you'
- ' have the correct privileges to run Docker'
- ' (root)', image.name)
- image.status = STATUS_CONNECTION_ERROR
- except Exception:
- LOG.exception('%s:Unknown error when pushing', image.name)
- image.status = STATUS_PUSH_ERROR
- finally:
- if (image.status not in STATUS_ERRORS
- and image.status != STATUS_UNPROCESSED):
- LOG.info('%s:Pushed successfully', image.name)
- self.success = True
- else:
- self.success = False
-
- def push_image(self, image):
- image.push_logs.clear()
- for response in self.dc.push(image.canonical_name,
- stream=True,
- insecure_registry=True):
- stream = json.loads(response)
- if 'stream' in stream:
- image.push_logs.write(image.logs)
- image.push_logs.write(stream['stream'])
- LOG.info('%s', stream['stream'])
- elif 'errorDetail' in stream:
- image.status = STATUS_ERROR
- LOG.error(stream['errorDetail']['message'])
-
-
-class BuildTask(task.Task):
- """Task that builds out an image."""
-
- def __init__(self, conf, image, push_queue):
- super(BuildTask, self).__init__()
- self.conf = conf
- self.image = image
- self.dc = docker_client()
- self.push_queue = push_queue
- self.forcerm = not conf.keep
-
- @property
- def name(self):
- return 'BuildTask(%s)' % self.image.name
-
- def run(self):
- self.builder(self.image)
- if self.image.status == STATUS_BUILT:
- self.success = True
-
- @property
- def followups(self):
- followups = []
- if self.conf.push and self.success:
- followups.extend([
- # If we are supposed to push the image into a docker
- # repository, then make sure we do that...
- PushIntoQueueTask(
- PushTask(self.conf, self.image),
- self.push_queue),
- ])
- if self.image.children and self.success:
- for image in self.image.children:
- followups.append(BuildTask(self.conf, image, self.push_queue))
- return followups
-
- def process_source(self, image, source):
- dest_archive = os.path.join(image.path, source['name'] + '-archive')
-
- if source.get('type') == 'url':
- LOG.debug("%s:Getting archive from %s", image.name,
- source['source'])
- try:
- r = requests.get(source['source'], timeout=self.conf.timeout)
- except requests_exc.Timeout:
- LOG.exception('Request timed out while getting archive'
- ' from %s', source['source'])
- image.status = STATUS_ERROR
- image.logs.clear()
- return
-
- if r.status_code == 200:
- with open(dest_archive, 'wb') as f:
- f.write(r.content)
- else:
- LOG.error('%s:Failed to download archive: status_code %s',
- image.name, r.status_code)
- image.status = STATUS_ERROR
- return
-
- elif source.get('type') == 'git':
- clone_dir = '{}-{}'.format(dest_archive,
- source['reference'].replace('/', '-'))
- try:
- LOG.debug("%s:Cloning from %s", image.name,
- source['source'])
- git.Git().clone(source['source'], clone_dir)
- git.Git(clone_dir).checkout(source['reference'])
- reference_sha = git.Git(clone_dir).rev_parse('HEAD')
- LOG.debug("%s:Git checkout by reference %s (%s)",
- image.name, source['reference'], reference_sha)
- except Exception as e:
- LOG.error("%s:Failed to get source from git", image.name)
- LOG.error("%s:Error:%s", image.name, str(e))
- # clean-up clone folder to retry
- shutil.rmtree(clone_dir)
- image.status = STATUS_ERROR
- return
-
- with tarfile.open(dest_archive, 'w') as tar:
- tar.add(clone_dir, arcname=os.path.basename(clone_dir))
-
- elif source.get('type') == 'local':
- LOG.debug("%s:Getting local archive from %s", image.name,
- source['source'])
- if os.path.isdir(source['source']):
- with tarfile.open(dest_archive, 'w') as tar:
- tar.add(source['source'],
- arcname=os.path.basename(source['source']))
- else:
- shutil.copyfile(source['source'], dest_archive)
-
- else:
- LOG.error("%s:Wrong source type '%s'", image.name,
- source.get('type'))
- image.status = STATUS_ERROR
- return
-
- # Set time on destination archive to epoch 0
- os.utime(dest_archive, (0, 0))
-
- return dest_archive
-
- def update_buildargs(self):
- buildargs = dict()
- if self.conf.build_args:
- buildargs = dict(self.conf.build_args)
-
- proxy_vars = ('HTTP_PROXY', 'http_proxy', 'HTTPS_PROXY',
- 'https_proxy', 'FTP_PROXY', 'ftp_proxy',
- 'NO_PROXY', 'no_proxy')
-
- for proxy_var in proxy_vars:
- if proxy_var in os.environ and proxy_var not in buildargs:
- buildargs[proxy_var] = os.environ.get(proxy_var)
-
- if not buildargs:
- return None
- return buildargs
-
- def builder(self, image):
- LOG.debug('%s:Processing', image.name)
- if image.status == STATUS_UNMATCHED:
- return
-
- if (image.parent is not None and
- image.parent.status in STATUS_ERRORS):
- LOG.error('%s:Parent image error\'d with message "%s"',
- image.name, image.parent.status)
- image.status = STATUS_PARENT_ERROR
- return
-
- image.status = STATUS_BUILDING
- LOG.info('%s:Building', image.name)
-
- if image.source and 'source' in image.source:
- self.process_source(image, image.source)
- if image.status in STATUS_ERRORS:
- return
-
- plugin_archives = list()
- plugins_path = os.path.join(image.path, 'plugins')
- for plugin in image.plugins:
- archive_path = self.process_source(image, plugin)
- if image.status in STATUS_ERRORS:
- return
- plugin_archives.append(archive_path)
- if plugin_archives:
- for plugin_archive in plugin_archives:
- with tarfile.open(plugin_archive, 'r') as plugin_archive_tar:
- plugin_archive_tar.extractall(path=plugins_path)
- else:
- try:
- os.mkdir(plugins_path)
- except OSError as e:
- if e.errno == errno.EEXIST:
- LOG.info('Directory %s already exist. Skipping.',
- plugins_path)
- else:
- LOG.error('Failed to create directory %s: %s',
- plugins_path, e)
- image.status = STATUS_CONNECTION_ERROR
- return
- with tarfile.open(os.path.join(image.path, 'plugins-archive'),
- 'w') as tar:
- tar.add(plugins_path, arcname='plugins')
-
- # Pull the latest image for the base distro only
- pull = True if image.parent is None else False
-
- image.logs.clear()
- buildargs = self.update_buildargs()
- for response in self.dc.build(path=image.path,
- tag=image.canonical_name,
- nocache=not self.conf.cache,
- rm=True,
- pull=pull,
- forcerm=self.forcerm,
- buildargs=buildargs):
- stream = json.loads(response.decode('utf-8'))
-
- if 'stream' in stream:
- image.logs.write(stream['stream'])
- for line in stream['stream'].split('\n'):
- if line:
- LOG.info('%s:%s', image.name, line)
-
- if 'errorDetail' in stream:
- image.status = STATUS_ERROR
- LOG.error('%s:Error\'d with the following message',
- image.name)
- for line in stream['errorDetail']['message'].split('\n'):
- if line:
- LOG.error('%s:%s', image.name, line)
- return
-
- image.status = STATUS_BUILT
-
- LOG.info('%s:Built', image.name)
-
-
-class WorkerThread(threading.Thread):
- """Thread that executes tasks until the queue provides a tombstone."""
-
- #: Object to be put on worker queues to get them to die.
- tombstone = object()
-
- def __init__(self, conf, queue):
- super(WorkerThread, self).__init__()
- self.queue = queue
- self.conf = conf
-
- def run(self):
- while True:
- task = self.queue.get()
- if task is self.tombstone:
- # Ensure any other threads also get the tombstone.
- self.queue.put(task)
- break
- try:
- for attempt in six.moves.range(self.conf.retries + 1):
- if attempt > 0:
- LOG.debug("Attempting to run task %s for the %s time",
- task.name, attempt + 1)
- else:
- LOG.debug("Attempting to run task %s for the first"
- " time", task.name)
- try:
- task.run()
- if task.success:
- break
- except Exception:
- LOG.exception('Unhandled error when running %s',
- task.name)
- # try again...
- task.reset()
- if task.success:
- for next_task in task.followups:
- LOG.debug('Added next task %s to queue',
- next_task.name)
- self.queue.put(next_task)
- finally:
- self.queue.task_done()
-
-
-class KollaWorker(object):
-
- def __init__(self, conf):
- self.conf = conf
- self.images_dir = self._get_images_dir()
- self.registry = conf.registry
- if self.registry:
- self.namespace = self.registry + '/' + conf.namespace
- else:
- self.namespace = conf.namespace
- self.base = conf.base
- self.base_tag = conf.base_tag
- self.install_type = conf.install_type
- self.tag = conf.tag
- self.images = list()
- rpm_setup_config = filter(None, conf.rpm_setup_config)
- self.rpm_setup = self.build_rpm_setup(rpm_setup_config)
-
- rh_base = ['fedora', 'centos', 'oraclelinux', 'rhel']
- rh_type = ['source', 'binary', 'rdo', 'rhos']
- deb_base = ['ubuntu', 'debian']
- deb_type = ['source', 'binary']
-
- if not ((self.base in rh_base and self.install_type in rh_type) or
- (self.base in deb_base and self.install_type in deb_type)):
- raise KollaMismatchBaseTypeException(
- '{} is unavailable for {}'.format(self.install_type, self.base)
- )
-
- if self.install_type == 'binary':
- self.install_metatype = 'rdo'
- elif self.install_type == 'source':
- self.install_metatype = 'mixed'
- elif self.install_type == 'rdo':
- self.install_type = 'binary'
- self.install_metatype = 'rdo'
- elif self.install_type == 'rhos':
- self.install_type = 'binary'
- self.install_metatype = 'rhos'
- else:
- raise KollaUnknownBuildTypeException(
- 'Unknown install type'
- )
-
- self.image_prefix = self.base + '-' + self.install_type + '-'
-
- self.include_header = conf.include_header
- self.include_footer = conf.include_footer
- self.regex = conf.regex
- self.image_statuses_bad = dict()
- self.image_statuses_good = dict()
- self.image_statuses_unmatched = dict()
- self.maintainer = conf.maintainer
-
- def _get_images_dir(self):
- possible_paths = (
- PROJECT_ROOT,
- os.path.join(sys.prefix, 'share/kolla'),
- os.path.join(sys.prefix, 'local/share/kolla'))
-
- for path in possible_paths:
- image_path = os.path.join(path, 'docker')
- # NOTE(SamYaple): We explicty check for the base folder to ensure
- # this is the correct path
- # TODO(SamYaple): Improve this to make this safer
- if os.path.exists(os.path.join(image_path, 'base')):
- LOG.info('Found the docker image folder at %s', image_path)
- return image_path
- else:
- raise KollaDirNotFoundException('Image dir can not be found')
-
- def build_rpm_setup(self, rpm_setup_config):
- """Generates a list of docker commands based on provided configuration.
-
- :param rpm_setup_config: A list of .rpm or .repo paths or URLs
- :return: A list of docker commands
- """
- rpm_setup = list()
-
- for config in rpm_setup_config:
- if config.endswith('.rpm'):
- # RPM files can be installed with yum from file path or url
- cmd = "RUN yum -y install {}".format(config)
- elif config.endswith('.repo'):
- if config.startswith('http'):
- # Curl http://url/etc.repo to /etc/yum.repos.d/etc.repo
- name = config.split('/')[-1]
- cmd = "RUN curl {} -o /etc/yum.repos.d/{}".format(config,
- name)
- else:
- # Copy .repo file from filesystem
- cmd = "COPY {} /etc/yum.repos.d/".format(config)
- else:
- raise KollaRpmSetupUnknownConfig(
- 'RPM setup must be provided as .rpm or .repo files.'
- ' Attempted configuration was {}'.format(config)
- )
-
- rpm_setup.append(cmd)
-
- return rpm_setup
-
- def copy_apt_files(self):
- if self.conf.apt_sources_list:
- shutil.copyfile(
- self.conf.apt_sources_list,
- os.path.join(self.working_dir, "base", "sources.list")
- )
-
- if self.conf.apt_preferences:
- shutil.copyfile(
- self.conf.apt_preferences,
- os.path.join(self.working_dir, "base", "apt_preferences")
- )
-
- def setup_working_dir(self):
- """Creates a working directory for use while building"""
- ts = time.time()
- ts = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M-%S_')
- self.temp_dir = tempfile.mkdtemp(prefix='kolla-' + ts)
- self.working_dir = os.path.join(self.temp_dir, 'docker')
- shutil.copytree(self.images_dir, self.working_dir)
- self.copy_apt_files()
- LOG.debug('Created working dir: %s', self.working_dir)
-
- def set_time(self):
- for root, dirs, files in os.walk(self.working_dir):
- for file_ in files:
- os.utime(os.path.join(root, file_), (0, 0))
- for dir_ in dirs:
- os.utime(os.path.join(root, dir_), (0, 0))
- LOG.debug('Set atime and mtime to 0 for all content in working dir')
-
- def create_dockerfiles(self):
- kolla_version = version.version_info.cached_version_string()
- supported_distro_release = common_config.DISTRO_RELEASE.get(
- self.base)
- for path in self.docker_build_paths:
- template_name = "Dockerfile.j2"
- env = jinja2.Environment( # nosec: not used to render HTML
- loader=jinja2.FileSystemLoader(path))
- template = env.get_template(template_name)
- values = {'base_distro': self.base,
- 'base_image': self.conf.base_image,
- 'base_distro_tag': self.base_tag,
- 'supported_distro_release': supported_distro_release,
- 'install_metatype': self.install_metatype,
- 'image_prefix': self.image_prefix,
- 'install_type': self.install_type,
- 'namespace': self.namespace,
- 'tag': self.tag,
- 'maintainer': self.maintainer,
- 'kolla_version': kolla_version,
- 'rpm_setup': self.rpm_setup}
- if self.include_header:
- with open(self.include_header, 'r') as f:
- values['include_header'] = f.read()
- if self.include_footer:
- with open(self.include_footer, 'r') as f:
- values['include_footer'] = f.read()
- content = template.render(values)
- with open(os.path.join(path, 'Dockerfile'), 'w') as f:
- f.write(content)
-
- def find_dockerfiles(self):
- """Recursive search for Dockerfiles in the working directory"""
- self.docker_build_paths = list()
- path = self.working_dir
- filename = 'Dockerfile.j2'
-
- for root, dirs, names in os.walk(path):
- if filename in names:
- self.docker_build_paths.append(root)
- LOG.debug('Found %s', root.split(self.working_dir)[1])
-
- LOG.debug('Found %d Dockerfiles', len(self.docker_build_paths))
-
- def cleanup(self):
- """Remove temp files"""
- shutil.rmtree(self.temp_dir)
-
- def filter_images(self):
- """Filter which images to build"""
- filter_ = list()
-
- if self.regex:
- filter_ += self.regex
- elif self.conf.profile:
- for profile in self.conf.profile:
- if profile not in self.conf.profiles:
- self.conf.register_opt(cfg.ListOpt(profile,
- default=[]),
- 'profiles')
- if len(self.conf.profiles[profile]) == 0:
- msg = 'Profile: {} does not exist'.format(profile)
- raise ValueError(msg)
- else:
- filter_ += self.conf.profiles[profile]
-
- if filter_:
- patterns = re.compile(r"|".join(filter_).join('()'))
- for image in self.images:
- if image.status == STATUS_MATCHED:
- continue
- if re.search(patterns, image.name):
- image.status = STATUS_MATCHED
- while (image.parent is not None and
- image.parent.status != STATUS_MATCHED):
- image = image.parent
- image.status = STATUS_MATCHED
- LOG.debug('%s:Matched regex', image.name)
- else:
- image.status = STATUS_UNMATCHED
- else:
- for image in self.images:
- image.status = STATUS_MATCHED
-
- def summary(self):
- """Walk the dictionary of images statuses and print results"""
- # For debug we print the logs again if the image error'd. This is to
- # to help us debug and it will be extra helpful in the gate.
- for image in self.images:
- if image.status in STATUS_ERRORS:
- LOG.debug("%s:Failed with the following logs", image.name)
- for line in image.logs:
- if line:
- LOG.debug("%s:%s", image.name, line)
-
- self.get_image_statuses()
-
- if self.image_statuses_good:
- LOG.info("Successfully built images")
- LOG.info("=========================")
- for name in self.image_statuses_good.keys():
- LOG.info(name)
-
- if self.image_statuses_bad:
- LOG.info("Images that failed to build")
- LOG.info("===========================")
- for name, status in self.image_statuses_bad.items():
- LOG.error('%s Failed with status: %s', name, status)
-
- if self.image_statuses_unmatched:
- LOG.debug("Images not matched for build by regex")
- LOG.debug("=====================================")
- for name in self.image_statuses_unmatched.keys():
- LOG.debug(name)
-
- def get_image_statuses(self):
- if any([self.image_statuses_bad,
- self.image_statuses_good,
- self.image_statuses_unmatched]):
- return (self.image_statuses_bad,
- self.image_statuses_good,
- self.image_statuses_unmatched)
- for image in self.images:
- if image.status == STATUS_BUILT:
- self.image_statuses_good[image.name] = image.status
- elif image.status == STATUS_UNMATCHED:
- self.image_statuses_unmatched[image.name] = image.status
- else:
- self.image_statuses_bad[image.name] = image.status
- return (self.image_statuses_bad,
- self.image_statuses_good,
- self.image_statuses_unmatched)
-
- def build_image_list(self):
- def process_source_installation(image, section):
- installation = dict()
- # NOTE(jeffrey4l): source is not needed when the type is None
- if self.conf._get('type', self.conf._get_group(section)) is None:
- LOG.debug('%s:No source location found', section)
- else:
- installation['type'] = self.conf[section]['type']
- installation['source'] = self.conf[section]['location']
- installation['name'] = section
- if installation['type'] == 'git':
- installation['reference'] = self.conf[section]['reference']
- return installation
-
- for path in self.docker_build_paths:
- # Reading parent image name
- with open(os.path.join(path, 'Dockerfile')) as f:
- content = f.read()
-
- image_name = os.path.basename(path)
- canonical_name = (self.namespace + '/' + self.image_prefix +
- image_name + ':' + self.tag)
- image = Image(image_name, canonical_name, path,
- parent_name=content.split(' ')[1].split('\n')[0])
-
- if self.install_type == 'source':
- # NOTE(jeffrey4l): register the opts if the section didn't
- # register in the kolla/common/config.py file
- if image.name not in self.conf._groups:
- self.conf.register_opts(common_config.get_source_opts(),
- image.name)
- image.source = process_source_installation(image, image.name)
- for plugin in [match.group(0) for match in
- (re.search('{}-plugin-.+'.format(image.name),
- section) for section in
- self.conf.list_all_sections()) if match]:
- self.conf.register_opts(common_config.get_source_opts(),
- plugin)
- image.plugins.append(
- process_source_installation(image, plugin))
-
- self.images.append(image)
-
- def save_dependency(self, to_file):
- dot = graphviz.Digraph(comment='Docker Images Dependency')
- dot.body.extend(['rankdir=LR'])
- for image in self.images:
- if image.status not in [STATUS_MATCHED]:
- continue
- dot.node(image.name)
- if image.parent is not None:
- dot.edge(image.parent.name, image.name)
-
- with open(to_file, 'w') as f:
- f.write(dot.source)
-
- def list_images(self):
- for count, image in enumerate(self.images):
- print(count + 1, ':', image.name)
-
- def list_dependencies(self):
- match = False
- for image in self.images:
- if image.status in [STATUS_MATCHED]:
- match = True
- if image.parent is None:
- base = image
- if not match:
- print('Nothing matched!')
- return
-
- def list_children(images, ancestry):
- children = ancestry.values()[0]
- for item in images:
- if item.status not in [STATUS_MATCHED]:
- continue
- if not item.children:
- children.append(item.name)
- else:
- newparent = {item.name: []}
- children.append(newparent)
- list_children(item.children, newparent)
-
- ancestry = {base.name: []}
- list_children(base.children, ancestry)
- pprint.pprint(ancestry)
-
- def find_parents(self):
- """Associate all images with parents and children"""
- sort_images = dict()
-
- for image in self.images:
- sort_images[image.canonical_name] = image
-
- for parent_name, parent in sort_images.items():
- for image in sort_images.values():
- if image.parent_name == parent_name:
- parent.children.append(image)
- image.parent = parent
-
- def build_queue(self, push_queue):
- """Organizes Queue list
-
- Return a list of Queues that have been organized into a hierarchy
- based on dependencies
- """
- self.build_image_list()
- self.find_parents()
- self.filter_images()
-
- queue = six.moves.queue.Queue()
-
- for image in self.images:
- if image.parent is None:
- queue.put(BuildTask(self.conf, image, push_queue))
- LOG.debug('%s:Added image to queue', image.name)
-
- return queue
-
-
-def run_build():
- """Build container images.
-
- :return: A 3-tuple containing bad, good, and unmatched container image
- status dicts, or None if no images were built.
- """
- conf = cfg.ConfigOpts()
- common_config.parse(conf, sys.argv[1:], prog='kolla-build')
-
- if conf.debug:
- LOG.setLevel(logging.DEBUG)
-
- kolla = KollaWorker(conf)
- kolla.setup_working_dir()
- kolla.find_dockerfiles()
- kolla.create_dockerfiles()
-
- if conf.template_only:
- LOG.info('Dockerfiles are generated in %s', kolla.working_dir)
- return
-
- # We set the atime and mtime to 0 epoch to preserve allow the Docker cache
- # to work like we want. A different size or hash will still force a rebuild
- kolla.set_time()
-
- if conf.save_dependency:
- kolla.save_dependency(conf.save_dependency)
- LOG.info('Docker images dependency is saved in %s',
- conf.save_dependency)
- return
- if conf.list_images:
- kolla.list_images()
- return
- if conf.list_dependencies:
- kolla.list_dependencies()
- return
-
- push_queue = six.moves.queue.Queue()
- queue = kolla.build_queue(push_queue)
- workers = []
-
- for x in six.moves.range(conf.threads):
- worker = WorkerThread(conf, queue)
- worker.setDaemon(True)
- worker.start()
- workers.append(worker)
-
- for x in six.moves.range(conf.push_threads):
- worker = WorkerThread(conf, push_queue)
- worker.start()
- workers.append(worker)
-
- # sleep until queue is empty
- while queue.unfinished_tasks or push_queue.unfinished_tasks:
- time.sleep(3)
-
- kolla.summary()
- kolla.cleanup()
-
- # ensure all threads exited happily
- queue.put(WorkerThread.tombstone)
- push_queue.put(WorkerThread.tombstone)
- for w in workers:
- w.join()
-
- return kolla.get_image_statuses()
+from kolla.image import build
def main():
- statuses = run_build()
+ statuses = build.run_build()
if statuses:
bad_results, good_results, unmatched_results = statuses
if bad_results:
diff --git a/kolla/image/__init__.py b/kolla/image/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/kolla/image/build.py b/kolla/image/build.py
new file mode 100755
index 000000000..40601f0e8
--- /dev/null
+++ b/kolla/image/build.py
@@ -0,0 +1,924 @@
+#!/usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import datetime
+import errno
+import graphviz
+import json
+import logging
+import os
+import pprint
+import re
+import requests
+import shutil
+import signal
+import sys
+import tarfile
+import tempfile
+import threading
+import time
+
+import docker
+import git
+import jinja2
+from oslo_config import cfg
+from requests import exceptions as requests_exc
+import six
+
+PROJECT_ROOT = os.path.abspath(os.path.join(
+ os.path.dirname(os.path.realpath(__file__)), '../..'))
+
+# NOTE(SamYaple): Update the search patch to prefer PROJECT_ROOT as the source
+# of packages to import if we are using local tools/build.py
+# instead of pip installed kolla-build tool
+if PROJECT_ROOT not in sys.path:
+ sys.path.insert(0, PROJECT_ROOT)
+
+from kolla.common import config as common_config
+from kolla.common import task
+from kolla import version
+
+logging.basicConfig()
+LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.INFO)
+
+
+def handle_ctrlc(single, frame):
+ kollaobj = frame.f_locals['kolla']
+ kollaobj.cleanup()
+ sys.exit(1)
+
+signal.signal(signal.SIGINT, handle_ctrlc)
+
+
+class KollaDirNotFoundException(Exception):
+ pass
+
+
+class KollaUnknownBuildTypeException(Exception):
+ pass
+
+
+class KollaMismatchBaseTypeException(Exception):
+ pass
+
+
+class KollaRpmSetupUnknownConfig(Exception):
+ pass
+
+
+# Image status constants.
+#
+# TODO(harlowja): use enum lib in the future??
+STATUS_CONNECTION_ERROR = 'connection_error'
+STATUS_PUSH_ERROR = 'push_error'
+STATUS_ERROR = 'error'
+STATUS_PARENT_ERROR = 'parent_error'
+STATUS_BUILT = 'built'
+STATUS_BUILDING = 'building'
+STATUS_UNMATCHED = 'unmatched'
+STATUS_MATCHED = 'matched'
+STATUS_UNPROCESSED = 'unprocessed'
+
+# All error status constants.
+STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR,
+ STATUS_ERROR, STATUS_PARENT_ERROR)
+
+
+class Recorder(object):
+ """Recorder/buffer of (unicode) log lines for eventual display."""
+
+ def __init__(self):
+ self._lines = []
+
+ def write(self, text=""):
+ if isinstance(text, six.text_type):
+ self._lines.append(text)
+ elif isinstance(text, six.binary_type):
+ self._lines.append(text.decode('utf8'))
+ elif isinstance(text, Recorder):
+ self._lines.extend(text._lines)
+ else:
+ self.write(text=str(text))
+
+ def clear(self):
+ self._lines = []
+
+ def __iter__(self):
+ for line in self._lines:
+ yield line
+
+ def __str__(self):
+ return u"\n".join(self._lines)
+
+
+def docker_client():
+ try:
+ docker_kwargs = docker.utils.kwargs_from_env()
+ return docker.Client(version='auto', **docker_kwargs)
+ except docker.errors.DockerException:
+ LOG.exception('Can not communicate with docker service.'
+ 'Please check docker service is running without errors')
+ sys.exit(1)
+
+
+class Image(object):
+ def __init__(self, name, canonical_name, path, parent_name='',
+ status=STATUS_UNPROCESSED, parent=None, source=None):
+ self.name = name
+ self.canonical_name = canonical_name
+ self.path = path
+ self.status = status
+ self.parent = parent
+ self.source = source
+ self.parent_name = parent_name
+ self.logs = Recorder()
+ self.push_logs = Recorder()
+ self.children = []
+ self.plugins = []
+
+ def __repr__(self):
+ return ("Image(%s, %s, %s, parent_name=%s,"
+ " status=%s, parent=%s, source=%s)") % (
+ self.name, self.canonical_name, self.path,
+ self.parent_name, self.status, self.parent, self.source)
+
+
+class PushIntoQueueTask(task.Task):
+ """Task that pushes some other task into a queue."""
+
+ def __init__(self, push_task, push_queue):
+ super(PushIntoQueueTask, self).__init__()
+ self.push_task = push_task
+ self.push_queue = push_queue
+
+ @property
+ def name(self):
+ return 'PushIntoQueueTask(%s=>%s)' % (self.push_task.name,
+ self.push_queue)
+
+ def run(self):
+ self.push_queue.put(self.push_task)
+ self.success = True
+
+
+class PushTask(task.Task):
+ """Task that pushes a image to a docker repository."""
+
+ def __init__(self, conf, image):
+ super(PushTask, self).__init__()
+ self.dc = docker_client()
+ self.conf = conf
+ self.image = image
+
+ @property
+ def name(self):
+ return 'PushTask(%s)' % self.image.name
+
+ def run(self):
+ image = self.image
+ try:
+ LOG.debug('%s:Try to push the image', image.name)
+ self.push_image(image)
+ except requests_exc.ConnectionError:
+ LOG.exception('%s:Make sure Docker is running and that you'
+ ' have the correct privileges to run Docker'
+ ' (root)', image.name)
+ image.status = STATUS_CONNECTION_ERROR
+ except Exception:
+ LOG.exception('%s:Unknown error when pushing', image.name)
+ image.status = STATUS_PUSH_ERROR
+ finally:
+ if (image.status not in STATUS_ERRORS
+ and image.status != STATUS_UNPROCESSED):
+ LOG.info('%s:Pushed successfully', image.name)
+ self.success = True
+ else:
+ self.success = False
+
+ def push_image(self, image):
+ image.push_logs.clear()
+ for response in self.dc.push(image.canonical_name,
+ stream=True,
+ insecure_registry=True):
+ stream = json.loads(response)
+ if 'stream' in stream:
+ image.push_logs.write(image.logs)
+ image.push_logs.write(stream['stream'])
+ LOG.info('%s', stream['stream'])
+ elif 'errorDetail' in stream:
+ image.status = STATUS_ERROR
+ LOG.error(stream['errorDetail']['message'])
+
+
+class BuildTask(task.Task):
+ """Task that builds out an image."""
+
+ def __init__(self, conf, image, push_queue):
+ super(BuildTask, self).__init__()
+ self.conf = conf
+ self.image = image
+ self.dc = docker_client()
+ self.push_queue = push_queue
+ self.nocache = not conf.cache or conf.no_cache
+ self.forcerm = not conf.keep
+
+ @property
+ def name(self):
+ return 'BuildTask(%s)' % self.image.name
+
+ def run(self):
+ self.builder(self.image)
+ if self.image.status == STATUS_BUILT:
+ self.success = True
+
+ @property
+ def followups(self):
+ followups = []
+ if self.conf.push and self.success:
+ followups.extend([
+ # If we are supposed to push the image into a docker
+ # repository, then make sure we do that...
+ PushIntoQueueTask(
+ PushTask(self.conf, self.image),
+ self.push_queue),
+ ])
+ if self.image.children and self.success:
+ for image in self.image.children:
+ followups.append(BuildTask(self.conf, image, self.push_queue))
+ return followups
+
+ def process_source(self, image, source):
+ dest_archive = os.path.join(image.path, source['name'] + '-archive')
+
+ if source.get('type') == 'url':
+ LOG.debug("%s:Getting archive from %s", image.name,
+ source['source'])
+ try:
+ r = requests.get(source['source'], timeout=self.conf.timeout)
+ except requests_exc.Timeout:
+ LOG.exception('Request timed out while getting archive'
+ ' from %s', source['source'])
+ image.status = STATUS_ERROR
+ image.logs.clear()
+ return
+
+ if r.status_code == 200:
+ with open(dest_archive, 'wb') as f:
+ f.write(r.content)
+ else:
+ LOG.error('%s:Failed to download archive: status_code %s',
+ image.name, r.status_code)
+ image.status = STATUS_ERROR
+ return
+
+ elif source.get('type') == 'git':
+ clone_dir = '{}-{}'.format(dest_archive,
+ source['reference'].replace('/', '-'))
+ try:
+ LOG.debug("%s:Cloning from %s", image.name,
+ source['source'])
+ git.Git().clone(source['source'], clone_dir)
+ git.Git(clone_dir).checkout(source['reference'])
+ reference_sha = git.Git(clone_dir).rev_parse('HEAD')
+ LOG.debug("%s:Git checkout by reference %s (%s)",
+ image.name, source['reference'], reference_sha)
+ except Exception as e:
+ LOG.error("%s:Failed to get source from git", image.name)
+ LOG.error("%s:Error:%s", image.name, str(e))
+ # clean-up clone folder to retry
+ shutil.rmtree(clone_dir)
+ image.status = STATUS_ERROR
+ return
+
+ with tarfile.open(dest_archive, 'w') as tar:
+ tar.add(clone_dir, arcname=os.path.basename(clone_dir))
+
+ elif source.get('type') == 'local':
+ LOG.debug("%s:Getting local archive from %s", image.name,
+ source['source'])
+ if os.path.isdir(source['source']):
+ with tarfile.open(dest_archive, 'w') as tar:
+ tar.add(source['source'],
+ arcname=os.path.basename(source['source']))
+ else:
+ shutil.copyfile(source['source'], dest_archive)
+
+ else:
+ LOG.error("%s:Wrong source type '%s'", image.name,
+ source.get('type'))
+ image.status = STATUS_ERROR
+ return
+
+ # Set time on destination archive to epoch 0
+ os.utime(dest_archive, (0, 0))
+
+ return dest_archive
+
+ def update_buildargs(self):
+ buildargs = dict()
+ if self.conf.build_args:
+ buildargs = dict(self.conf.build_args)
+
+ proxy_vars = ('HTTP_PROXY', 'http_proxy', 'HTTPS_PROXY',
+ 'https_proxy', 'FTP_PROXY', 'ftp_proxy',
+ 'NO_PROXY', 'no_proxy')
+
+ for proxy_var in proxy_vars:
+ if proxy_var in os.environ and proxy_var not in buildargs:
+ buildargs[proxy_var] = os.environ.get(proxy_var)
+
+ if not buildargs:
+ return None
+ return buildargs
+
+ def builder(self, image):
+ LOG.debug('%s:Processing', image.name)
+ if image.status == STATUS_UNMATCHED:
+ return
+
+ if (image.parent is not None and
+ image.parent.status in STATUS_ERRORS):
+ LOG.error('%s:Parent image error\'d with message "%s"',
+ image.name, image.parent.status)
+ image.status = STATUS_PARENT_ERROR
+ return
+
+ image.status = STATUS_BUILDING
+ LOG.info('%s:Building', image.name)
+
+ if image.source and 'source' in image.source:
+ self.process_source(image, image.source)
+ if image.status in STATUS_ERRORS:
+ return
+
+ plugin_archives = list()
+ plugins_path = os.path.join(image.path, 'plugins')
+ for plugin in image.plugins:
+ archive_path = self.process_source(image, plugin)
+ if image.status in STATUS_ERRORS:
+ return
+ plugin_archives.append(archive_path)
+ if plugin_archives:
+ for plugin_archive in plugin_archives:
+ with tarfile.open(plugin_archive, 'r') as plugin_archive_tar:
+ plugin_archive_tar.extractall(path=plugins_path)
+ else:
+ try:
+ os.mkdir(plugins_path)
+ except OSError as e:
+ if e.errno == errno.EEXIST:
+ LOG.info('Directory %s already exist. Skipping.',
+ plugins_path)
+ else:
+ LOG.error('Failed to create directory %s: %s',
+ plugins_path, e)
+ image.status = STATUS_CONNECTION_ERROR
+ return
+ with tarfile.open(os.path.join(image.path, 'plugins-archive'),
+ 'w') as tar:
+ tar.add(plugins_path, arcname='plugins')
+
+ # Pull the latest image for the base distro only
+ pull = True if image.parent is None else False
+
+ image.logs.clear()
+ buildargs = self.update_buildargs()
+ for response in self.dc.build(path=image.path,
+ tag=image.canonical_name,
+ nocache=not self.conf.cache,
+ rm=True,
+ pull=pull,
+ forcerm=self.forcerm,
+ buildargs=buildargs):
+ stream = json.loads(response.decode('utf-8'))
+
+ if 'stream' in stream:
+ image.logs.write(stream['stream'])
+ for line in stream['stream'].split('\n'):
+ if line:
+ LOG.info('%s:%s', image.name, line)
+
+ if 'errorDetail' in stream:
+ image.status = STATUS_ERROR
+ LOG.error('%s:Error\'d with the following message',
+ image.name)
+ for line in stream['errorDetail']['message'].split('\n'):
+ if line:
+ LOG.error('%s:%s', image.name, line)
+ return
+
+ image.status = STATUS_BUILT
+
+ LOG.info('%s:Built', image.name)
+
+
+class WorkerThread(threading.Thread):
+ """Thread that executes tasks until the queue provides a tombstone."""
+
+ #: Object to be put on worker queues to get them to die.
+ tombstone = object()
+
+ def __init__(self, conf, queue):
+ super(WorkerThread, self).__init__()
+ self.queue = queue
+ self.conf = conf
+
+ def run(self):
+ while True:
+ task = self.queue.get()
+ if task is self.tombstone:
+ # Ensure any other threads also get the tombstone.
+ self.queue.put(task)
+ break
+ try:
+ for attempt in six.moves.range(self.conf.retries + 1):
+ if attempt > 0:
+ LOG.debug("Attempting to run task %s for the %s time",
+ task.name, attempt + 1)
+ else:
+ LOG.debug("Attempting to run task %s for the first"
+ " time", task.name)
+ try:
+ task.run()
+ if task.success:
+ break
+ except Exception:
+ LOG.exception('Unhandled error when running %s',
+ task.name)
+ # try again...
+ task.reset()
+ if task.success:
+ for next_task in task.followups:
+ LOG.debug('Added next task %s to queue',
+ next_task.name)
+ self.queue.put(next_task)
+ finally:
+ self.queue.task_done()
+
+
+class KollaWorker(object):
+
+ def __init__(self, conf):
+ self.conf = conf
+ self.images_dir = self._get_images_dir()
+ self.registry = conf.registry
+ if self.registry:
+ self.namespace = self.registry + '/' + conf.namespace
+ else:
+ self.namespace = conf.namespace
+ self.base = conf.base
+ self.base_tag = conf.base_tag
+ self.install_type = conf.install_type
+ self.tag = conf.tag
+ self.images = list()
+ rpm_setup_config = filter(None, conf.rpm_setup_config)
+ self.rpm_setup = self.build_rpm_setup(rpm_setup_config)
+
+ rh_base = ['fedora', 'centos', 'oraclelinux', 'rhel']
+ rh_type = ['source', 'binary', 'rdo', 'rhos']
+ deb_base = ['ubuntu', 'debian']
+ deb_type = ['source', 'binary']
+
+ if not ((self.base in rh_base and self.install_type in rh_type) or
+ (self.base in deb_base and self.install_type in deb_type)):
+ raise KollaMismatchBaseTypeException(
+ '{} is unavailable for {}'.format(self.install_type, self.base)
+ )
+
+ if self.install_type == 'binary':
+ self.install_metatype = 'rdo'
+ elif self.install_type == 'source':
+ self.install_metatype = 'mixed'
+ elif self.install_type == 'rdo':
+ self.install_type = 'binary'
+ self.install_metatype = 'rdo'
+ elif self.install_type == 'rhos':
+ self.install_type = 'binary'
+ self.install_metatype = 'rhos'
+ else:
+ raise KollaUnknownBuildTypeException(
+ 'Unknown install type'
+ )
+
+ self.image_prefix = self.base + '-' + self.install_type + '-'
+
+ self.include_header = conf.include_header
+ self.include_footer = conf.include_footer
+ self.regex = conf.regex
+ self.image_statuses_bad = dict()
+ self.image_statuses_good = dict()
+ self.image_statuses_unmatched = dict()
+ self.maintainer = conf.maintainer
+
+ def _get_images_dir(self):
+ possible_paths = (
+ PROJECT_ROOT,
+ os.path.join(sys.prefix, 'share/kolla'),
+ os.path.join(sys.prefix, 'local/share/kolla'))
+
+ for path in possible_paths:
+ image_path = os.path.join(path, 'docker')
+ # NOTE(SamYaple): We explicty check for the base folder to ensure
+ # this is the correct path
+ # TODO(SamYaple): Improve this to make this safer
+ if os.path.exists(os.path.join(image_path, 'base')):
+ LOG.info('Found the docker image folder at %s', image_path)
+ return image_path
+ else:
+ raise KollaDirNotFoundException('Image dir can not be found')
+
+ def build_rpm_setup(self, rpm_setup_config):
+ """Generates a list of docker commands based on provided configuration.
+
+ :param rpm_setup_config: A list of .rpm or .repo paths or URLs
+ :return: A list of docker commands
+ """
+ rpm_setup = list()
+
+ for config in rpm_setup_config:
+ if config.endswith('.rpm'):
+ # RPM files can be installed with yum from file path or url
+ cmd = "RUN yum -y install {}".format(config)
+ elif config.endswith('.repo'):
+ if config.startswith('http'):
+ # Curl http://url/etc.repo to /etc/yum.repos.d/etc.repo
+ name = config.split('/')[-1]
+ cmd = "RUN curl {} -o /etc/yum.repos.d/{}".format(config,
+ name)
+ else:
+ # Copy .repo file from filesystem
+ cmd = "COPY {} /etc/yum.repos.d/".format(config)
+ else:
+ raise KollaRpmSetupUnknownConfig(
+ 'RPM setup must be provided as .rpm or .repo files.'
+ ' Attempted configuration was {}'.format(config)
+ )
+
+ rpm_setup.append(cmd)
+
+ return rpm_setup
+
+ def copy_apt_files(self):
+ if self.conf.apt_sources_list:
+ shutil.copyfile(
+ self.conf.apt_sources_list,
+ os.path.join(self.working_dir, "base", "sources.list")
+ )
+
+ if self.conf.apt_preferences:
+ shutil.copyfile(
+ self.conf.apt_preferences,
+ os.path.join(self.working_dir, "base", "apt_preferences")
+ )
+
+ def setup_working_dir(self):
+ """Creates a working directory for use while building"""
+ ts = time.time()
+ ts = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M-%S_')
+ self.temp_dir = tempfile.mkdtemp(prefix='kolla-' + ts)
+ self.working_dir = os.path.join(self.temp_dir, 'docker')
+ shutil.copytree(self.images_dir, self.working_dir)
+ self.copy_apt_files()
+ LOG.debug('Created working dir: %s', self.working_dir)
+
+ def set_time(self):
+ for root, dirs, files in os.walk(self.working_dir):
+ for file_ in files:
+ os.utime(os.path.join(root, file_), (0, 0))
+ for dir_ in dirs:
+ os.utime(os.path.join(root, dir_), (0, 0))
+ LOG.debug('Set atime and mtime to 0 for all content in working dir')
+
+ def create_dockerfiles(self):
+ kolla_version = version.version_info.cached_version_string()
+ supported_distro_release = common_config.DISTRO_RELEASE.get(
+ self.base)
+ for path in self.docker_build_paths:
+ template_name = "Dockerfile.j2"
+ env = jinja2.Environment( # nosec: not used to render HTML
+ loader=jinja2.FileSystemLoader(path))
+ template = env.get_template(template_name)
+ values = {'base_distro': self.base,
+ 'base_image': self.conf.base_image,
+ 'base_distro_tag': self.base_tag,
+ 'supported_distro_release': supported_distro_release,
+ 'install_metatype': self.install_metatype,
+ 'image_prefix': self.image_prefix,
+ 'install_type': self.install_type,
+ 'namespace': self.namespace,
+ 'tag': self.tag,
+ 'maintainer': self.maintainer,
+ 'kolla_version': kolla_version,
+ 'rpm_setup': self.rpm_setup}
+ if self.include_header:
+ with open(self.include_header, 'r') as f:
+ values['include_header'] = f.read()
+ if self.include_footer:
+ with open(self.include_footer, 'r') as f:
+ values['include_footer'] = f.read()
+ content = template.render(values)
+ with open(os.path.join(path, 'Dockerfile'), 'w') as f:
+ f.write(content)
+
+ def find_dockerfiles(self):
+ """Recursive search for Dockerfiles in the working directory"""
+ self.docker_build_paths = list()
+ path = self.working_dir
+ filename = 'Dockerfile.j2'
+
+ for root, dirs, names in os.walk(path):
+ if filename in names:
+ self.docker_build_paths.append(root)
+ LOG.debug('Found %s', root.split(self.working_dir)[1])
+
+ LOG.debug('Found %d Dockerfiles', len(self.docker_build_paths))
+
+ def cleanup(self):
+ """Remove temp files"""
+ shutil.rmtree(self.temp_dir)
+
+ def filter_images(self):
+ """Filter which images to build"""
+ filter_ = list()
+
+ if self.regex:
+ filter_ += self.regex
+ elif self.conf.profile:
+ for profile in self.conf.profile:
+ if profile not in self.conf.profiles:
+ self.conf.register_opt(cfg.ListOpt(profile,
+ default=[]),
+ 'profiles')
+ if len(self.conf.profiles[profile]) == 0:
+ msg = 'Profile: {} does not exist'.format(profile)
+ raise ValueError(msg)
+ else:
+ filter_ += self.conf.profiles[profile]
+
+ if filter_:
+ patterns = re.compile(r"|".join(filter_).join('()'))
+ for image in self.images:
+ if image.status == STATUS_MATCHED:
+ continue
+ if re.search(patterns, image.name):
+ image.status = STATUS_MATCHED
+ while (image.parent is not None and
+ image.parent.status != STATUS_MATCHED):
+ image = image.parent
+ image.status = STATUS_MATCHED
+ LOG.debug('%s:Matched regex', image.name)
+ else:
+ image.status = STATUS_UNMATCHED
+ else:
+ for image in self.images:
+ image.status = STATUS_MATCHED
+
+ def summary(self):
+ """Walk the dictionary of images statuses and print results"""
+ # For debug we print the logs again if the image error'd. This is to
+ # to help us debug and it will be extra helpful in the gate.
+ for image in self.images:
+ if image.status in STATUS_ERRORS:
+ LOG.debug("%s:Failed with the following logs", image.name)
+ for line in image.logs:
+ if line:
+ LOG.debug("%s:%s", image.name, line)
+
+ self.get_image_statuses()
+
+ if self.image_statuses_good:
+ LOG.info("Successfully built images")
+ LOG.info("=========================")
+ for name in self.image_statuses_good.keys():
+ LOG.info(name)
+
+ if self.image_statuses_bad:
+ LOG.info("Images that failed to build")
+ LOG.info("===========================")
+ for name, status in self.image_statuses_bad.items():
+ LOG.error('%s Failed with status: %s', name, status)
+
+ if self.image_statuses_unmatched:
+ LOG.debug("Images not matched for build by regex")
+ LOG.debug("=====================================")
+ for name in self.image_statuses_unmatched.keys():
+ LOG.debug(name)
+
+ def get_image_statuses(self):
+ if any([self.image_statuses_bad,
+ self.image_statuses_good,
+ self.image_statuses_unmatched]):
+ return (self.image_statuses_bad,
+ self.image_statuses_good,
+ self.image_statuses_unmatched)
+ for image in self.images:
+ if image.status == STATUS_BUILT:
+ self.image_statuses_good[image.name] = image.status
+ elif image.status == STATUS_UNMATCHED:
+ self.image_statuses_unmatched[image.name] = image.status
+ else:
+ self.image_statuses_bad[image.name] = image.status
+ return (self.image_statuses_bad,
+ self.image_statuses_good,
+ self.image_statuses_unmatched)
+
+ def build_image_list(self):
+ def process_source_installation(image, section):
+ installation = dict()
+ # NOTE(jeffrey4l): source is not needed when the type is None
+ if self.conf._get('type', self.conf._get_group(section)) is None:
+ LOG.debug('%s:No source location found', section)
+ else:
+ installation['type'] = self.conf[section]['type']
+ installation['source'] = self.conf[section]['location']
+ installation['name'] = section
+ if installation['type'] == 'git':
+ installation['reference'] = self.conf[section]['reference']
+ return installation
+
+ for path in self.docker_build_paths:
+ # Reading parent image name
+ with open(os.path.join(path, 'Dockerfile')) as f:
+ content = f.read()
+
+ image_name = os.path.basename(path)
+ canonical_name = (self.namespace + '/' + self.image_prefix +
+ image_name + ':' + self.tag)
+ image = Image(image_name, canonical_name, path,
+ parent_name=content.split(' ')[1].split('\n')[0])
+
+ if self.install_type == 'source':
+ # NOTE(jeffrey4l): register the opts if the section didn't
+ # register in the kolla/common/config.py file
+ if image.name not in self.conf._groups:
+ self.conf.register_opts(common_config.get_source_opts(),
+ image.name)
+ image.source = process_source_installation(image, image.name)
+ for plugin in [match.group(0) for match in
+ (re.search('{}-plugin-.+'.format(image.name),
+ section) for section in
+ self.conf.list_all_sections()) if match]:
+ self.conf.register_opts(common_config.get_source_opts(),
+ plugin)
+ image.plugins.append(
+ process_source_installation(image, plugin))
+
+ self.images.append(image)
+
+ def save_dependency(self, to_file):
+ dot = graphviz.Digraph(comment='Docker Images Dependency')
+ dot.body.extend(['rankdir=LR'])
+ for image in self.images:
+ if image.status not in [STATUS_MATCHED]:
+ continue
+ dot.node(image.name)
+ if image.parent is not None:
+ dot.edge(image.parent.name, image.name)
+
+ with open(to_file, 'w') as f:
+ f.write(dot.source)
+
+ def list_images(self):
+ for count, image in enumerate(self.images):
+ print(count + 1, ':', image.name)
+
+ def list_dependencies(self):
+ match = False
+ for image in self.images:
+ if image.status in [STATUS_MATCHED]:
+ match = True
+ if image.parent is None:
+ base = image
+ if not match:
+ print('Nothing matched!')
+ return
+
+ def list_children(images, ancestry):
+ children = ancestry.values()[0]
+ for item in images:
+ if item.status not in [STATUS_MATCHED]:
+ continue
+ if not item.children:
+ children.append(item.name)
+ else:
+ newparent = {item.name: []}
+ children.append(newparent)
+ list_children(item.children, newparent)
+
+ ancestry = {base.name: []}
+ list_children(base.children, ancestry)
+ pprint.pprint(ancestry)
+
+ def find_parents(self):
+ """Associate all images with parents and children"""
+ sort_images = dict()
+
+ for image in self.images:
+ sort_images[image.canonical_name] = image
+
+ for parent_name, parent in sort_images.items():
+ for image in sort_images.values():
+ if image.parent_name == parent_name:
+ parent.children.append(image)
+ image.parent = parent
+
+ def build_queue(self, push_queue):
+ """Organizes Queue list
+
+ Return a list of Queues that have been organized into a hierarchy
+ based on dependencies
+ """
+ self.build_image_list()
+ self.find_parents()
+ self.filter_images()
+
+ queue = six.moves.queue.Queue()
+
+ for image in self.images:
+ if image.parent is None:
+ queue.put(BuildTask(self.conf, image, push_queue))
+ LOG.debug('%s:Added image to queue', image.name)
+
+ return queue
+
+
+def run_build():
+ """Build container images.
+
+ :return: A 3-tuple containing bad, good, and unmatched container image
+ status dicts, or None if no images were built.
+ """
+ conf = cfg.ConfigOpts()
+ common_config.parse(conf, sys.argv[1:], prog='kolla-build')
+
+ if conf.debug:
+ LOG.setLevel(logging.DEBUG)
+
+ kolla = KollaWorker(conf)
+ kolla.setup_working_dir()
+ kolla.find_dockerfiles()
+ kolla.create_dockerfiles()
+
+ if conf.template_only:
+ LOG.info('Dockerfiles are generated in %s', kolla.working_dir)
+ return
+
+ # We set the atime and mtime to 0 epoch to preserve allow the Docker cache
+ # to work like we want. A different size or hash will still force a rebuild
+ kolla.set_time()
+
+ if conf.save_dependency:
+ kolla.save_dependency(conf.save_dependency)
+ LOG.info('Docker images dependency is saved in %s',
+ conf.save_dependency)
+ return
+ if conf.list_images:
+ kolla.list_images()
+ return
+ if conf.list_dependencies:
+ kolla.list_dependencies()
+ return
+
+ push_queue = six.moves.queue.Queue()
+ queue = kolla.build_queue(push_queue)
+ workers = []
+
+ for x in six.moves.range(conf.threads):
+ worker = WorkerThread(conf, queue)
+ worker.setDaemon(True)
+ worker.start()
+ workers.append(worker)
+
+ for x in six.moves.range(conf.push_threads):
+ worker = WorkerThread(conf, push_queue)
+ worker.start()
+ workers.append(worker)
+
+ # sleep until queue is empty
+ while queue.unfinished_tasks or push_queue.unfinished_tasks:
+ time.sleep(3)
+
+ kolla.summary()
+ kolla.cleanup()
+
+ # ensure all threads exited happily
+ queue.put(WorkerThread.tombstone)
+ push_queue.put(WorkerThread.tombstone)
+ for w in workers:
+ w.join()
+
+ return kolla.get_image_statuses()
diff --git a/kolla/tests/base.py b/kolla/tests/base.py
index 0ac1467cc..1c43b7619 100644
--- a/kolla/tests/base.py
+++ b/kolla/tests/base.py
@@ -36,7 +36,7 @@ class TestCase(oslotest_base.BaseTestCase):
# NOTE(jeffrey4l): mock the _get_image_dir method to return a fake
# docker images dir
self.useFixture(fixtures.MockPatch(
- 'kolla.cmd.build.KollaWorker._get_images_dir',
+ 'kolla.image.build.KollaWorker._get_images_dir',
mock.Mock(return_value=os.path.join(TESTS_ROOT, 'docker'))))
def get_default_config_files(self):
diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py
index 98c240d6e..98ffebf99 100644
--- a/kolla/tests/test_build.py
+++ b/kolla/tests/test_build.py
@@ -17,7 +17,8 @@ import mock
import os
import requests
-from kolla.cmd import build
+from kolla.cmd import build as build_cmd
+from kolla.image import build
from kolla.tests import base
@@ -233,22 +234,22 @@ class MainTest(base.TestCase):
def test_images_built(self, mock_run_build):
image_statuses = ({}, {'img': 'built'}, {})
mock_run_build.return_value = image_statuses
- result = build.main()
+ result = build_cmd.main()
self.assertEqual(0, result)
def test_images_unmatched(self, mock_run_build):
image_statuses = ({}, {}, {'img': 'unmatched'})
mock_run_build.return_value = image_statuses
- result = build.main()
+ result = build_cmd.main()
self.assertEqual(0, result)
def test_no_images_built(self, mock_run_build):
mock_run_build.return_value = None
- result = build.main()
+ result = build_cmd.main()
self.assertEqual(0, result)
def test_bad_images(self, mock_run_build):
image_statuses = ({'img': 'error'}, {}, {})
mock_run_build.return_value = image_statuses
- result = build.main()
+ result = build_cmd.main()
self.assertEqual(1, result)
diff --git a/tests/test_build.py b/tests/test_build.py
index f2e802bf6..075063f87 100644
--- a/tests/test_build.py
+++ b/tests/test_build.py
@@ -23,7 +23,7 @@ import testtools
sys.path.append(
os.path.abspath(os.path.join(os.path.dirname(__file__), '../tools')))
-from kolla.cmd import build
+from kolla.image import build
LOG = logging.getLogger(__name__)
--
GitLab