#!/usr/bin/env python3 # # Copyright (C) 2021 Red Hat, Inc. # SPDX-License-Identifier: LGPL-2.1-or-later import argparse import os import pathlib import pty import shutil import subprocess import sys import textwrap import util class Parser: def __init__(self): # Options that are common to all actions that use containers containerparser = argparse.ArgumentParser(add_help=False) containerparser.add_argument( "target", help="perform action on target OS", ) containerparser.add_argument( "--engine", choices=["auto", "podman", "docker"], default="auto", help="container engine to use", ) containerparser.add_argument( "--login", default=os.getlogin(), # exempt from syntax-check help="login to use inside the container", ) containerparser.add_argument( "--image-prefix", default="registry.gitlab.com/libvirt/libvirt/ci-", help="use container images from non-default location", ) containerparser.add_argument( "--image-tag", default=":latest", help="use container images with non-default tags", ) # Options that are common to all actions that call the # project's build system mesonparser = argparse.ArgumentParser(add_help=False) mesonparser.add_argument( "--meson-args", default="", help="additional arguments passed to meson " "(eg --meson-args='-Dopt1=enabled -Dopt2=disabled')", ) mesonparser.add_argument( "--ninja-args", default="", help="additional arguments passed to ninja", ) # Options that are common to all actions that use lcitool lcitoolparser = argparse.ArgumentParser(add_help=False) lcitoolparser.add_argument( "--lcitool", metavar="PATH", default="lcitool", help="path to lcitool binary", ) # Options that are common to actions communicating with a GitLab # instance gitlabparser = argparse.ArgumentParser(add_help=False) gitlabparser.add_argument( "--namespace", default="libvirt/libvirt", help="GitLab project namespace" ) gitlabparser.add_argument( "--gitlab-uri", default="https://gitlab.com", help="base GitLab URI" ) # Main parser self._parser = argparse.ArgumentParser() subparsers = self._parser.add_subparsers( dest="action", metavar="ACTION", ) subparsers.required = True # build action buildparser = subparsers.add_parser( "build", help="run a build in a container", parents=[containerparser, mesonparser], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) buildparser.set_defaults(func=Application._action_build) # test action testparser = subparsers.add_parser( "test", help="run a build in a container (including tests)", parents=[containerparser, mesonparser], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) testparser.set_defaults(func=Application._action_test) # shell action shellparser = subparsers.add_parser( "shell", help="start a shell in a container", parents=[containerparser], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) shellparser.set_defaults(func=Application._action_shell) # list-images action listimagesparser = subparsers.add_parser( "list-images", help="list known container images", parents=[gitlabparser], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) listimagesparser.set_defaults(func=Application._action_list_images) # refresh action refreshparser = subparsers.add_parser( "refresh", help="refresh data generated with lcitool", parents=[lcitoolparser, gitlabparser], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) refreshparser.add_argument( "--quiet", action="store_true", default=False, help="refresh data silently" ) refreshparser.add_argument( "--check-stale", action="store", choices=["yes", "no"], default="yes", help="check for existence of stale images on the GitLab instance" ) refreshparser.set_defaults(func=Application._action_refresh) def parse(self): return self._parser.parse_args() class Application: def __init__(self): self._basedir = pathlib.Path(__file__).resolve().parent self._args = Parser().parse() if self._args.action == "refresh": if not shutil.which(self._args.lcitool): sys.exit("error: 'lcitool' not installed") def _make_run(self, target): args = [ "-C", self._basedir, target, ] if self._args.action in ["build", "test", "shell"]: args.extend([ f"CI_ENGINE={self._args.engine}", f"CI_USER_LOGIN={self._args.login}", f"CI_IMAGE_PREFIX={self._args.image_prefix}", f"CI_IMAGE_TAG={self._args.image_tag}", ]) if self._args.action in ["build", "test"]: args.extend([ f"CI_MESON_ARGS={self._args.meson_args}", f"CI_NINJA_ARGS={self._args.ninja_args}", ]) if pty.spawn(["make"] + args) != 0: sys.exit("error: 'make' failed") def _lcitool_run(self, args): output = subprocess.check_output([self._args.lcitool] + args) return output.decode("utf-8") def _lcitool_get_hosts(self): output = self._lcitool_run(["hosts"]) return output.splitlines() def _generate_dockerfile(self, host, cross=None): args = ["dockerfile", host, "libvirt"] outdir = self._basedir.joinpath("containers") outfile = f"{host}.Dockerfile" if cross: args.extend(["--cross", cross]) outfile = f"{host}-cross-{cross}.Dockerfile" outpath = outdir.joinpath(outfile) if not self._args.quiet: print(outpath) output = self._lcitool_run(args) with open(outpath, "w") as f: f.write(output) def _generate_vars(self, host): args = ["variables", host, "libvirt"] outdir = self._basedir.joinpath("cirrus") outfile = f"{host}.vars" outpath = outdir.joinpath(outfile) if not self._args.quiet: print(outpath) output = self._lcitool_run(args) with open(outpath, "w") as f: f.write(output) def _refresh_containers(self): debian_cross = [ "aarch64", "armv6l", "armv7l", "i686", "mips", "mips64el", "mipsel", "ppc64le", "s390x", ] fedora_cross = [ "mingw32", "mingw64", ] for host in self._lcitool_get_hosts(): if host.startswith("freebsd-") or host.startswith("macos-"): continue self._generate_dockerfile(host) if host == "fedora-rawhide": for cross in fedora_cross: self._generate_dockerfile(host, cross) if host.startswith("debian-"): for cross in debian_cross: if host == "debian-sid" and cross == "mips": continue self._generate_dockerfile(host, cross) def _refresh_cirrus(self): for host in self._lcitool_get_hosts(): if not (host.startswith("freebsd-") or host.startswith("macos-")): continue self._generate_vars(host) def _check_stale_images(self): namespace = self._args.namespace gitlab_uri = self._args.gitlab_uri registry_uri = util.get_registry_uri(namespace, gitlab_uri) lcitool_hosts = self._lcitool_get_hosts() stale_images = util.get_registry_stale_images(registry_uri, lcitool_hosts) if stale_images: spacing = "\n" + 4 * " " stale_fmt = [f"{k} (ID: {v})" for k, v in stale_images.items()] stale_details = spacing.join(stale_fmt) stale_ids = ' '.join([str(id) for id in stale_images.values()]) registry_uri = util.get_registry_uri(namespace, gitlab_uri) msg = textwrap.dedent(f""" The following images are stale and can be purged from the registry: STALE_DETAILS You can delete the images listed above using this shell snippet: $ for image_id in {stale_ids}; do curl --request DELETE --header "PRIVATE-TOKEN: " \\ {registry_uri}/$image_id; done You can generate a personal access token here: {gitlab_uri}/-/profile/personal_access_tokens """) print(msg.replace("STALE_DETAILS", stale_details)) def _action_build(self): self._make_run(f"ci-build@{self._args.target}") def _action_test(self): self._make_run(f"ci-test@{self._args.target}") def _action_shell(self): self._make_run(f"ci-shell@{self._args.target}") def _action_list_images(self): registry_uri = util.get_registry_uri(self._args.namespace, self._args.gitlab_uri) images = util.get_registry_images(registry_uri) # skip the "ci-" prefix each of our container images' name has name_prefix = "ci-" names = [i["name"][len(name_prefix):] for i in images] names.sort() native = [name for name in names if "-cross-" not in name] cross = [name for name in names if "-cross-" in name] spacing = 4 * " " print("Available x86 container images:\n") print(spacing + ("\n" + spacing).join(native)) if cross: print() print("Available cross-compiler container images:\n") print(spacing + ("\n" + spacing).join(cross)) def _action_refresh(self): self._refresh_containers() self._refresh_cirrus() if self._args.check_stale == "yes" and not self._args.quiet: self._check_stale_images() def run(self): self._args.func(self) if __name__ == "__main__": Application().run()