libvirt/ci/helper

#!/usr/bin/env python3
#
# Copyright (C) 2021 Red Hat, Inc.
# SPDX-License-Identifier: LGPL-2.1-or-later

import argparse
import os
import pathlib
import pty
import shutil
import subprocess
import sys
import textwrap

import util


class Parser:
    def __init__(self):
        # Options that are common to all actions that use containers
        containerparser = argparse.ArgumentParser(add_help=False)
        containerparser.add_argument(
            "target",
            help="perform action on target OS",
        )
        containerparser.add_argument(
            "--engine",
            choices=["auto", "podman", "docker"],
            default="auto",
            help="container engine to use",
        )
        containerparser.add_argument(
            "--login",
            default=os.getlogin(),  # exempt from syntax-check
            help="login to use inside the container",
        )
        containerparser.add_argument(
            "--image-prefix",
            default="registry.gitlab.com/libvirt/libvirt/ci-",
            help="use container images from non-default location",
        )
        containerparser.add_argument(
            "--image-tag",
            default=":latest",
            help="use container images with non-default tags",
        )

        # Options that are common to all actions that call the
        # project's build system
        mesonparser = argparse.ArgumentParser(add_help=False)
        mesonparser.add_argument(
            "--meson-args",
            default="",
            help="additional arguments passed to meson "
                 "(eg --meson-args='-Dopt1=enabled -Dopt2=disabled')",
        )
        mesonparser.add_argument(
            "--ninja-args",
            default="",
            help="additional arguments passed to ninja",
        )

        # Options that are common to all actions that use lcitool
        lcitoolparser = argparse.ArgumentParser(add_help=False)
        lcitoolparser.add_argument(
            "--lcitool",
            metavar="PATH",
            default="lcitool",
            help="path to lcitool binary",
        )

        # Options that are common to actions communicating with a GitLab
        # instance
        gitlabparser = argparse.ArgumentParser(add_help=False)
        gitlabparser.add_argument(
            "--namespace",
            default="libvirt/libvirt",
            help="GitLab project namespace"
        )
        gitlabparser.add_argument(
            "--gitlab-uri",
            default="https://gitlab.com",
            help="base GitLab URI"
        )

        # Main parser
        self.parser = argparse.ArgumentParser()
        subparsers = self.parser.add_subparsers(
            dest="action",
            metavar="ACTION",
        )
        subparsers.required = True

        # build action
        buildparser = subparsers.add_parser(
            "build",
            help="run a build in a container",
            parents=[containerparser, mesonparser],
            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        )
        buildparser.set_defaults(func=Application.action_build)

        # test action
        testparser = subparsers.add_parser(
            "test",
            help="run a build in a container (including tests)",
            parents=[containerparser, mesonparser],
            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        )
        testparser.set_defaults(func=Application.action_test)

        # shell action
        shellparser = subparsers.add_parser(
            "shell",
            help="start a shell in a container",
            parents=[containerparser],
            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        )
        shellparser.set_defaults(func=Application.action_shell)

        # list-images action
        listimagesparser = subparsers.add_parser(
            "list-images",
            help="list known container images",
            parents=[gitlabparser],
            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        )
        listimagesparser.set_defaults(func=Application.action_list_images)

        # refresh action
        refreshparser = subparsers.add_parser(
            "refresh",
            help="refresh data generated with lcitool",
            parents=[lcitoolparser, gitlabparser],
            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        )
        refreshparser.add_argument(
            "--quiet",
            action="store_true",
            default=False,
            help="refresh data silently"
        )
        refreshparser.add_argument(
            "--check-stale",
            action="store",
            choices=["yes", "no"],
            default="yes",
            help="check for existence of stale images on the GitLab instance"
        )
        refreshparser.set_defaults(func=Application.action_refresh)

    def parse(self):
        return self.parser.parse_args()


class Application:
    def __init__(self):
        self.basedir = pathlib.Path(__file__).resolve().parent
        self.args = Parser().parse()

        if self.args.action == "refresh":
            if not shutil.which(self.args.lcitool):
                sys.exit("error: 'lcitool' not installed")

    def make_run(self, target):
        args = [
            "-C",
            self.basedir,
            target,
        ]

        if self.args.action in ["build", "test", "shell"]:
            args.extend([
                f"CI_ENGINE={self.args.engine}",
                f"CI_USER_LOGIN={self.args.login}",
                f"CI_IMAGE_PREFIX={self.args.image_prefix}",
                f"CI_IMAGE_TAG={self.args.image_tag}",
            ])

        if self.args.action in ["build", "test"]:
            args.extend([
                f"CI_MESON_ARGS={self.args.meson_args}",
                f"CI_NINJA_ARGS={self.args.ninja_args}",
            ])

        if pty.spawn(["make"] + args) != 0:
            sys.exit("error: 'make' failed")

    def lcitool_run(self, args):
        output = subprocess.check_output([self.args.lcitool] + args)
        return output.decode("utf-8")

    def lcitool_get_hosts(self):
        output = self.lcitool_run(["hosts"])
        return output.splitlines()

    def generate_dockerfile(self, host, cross=None):
        args = ["dockerfile", host, "libvirt"]
        outdir = self.basedir.joinpath("containers")
        outfile = f"ci-{host}.Dockerfile"

        if cross:
            args.extend(["--cross", cross])
            outfile = f"ci-{host}-cross-{cross}.Dockerfile"

        outpath = outdir.joinpath(outfile)
        if not self.args.quiet:
            print(outpath)

        output = self.lcitool_run(args)
        with open(outpath, "w") as f:
            f.write(output)

    def generate_vars(self, host):
        args = ["variables", host, "libvirt"]
        outdir = self.basedir.joinpath("cirrus")
        outfile = f"{host}.vars"

        outpath = outdir.joinpath(outfile)
        if not self.args.quiet:
            print(outpath)

        output = self.lcitool_run(args)
        with open(outpath, "w") as f:
            f.write(output)

    def refresh_containers(self):
        debian_cross = [
            "aarch64",
            "armv6l",
            "armv7l",
            "i686",
            "mips",
            "mips64el",
            "mipsel",
            "ppc64le",
            "s390x",
        ]
        fedora_cross = [
            "mingw32",
            "mingw64",
        ]

        for host in self.lcitool_get_hosts():
            if host.startswith("freebsd-") or host.startswith("macos-"):
                continue

            self.generate_dockerfile(host)

            if host == "fedora-rawhide":
                for cross in fedora_cross:
                    self.generate_dockerfile(host, cross)

            if host.startswith("debian-"):
                for cross in debian_cross:
                    if host == "debian-sid" and cross == "mips":
                        continue
                    self.generate_dockerfile(host, cross)

    def refresh_cirrus(self):
        for host in self.lcitool_get_hosts():
            if not (host.startswith("freebsd-") or host.startswith("macos-")):
                continue

            self.generate_vars(host)

    def check_stale_images(self):
        namespace = self.args.namespace
        gitlab_uri = self.args.gitlab_uri
        registry_uri = util.get_registry_uri(namespace, gitlab_uri)
        lcitool_hosts = self.lcitool_get_hosts()

        stale_images = util.get_registry_stale_images(registry_uri,
                                                      lcitool_hosts)
        if stale_images:
            spacing = "\n" + 4 * " "
            stale_fmt = [f"{k} (ID: {v})" for k, v in stale_images.items()]
            stale_details = spacing.join(stale_fmt)
            stale_ids = ' '.join([str(id) for id in stale_images.values()])
            registry_uri = util.get_registry_uri(namespace, gitlab_uri)

            msg = textwrap.dedent(f"""
                The following images are stale and can be purged from the registry:

                    STALE_DETAILS

                You can delete the images listed above using this shell snippet:

                    $ for image_id in {stale_ids}; do
                          curl --request DELETE --header "PRIVATE-TOKEN: <access_token>" \\
                               {registry_uri}/$image_id;
                      done

                You can generate a personal access token here:

                    {gitlab_uri}/-/profile/personal_access_tokens
            """)
            print(msg.replace("STALE_DETAILS", stale_details))

    def action_build(self):
        self.make_run(f"ci-build@{self.args.target}")

    def action_test(self):
        self.make_run(f"ci-test@{self.args.target}")

    def action_shell(self):
        self.make_run(f"ci-shell@{self.args.target}")

    def action_list_images(self):
        registry_uri = util.get_registry_uri(self.args.namespace,
                                             self.args.gitlab_uri)
        images = util.get_registry_images(registry_uri)

        # skip the "ci-" prefix each of our container images' name has
        name_prefix = "ci-"
        names = [i["name"][len(name_prefix):] for i in images]
        names.sort()

        native = [name for name in names if "-cross-" not in name]
        cross = [name for name in names if "-cross-" in name]

        spacing = 4 * " "
        print("Available x86 container images:\n")
        print(spacing + ("\n" + spacing).join(native))

        if cross:
            print()
            print("Available cross-compiler container images:\n")
            print(spacing + ("\n" + spacing).join(cross))

    def action_refresh(self):
        self.refresh_containers()
        self.refresh_cirrus()

        if self.args.check_stale == "yes" and not self.args.quiet:
            self.check_stale_images()

    def run(self):
        self.args.func(self)


if __name__ == "__main__":
    Application().run()