mirror of
https://gitlab.com/libvirt/libvirt.git
synced 2024-12-25 07:05:28 +00:00
792843c05e
Since the string "ci" is already contained in the path, it seems unnecessary to include it into the filename too: in fact, we only do that for Dockerfiles and not for files in ci/cirrus, even though those are generated the very same way. Signed-off-by: Andrea Bolognani <abologna@redhat.com> Reviewed-by: Erik Skultety <eskultet@redhat.com>
344 lines
11 KiB
Python
Executable File
344 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (C) 2021 Red Hat, Inc.
|
|
# SPDX-License-Identifier: LGPL-2.1-or-later
|
|
|
|
import argparse
|
|
import os
|
|
import pathlib
|
|
import pty
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import textwrap
|
|
|
|
import util
|
|
|
|
|
|
class Parser:
|
|
def __init__(self):
|
|
# Options that are common to all actions that use containers
|
|
containerparser = argparse.ArgumentParser(add_help=False)
|
|
containerparser.add_argument(
|
|
"target",
|
|
help="perform action on target OS",
|
|
)
|
|
containerparser.add_argument(
|
|
"--engine",
|
|
choices=["auto", "podman", "docker"],
|
|
default="auto",
|
|
help="container engine to use",
|
|
)
|
|
containerparser.add_argument(
|
|
"--login",
|
|
default=os.getlogin(), # exempt from syntax-check
|
|
help="login to use inside the container",
|
|
)
|
|
containerparser.add_argument(
|
|
"--image-prefix",
|
|
default="registry.gitlab.com/libvirt/libvirt/ci-",
|
|
help="use container images from non-default location",
|
|
)
|
|
containerparser.add_argument(
|
|
"--image-tag",
|
|
default=":latest",
|
|
help="use container images with non-default tags",
|
|
)
|
|
|
|
# Options that are common to all actions that call the
|
|
# project's build system
|
|
mesonparser = argparse.ArgumentParser(add_help=False)
|
|
mesonparser.add_argument(
|
|
"--meson-args",
|
|
default="",
|
|
help="additional arguments passed to meson "
|
|
"(eg --meson-args='-Dopt1=enabled -Dopt2=disabled')",
|
|
)
|
|
mesonparser.add_argument(
|
|
"--ninja-args",
|
|
default="",
|
|
help="additional arguments passed to ninja",
|
|
)
|
|
|
|
# Options that are common to all actions that use lcitool
|
|
lcitoolparser = argparse.ArgumentParser(add_help=False)
|
|
lcitoolparser.add_argument(
|
|
"--lcitool",
|
|
metavar="PATH",
|
|
default="lcitool",
|
|
help="path to lcitool binary",
|
|
)
|
|
|
|
# Options that are common to actions communicating with a GitLab
|
|
# instance
|
|
gitlabparser = argparse.ArgumentParser(add_help=False)
|
|
gitlabparser.add_argument(
|
|
"--namespace",
|
|
default="libvirt/libvirt",
|
|
help="GitLab project namespace"
|
|
)
|
|
gitlabparser.add_argument(
|
|
"--gitlab-uri",
|
|
default="https://gitlab.com",
|
|
help="base GitLab URI"
|
|
)
|
|
|
|
# Main parser
|
|
self._parser = argparse.ArgumentParser()
|
|
subparsers = self._parser.add_subparsers(
|
|
dest="action",
|
|
metavar="ACTION",
|
|
)
|
|
subparsers.required = True
|
|
|
|
# build action
|
|
buildparser = subparsers.add_parser(
|
|
"build",
|
|
help="run a build in a container",
|
|
parents=[containerparser, mesonparser],
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
)
|
|
buildparser.set_defaults(func=Application._action_build)
|
|
|
|
# test action
|
|
testparser = subparsers.add_parser(
|
|
"test",
|
|
help="run a build in a container (including tests)",
|
|
parents=[containerparser, mesonparser],
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
)
|
|
testparser.set_defaults(func=Application._action_test)
|
|
|
|
# shell action
|
|
shellparser = subparsers.add_parser(
|
|
"shell",
|
|
help="start a shell in a container",
|
|
parents=[containerparser],
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
)
|
|
shellparser.set_defaults(func=Application._action_shell)
|
|
|
|
# list-images action
|
|
listimagesparser = subparsers.add_parser(
|
|
"list-images",
|
|
help="list known container images",
|
|
parents=[gitlabparser],
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
)
|
|
listimagesparser.set_defaults(func=Application._action_list_images)
|
|
|
|
# refresh action
|
|
refreshparser = subparsers.add_parser(
|
|
"refresh",
|
|
help="refresh data generated with lcitool",
|
|
parents=[lcitoolparser, gitlabparser],
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
)
|
|
refreshparser.add_argument(
|
|
"--quiet",
|
|
action="store_true",
|
|
default=False,
|
|
help="refresh data silently"
|
|
)
|
|
refreshparser.add_argument(
|
|
"--check-stale",
|
|
action="store",
|
|
choices=["yes", "no"],
|
|
default="yes",
|
|
help="check for existence of stale images on the GitLab instance"
|
|
)
|
|
refreshparser.set_defaults(func=Application._action_refresh)
|
|
|
|
def parse(self):
|
|
return self._parser.parse_args()
|
|
|
|
|
|
class Application:
|
|
def __init__(self):
|
|
self._basedir = pathlib.Path(__file__).resolve().parent
|
|
self._args = Parser().parse()
|
|
|
|
if self._args.action == "refresh":
|
|
if not shutil.which(self._args.lcitool):
|
|
sys.exit("error: 'lcitool' not installed")
|
|
|
|
def _make_run(self, target):
|
|
args = [
|
|
"-C",
|
|
self._basedir,
|
|
target,
|
|
]
|
|
|
|
if self._args.action in ["build", "test", "shell"]:
|
|
args.extend([
|
|
f"CI_ENGINE={self._args.engine}",
|
|
f"CI_USER_LOGIN={self._args.login}",
|
|
f"CI_IMAGE_PREFIX={self._args.image_prefix}",
|
|
f"CI_IMAGE_TAG={self._args.image_tag}",
|
|
])
|
|
|
|
if self._args.action in ["build", "test"]:
|
|
args.extend([
|
|
f"CI_MESON_ARGS={self._args.meson_args}",
|
|
f"CI_NINJA_ARGS={self._args.ninja_args}",
|
|
])
|
|
|
|
if pty.spawn(["make"] + args) != 0:
|
|
sys.exit("error: 'make' failed")
|
|
|
|
def _lcitool_run(self, args):
|
|
output = subprocess.check_output([self._args.lcitool] + args)
|
|
return output.decode("utf-8")
|
|
|
|
def _lcitool_get_hosts(self):
|
|
output = self._lcitool_run(["hosts"])
|
|
return output.splitlines()
|
|
|
|
def _generate_dockerfile(self, host, cross=None):
|
|
args = ["dockerfile", host, "libvirt"]
|
|
outdir = self._basedir.joinpath("containers")
|
|
outfile = f"{host}.Dockerfile"
|
|
|
|
if cross:
|
|
args.extend(["--cross", cross])
|
|
outfile = f"{host}-cross-{cross}.Dockerfile"
|
|
|
|
outpath = outdir.joinpath(outfile)
|
|
if not self._args.quiet:
|
|
print(outpath)
|
|
|
|
output = self._lcitool_run(args)
|
|
with open(outpath, "w") as f:
|
|
f.write(output)
|
|
|
|
def _generate_vars(self, host):
|
|
args = ["variables", host, "libvirt"]
|
|
outdir = self._basedir.joinpath("cirrus")
|
|
outfile = f"{host}.vars"
|
|
|
|
outpath = outdir.joinpath(outfile)
|
|
if not self._args.quiet:
|
|
print(outpath)
|
|
|
|
output = self._lcitool_run(args)
|
|
with open(outpath, "w") as f:
|
|
f.write(output)
|
|
|
|
def _refresh_containers(self):
|
|
debian_cross = [
|
|
"aarch64",
|
|
"armv6l",
|
|
"armv7l",
|
|
"i686",
|
|
"mips",
|
|
"mips64el",
|
|
"mipsel",
|
|
"ppc64le",
|
|
"s390x",
|
|
]
|
|
fedora_cross = [
|
|
"mingw32",
|
|
"mingw64",
|
|
]
|
|
|
|
for host in self._lcitool_get_hosts():
|
|
if host.startswith("freebsd-") or host.startswith("macos-"):
|
|
continue
|
|
|
|
self._generate_dockerfile(host)
|
|
|
|
if host == "fedora-rawhide":
|
|
for cross in fedora_cross:
|
|
self._generate_dockerfile(host, cross)
|
|
|
|
if host.startswith("debian-"):
|
|
for cross in debian_cross:
|
|
if host == "debian-sid" and cross == "mips":
|
|
continue
|
|
self._generate_dockerfile(host, cross)
|
|
|
|
def _refresh_cirrus(self):
|
|
for host in self._lcitool_get_hosts():
|
|
if not (host.startswith("freebsd-") or host.startswith("macos-")):
|
|
continue
|
|
|
|
self._generate_vars(host)
|
|
|
|
def _check_stale_images(self):
|
|
namespace = self._args.namespace
|
|
gitlab_uri = self._args.gitlab_uri
|
|
registry_uri = util.get_registry_uri(namespace, gitlab_uri)
|
|
lcitool_hosts = self._lcitool_get_hosts()
|
|
|
|
stale_images = util.get_registry_stale_images(registry_uri,
|
|
lcitool_hosts)
|
|
if stale_images:
|
|
spacing = "\n" + 4 * " "
|
|
stale_fmt = [f"{k} (ID: {v})" for k, v in stale_images.items()]
|
|
stale_details = spacing.join(stale_fmt)
|
|
stale_ids = ' '.join([str(id) for id in stale_images.values()])
|
|
registry_uri = util.get_registry_uri(namespace, gitlab_uri)
|
|
|
|
msg = textwrap.dedent(f"""
|
|
The following images are stale and can be purged from the registry:
|
|
|
|
STALE_DETAILS
|
|
|
|
You can delete the images listed above using this shell snippet:
|
|
|
|
$ for image_id in {stale_ids}; do
|
|
curl --request DELETE --header "PRIVATE-TOKEN: <access_token>" \\
|
|
{registry_uri}/$image_id;
|
|
done
|
|
|
|
You can generate a personal access token here:
|
|
|
|
{gitlab_uri}/-/profile/personal_access_tokens
|
|
""")
|
|
print(msg.replace("STALE_DETAILS", stale_details))
|
|
|
|
def _action_build(self):
|
|
self._make_run(f"ci-build@{self._args.target}")
|
|
|
|
def _action_test(self):
|
|
self._make_run(f"ci-test@{self._args.target}")
|
|
|
|
def _action_shell(self):
|
|
self._make_run(f"ci-shell@{self._args.target}")
|
|
|
|
def _action_list_images(self):
|
|
registry_uri = util.get_registry_uri(self._args.namespace,
|
|
self._args.gitlab_uri)
|
|
images = util.get_registry_images(registry_uri)
|
|
|
|
# skip the "ci-" prefix each of our container images' name has
|
|
name_prefix = "ci-"
|
|
names = [i["name"][len(name_prefix):] for i in images]
|
|
names.sort()
|
|
|
|
native = [name for name in names if "-cross-" not in name]
|
|
cross = [name for name in names if "-cross-" in name]
|
|
|
|
spacing = 4 * " "
|
|
print("Available x86 container images:\n")
|
|
print(spacing + ("\n" + spacing).join(native))
|
|
|
|
if cross:
|
|
print()
|
|
print("Available cross-compiler container images:\n")
|
|
print(spacing + ("\n" + spacing).join(cross))
|
|
|
|
def _action_refresh(self):
|
|
self._refresh_containers()
|
|
self._refresh_cirrus()
|
|
|
|
if self._args.check_stale == "yes" and not self._args.quiet:
|
|
self._check_stale_images()
|
|
|
|
def run(self):
|
|
self._args.func(self)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
Application().run()
|