libvirt/ci/helper
Erik Skultety 04b9118fe5 ci: helper: Rework _lcitool_run method logic
This method wasn't even utilized before this patch. This patch adds all
the necessary logic to successfully execute a container workload via
lcitool (which will later allow us to ditch ci/Makefile). Because
container executions via lcitool creates the following inside the
container:

    $ ls
    script datadir

where 'datadir' is the workload directory (in this case a local git
repo clone) and 'script' is the code that runs whatever the workload is
over 'datadir'.

In order to satisfy the ^above, our helper generates a trivial
temporary 'script' that will source ci/build.sh and run whatever was
specified as --job essentially to simulate the exact steps a GitLab
pipeline job would go through.

Signed-off-by: Erik Skultety <eskultet@redhat.com>
Reviewed-by: Daniel P. Berrangé <berrange@redhat.com>
2023-09-12 11:36:03 +02:00

329 lines
10 KiB
Python
Executable File

#!/usr/bin/env python3
#
# Copyright (C) 2021 Red Hat, Inc.
# SPDX-License-Identifier: LGPL-2.1-or-later
import argparse
import os
import pathlib
import pty
import subprocess
import sys
import textwrap
from pathlib import Path
from tempfile import TemporaryDirectory
import util
def required_deps(*deps):
module2pkg = {
"git": "GitPython"
}
def inner_decorator(func):
def wrapped(*args, **kwargs):
cmd = func.__name__[len('_action_'):]
for dep in deps:
try:
import importlib
importlib.import_module(dep)
except ImportError:
pkg = module2pkg[dep]
msg = f"'{pkg}' not found (required by the '{cmd}' command)"
print(msg, file=sys.stderr)
sys.exit(1)
func(*args, **kwargs)
return wrapped
return inner_decorator
class Parser:
def __init__(self):
# Options that are common to all actions that use containers
containerparser = argparse.ArgumentParser(add_help=False)
containerparser.add_argument(
"target",
help="perform action on target OS",
)
containerparser.add_argument(
"--engine",
choices=["auto", "podman", "docker"],
default="auto",
help="container engine to use",
)
containerparser.add_argument(
"--login",
default=os.getlogin(), # exempt from syntax-check
help="login to use inside the container",
)
containerparser.add_argument(
"--image-prefix",
default="registry.gitlab.com/libvirt/libvirt/ci-",
help="use container images from non-default location",
)
containerparser.add_argument(
"--image-tag",
default="latest",
help="use container images with non-default tags",
)
containerparser.add_argument(
"--lcitool-path",
dest="lcitool",
default="lcitool",
help="path to lcitool (default: $PATH)",
)
# Options that are common to all actions that call the
# project's build system
mesonparser = argparse.ArgumentParser(add_help=False)
mesonparser.add_argument(
"--meson-args",
default="",
help="additional arguments passed to meson "
"(eg --meson-args='-Dopt1=enabled -Dopt2=disabled')",
)
mesonparser.add_argument(
"--ninja-args",
default="",
help="additional arguments passed to ninja",
)
# Options that are common to actions communicating with a GitLab
# instance
gitlabparser = argparse.ArgumentParser(add_help=False)
gitlabparser.add_argument(
"--namespace",
default="libvirt/libvirt",
help="GitLab project namespace"
)
gitlabparser.add_argument(
"--gitlab-uri",
default="https://gitlab.com",
help="base GitLab URI"
)
# Main parser
self._parser = argparse.ArgumentParser()
subparsers = self._parser.add_subparsers(
dest="action",
metavar="ACTION",
)
subparsers.required = True
# build action
buildparser = subparsers.add_parser(
"build",
help="run a build in a container",
parents=[containerparser, mesonparser],
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
buildparser.set_defaults(func=Application._action_build)
# test action
testparser = subparsers.add_parser(
"test",
help="run a build in a container (including tests)",
parents=[containerparser, mesonparser],
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
testparser.set_defaults(func=Application._action_test)
# shell action
shellparser = subparsers.add_parser(
"shell",
help="start a shell in a container",
parents=[containerparser],
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
shellparser.set_defaults(func=Application._action_shell)
# list-images action
listimagesparser = subparsers.add_parser(
"list-images",
help="list known container images",
parents=[gitlabparser],
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
listimagesparser.set_defaults(func=Application._action_list_images)
# check_stale action
check_staleparser = subparsers.add_parser(
"check-stale",
help="check for existence of stale images on the GitLab instance",
parents=[gitlabparser],
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
check_staleparser.set_defaults(func=Application._action_check_stale)
def parse(self):
return self._parser.parse_args()
class Application:
@property
def repo(self):
if self._repo is None:
from git import Repo
self._repo = Repo(search_parent_directories=True)
return self._repo
def __init__(self):
self._basedir = pathlib.Path(__file__).resolve().parent
self._args = Parser().parse()
self._repo = None
def _make_run(self, target):
args = [
"-C",
self._basedir,
target,
]
if self._args.action in ["build", "test", "shell"]:
args.extend([
f"CI_ENGINE={self._args.engine}",
f"CI_USER_LOGIN={self._args.login}",
f"CI_IMAGE_PREFIX={self._args.image_prefix}",
f"CI_IMAGE_TAG={self._args.image_tag}",
])
if self._args.action in ["build", "test"]:
args.extend([
f"MESON_ARGS={self._args.meson_args}",
f"NINJA_ARGS={self._args.ninja_args}",
])
if pty.spawn(["make"] + args) != 0:
sys.exit("error: 'make' failed")
@staticmethod
def _prepare_repo_copy(repo, dest):
return repo.clone(dest, local=True)
def _lcitool_run(self, args):
positional_args = ["container"]
opts = ["--user", self._args.login]
tmpdir = TemporaryDirectory(prefix="scratch",
dir=Path(self.repo.working_dir, "ci"))
repo_dest_path = Path(tmpdir.name, "libvirt.git").as_posix()
repo_clone = self._prepare_repo_copy(self.repo, repo_dest_path)
opts.extend(["--workload-dir", repo_clone.working_dir])
if self._args.job == "shell":
positional_args.append("shell")
else:
job2func = {
"test": "run_test",
"build": "run_build",
"codestyle": "run_codestyle",
"potfile": "run_potfile",
"rpmbuild": "run_rpmbuild",
"website": "run_website_build",
}
if self._args.engine != "auto":
positional_args.extend(["--engine", self._args.engine])
with open(Path(tmpdir.name, "script"), "w") as f:
script_path = f.name
contents = textwrap.dedent(f"""\
#!/bin/sh
cd datadir
. ci/jobs.sh
{job2func[self._args.job]}
""")
f.write(contents)
positional_args.append("run")
opts.extend(["--script", script_path])
opts.append(f"{self._args.image_prefix}{self._args.target}:{self._args.image_tag}")
proc = None
try:
proc = subprocess.run([self._args.lcitool] + positional_args + opts)
except KeyboardInterrupt:
sys.exit(1)
finally:
# this will take care of the generated script file above as well
tmpdir.cleanup()
return proc.returncode
def _check_stale_images(self):
namespace = self._args.namespace
gitlab_uri = self._args.gitlab_uri
registry_uri = util.get_registry_uri(namespace, gitlab_uri)
stale_images = util.get_registry_stale_images(registry_uri, self._basedir)
if stale_images:
spacing = "\n" + 4 * " "
stale_fmt = [f"{k} (ID: {v})" for k, v in stale_images.items()]
stale_details = spacing.join(stale_fmt)
stale_ids = ' '.join([str(id) for id in stale_images.values()])
registry_uri = util.get_registry_uri(namespace, gitlab_uri)
msg = textwrap.dedent(f"""
The following images are stale and can be purged from the registry:
STALE_DETAILS
You can delete the images listed above using this shell snippet:
$ for image_id in {stale_ids}; do
curl --request DELETE --header "PRIVATE-TOKEN: <access_token>" \\
{registry_uri}/$image_id;
done
You can generate a personal access token here:
{gitlab_uri}/-/profile/personal_access_tokens
""")
print(msg.replace("STALE_DETAILS", stale_details))
def _action_build(self):
self._make_run(f"ci-build@{self._args.target}")
def _action_test(self):
self._make_run(f"ci-test@{self._args.target}")
def _action_shell(self):
self._make_run(f"ci-shell@{self._args.target}")
def _action_list_images(self):
registry_uri = util.get_registry_uri(self._args.namespace,
self._args.gitlab_uri)
images = util.get_registry_images(registry_uri)
# skip the "ci-" prefix each of our container images' name has
name_prefix = "ci-"
names = [i["name"][len(name_prefix):] for i in images]
names.sort()
native = [name for name in names if "-cross-" not in name]
cross = [name for name in names if "-cross-" in name]
spacing = 4 * " "
print("Available x86 container images:\n")
print(spacing + ("\n" + spacing).join(native))
if cross:
print()
print("Available cross-compiler container images:\n")
print(spacing + ("\n" + spacing).join(cross))
def _action_check_stale(self):
self._check_stale_images()
def run(self):
self._args.func(self)
if __name__ == "__main__":
Application().run()