From f7b6623b25be07547074989350247dd01247d817 Mon Sep 17 00:00:00 2001 From: Lukas Greve Date: Wed, 23 Sep 2026 20:04:32 +0200 Subject: [PATCH] ci: smoke-test built images; build on labeled PRs, weekly and on demand smoke-test.sh checks an image from build-image.sh in two steps: - static, read-only: if the image declares SELinux, no kernel command line (/etc/kernel/cmdline, ESP and /boot loader entries) may carry selinux=0 and files must be labeled. This catches images built on a host with SELinux disabled, where anaconda inherits the host's selinux=0 and overrides the kickstart's `selinux --enforcing` (CI run 237); - boot: KVM with -snapshot, no network, waits for qemu-guest-agent to answer, so it does not depend on a serial console in the image. Verified on the runner: the run-237 image fails the static check, a fresh build passes both (agent answered after 27s). build-image.yaml now smoke-tests each edition before compressing it, and besides v*.*.* tags also runs: - on PRs into main labeled `build-image` (the review gate: Gitea does not let authors approve their own PRs), re-running on new pushes while the label stays, only for image-affecting paths, and not when an unrelated label is added; - weekly (Mondays 03:00), to catch upstream Fedora breakage; - via workflow_dispatch. Non-tag runs keep the images as 7-day artifacts. README updated, including the stale "every push to main" sentence. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/build-image.yaml | 57 ++++++++- README.md | 11 +- smoke-test.sh | 206 ++++++++++++++++++++++++++++++ 3 files changed, 267 insertions(+), 7 deletions(-) create mode 100755 smoke-test.sh diff --git a/.gitea/workflows/build-image.yaml b/.gitea/workflows/build-image.yaml index 314cf75..ecd52f4 100644 --- a/.gitea/workflows/build-image.yaml +++ b/.gitea/workflows/build-image.yaml @@ -1,11 +1,29 @@ name: build-image -# Only a release tag builds images: a full build takes ~25 min on the runner, -# too costly to repeat on every push to main. Recipes are still linted and -# validated on every push by ci.yml. +# A full build takes ~25 min on the runner and executes the checked-out code +# as root there, so it never runs on a plain push. Recipes are still linted +# and validated on every push and PR by ci.yml. Images are built: +# - on a release tag (v*.*.*): build, smoke test, publish as a release; +# - on a PR into main carrying the `build-image` label, i.e. once a +# maintainer has reviewed it (Gitea does not let authors approve their +# own PRs, so approval cannot be the gate). Re-runs on every new push to +# the PR while the label stays. Only for changes that affect images; +# - weekly, to catch breakage from upstream Fedora package changes; +# - by hand (workflow_dispatch) on any branch. on: push: tags: ["v*.*.*"] + pull_request: + branches: [main] + types: [labeled, synchronize, reopened] + paths: + - "cook/**" + - "build-image.sh" + - "smoke-test.sh" + - ".gitea/workflows/build-image.yaml" + schedule: + - cron: "0 3 * * 1" # Mondays 03:00 + workflow_dispatch: # Builds raw disk images of the two default-tier editions (see # cook/recipes_manifest.yaml) with build-image.sh, i.e. @@ -15,10 +33,19 @@ on: # The build job runs on the registered `fedora:host` runner (a Fedora VM, jobs # run as root), not in a container: --make-disk needs real loop devices and a # live systemd-udevd. The runner needs lorax-lmc-novirt, qemu-img, make, -# python3-pip, pykickstart and xz installed. +# python3-pip, pykickstart and xz installed, plus qemu-system-x86_64, +# /dev/kvm and edk2-ovmf for smoke-test.sh. It must run with SELinux enabled: +# anaconda inherits the runner's own selinux=0, if any (smoke-test.sh fails +# such images). jobs: validate: + # On PRs, only once the `build-image` label is set (see `on:` above), and + # not again when some other label is added to an already-labeled PR. + if: >- + github.event_name != 'pull_request' || + (contains(github.event.pull_request.labels.*.name, 'build-image') && + (github.event.action != 'labeled' || github.event.label.name == 'build-image')) runs-on: fedora defaults: run: @@ -68,10 +95,12 @@ jobs: - name: Check runner prerequisites run: | missing=0 - for tool in livemedia-creator qemu-img make pip xz sudo; do + for tool in livemedia-creator qemu-img make pip xz sudo qemu-system-x86_64 debugfs; do command -v "$tool" >/dev/null || { echo "missing on runner: $tool"; missing=1; } done [ -e /dev/loop-control ] || { echo "missing on runner: /dev/loop-control"; missing=1; } + [ -e /dev/kvm ] || { echo "missing on runner: /dev/kvm"; missing=1; } + [ -e /usr/share/edk2/ovmf/OVMF_CODE.fd ] || { echo "missing on runner: edk2-ovmf"; missing=1; } [ "$missing" = 0 ] || exit 1 - name: Install Python deps @@ -85,7 +114,23 @@ jobs: dish="$(basename "$(ls cook/dishes/${{ matrix.edition }}_*.cfg)" .cfg)" echo "DISH=$dish" >> "$GITHUB_ENV" ./build-image.sh --dish "$dish" --root-size ${{ matrix.root_size }} - xz -T0 "build/$dish.img" + + - name: Smoke test (static checks + boot) + run: ./smoke-test.sh "build/$DISH.img" + + - name: Compress image + run: xz -T0 "build/$DISH.img" + + - name: Upload image and kickstart as artifact + if: "!startsWith(github.ref, 'refs/tags/')" + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.edition }} + path: | + build/${{ env.DISH }}.img.xz + cook/dishes/${{ env.DISH }}.cfg + if-no-files-found: error + retention-days: 7 - name: Publish image and kickstart as release if: startsWith(github.ref, 'refs/tags/') diff --git a/README.md b/README.md index 350e216..92b8648 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,16 @@ bootable ISO: ``` It picks a dish the same way `deploy.sh` does (or takes `--dish NAME`) and writes the result under -`./build/`. The `build-image` CI workflow runs it for both default editions on every push to `main`, and on `v*.*.*` tags it attaches the compressed raw images (`.img.xz`) and the flattened kickstart files to the release. Requires `lorax-lmc-novirt` (heavier than `deploy.sh`'s plain +`./build/`. `smoke-test.sh IMAGE` then checks the result: read-only static checks (no `selinux=0` on any kernel command line and labeled files when the image declares SELinux; builds on a host with SELinux disabled fail this), then a KVM boot with `-snapshot` that waits for qemu-guest-agent to answer. + +The `build-image` CI workflow builds, smoke-tests and compresses both default editions. It does not run on plain pushes (a build takes ~25 min and runs as root on the runner); `ci.yml` lints and validates every push instead. It runs on: + +* `v*.*.*` tags: attaches the compressed raw images (`.img.xz`) and the flattened kickstart files to the release; +* PRs into `main` labeled `build-image` (set it once the PR is reviewed; it re-runs on every new push while the label stays), for changes under `cook/`, to `build-image.sh`, `smoke-test.sh` or the workflow itself; +* a weekly schedule (Mondays 03:00), to catch breakage from upstream Fedora packages; +* manual dispatch from the Actions tab, on any branch. + +Non-tag runs keep the images as 7-day workflow artifacts. Requires `lorax-lmc-novirt` (heavier than `deploy.sh`'s plain QEMU/libvirt prerequisites, not installed by `deploy/install-prerequisites-on-linux.sh`) and root — the `--no-virt` disk install needs real loop-device access (`/dev/loop-control`), which a diff --git a/smoke-test.sh b/smoke-test.sh new file mode 100755 index 0000000..50141d4 --- /dev/null +++ b/smoke-test.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# Smoke-test a disk image produced by build-image.sh. +# +# Usage: ./smoke-test.sh [--timeout SECONDS] IMAGE +# +# 1. Static checks, read-only, without booting: if the image declares +# SELinux (SELINUX= in /etc/selinux/config is not "disabled"), no boot +# entry and not /etc/kernel/cmdline may carry selinux=0, and files must be +# labeled. Catches builds on a host with SELinux disabled: anaconda then +# inherits the host's selinux=0 and overrides the kickstart's +# `selinux --enforcing` (see devices/runner.md in inventory-of-devices). +# 2. Boot test: boots the image under KVM with -snapshot (the image is never +# written) and no network, and waits for qemu-guest-agent to answer +# guest-sync. The agent only starts once userspace is up, so this proves +# firmware, bootloader, kernel, initramfs, root mount and systemd all work, +# without depending on a serial console being configured in the image. +# +# Requires: root (loop devices, mount), qemu-system-x86_64 with /dev/kvm, +# edk2-ovmf (UEFI images), e2fsprogs (debugfs), python3. The image must +# include qemu-guest-agent (the guest-agents ingredient). + +set -euo pipefail + +TIMEOUT=300 +while [[ $# -gt 0 ]]; do + case "$1" in + --timeout) + TIMEOUT="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [--timeout SECONDS] IMAGE" + exit 0 + ;; + *) + break + ;; + esac +done +IMAGE="${1:?Usage: $0 [--timeout SECONDS] IMAGE}" +[ -f "$IMAGE" ] || { echo "no such image: $IMAGE"; exit 1; } + +OVMF_CODE=/usr/share/edk2/ovmf/OVMF_CODE.fd +OVMF_VARS=/usr/share/edk2/ovmf/OVMF_VARS.fd + +WORK="$(mktemp -d "${TMPDIR:-/tmp}/smoke-test.XXXXXX")" +LOOP="" +QEMU_PID="" +cleanup() { + [ -n "$QEMU_PID" ] && kill "$QEMU_PID" 2>/dev/null || true + mountpoint -q "$WORK/esp" && umount "$WORK/esp" || true + [ -n "$LOOP" ] && losetup -d "$LOOP" || true + rm -rf "$WORK" +} +trap cleanup EXIT + +fail() { echo "FAIL: $*"; exit 1; } + +# --- 1. static checks --------------------------------------------------------- + +LOOP="$(losetup --read-only --partscan --find --show "$IMAGE")" +udevadm settle +# Filesystem labels come from the storage ingredients (root, boot); the ESP is +# matched by partition type instead, as its FAT label ends up upper-cased. +part_by_label() { lsblk -rno PATH,LABEL "$LOOP" | awk -v l="$1" '$2 == l { print $1; exit }'; } +ROOT_PART="$(part_by_label root)" +BOOT_PART="$(part_by_label boot)" +ESP_PART="$(lsblk -rno PATH,PARTTYPE "$LOOP" | awk '$2 == "c12a7328-f81f-11d2-ba4b-00a0c93ec93b" { print $1; exit }')" +[ -n "$ROOT_PART" ] || fail "no partition labeled 'root' in $IMAGE" + +rootcat() { debugfs -R "cat $1" "$ROOT_PART" 2>/dev/null; } +selinux_mode="$(rootcat /etc/selinux/config | sed -n 's/^SELINUX=//p')" +echo "SELINUX=${selinux_mode:-} in /etc/selinux/config" + +# Collect every kernel command line the image boots with. +cmdlines="$WORK/cmdlines" +: > "$cmdlines" +{ echo "== /etc/kernel/cmdline"; rootcat /etc/kernel/cmdline; } >> "$cmdlines" +if [ -n "$ESP_PART" ]; then + mkdir -p "$WORK/esp" + mount -o ro "$ESP_PART" "$WORK/esp" + for f in "$WORK"/esp/loader/entries/*.conf; do + [ -e "$f" ] && { echo "== ESP ${f#"$WORK"/esp/}"; grep '^options' "$f"; } >> "$cmdlines" + done +fi +if [ -n "$BOOT_PART" ]; then + for f in $(debugfs -R "ls /loader/entries" "$BOOT_PART" 2>/dev/null | grep -o '[^ ]*\.conf'); do + { echo "== /boot/loader/entries/$f"; debugfs -R "cat /loader/entries/$f" "$BOOT_PART" 2>/dev/null | grep '^options'; } >> "$cmdlines" + done +fi +cat "$cmdlines" + +if [ -n "$selinux_mode" ] && [ "$selinux_mode" != disabled ]; then + if grep -qw 'selinux=0' "$cmdlines"; then + fail "image declares SELINUX=$selinux_mode but boots with selinux=0 (built on a host with SELinux disabled?)" + fi + for f in /etc/shadow /usr/bin/bash; do + label="$(debugfs -R "ea_get $f security.selinux" "$ROOT_PART" 2>/dev/null | sed -n 's/.*= "\(.*\)\\000"$/\1/p')" + echo "label $f: ${label:-}" + [ -n "$label" ] || fail "$f has no SELinux label (built on a host with SELinux disabled?)" + done +fi +echo "static checks: OK" + +mountpoint -q "$WORK/esp" && umount "$WORK/esp" +losetup -d "$LOOP" +LOOP="" + +# --- 2. boot test ------------------------------------------------------------- + +firmware=() +if [ -n "$ESP_PART" ]; then + cp "$OVMF_VARS" "$WORK/vars.fd" + firmware=(-drive "if=pflash,format=raw,readonly=on,file=$OVMF_CODE" + -drive "if=pflash,format=raw,file=$WORK/vars.fd") +fi + +qemu-system-x86_64 \ + -name smoke-test -machine q35,accel=kvm -cpu host -smp 2 -m 2048 \ + "${firmware[@]}" \ + -drive "file=$IMAGE,format=raw,if=virtio,snapshot=on" \ + -nic none -display none -monitor none \ + -serial "file:$WORK/serial.log" \ + -chardev "socket,id=qga0,path=$WORK/qga.sock,server=on,wait=off" \ + -device virtio-serial \ + -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 & +QEMU_PID=$! + +echo "booting (timeout ${TIMEOUT}s), waiting for qemu-guest-agent..." +if python3 - "$WORK/qga.sock" "$TIMEOUT" "$QEMU_PID" <<'EOF' +import json, os, socket, sys, time + +path, timeout, qemu_pid = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]) +deadline = time.monotonic() + timeout + +def qemu_alive(): + try: + os.kill(qemu_pid, 0) + return True + except ProcessLookupError: + return False + +while not os.path.exists(path): + if not qemu_alive(): + sys.exit("qemu exited before creating the guest agent socket") + if time.monotonic() > deadline: + sys.exit("qemu did not create the guest agent socket within %ds" % timeout) + time.sleep(0.5) +sock = socket.socket(socket.AF_UNIX) +sock.connect(path) +buf = b"" +sync_id = 0 +while time.monotonic() < deadline: + if not qemu_alive(): + sys.exit("qemu exited before the guest agent answered") + # guest-sync with a fresh id flushes anything stale in the channel; the + # agent echoes the id back once it runs. + sync_id += 1 + sock.sendall(json.dumps({"execute": "guest-sync", "arguments": {"id": sync_id}}).encode() + b"\n") + sock.settimeout(5) + try: + while b"\n" not in buf: + chunk = sock.recv(4096) + if not chunk: + break + buf += chunk + except socket.timeout: + continue + line, _, buf = buf.partition(b"\n") + try: + reply = json.loads(line) + except ValueError: + continue + # Replies can lag behind: syncs sent while the agent was still starting + # are answered in order, so accept any id we have sent. + ret = reply.get("return") if isinstance(reply, dict) else None + if isinstance(ret, int) and 1 <= ret <= sync_id: + # Drain whatever else is queued before asking for osinfo. + sock.settimeout(1) + try: + while True: + chunk = sock.recv(4096) + if not chunk: + break + except socket.timeout: + pass + buf = b"" + sock.sendall(b'{"execute": "guest-get-osinfo"}\n') + sock.settimeout(10) + while b"\n" not in buf: + buf += sock.recv(4096) + info = json.loads(buf.partition(b"\n")[0]).get("return", {}) + print("guest agent answered after %ds: %s, kernel %s" + % (timeout - (deadline - time.monotonic()), info.get("pretty-name"), info.get("kernel-release"))) + sys.exit(0) +sys.exit("guest agent did not answer within %ds" % timeout) +EOF +then + echo "boot test: OK" +else + echo "--- qemu-guest-agent on the serial console:" + grep -a 'qemu-guest-agent\|Guest Agent' "$WORK/serial.log" 2>/dev/null || echo "(no mention)" + echo "--- last 40 lines of the serial console:" + tail -40 "$WORK/serial.log" 2>/dev/null || true + fail "boot test" +fi