Merge pull request 'ci: smoke-test built images; build on labeled PRs, weekly and on demand' (#9) from ci/pr-label-builds into main

Reviewed-on: #9

.gitea/workflows/build-image.yaml

@@ -1,11 +1,29 @@
 name: build-image
 
-# Only a release tag builds images: a full build takes ~25 min on the runner,
+# A full build takes ~25 min on the runner and executes the checked-out code
-# too costly to repeat on every push to main. Recipes are still linted and
+# as root there, so it never runs on a plain push. Recipes are still linted
-# validated on every push by ci.yml.
+# and validated on every push and PR by ci.yml. Images are built:
+#   - on a release tag (v*.*.*): build, smoke test, publish as a release;
+#   - on a PR into main carrying the `build-image` label, i.e. once a
+#     maintainer has reviewed it (Gitea does not let authors approve their
+#     own PRs, so approval cannot be the gate). Re-runs on every new push to
+#     the PR while the label stays. Only for changes that affect images;
+#   - weekly, to catch breakage from upstream Fedora package changes;
+#   - by hand (workflow_dispatch) on any branch.
 on:
   push:
     tags: ["v*.*.*"]
+  pull_request:
+    branches: [main]
+    types: [labeled, synchronize, reopened]
+    paths:
+      - "cook/**"
+      - "build-image.sh"
+      - "smoke-test.sh"
+      - ".gitea/workflows/build-image.yaml"
+  schedule:
+    - cron: "0 3 * * 1" # Mondays 03:00
+  workflow_dispatch:
 
 # Builds raw disk images of the two default-tier editions (see
 # cook/recipes_manifest.yaml) with build-image.sh, i.e.
@@ -15,10 +33,19 @@ on:
 # The build job runs on the registered `fedora:host` runner (a Fedora VM, jobs
 # run as root), not in a container: --make-disk needs real loop devices and a
 # live systemd-udevd. The runner needs lorax-lmc-novirt, qemu-img, make,
-# python3-pip, pykickstart and xz installed.
+# python3-pip, pykickstart and xz installed, plus qemu-system-x86_64,
+# /dev/kvm and edk2-ovmf for smoke-test.sh. It must run with SELinux enabled:
+# anaconda inherits the runner's own selinux=0, if any (smoke-test.sh fails
+# such images).
 
 jobs:
   validate:
+    # On PRs, only once the `build-image` label is set (see `on:` above), and
+    # not again when some other label is added to an already-labeled PR.
+    if: >-
+      github.event_name != 'pull_request' ||
+      (contains(github.event.pull_request.labels.*.name, 'build-image') &&
+       (github.event.action != 'labeled' || github.event.label.name == 'build-image'))
     runs-on: fedora
     defaults:
       run:
@@ -68,10 +95,12 @@ jobs:
       - name: Check runner prerequisites
         run: |
           missing=0
-          for tool in livemedia-creator qemu-img make pip xz sudo; do
+          for tool in livemedia-creator qemu-img make pip xz sudo qemu-system-x86_64 debugfs; do
             command -v "$tool" >/dev/null || { echo "missing on runner: $tool"; missing=1; }
           done
           [ -e /dev/loop-control ] || { echo "missing on runner: /dev/loop-control"; missing=1; }
+          [ -e /dev/kvm ] || { echo "missing on runner: /dev/kvm"; missing=1; }
+          [ -e /usr/share/edk2/ovmf/OVMF_CODE.fd ] || { echo "missing on runner: edk2-ovmf"; missing=1; }
           [ "$missing" = 0 ] || exit 1
 
       - name: Install Python deps
@@ -85,7 +114,23 @@ jobs:
           dish="$(basename "$(ls cook/dishes/${{ matrix.edition }}_*.cfg)" .cfg)"
           echo "DISH=$dish" >> "$GITHUB_ENV"
           ./build-image.sh --dish "$dish" --root-size ${{ matrix.root_size }}
-          xz -T0 "build/$dish.img"
+
+      - name: Smoke test (static checks + boot)
+        run: ./smoke-test.sh "build/$DISH.img"
+
+      - name: Compress image
+        run: xz -T0 "build/$DISH.img"
+
+      - name: Upload image and kickstart as artifact
+        if: "!startsWith(github.ref, 'refs/tags/')"
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.edition }}
+          path: |
+            build/${{ env.DISH }}.img.xz
+            cook/dishes/${{ env.DISH }}.cfg
+          if-no-files-found: error
+          retention-days: 7
 
       - name: Publish image and kickstart as release
         if: startsWith(github.ref, 'refs/tags/')

README.md

@@ -71,7 +71,16 @@ bootable ISO:
 ```
 
 It picks a dish the same way `deploy.sh` does (or takes `--dish NAME`) and writes the result under
-`./build/`. The `build-image` CI workflow runs it for both default editions on every push to `main`, and on `v*.*.*` tags it attaches the compressed raw images (`.img.xz`) and the flattened kickstart files to the release. Requires `lorax-lmc-novirt` (heavier than `deploy.sh`'s plain
+`./build/`. `smoke-test.sh IMAGE` then checks the result: read-only static checks (no `selinux=0` on any kernel command line and labeled files when the image declares SELinux; builds on a host with SELinux disabled fail this), then a KVM boot with `-snapshot` that waits for qemu-guest-agent to answer.
+
+The `build-image` CI workflow builds, smoke-tests and compresses both default editions. It does not run on plain pushes (a build takes ~25 min and runs as root on the runner); `ci.yml` lints and validates every push instead. It runs on:
+
+* `v*.*.*` tags: attaches the compressed raw images (`.img.xz`) and the flattened kickstart files to the release;
+* PRs into `main` labeled `build-image` (set it once the PR is reviewed; it re-runs on every new push while the label stays), for changes under `cook/`, to `build-image.sh`, `smoke-test.sh` or the workflow itself;
+* a weekly schedule (Mondays 03:00), to catch breakage from upstream Fedora packages;
+* manual dispatch from the Actions tab, on any branch.
+
+Non-tag runs keep the images as 7-day workflow artifacts. Requires `lorax-lmc-novirt` (heavier than `deploy.sh`'s plain
 QEMU/libvirt prerequisites, not installed by
 `deploy/install-prerequisites-on-linux.sh`) and root — the `--no-virt` disk
 install needs real loop-device access (`/dev/loop-control`), which a

smoke-test.sh

@@ -0,0 +1,206 @@
+#!/bin/bash
+# Smoke-test a disk image produced by build-image.sh.
+#
+# Usage: ./smoke-test.sh [--timeout SECONDS] IMAGE
+#
+# 1. Static checks, read-only, without booting: if the image declares
+#    SELinux (SELINUX= in /etc/selinux/config is not "disabled"), no boot
+#    entry and not /etc/kernel/cmdline may carry selinux=0, and files must be
+#    labeled. Catches builds on a host with SELinux disabled: anaconda then
+#    inherits the host's selinux=0 and overrides the kickstart's
+#    `selinux --enforcing` (see devices/runner.md in inventory-of-devices).
+# 2. Boot test: boots the image under KVM with -snapshot (the image is never
+#    written) and no network, and waits for qemu-guest-agent to answer
+#    guest-sync. The agent only starts once userspace is up, so this proves
+#    firmware, bootloader, kernel, initramfs, root mount and systemd all work,
+#    without depending on a serial console being configured in the image.
+#
+# Requires: root (loop devices, mount), qemu-system-x86_64 with /dev/kvm,
+# edk2-ovmf (UEFI images), e2fsprogs (debugfs), python3. The image must
+# include qemu-guest-agent (the guest-agents ingredient).
+
+set -euo pipefail
+
+TIMEOUT=300
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --timeout)
+      TIMEOUT="$2"
+      shift 2
+      ;;
+    -h|--help)
+      echo "Usage: $0 [--timeout SECONDS] IMAGE"
+      exit 0
+      ;;
+    *)
+      break
+      ;;
+  esac
+done
+IMAGE="${1:?Usage: $0 [--timeout SECONDS] IMAGE}"
+[ -f "$IMAGE" ] || { echo "no such image: $IMAGE"; exit 1; }
+
+OVMF_CODE=/usr/share/edk2/ovmf/OVMF_CODE.fd
+OVMF_VARS=/usr/share/edk2/ovmf/OVMF_VARS.fd
+
+WORK="$(mktemp -d "${TMPDIR:-/tmp}/smoke-test.XXXXXX")"
+LOOP=""
+QEMU_PID=""
+cleanup() {
+  [ -n "$QEMU_PID" ] && kill "$QEMU_PID" 2>/dev/null || true
+  mountpoint -q "$WORK/esp" && umount "$WORK/esp" || true
+  [ -n "$LOOP" ] && losetup -d "$LOOP" || true
+  rm -rf "$WORK"
+}
+trap cleanup EXIT
+
+fail() { echo "FAIL: $*"; exit 1; }
+
+# --- 1. static checks ---------------------------------------------------------
+
+LOOP="$(losetup --read-only --partscan --find --show "$IMAGE")"
+udevadm settle
+# Filesystem labels come from the storage ingredients (root, boot); the ESP is
+# matched by partition type instead, as its FAT label ends up upper-cased.
+part_by_label() { lsblk -rno PATH,LABEL "$LOOP" | awk -v l="$1" '$2 == l { print $1; exit }'; }
+ROOT_PART="$(part_by_label root)"
+BOOT_PART="$(part_by_label boot)"
+ESP_PART="$(lsblk -rno PATH,PARTTYPE "$LOOP" | awk '$2 == "c12a7328-f81f-11d2-ba4b-00a0c93ec93b" { print $1; exit }')"
+[ -n "$ROOT_PART" ] || fail "no partition labeled 'root' in $IMAGE"
+
+rootcat() { debugfs -R "cat $1" "$ROOT_PART" 2>/dev/null; }
+selinux_mode="$(rootcat /etc/selinux/config | sed -n 's/^SELINUX=//p')"
+echo "SELINUX=${selinux_mode:-<unset>} in /etc/selinux/config"
+
+# Collect every kernel command line the image boots with.
+cmdlines="$WORK/cmdlines"
+: > "$cmdlines"
+{ echo "== /etc/kernel/cmdline"; rootcat /etc/kernel/cmdline; } >> "$cmdlines"
+if [ -n "$ESP_PART" ]; then
+  mkdir -p "$WORK/esp"
+  mount -o ro "$ESP_PART" "$WORK/esp"
+  for f in "$WORK"/esp/loader/entries/*.conf; do
+    [ -e "$f" ] && { echo "== ESP ${f#"$WORK"/esp/}"; grep '^options' "$f"; } >> "$cmdlines"
+  done
+fi
+if [ -n "$BOOT_PART" ]; then
+  for f in $(debugfs -R "ls /loader/entries" "$BOOT_PART" 2>/dev/null | grep -o '[^ ]*\.conf'); do
+    { echo "== /boot/loader/entries/$f"; debugfs -R "cat /loader/entries/$f" "$BOOT_PART" 2>/dev/null | grep '^options'; } >> "$cmdlines"
+  done
+fi
+cat "$cmdlines"
+
+if [ -n "$selinux_mode" ] && [ "$selinux_mode" != disabled ]; then
+  if grep -qw 'selinux=0' "$cmdlines"; then
+    fail "image declares SELINUX=$selinux_mode but boots with selinux=0 (built on a host with SELinux disabled?)"
+  fi
+  for f in /etc/shadow /usr/bin/bash; do
+    label="$(debugfs -R "ea_get $f security.selinux" "$ROOT_PART" 2>/dev/null | sed -n 's/.*= "\(.*\)\\000"$/\1/p')"
+    echo "label $f: ${label:-<none>}"
+    [ -n "$label" ] || fail "$f has no SELinux label (built on a host with SELinux disabled?)"
+  done
+fi
+echo "static checks: OK"
+
+mountpoint -q "$WORK/esp" && umount "$WORK/esp"
+losetup -d "$LOOP"
+LOOP=""
+
+# --- 2. boot test -------------------------------------------------------------
+
+firmware=()
+if [ -n "$ESP_PART" ]; then
+  cp "$OVMF_VARS" "$WORK/vars.fd"
+  firmware=(-drive "if=pflash,format=raw,readonly=on,file=$OVMF_CODE"
+            -drive "if=pflash,format=raw,file=$WORK/vars.fd")
+fi
+
+qemu-system-x86_64 \
+  -name smoke-test -machine q35,accel=kvm -cpu host -smp 2 -m 2048 \
+  "${firmware[@]}" \
+  -drive "file=$IMAGE,format=raw,if=virtio,snapshot=on" \
+  -nic none -display none -monitor none \
+  -serial "file:$WORK/serial.log" \
+  -chardev "socket,id=qga0,path=$WORK/qga.sock,server=on,wait=off" \
+  -device virtio-serial \
+  -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 &
+QEMU_PID=$!
+
+echo "booting (timeout ${TIMEOUT}s), waiting for qemu-guest-agent..."
+if python3 - "$WORK/qga.sock" "$TIMEOUT" "$QEMU_PID" <<'EOF'
+import json, os, socket, sys, time
+
+path, timeout, qemu_pid = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
+deadline = time.monotonic() + timeout
+
+def qemu_alive():
+    try:
+        os.kill(qemu_pid, 0)
+        return True
+    except ProcessLookupError:
+        return False
+
+while not os.path.exists(path):
+    if not qemu_alive():
+        sys.exit("qemu exited before creating the guest agent socket")
+    if time.monotonic() > deadline:
+        sys.exit("qemu did not create the guest agent socket within %ds" % timeout)
+    time.sleep(0.5)
+sock = socket.socket(socket.AF_UNIX)
+sock.connect(path)
+buf = b""
+sync_id = 0
+while time.monotonic() < deadline:
+    if not qemu_alive():
+        sys.exit("qemu exited before the guest agent answered")
+    # guest-sync with a fresh id flushes anything stale in the channel; the
+    # agent echoes the id back once it runs.
+    sync_id += 1
+    sock.sendall(json.dumps({"execute": "guest-sync", "arguments": {"id": sync_id}}).encode() + b"\n")
+    sock.settimeout(5)
+    try:
+        while b"\n" not in buf:
+            chunk = sock.recv(4096)
+            if not chunk:
+                break
+            buf += chunk
+    except socket.timeout:
+        continue
+    line, _, buf = buf.partition(b"\n")
+    try:
+        reply = json.loads(line)
+    except ValueError:
+        continue
+    # Replies can lag behind: syncs sent while the agent was still starting
+    # are answered in order, so accept any id we have sent.
+    ret = reply.get("return") if isinstance(reply, dict) else None
+    if isinstance(ret, int) and 1 <= ret <= sync_id:
+        # Drain whatever else is queued before asking for osinfo.
+        sock.settimeout(1)
+        try:
+            while True:
+                chunk = sock.recv(4096)
+                if not chunk:
+                    break
+        except socket.timeout:
+            pass
+        buf = b""
+        sock.sendall(b'{"execute": "guest-get-osinfo"}\n')
+        sock.settimeout(10)
+        while b"\n" not in buf:
+            buf += sock.recv(4096)
+        info = json.loads(buf.partition(b"\n")[0]).get("return", {})
+        print("guest agent answered after %ds: %s, kernel %s"
+              % (timeout - (deadline - time.monotonic()), info.get("pretty-name"), info.get("kernel-release")))
+        sys.exit(0)
+sys.exit("guest agent did not answer within %ds" % timeout)
+EOF
+then
+  echo "boot test: OK"
+else
+  echo "--- qemu-guest-agent on the serial console:"
+  grep -a 'qemu-guest-agent\|Guest Agent' "$WORK/serial.log" 2>/dev/null || echo "(no mention)"
+  echo "--- last 40 lines of the serial console:"
+  tail -40 "$WORK/serial.log" 2>/dev/null || true
+  fail "boot test"
+fi