roots/phyllomeos
1
0
Fork 1
Code Pull Requests Activity

refactor(cook): data-driven recipe generation with in-process flatten and lint

Browse Source 
Replace the hardcoded TEMPLATES dict and the ksflatten-relative wrapper
with a single data-driven pipeline:

- recipe_templates.yaml becomes the single source of truth with three
  sections: base (always included), choices (exactly-one per category,
  the invariant pykickstart cannot check) and features (additive flags
  or one-of values)
- generate_recipe.py is a pure YAML consumer: it resolves ingredients,
  renders dishes via pykickstart in-process (no ksflatten binary, no
  %include path munging), lints the invariants pykickstart cannot check
  (exactly-one per choice category, known keys, existing fragments,
  unique filenames) and validates every generated dish
- dish names are derived by a generic rule with canonical category order
  from the templates, independent of YAML key order; no committed
  generated files means no name burn-in
- recipes/ and dishes/ become gitignored build products; the 68 tracked
  generated files (already out of sync with the manifest) are removed
- delete dead ingredients (section-data/, validation/, initial-setup/
  desktop/, live/hypervisor.ks, rpmfusion-nonfree.ks), the stale
  all-ingredients.cfg (replaced by 'make inventory') and the
  bin/generate-recipe and bin/ksflatten-relative wrappers
- Makefile: single 'make all' step plus lint, validate, inventory, test
- add 16 pytest tests covering expansion, selection, naming, lint and
  flatten/validate round-trips
This commit is contained in:
Lukas Greve
2026-08-30 12:09:42 +02:00
parent 842f754681
commit ae6eef4045
87 changed files with 686 additions and 6127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cook/generate_recipe.py
+308 -342
View File
@@ -1,7 +1,19 @@
#!/usr/bin/env python3
"""
Recipe generator for Phyllome OS.
Reads recipes_manifest.yaml and generates kickstart recipe files.
Recipe and dish generator for Phyllome OS.
Reads recipes_manifest.yaml (the variant matrix) and recipe_templates.yaml
(the ingredient wiring) to:
1. compose kickstart "recipes" (%include lists) under recipes/,
2. flatten them into standalone "dishes" under dishes/ using pykickstart
in-process (no external ksflatten binary), and
3. lint the invariants pykickstart cannot check (exactly-one per choice
category, known keys, existing fragments, unique filenames) and
validate every generated dish.
Everything is derived from YAML; ingredient names, order and filenames are
pure functions of the manifest and templates.
"""
import argparse
@@ -11,103 +23,30 @@ import sys
import yaml
from pykickstart.version import DEVEL, returnClassForVersion
from pykickstart.parser import KickstartParser
# Template mappings from recipe_templates.yaml
TEMPLATES = {
'required': {
'repository': {
'43': 'repo/fedora-43-mirrors.ks',
'rawhide': 'repo/rawhide-mirrors.ks',
},
'core': {
'base': 'core/base.ks',
},
'bootloader': {
'grub': 'bootloader/grub.ks',
'systemd-boot': 'bootloader/systemd-boot.ks',
},
'storage': {
'standard': 'storage/standard.ks',
'encrypted': 'storage/encrypted.ks',
},
'locale': {
'default': 'core/locale.ks',
},
'services': {
'minimal': 'core/services.ks',
},
'network': {
'network-manager': 'core/network.ks',
},
'packages': {
'core': 'packages/core.ks',
},
'brand': {
'fedora-remix': 'packages/fedora-remix.ks',
},
'extra': {
'extra': 'packages/hand-picked.ks',
},
'security': {
'secure': 'core/security/enabled.ks',
'insecure': 'core/security/disabled.ks',
},
'initial-setup': {
'server': 'initial-setup/server/config.ks',
'gnome': 'initial-setup/gnome/config.ks',
'generic-wayland': 'initial-setup/generic-wayland/config.ks',
},
},
'optional': {
'hardware-support': 'packages/hardware-support.ks',
'guest-agents': 'guest-agents/base.ks',
'hypervisor': {
'base': [
'hypervisor/base/packages.ks',
'hypervisor/base/services.ks',
'hypervisor/base/post-scripts.ks',
],
'desktop': [
'packages/virtual-machine-manager/packages.ks',
'packages/virtual-machine-manager/post-scripts.ks',
],
},
'desktop': {
'gnome': [
'desktop/gnome/config.ks',
'desktop/gnome/packages.ks',
'desktop/gnome/post-scripts.ks',
],
'labwc': [
'desktop/labwc/config.ks',
],
},
'hypervisor_type': {
'amdcpu': 'hypervisor/amdcpu.ks',
'intelcpu': 'hypervisor/intelcpu.ks',
'intelgpu': 'hypervisor/intelgpu.ks',
},
},
'live': {
'core': [
'live/core/base.ks',
'live/core/storage.ks',
'live/core/packages.ks',
],
'bootloader': {
'grub': 'live/core/bootloader/grub.ks',
'systemd-boot': 'live/core/bootloader/systemd-boot.ks',
},
'post': [
'live/post/base.ks',
'live/post/session.ks',
],
},
}
HEADER = """# __ ____ ____ _____
# ____ / /_ __ __/ / /___ ____ ___ ___ / __ \\/ ___/
# / __ \\/ __ \\/ / / / / / __ \\/ __ `__ \\/ _ \\ / / / /\\__ \\
# / /_/ / / / / /_/ / / / /_/ / / / / / / __/ / /_/ /___/ /
# / .___/_/ /_/\\__, /_/_/\\____/_/ /_/ /_/\\___/ \\____//____/
# /_/ /____/
"""
def load_yaml(path):
with open(path) as f:
return yaml.safe_load(f)
def expand_variants(variant_config):
"""Expand variant config with list values into cartesian product."""
"""Expand a variant config into its cartesian product.
Keys holding a list of values expand to one variant per combination;
scalar and boolean keys are fixed across every combination.
"""
keys = variant_config.keys()
values = []
for key in keys:
@@ -116,268 +55,295 @@ def expand_variants(variant_config):
values.append([(key, v) for v in val])
else:
values.append([(key, val)])
combos = itertools.product(*values)
return [dict(combo) for combo in combos]
def get_required_ingredients(variant):
"""Get required ingredients for a variant."""
ingredients = []
# Build variant dict with special mappings
variant_map = dict(variant)
for category, mapping in TEMPLATES['required'].items():
key = variant_map.get(category, list(mapping.keys())[0])
if category == 'version':
key = str(key)
path = mapping.get(key)
if path:
ingredients.append(path)
return ingredients
def default_choice(choices):
"""Return the first declared value of a choices category."""
return next(iter(choices))
def get_optional_ingredients(variant):
"""Get optional ingredients based on variant flags."""
ingredients = []
# Handle boolean flags
for opt_key in ['hardware-support', 'guest-agents']:
if variant.get(opt_key):
path = TEMPLATES['optional'].get(opt_key)
if path:
ingredients.append(path)
# Handle live flag
if variant.get('live'):
ingredients.extend(TEMPLATES['live']['core'])
bootloader = variant.get('bootloader', 'grub')
ingredients.append(TEMPLATES['live']['bootloader'][bootloader])
ingredients.extend(TEMPLATES['live']['post'])
# Handle desktop
desktop = variant.get('desktop')
if desktop:
desktop_map = TEMPLATES['optional']['desktop']
if desktop in desktop_map:
ingredients.extend(desktop_map[desktop])
# Handle hypervisor
hypervisor = variant.get('hypervisor')
if hypervisor:
hypervisor_map = TEMPLATES['optional']['hypervisor']
if hypervisor in hypervisor_map:
ingredients.extend(hypervisor_map[hypervisor])
# Handle hypervisor_type
hypervisor_type = variant.get('hypervisor_type')
if hypervisor_type:
path = TEMPLATES['optional']['hypervisor_type'].get(hypervisor_type)
if path:
ingredients.append(path)
return ingredients
def collect_fragments(templates, variant):
"""Resolve a variant to the ordered list of ingredient fragments.
Returns (fragments, problems) where problems is a list of human-readable
lint messages. Choices default to their first declared value when the
variant omits them.
"""
fragments = list(templates.get('base', []))
problems = []
def generate_filename(variant, group_name=None):
"""Generate filename from variant configuration."""
parts = []
for category, mapping in templates.get('choices', {}).items():
value = variant.get(category)
if value is None:
value = default_choice(mapping)
path = mapping.get(value)
if path is None:
problems.append(
"choice '%s': unknown value '%s' (expected one of %s)"
% (category, value, ', '.join(sorted(mapping))))
else:
fragments.append(path)
# Guest agents
if variant.get('guest-agents'):
parts.append('virtual')
# Desktop
desktop = variant.get('desktop')
if desktop:
parts.append(desktop)
# Hypervisor type
if variant.get('hypervisor'):
parts.append(f"hypervisor-{variant['hypervisor']}")
if variant.get('hypervisor_type'):
parts.append(variant['hypervisor_type'])
# Storage
if variant.get('storage') == 'encrypted':
parts.append('encrypted')
# Bootloader
if variant.get('bootloader') == 'systemd-boot':
parts.append('systemd-boot')
# Hardware support
if variant.get('hardware-support'):
parts.append('hardware-support')
# Security (only if not default)
if variant.get('security') == 'disabled':
parts.append('security-disabled')
# Add group name if specified (for uniqueness)
if group_name and group_name in ['desktop-live', 'server-live']:
parts.append(group_name.replace('live', ''))
# Version
parts.append(str(variant.get('repository', '43')))
return '_'.join(parts) + '.cfg'
def write_recipe_file(ingredients, variant, output_dir, group_name=None):
"""Write recipe file with header and includes."""
os.makedirs(output_dir, exist_ok=True)
filename = generate_filename(variant, group_name)
filepath = os.path.join(output_dir, filename)
header = f"""# __ ____ ____ _____
# ____ / /_ __ __/ / /___ ____ ___ ___ / __ \\/ ___/
# / __ \\/ __ \\/ / / / / / __ \\/ __ `__ \\/ _ \\ / / / /\\__ \\
# / /_/ / / / / /_/ / / / /_/ / / / / / / __/ / /_/ /___/ /
# / .___/_/ /_/\\__, /_/_/\\____/_/ /_/ /_/\\___/ \\____//____/
# /_/ /____/
# Universal template to generate kickstart recipes for a server, a desktop or a hypervisor system
"""
# Use ingredient paths as-is (simple paths relative to project root)
# ksflatten-relative will handle conversion during flattening
seen = set()
unique_ingredients = []
for ing in ingredients:
if ing not in seen:
seen.add(ing)
unique_ingredients.append(ing)
includes = '\n'.join(f'%include {ing}' for ing in unique_ingredients)
content = header + includes + '\n'
with open(filepath, 'w') as f:
f.write(content)
return filepath
def generate_recipes(manifest_path, output_dir):
"""Generate all recipes from manifest."""
with open(manifest_path) as f:
manifest = yaml.safe_load(f)
generated = []
for recipe_group in manifest.get('recipes', []):
group_name = recipe_group.get('name', 'unknown')
for variant_config in recipe_group.get('variants', []):
for variant in expand_variants(variant_config):
required = get_required_ingredients(variant)
optional = get_optional_ingredients(variant)
all_ingredients = required + optional
filepath = write_recipe_file(all_ingredients, variant, output_dir, group_name)
generated.append(filepath)
return generated
def validate_recipe(recipe_path):
"""Validate a recipe file using pykickstart."""
try:
from pykickstart.parser import KickstartParser
from pykickstart.version import returnClassForVersion
ks_class = returnClassForVersion()
parser = KickstartParser(ks_class)
# Get absolute path and change to that directory for include resolution
abs_path = os.path.abspath(recipe_path)
recipe_dir = os.path.dirname(abs_path)
original_dir = os.getcwd()
os.chdir(recipe_dir)
parser.readKickstart(abs_path)
os.chdir(original_dir)
return True, None
except Exception as e:
os.chdir(original_dir)
return False, str(e)
def validate_all(recipes_dir):
"""Validate all recipes in directory."""
if not os.path.isdir(recipes_dir):
print(f"Error: {recipes_dir} is not a directory", file=sys.stderr)
return False
errors = []
for filename in sorted(os.listdir(recipes_dir)):
if filename.endswith('.cfg'):
filepath = os.path.join(recipes_dir, filename)
valid, error = validate_recipe(filepath)
if valid:
print(f"✓ {filename}")
for feature, body in templates.get('features', {}).items():
value = variant.get(feature)
if value is None or isinstance(value, bool) and not value:
continue
if isinstance(body, str):
fragments.append(body)
else:
rendered = body.get(value)
if rendered is None:
problems.append(
"feature '%s': unknown value '%s' (expected one of %s)"
% (feature, value, ', '.join(sorted(body))))
elif isinstance(rendered, list):
fragments.extend(rendered)
else:
print(f"✗ {filename}: {error}", file=sys.stderr)
errors.append((filename, error))
return len(errors) == 0
fragments.append(rendered)
fragments = list(dict.fromkeys(fragments))
return fragments, problems
def main():
parser = argparse.ArgumentParser(description='Generate Phyllome OS recipes')
def render_filename(templates, group_name, variant):
"""Build a deterministic, collision-free dish name for a variant.
Tokens follow the canonical category order from the templates (choices
first, then features), independent of the key order the manifest uses, so
reordering the manifest -- or YAML tools that sort keys -- never renames
dishes. Boolean True renders the key itself, boolean False is omitted,
other values render str(value). The group name prefixes every dish,
making names unique across groups.
"""
order = list(templates.get('choices', {})) + list(templates.get('features', {}))
tokens = []
for key in order:
if key not in variant:
continue
value = variant[key]
if isinstance(value, bool):
if value:
tokens.append(key)
else:
tokens.append(str(value))
return "%s_%s.cfg" % (group_name, '_'.join(tokens))
def render_recipe(fragments):
"""Render a recipe (%include list) for the given ingredient fragments."""
includes = '\n'.join(f'%include ../ingredients/{slug}' for slug in fragments)
return HEADER + includes + '\n'
def lint_manifest(manifest, templates, ingredients_root):
"""Return a list of lint problems across the whole manifest."""
problems = []
group_names = set()
for group in manifest.get('recipes', []):
group_name = group.get('name', 'unknown')
if group_name in group_names:
problems.append("duplicate group name '%s'" % group_name)
group_names.add(group_name)
for config in group.get('variants', []):
for variant in expand_variants(config):
known = set(templates.get('choices')) | set(templates.get('features'))
for key in variant:
if key not in known:
problems.append(
"unknown variant key '%s' (group '%s')" % (key, group_name))
_, probs = collect_fragments(templates, variant)
problems.extend(probs)
name = render_filename(templates, group_name, variant)
problems.extend(check_fragments_exist(templates, variant, ingredients_root))
filenames = []
for group in manifest.get('recipes', []):
group_name = group.get('name', 'unknown')
for config in group.get('variants', []):
for variant in expand_variants(config):
filenames.append(render_filename(templates, group_name, variant))
seen = set()
for name in filenames:
if name in seen:
problems.append("duplicate dish name '%s'" % name)
seen.add(name)
return problems
def check_fragments_exist(templates, variant, ingredients_root):
fragments, _ = collect_fragments(templates, variant)
problems = []
for slug in fragments:
path = os.path.join(ingredients_root, slug)
if not os.path.isfile(path):
problems.append("missing ingredient fragment '%s'" % slug)
return problems
def flatten_recipe(recipe_path):
"""Flatten a recipe (%include list) into a standalone kickstart string."""
handler = returnClassForVersion(DEVEL)()
parser = KickstartParser(handler)
parser.readKickstart(recipe_path)
return str(handler)
def validate_dish(dish_path):
"""Return (ok, error) for a standalone kickstart dish."""
try:
handler = returnClassForVersion(DEVEL)()
parser = KickstartParser(handler)
parser.readKickstart(dish_path)
return True, None
except Exception as exc: # noqa: BLE001 - report any parse failure
return False, str(exc)
def generate(manifest_path, templates_path, recipes_dir, dishes_dir,
ingredients_dir, do_generate=True, do_lint=True, do_validate=True):
"""Full cooking pipeline. Returns an exit code (0 on success)."""
manifest = load_yaml(manifest_path)
templates = load_yaml(templates_path)
ingredients_root = os.path.join(os.path.dirname(os.path.abspath(manifest_path)),
ingredients_dir)
if do_lint:
problems = lint_manifest(manifest, templates, ingredients_root)
else:
problems = []
failures = []
generated = []
recipe_dir = os.path.abspath(recipes_dir)
dish_dir = os.path.abspath(dishes_dir)
if do_generate:
os.makedirs(recipe_dir, exist_ok=True)
os.makedirs(dish_dir, exist_ok=True)
for group in manifest.get('recipes', []):
group_name = group.get('name', 'unknown')
for config in group.get('variants', []):
for variant in expand_variants(config):
fragments, probs = collect_fragments(templates, variant)
problems.extend(probs)
name = render_filename(templates, group_name, variant)
generated.append(name)
if not do_generate:
continue
recipe_path = os.path.join(recipe_dir, name)
with open(recipe_path, 'w') as f:
f.write(render_recipe(fragments))
dish_path = os.path.join(dish_dir, name)
try:
dish = flatten_recipe(recipe_path)
with open(dish_path, 'w') as f:
f.write(dish)
except Exception as exc: # noqa: BLE001
failures.append('%s: flatten failed: %s' % (name, exc))
if do_validate and do_generate:
for name in generated:
dish_path = os.path.join(dish_dir, name)
if not os.path.isfile(dish_path):
continue
ok, error = validate_dish(dish_path)
if not ok:
problems.append('%s: invalid dish: %s' % (name, error))
if do_generate:
print("Generated %d recipes and %d dishes" % (len(generated), len(generated)))
if problems:
for problem in problems:
print("lint: %s" % problem, file=sys.stderr)
if failures:
for failure in failures:
print(failure, file=sys.stderr)
return 1 if (problems or failures) else 0
def print_inventory(templates_path, ingredients_dir):
templates = load_yaml(templates_path)
root = os.path.abspath(ingredients_dir)
out = []
out.append("Ingredient inventory for Phyllome OS (from %s)" % templates_path)
out.append("=" * 60)
out.append("\nBase (always included, in order):")
for slug in templates.get('base', []):
out.append(" %-40s %s" % (slug, os.path.exists(os.path.join(root, slug)) and "ok" or "MISSING"))
out.append("\nChoices (exactly one per category):")
for category, mapping in templates.get('choices', {}).items():
out.append(" %s:" % category)
for value, slug in mapping.items():
out.append(" %-12s %-30s %s" % (value, slug,
os.path.exists(os.path.join(root, slug)) and "ok" or "MISSING"))
out.append("\nFeatures (additive):")
for feature, body in templates.get('features', {}).items():
out.append(" %s:" % feature)
if isinstance(body, str):
out.append(" (flag) %-30s %s" % (body,
os.path.exists(os.path.join(root, body)) and "ok" or "MISSING"))
else:
for value, slugs in body.items():
if isinstance(slugs, list):
for slug in slugs:
out.append(" %-12s %-30s %s" % (value, slug,
os.path.exists(os.path.join(root, slug)) and "ok" or "MISSING"))
else:
out.append(" %-12s %-30s %s" % (value, slugs,
os.path.exists(os.path.join(root, slugs)) and "ok" or "MISSING"))
print('\n'.join(out))
def main(argv=None):
parser = argparse.ArgumentParser(
description='Generate Phyllome OS recipes and dishes')
parser.add_argument('--manifest', default='recipes_manifest.yaml',
help='Path to recipes manifest YAML')
parser.add_argument('--output-dir', default='recipes/',
help='Output directory for recipe files')
parser.add_argument('--validate', action='store_true',
help='Validate generated recipes')
parser.add_argument('files', nargs='*', default=[],
help='Recipe files to validate')
args = parser.parse_args()
# Generate recipes from cook/ directory
help='recipe variant manifest (default: recipes_manifest.yaml)')
parser.add_argument('--templates', default='recipe_templates.yaml',
help='ingredient templates (default: recipe_templates.yaml)')
parser.add_argument('--recipes-dir', default='recipes',
help='output directory for recipe include lists (default: recipes)')
parser.add_argument('--dishes-dir', default='dishes',
help='output directory for flattened dishes (default: dishes)')
parser.add_argument('--ingredients-dir', default='ingredients',
help='ingredient fragments directory (default: ingredients)')
parser.add_argument('--no-generate', action='store_true',
help='do not write recipes or dishes (lint/validate only)')
parser.add_argument('--no-lint', action='store_true',
help='skip manifest linting')
parser.add_argument('--no-validate', action='store_true',
help='skip dish validation')
parser.add_argument('--inventory', action='store_true',
help='print the ingredient inventory from templates and exit')
args = parser.parse_args(argv)
script_dir = os.path.dirname(os.path.abspath(__file__))
os.chdir(script_dir)
# Paths relative to cook/ directory
manifest_path = args.manifest
if not os.path.isabs(args.manifest):
manifest_path = os.path.join(script_dir, args.manifest)
output_dir = args.output_dir
if not os.path.isabs(args.output_dir):
output_dir = os.path.join(script_dir, args.output_dir)
print(f"Generating recipes from {manifest_path} to {output_dir}...")
recipes = generate_recipes(manifest_path, output_dir)
print(f"✓ Generated {len(recipes)} recipe files")
# Validate if requested (from command line files or directory)
if args.validate or args.files:
print("\nValidating recipes...")
if args.files:
# Validate specific files
all_valid = True
for filepath in args.files:
valid, error = validate_recipe(filepath)
if valid:
print(f"✓ {os.path.basename(filepath)}")
else:
print(f"✗ {os.path.basename(filepath)}: {error}", file=sys.stderr)
all_valid = False
if not all_valid:
sys.exit(1)
else:
# Validate output directory
if not validate_all(output_dir):
sys.exit(1)
print("✓ All recipes validated")
manifest_path = args.manifest if os.path.isabs(args.manifest) \
else os.path.join(script_dir, args.manifest)
templates_path = args.templates if os.path.isabs(args.templates) \
else os.path.join(script_dir, args.templates)
if args.inventory:
print_inventory(templates_path, args.ingredients_dir)
return 0
return generate(
manifest_path, templates_path, args.recipes_dir, args.dishes_dir,
args.ingredients_dir,
do_generate=not args.no_generate,
do_lint=not args.no_lint,
do_validate=not args.no_validate)
if __name__ == '__main__':
main()
sys.exit(main())