diff --git a/.gitea/workflows/test-generation.yaml b/.gitea/workflows/test-generation.yaml index 37dda99..d02ac8a 100644 --- a/.gitea/workflows/test-generation.yaml +++ b/.gitea/workflows/test-generation.yaml @@ -3,13 +3,13 @@ name: test-generation on: push: paths: - - 'scripts/**/*.py' - - 'scripts/**/*.yaml' + - 'recipe-generator/**/*.py' + - 'recipe-generator/**/*.yaml' - 'fragments/**/*.ks' pull_request: paths: - - 'scripts/**/*.py' - - 'scripts/**/*.yaml' + - 'recipe-generator/**/*.py' + - 'recipe-generator/**/*.yaml' jobs: generate: @@ -28,7 +28,7 @@ jobs: - name: Generate all variants run: | - cd scripts + cd recipe-generator python3 generate_recipe.py \ --manifest recipes_manifest.yaml \ --output-dir ../recipes/ diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index aa5ce02..2f4a4b4 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -21,7 +21,7 @@ Phyllome OS uses a **fragment-driven** kickstart generation system: ``` fragments/ (54 .ks files) ↓ (modular snippets) -scripts/generate_recipe.py +recipe-generator/generate_recipe.py ↓ (YAML templates + manifest) recipes/ (16 auto-generated .cfg) ↓ (ksflatten) @@ -40,14 +40,16 @@ VMs and ISO images | `recipes/` | Generated recipes | Manifest-driven compositions | | `dishes/` | Flattened kickstarts | Ready-to-deploy artifacts | | `ingredients/` | Legacy building blocks | 35 `.cfg` files (legacy) | -| `scripts/` | Automation tools | `generate_recipe.py`, Makefile | +| `recipe-generator/` | Recipe generation | `generate_recipe.py`, YAML configs, Makefile | +| `deploy/` | Deployment scripts | Bash automation tools | +| `bin/` | Executables | Wrapper scripts (e.g., `generate-recipe`) | ### Data Flow 1. **Fragments** (`fragments/**/*.ks`) - Small, reusable kickstart snippets -2. **Templates** (`scripts/recipe_templates.yaml`) - Define recipe structures -3. **Manifest** (`scripts/recipes_manifest.yaml`) - Specify variants (version, desktop, storage, etc.) -4. **Generator** (`scripts/generate_recipe.py`) - Composes fragments via `%ksappend` directives +2. **Templates** (`recipe-generator/recipe_templates.yaml`) - Define recipe structures +3. **Manifest** (`recipe-generator/recipes_manifest.yaml`) - Specify variants (version, desktop, storage, etc.) +4. **Generator** (`recipe-generator/generate_recipe.py`) - Composes fragments via `%ksappend` directives 5. **Recipes** (`recipes/*.cfg`) - Generated kickstart files with fragment references 6. **Flattening** (`ksflatten`) - Resolves `%ksappend` into single dish file 7. **Deployment** (`virt-install`) - Creates VMs from dish files @@ -169,7 +171,7 @@ EOF ### Fragment Validation Script -Create `scripts/validate-fragment.sh`: +Create `deploy/validate-fragment.sh`: ```bash #!/bin/bash @@ -205,7 +207,7 @@ Recipes are generated from templates and the manifest file. This section covers ### Manifest Editing -**File:** `scripts/recipes_manifest.yaml` +**File:** `recipe-generator/recipes_manifest.yaml` The manifest defines all recipe variants using modifiers from templates. @@ -252,7 +254,7 @@ recipes: ### Template Editing -**File:** `scripts/recipe_templates.yaml` +**File:** `recipe-generator/recipe_templates.yaml` Templates define the structure and fragment composition for each recipe type. @@ -551,7 +553,7 @@ done **File:** `.gitea/workflows/test-generation.yaml` **Triggers:** -- Push to `scripts/**/*.py` or `scripts/**/*.yaml` +- Push to `recipe-generator/**/*.py` or `recipe-generator/**/*.yaml` - Pull request with script changes **Steps:** @@ -609,12 +611,12 @@ luanti EOF # Step 2: Add to recipe template -# Edit scripts/recipe_templates.yaml +# Edit recipe-generator/recipe_templates.yaml # Add to 'required' section: # - luanti: fragments/shared/packages/luanti.ks # Step 3: Regenerate recipes -cd scripts +cd recipe-generator make generate-recipes # Step 4: Validate @@ -637,12 +639,12 @@ plasma-workspace EOF # Step 2: Add to template -# Edit scripts/recipe_templates.yaml +# Edit recipe-generator/recipe_templates.yaml # Add to optional/desktop section: # kde: fragments/shared/desktop/kde/packages.ks # Step 3: Add variant to manifest -# Edit scripts/recipes_manifest.yaml +# Edit recipe-generator/recipes_manifest.yaml # Add variant: # - version: 43 # desktop: kde @@ -660,7 +662,7 @@ make test ```bash # Step 1: Add template to recipe_templates.yaml -cat >> scripts/recipe_templates.yaml << 'EOF' +cat >> recipe-generator/recipe_templates.yaml << 'EOF' minimal-server: description: "A minimal server recipe" @@ -681,7 +683,7 @@ cat >> scripts/recipe_templates.yaml << 'EOF' EOF # Step 2: Add variant to recipes_manifest.yaml -cat >> scripts/recipes_manifest.yaml << 'EOF' +cat >> recipe-generator/recipes_manifest.yaml << 'EOF' - name: minimal-server variants: @@ -729,7 +731,7 @@ luanti %end ``` -**Template:** `scripts/recipe_templates.yaml` +**Template:** `recipe-generator/recipe_templates.yaml` ```yaml templates: virtual-desktop: diff --git a/DEVELOPMENT_QUICK.md b/DEVELOPMENT_QUICK.md index d856610..c6a3df8 100644 --- a/DEVELOPMENT_QUICK.md +++ b/DEVELOPMENT_QUICK.md @@ -10,7 +10,7 @@ sudo dnf install qemu libvirt virt-install pykickstart pip install PyYAML pytest # Verify setup -cd scripts && make generate-recipes && make test +cd recipe-generator && make generate-recipes && make test ``` ## Core Workflows @@ -28,17 +28,17 @@ new-package ### Add to Recipe ```bash -# Edit scripts/recipe_templates.yaml to include fragment -# Edit scripts/recipes_manifest.yaml to add variant +# Edit recipe-generator/recipe_templates.yaml to include fragment +# Edit recipe-generator/recipes_manifest.yaml to add variant # Regenerate -cd scripts && make generate-recipes && make validate-recipes +cd recipe-generator && make generate-recipes && make validate-recipes ``` ### Run Tests ```bash -cd scripts +cd recipe-generator make test # All tests make test-integration # Integration only make test-container # Containerized @@ -60,7 +60,7 @@ done ## Architecture ``` -fragments/ (54 .ks) → generate_recipe.py → recipes/ (16 .cfg) → ksflatten → dishes/ (28 .cfg) +fragments/ (54 .ks) → recipe-generator/generate_recipe.py → recipes/ (16 .cfg) → ksflatten → dishes/ (28 .cfg) ``` See `DEVELOPMENT.md` Section 1 for detailed architecture overview. diff --git a/README.md b/README.md index 7652bec..1d61d03 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ chmod +x deploy-vm.sh ``` ./deploy-vm.sh -Executing: ./scripts/core-count.sh +Executing: ./deploy/core-count.sh System has more than 2 core (nproc --all: 6). [...] 10. virtual-desktop-hypervisor diff --git a/bin/generate-recipe b/bin/generate-recipe new file mode 100755 index 0000000..8a6cca7 --- /dev/null +++ b/bin/generate-recipe @@ -0,0 +1,2 @@ +#!/bin/bash +exec "$(dirname "$0")/../recipe-generator/generate_recipe.py" "$@" diff --git a/deploy.sh b/deploy.sh index bbd5ab7..c1e80f1 100755 --- a/deploy.sh +++ b/deploy.sh @@ -13,10 +13,10 @@ execute_script() { # Array of scripts scripts=( - "./scripts/install-prerequisites-on-linux.sh" - "./scripts/core-count.sh" - "./scripts/system-memory.sh" - "./scripts/deploy-distro.sh" + "./deploy/install-prerequisites-on-linux.sh" + "./deploy/core-count.sh" + "./deploy/system-memory.sh" + "./deploy/deploy-distro.sh" ) # Iterate through the scripts and execute them diff --git a/scripts/core-count.sh b/deploy/core-count.sh similarity index 100% rename from scripts/core-count.sh rename to deploy/core-count.sh diff --git a/scripts/deploy-distro.sh b/deploy/deploy-distro.sh similarity index 100% rename from scripts/deploy-distro.sh rename to deploy/deploy-distro.sh diff --git a/scripts/install-prerequisites-on-linux.sh b/deploy/install-prerequisites-on-linux.sh similarity index 100% rename from scripts/install-prerequisites-on-linux.sh rename to deploy/install-prerequisites-on-linux.sh diff --git a/scripts/system-memory.sh b/deploy/system-memory.sh similarity index 100% rename from scripts/system-memory.sh rename to deploy/system-memory.sh diff --git a/scripts/Makefile b/recipe-generator/Makefile similarity index 87% rename from scripts/Makefile rename to recipe-generator/Makefile index ec1a690..bfdce20 100644 --- a/scripts/Makefile +++ b/recipe-generator/Makefile @@ -25,13 +25,13 @@ validate-recipes: --validate ../recipes/*.cfg test: - python3 -m pytest tests/ -v --tb=short + python3 -m pytest ../tests/ -v --tb=short test-integration: - python3 -m pytest tests/integration/ -v --tb=short + python3 -m pytest ../tests/integration/ -v --tb=short test-container: - podman build -t phyllo/test-runner tests/container/ + podman build -t phyllo/test-runner ../tests/container/ podman run --rm -v .:/phyllomeos:ro phyllo/test-runner clean: diff --git a/scripts/__pycache__/generate_recipe.cpython-314.pyc b/recipe-generator/__pycache__/generate_recipe.cpython-314.pyc similarity index 66% rename from scripts/__pycache__/generate_recipe.cpython-314.pyc rename to recipe-generator/__pycache__/generate_recipe.cpython-314.pyc index 0121269..3987753 100644 Binary files a/scripts/__pycache__/generate_recipe.cpython-314.pyc and b/recipe-generator/__pycache__/generate_recipe.cpython-314.pyc differ diff --git a/scripts/generate_recipe.py b/recipe-generator/generate_recipe.py old mode 100644 new mode 100755 similarity index 98% rename from scripts/generate_recipe.py rename to recipe-generator/generate_recipe.py index 010b144..cf8c3b2 --- a/scripts/generate_recipe.py +++ b/recipe-generator/generate_recipe.py @@ -69,7 +69,6 @@ class RecipeGenerator: """Generate kickstart recipes from templates and modifiers.""" def __init__(self, ingredients_dir: Path, templates_file: Path): - # Resolve ingredients_dir relative to the project root (parent of scripts/) self.project_root = Path(__file__).parent.parent self.ingredients_dir = self.project_root / ingredients_dir self.templates = self.load_templates(templates_file) @@ -90,7 +89,7 @@ class RecipeGenerator: if path.is_absolute(): template_path = path else: - template_path = self.project_root / 'scripts' / path + template_path = self.project_root / path with open(template_path) as f: data = yaml.safe_load(f) return data['templates'] @@ -419,7 +418,6 @@ def main(): ) # Global options - # Use __file__ to find the scripts directory, then go up to project root SCRIPTS_DIR = Path(__file__).resolve().parent PROJECT_ROOT = SCRIPTS_DIR.parent @@ -428,14 +426,14 @@ def main(): help='Ingredients directory (default: parent/ingredients)') parser.add_argument('--templates', '-t', type=Path, default=SCRIPTS_DIR / 'recipe_templates.yaml', - help='Templates YAML file (default: parent/recipe_templates.yaml)') + help='Templates YAML file (default: ./recipe_templates.yaml)') # Batch mode parser.add_argument('--manifest', '-m', type=Path, help='Manifest YAML for batch generation') parser.add_argument('--output-dir', '-d', - type=Path, default=Path(__file__).parent / 'recipes', - help='Output directory (batch generation, default: parent/recipes)') + type=Path, default=SCRIPTS_DIR / 'recipes', + help='Output directory (batch generation, default: ./recipes)') parser.add_argument('--dry-run', '-n', action='store_true', help='Show what would be generated without writing files') diff --git a/ingredients/python-update-package-names.py b/recipe-generator/python-update-package-names.py similarity index 100% rename from ingredients/python-update-package-names.py rename to recipe-generator/python-update-package-names.py diff --git a/scripts/recipe_templates.yaml b/recipe-generator/recipe_templates.yaml similarity index 100% rename from scripts/recipe_templates.yaml rename to recipe-generator/recipe_templates.yaml diff --git a/scripts/recipes_manifest.yaml b/recipe-generator/recipes_manifest.yaml similarity index 100% rename from scripts/recipes_manifest.yaml rename to recipe-generator/recipes_manifest.yaml diff --git a/scripts/requirements.txt b/recipe-generator/requirements.txt similarity index 100% rename from scripts/requirements.txt rename to recipe-generator/requirements.txt diff --git a/tests/__pycache__/test_recipe_generator.cpython-314-pytest-8.4.2.pyc b/tests/__pycache__/test_recipe_generator.cpython-314-pytest-8.4.2.pyc index 19fb265..215ce4b 100644 Binary files a/tests/__pycache__/test_recipe_generator.cpython-314-pytest-8.4.2.pyc and b/tests/__pycache__/test_recipe_generator.cpython-314-pytest-8.4.2.pyc differ diff --git a/tests/integration/__pycache__/test_fragments.cpython-314-pytest-8.4.2.pyc b/tests/integration/__pycache__/test_fragments.cpython-314-pytest-8.4.2.pyc deleted file mode 100644 index bae8055..0000000 Binary files a/tests/integration/__pycache__/test_fragments.cpython-314-pytest-8.4.2.pyc and /dev/null differ diff --git a/tests/integration/__pycache__/test_golden_masters.cpython-314-pytest-8.4.2.pyc b/tests/integration/__pycache__/test_golden_masters.cpython-314-pytest-8.4.2.pyc deleted file mode 100644 index 944c7d1..0000000 Binary files a/tests/integration/__pycache__/test_golden_masters.cpython-314-pytest-8.4.2.pyc and /dev/null differ diff --git a/tests/integration/__pycache__/test_integration.cpython-314-pytest-8.4.2.pyc b/tests/integration/__pycache__/test_integration.cpython-314-pytest-8.4.2.pyc index 92d53bf..a2d8b72 100644 Binary files a/tests/integration/__pycache__/test_integration.cpython-314-pytest-8.4.2.pyc and b/tests/integration/__pycache__/test_integration.cpython-314-pytest-8.4.2.pyc differ diff --git a/tests/integration/__pycache__/test_recipe_composition.cpython-314-pytest-8.4.2.pyc b/tests/integration/__pycache__/test_recipe_composition.cpython-314-pytest-8.4.2.pyc deleted file mode 100644 index 7890c02..0000000 Binary files a/tests/integration/__pycache__/test_recipe_composition.cpython-314-pytest-8.4.2.pyc and /dev/null differ diff --git a/tests/integration/__pycache__/test_semantic_validation.cpython-314-pytest-8.4.2.pyc b/tests/integration/__pycache__/test_semantic_validation.cpython-314-pytest-8.4.2.pyc deleted file mode 100644 index 73c6409..0000000 Binary files a/tests/integration/__pycache__/test_semantic_validation.cpython-314-pytest-8.4.2.pyc and /dev/null differ diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 2624753..baf4708 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -11,7 +11,7 @@ import os # Use actual project root PROJECT_ROOT = Path('/home/lukas/Code/virt/phyllomeos') -SCRIPTS_DIR = PROJECT_ROOT / 'scripts' +RECIPE_GENERATOR_DIR = PROJECT_ROOT / 'recipe-generator' RECIPE_DIR = PROJECT_ROOT / 'recipes' FRAGMENTS_DIR = PROJECT_ROOT / 'fragments' CONTAINER_DIR = PROJECT_ROOT / 'tests' / 'container' @@ -19,8 +19,8 @@ CONTAINER_DIR = PROJECT_ROOT / 'tests' / 'container' def test_generate_recipes_from_manifest(): """Test generating all recipes from manifest.""" - os.chdir(SCRIPTS_DIR) - + os.chdir(RECIPE_GENERATOR_DIR) + result = subprocess.run( ['python3', 'generate_recipe.py', '--manifest', 'recipes_manifest.yaml', @@ -28,7 +28,7 @@ def test_generate_recipes_from_manifest(): capture_output=True, text=True ) - + assert result.returncode == 0, f"Recipe generation failed: {result.stderr}" assert 'Generating:' in result.stdout @@ -51,7 +51,7 @@ def test_make_targets(): # Test generate-recipes result = subprocess.run( ['make', 'generate-recipes'], - cwd=SCRIPTS_DIR, + cwd=RECIPE_GENERATOR_DIR, capture_output=True, text=True ) diff --git a/tests/test_recipe_generator.py b/tests/test_recipe_generator.py index 585f26c..5e1f548 100644 --- a/tests/test_recipe_generator.py +++ b/tests/test_recipe_generator.py @@ -3,11 +3,9 @@ import pytest from pathlib import Path import sys -import os -# Add scripts directory to path -SCRIPTS_DIR = Path(__file__).parent.parent / 'scripts' -sys.path.insert(0, str(SCRIPTS_DIR)) +RECIPE_GENERATOR_DIR = Path(__file__).parent.parent / 'recipe-generator' +sys.path.insert(0, str(RECIPE_GENERATOR_DIR)) from generate_recipe import RecipeGenerator @@ -19,7 +17,11 @@ class TestRecipeGenerator: def setup_method(self): """Set up test fixtures.""" - self.generator = RecipeGenerator(Path('ingredients'), Path('recipe_templates.yaml')) + project_root = Path(__file__).parent.parent + self.generator = RecipeGenerator( + project_root / 'ingredients', + project_root / 'recipe-generator' / 'recipe_templates.yaml' + ) def test_template_loading(self): """Test that templates are loaded correctly."""