From 383aa6b35fd671ae947798db3433060265b5e9f9 Mon Sep 17 00:00:00 2001 From: Damien Arnodo Date: Thu, 5 Feb 2026 08:48:34 +0000 Subject: [PATCH] docs: Add Git-as-backend architecture for InfraHub - Repository now serves as InfraHub backend (schema + data in Git) - Add data/ directory structure for infrastructure objects - Add transforms/ for Jinja2 config templates - Update architecture diagram to show Git-centric workflow - Add "Repository as InfraHub Backend" section explaining benefits - Simplify project structure to reflect new approach --- README.md | 229 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 134 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index bead79c..c98e624 100644 --- a/README.md +++ b/README.md @@ -21,17 +21,29 @@ Think `terraform plan` and `terraform apply`, but for your network fabric — po ``` ┌──────────────────────────────────────────────────────────────────────────────┐ │ INTENT LAYER │ -│ ┌─────────────────────────┐ ┌──────────────────────────────────────────┐ │ -│ │ InfraHub │ │ Git Repository │ │ -│ │ (Source of Truth) │◄──►│ - Schema definitions (YAML) │ │ -│ │ │ │ - Transforms (Jinja2/Python) │ │ -│ │ • Custom fabric schema │ │ - Version-controlled intent │ │ -│ │ • GraphQL API │ └──────────────────────────────────────────┘ │ -│ │ • Branch-based changes │ │ -│ └────────────┬────────────┘ │ -└───────────────┼──────────────────────────────────────────────────────────────┘ - │ GraphQL / SDK - ▼ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────────┐ │ +│ │ This Git Repository │ │ +│ │ │ │ +│ │ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────────────┐ │ │ +│ │ │ schemas/ │ │ data/ │ │ transforms/ │ │ │ +│ │ │ └─fabric.yml │ │ ├─devices.yml │ │ └─arista_config.j2 │ │ │ +│ │ │ (InfraHub │ │ ├─vlans.yml │ │ (Jinja2 templates) │ │ │ +│ │ │ schema) │ │ ├─bgp.yml │ │ │ │ │ +│ │ │ │ │ └─... │ │ │ │ │ +│ │ └─────────────────┘ └─────────────────┘ └─────────────────────────┘ │ │ +│ └─────────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────────────────┐ │ +│ │ InfraHub │ │ +│ │ (Runtime + GraphQL API) │ │ +│ │ Syncs schema & data from Git, exposes via GraphQL │ │ +│ └─────────────────────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────────────────┘ + │ + │ GraphQL / SDK + ▼ ┌──────────────────────────────────────────────────────────────────────────────┐ │ ORCHESTRATION LAYER (PREFECT) │ │ │ @@ -50,13 +62,9 @@ Think `terraform plan` and `terraform apply`, but for your network fabric — po │ │ │ (InfraHub→YANG) │ │ (Want vs Have) │ │ (pygnmi wrapper) ││ │ │ │ └─────────────────┘ └─────────────────┘ └─────────────────────────┘│ │ │ └────────────────────────────────────────────────────────────────────────┘ │ -│ │ -│ ┌────────────────────────────────────────────────────────────────────────┐ │ -│ │ Prefect Server (UI) │ Prefect .serve() │ Webhook Receiver │ │ -│ └────────────────────────────────────────────────────────────────────────┘ │ -└──────────────────────────┬───────────────────────────────────────────────────┘ - │ gNMI Get/Set/Subscribe - ▼ +└──────────────────────────────────────┬───────────────────────────────────────┘ + │ gNMI Get/Set/Subscribe + ▼ ┌──────────────────────────────────────────────────────────────────────────────┐ │ DEVICE LAYER │ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ @@ -86,9 +94,63 @@ We chose [InfraHub](https://github.com/opsmill/infrahub) over NetBox as Source o 3. **Transforms** - Generate device configs directly from InfraHub 4. **Branches** - Test fabric changes in isolated branches before merge -## 🎛 Why Prefect? +## 📦 Repository as InfraHub Backend -We chose [Prefect](https://prefect.io) as the orchestration engine for several reasons: +This repository serves as the **single source of truth** for both code and infrastructure data: + +``` +fabric-orchestrator/ +├── .infrahub.yml # InfraHub repository config +│ +├── schemas/ # InfraHub schema definitions +│ └── fabric.yml # Custom EVPN-VXLAN fabric schema +│ +├── data/ # Infrastructure objects (YAML) +│ ├── topology/ +│ │ ├── sites.yml +│ │ └── devices.yml # Spines, Leafs, VTEP pairs +│ ├── network/ +│ │ ├── vlans.yml # VLANs + L2VNI mappings +│ │ ├── vrfs.yml # VRFs + L3VNI mappings +│ │ └── interfaces.yml # Interface configs +│ └── routing/ +│ ├── bgp_sessions.yml # Underlay + EVPN overlay +│ └── evpn.yml # Route targets, RDs +│ +├── transforms/ # Jinja2 templates for config generation +│ └── arista/ +│ ├── base.j2 +│ ├── interfaces.j2 +│ ├── bgp.j2 +│ └── evpn.j2 +│ +└── src/ # Python orchestration code +``` + +### Workflow + +```bash +# 1. Edit data files (e.g., add a VLAN) +vim data/network/vlans.yml + +# 2. Commit & push +git commit -am "Add VLAN 100 for production" +git push + +# 3. InfraHub syncs automatically from Git +# → Data available via GraphQL + +# 4. Prefect flow detects change → reconciles fabric +``` + +### Benefits + +- **Reproductibility**: `git clone` → `docker compose up` → complete environment +- **Code Review**: Infrastructure changes go through PR review +- **History**: Full audit trail via Git +- **Testing**: Create a branch, test changes, merge when validated + +## 🎛 Why Prefect? | Feature | Benefit | |---------|---------| @@ -129,49 +191,62 @@ Progress is tracked via issues. See [all issues](https://gitea.arnodo.fr/Damien/ fabric-orchestrator/ ├── README.md ├── pyproject.toml +├── .infrahub.yml # InfraHub config (points to schemas/) +│ +├── schemas/ # InfraHub schema definitions +│ └── fabric.yml # Custom EVPN-VXLAN fabric schema +│ +├── data/ # Infrastructure data (YAML) +│ ├── topology/ +│ │ ├── sites.yml +│ │ └── devices.yml +│ ├── network/ +│ │ ├── vlans.yml +│ │ ├── vrfs.yml +│ │ └── interfaces.yml +│ └── routing/ +│ ├── bgp_sessions.yml +│ └── evpn.yml +│ +├── transforms/ # Jinja2 config templates +│ └── arista/ +│ └── *.j2 │ ├── src/ # Python package │ ├── __init__.py -│ ├── cli.py # CLI for YANG discovery (discover commands) +│ ├── cli.py # CLI for YANG discovery │ │ │ ├── flows/ # Prefect flows │ │ ├── __init__.py -│ │ ├── reconcile.py # @flow fabric_reconcile (plan/apply) +│ │ ├── reconcile.py # @flow fabric_reconcile │ │ ├── drift.py # @flow handle_drift │ │ └── remediation.py # @flow drift_remediation │ │ -│ ├── api/ # FastAPI webhook receiver -│ │ ├── __init__.py -│ │ └── webhooks.py # InfraHub webhook endpoint -│ │ -│ ├── services/ # Long-running services -│ │ ├── __init__.py -│ │ └── drift_monitor.py # gNMI Subscribe drift detection -│ │ │ ├── gnmi/ │ │ ├── __init__.py │ │ ├── client.py # gNMI client wrapper (pygnmi) │ │ └── README.md │ │ -│ ├── infrahub/ # InfraHub integration (TODO) +│ ├── infrahub/ # InfraHub integration │ │ ├── __init__.py -│ │ ├── client.py # InfraHub SDK client -│ │ └── models.py # Pydantic models for intent validation +│ │ ├── client.py # InfraHub SDK wrapper +│ │ └── queries.py # GraphQL queries │ │ │ └── yang/ │ ├── __init__.py │ ├── mapper.py # InfraHub intent → YANG paths │ ├── paths.py # YANG path definitions -│ └── dependencies.py # Dependency ordering graph -│ -├── schemas/ # InfraHub schema definitions (TODO) -│ └── fabric.yml # Custom fabric schema +│ └── mappers/ # Resource-specific mappers +│ ├── vlan.py +│ ├── interface.py +│ ├── bgp.py +│ └── vxlan.py │ ├── tests/ │ └── docs/ - ├── cli-user-guide.md # CLI documentation - └── yang-paths.md # Documented YANG paths + ├── cli-user-guide.md + └── yang-paths.md ``` ## 🛠️ Technology Stack @@ -179,8 +254,8 @@ fabric-orchestrator/ | Component | Technology | Purpose | |-----------|------------|---------| | Source of Truth | **InfraHub** | Intent definition via custom schema | +| Data Storage | **This Git repo** | Schema + data versioned together | | Orchestrator | **Prefect** | Python-native workflow orchestration | -| Webhooks | FastAPI | Receive InfraHub webhooks | | Transport | gNMI | Configuration and telemetry | | Data Models | YANG (OpenConfig + Arista) | Structured configuration | | Python Library | pygnmi + infrahub-sdk | gNMI/InfraHub interactions | @@ -194,7 +269,6 @@ fabric-orchestrator/ - [InfraHub](https://github.com/opsmill/infrahub) - Source of Truth platform - [InfraHub Schema Library](https://github.com/opsmill/schema-library) - Reference schemas - [Arista YANG Models](https://github.com/aristanetworks/yang/tree/master/EOS-4.35.0F) - EOS 4.35.0F YANG definitions -- [Prefect Documentation](https://docs.prefect.io) - Orchestration platform docs ## 📚 References @@ -202,13 +276,12 @@ fabric-orchestrator/ - [InfraHub Documentation](https://docs.infrahub.app) - [InfraHub Schema Guide](https://docs.infrahub.app/guides/create-schema) - [InfraHub Python SDK](https://github.com/opsmill/infrahub-sdk-python) +- [InfraHub .infrahub.yml Reference](https://docs.infrahub.app/reference/dotinfrahub) ### Prefect - [Prefect Documentation](https://docs.prefect.io) - [Prefect Flows](https://docs.prefect.io/latest/develop/write-flows/) - [Prefect Tasks](https://docs.prefect.io/latest/develop/write-tasks/) -- [Prefect Deployments](https://docs.prefect.io/latest/deploy/run-flows-in-local-processes/) -- [Prefect Secrets](https://docs.prefect.io/latest/develop/blocks/#secret) ### YANG / gNMI - [Arista gNMI Documentation](https://aristanetworks.github.io/openmgmt/configuration/gnmi/) @@ -217,7 +290,6 @@ fabric-orchestrator/ ### EVPN-VXLAN - [Arista BGP EVPN Configuration Example](https://overlaid.net/2019/01/27/arista-bgp-evpn-configuration-example/) -- [Arista EVPN Deployment Guide](https://www.arista.com/en/solutions/evpn-vxlan) ## 🚀 Getting Started @@ -225,21 +297,21 @@ fabric-orchestrator/ - Python 3.12+ - `uv` package manager -- Access to ContainerLab with cEOS images - Docker (for InfraHub) +- Access to ContainerLab with cEOS images ### Quick Start ```bash -# Clone the repository +# Clone the repository (includes schema + data) git clone https://gitea.arnodo.fr/Damien/fabric-orchestrator.git cd fabric-orchestrator # Install Python dependencies uv sync -# Start InfraHub (see InfraHub docs for full setup) -# docker compose -f infrahub-docker-compose.yml up -d +# Start InfraHub (loads schema & data from this repo) +docker compose up -d # Configure Prefect secrets python -c " @@ -247,28 +319,22 @@ from prefect.blocks.system import Secret from prefect.variables import Variable Secret(value='your-gnmi-password').save('gnmi-password', overwrite=True) -Secret(value='your-infrahub-token').save('infrahub-token', overwrite=True) - Variable.set('infrahub_url', 'http://localhost:8000') Variable.set('gnmi_username', 'admin') " -# Start Prefect server (optional, for UI) -prefect server start - -# Verify gNMI connectivity to your fabric +# Verify gNMI connectivity uv run fabric-orch discover capabilities --target leaf1:6030 -# Explore YANG paths -uv run fabric-orch discover get --target leaf1:6030 \ - --path "/interfaces/interface[name=Ethernet1]/state" +# Run reconciliation +uv run fabric-orch plan +uv run fabric-orch apply ``` ## Prefect Flow Example ```python from prefect import flow, task -from prefect.blocks.system import Secret from prefect.variables import Variable @@ -277,13 +343,9 @@ def get_fabric_intent(device: str | None = None) -> dict: """Retrieve fabric intent from InfraHub.""" from infrahub_sdk import InfrahubClient - infrahub_url = Variable.get("infrahub_url") - infrahub_token = Secret.load("infrahub-token").get() - - client = InfrahubClient(address=infrahub_url, api_token=infrahub_token) + client = InfrahubClient(address=Variable.get("infrahub_url")) # Query fabric intent via GraphQL - # ... - return intent + return client.query(...) @task @@ -293,46 +355,23 @@ def compute_diff(intent: dict, current: dict) -> list[dict]: return diff_engine(want=intent, have=current) -@task(retries=1) -def apply_changes(changes: list[dict], dry_run: bool = True) -> dict: - """Apply changes via gNMI Set.""" - if dry_run: - return {"applied": False, "changes": changes} - # Apply via gNMI... - return {"applied": True, "changes": changes} - - @flow(log_prints=True, name="fabric-reconcile") -def fabric_reconcile( - device: str | None = None, - auto_apply: bool = False, - dry_run: bool = True -) -> dict: +def fabric_reconcile(device: str | None = None, dry_run: bool = True) -> dict: """Reconcile fabric state with InfraHub intent.""" - print(f"🔄 Starting fabric reconciliation") - intent = get_fabric_intent(device) - current = get_current_state(devices) + current = get_current_state(device) changes = compute_diff(intent, current) if not changes: - print("✅ No changes detected - fabric is in sync") - return {"changes": [], "in_sync": True} + print("✅ Fabric is in sync") + return {"in_sync": True} - should_apply = auto_apply and not dry_run - result = apply_changes(changes, dry_run=not should_apply) + if not dry_run: + apply_changes(changes) - return {"changes": changes, "applied": should_apply} - - -if __name__ == "__main__": - fabric_reconcile.serve( - name="fabric-reconcile-scheduled", - cron="0 */6 * * *", - tags=["network", "fabric"] - ) + return {"changes": changes, "applied": not dry_run} ``` --- -**Status**: 🚧 Active Development - Migrating to InfraHub as Source of Truth +**Status**: 🚧 Active Development - Phase 2 (InfraHub Setup & Core Reconciler)