diff --git a/README.md b/README.md index 0800b72..727576e 100644 --- a/README.md +++ b/README.md @@ -2,59 +2,82 @@ **Declarative Network Infrastructure Management for Arista EVPN-VXLAN Fabrics** -A Terraform-like orchestration system that uses NetBox as Source of Truth and gNMI/YANG for atomic configuration management of Arista data center fabrics. +A workflow-based orchestration system that uses NetBox as Source of Truth, [Kestra](https://kestra.io) for orchestration, and gNMI/YANG for atomic configuration management of Arista data center fabrics. ## 🎯 Project Vision Transform network infrastructure management from imperative scripting to true declarative infrastructure-as-code, where: -- **Intent** is defined in NetBox (ConfigContexts, Custom Fields) +- **Intent** is defined in NetBox (Custom Fields, Native Models, BGP Plugin) +- **Orchestration** is handled by Kestra (declarative YAML workflows) - **State** is continuously monitored via gNMI Subscribe - **Changes** are computed as diffs and applied atomically via gNMI Set - **Drift** is detected and optionally auto-remediated -Think `terraform plan` and `terraform apply`, but for your network fabric. +Think `terraform plan` and `terraform apply`, but for your network fabric β€” powered by Kestra workflows. ## πŸ—οΈ Architecture ``` β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ INTENT LAYER β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ NetBox β”‚ β”‚ ConfigContexts β”‚ β”‚ Custom Fields / Tags β”‚ β”‚ -β”‚ β”‚ (SoT) │◄──►│ (Structured │◄──►│ (VLAN, VNI, VRF, BGP AS) β”‚ β”‚ -β”‚ β”‚ β”‚ β”‚ Intent Data) β”‚ β”‚ β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ Webhook / Polling - β–Ό +β”‚ INTENT LAYER β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ NetBox β”‚ β”‚ Custom Fields / β”‚ β”‚ netbox-bgp β”‚ β”‚ +β”‚ β”‚ (SoT) │◄───│ Native Models │◄───│ Plugin β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ Webhook / Polling + β–Ό β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ ORCHESTRATION LAYER β”‚ +β”‚ ORCHESTRATION LAYER (KESTRA) β”‚ +β”‚ β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Kestra Workflows (YAML) β”‚ β”‚ +β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ +β”‚ β”‚ β”‚ fabric-reconcileβ”‚ β”‚ drift-detection β”‚ β”‚ netbox-webhook-handler β”‚ β”‚ β”‚ +β”‚ β”‚ β”‚ (plan/apply) β”‚ β”‚ (subscribe) β”‚ β”‚ (event trigger) β”‚ β”‚ β”‚ +β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Python Tasks (containerized) β”‚ β”‚ +β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ +β”‚ β”‚ β”‚ Intent Parser β”‚ β”‚ Diff Engine β”‚ β”‚ gNMI Client (Get/Set) β”‚ β”‚ β”‚ +β”‚ β”‚ β”‚ (NetBoxβ†’YANG) β”‚ β”‚ (Want vs Have)β”‚ β”‚ (pygnmi wrapper) β”‚ β”‚ β”‚ +β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”β”‚ -β”‚ β”‚ State Reconciliation Engine β”‚β”‚ -β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚β”‚ -β”‚ β”‚ β”‚ Intent Parser β”‚ β”‚ Diff Engine β”‚ β”‚ Transaction Planner β”‚ β”‚β”‚ -β”‚ β”‚ β”‚ (NetBoxβ†’YANG) │──►│ (Want vs Have)│──►│ (Ordered gNMI SetReqs) β”‚ β”‚β”‚ -β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜β”‚ -β”‚ β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”β”‚ -β”‚ β”‚ Event Bus (Redis / NATS) β”‚β”‚ -β”‚ β”‚ β€’ config_drift_detected β€’ intent_changed β€’ apply_complete β”‚β”‚ +β”‚ β”‚ Triggers: Webhook (NetBox) β”‚ Schedule (cron) β”‚ Flow (event-driven) β”‚β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ gNMI Subscribe (Telemetry) β”‚ gNMI Set (Config) - β–Ό β–Ό + β”‚ gNMI Get/Set/Subscribe + β–Ό β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ DEVICE LAYER β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ spine1 β”‚ β”‚ spine2 β”‚ β”‚ leaf1 β”‚ β”‚ leaf2 β”‚ ... β”‚ -β”‚ β”‚ gNMI:6030 β”‚ β”‚ gNMI:6030 β”‚ β”‚ gNMI:6030 β”‚ β”‚ gNMI:6030 β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ DEVICE LAYER β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ spine1 β”‚ β”‚ spine2 β”‚ β”‚ leaf1 β”‚ β”‚ leaf2 β”‚ ... β”‚ +β”‚ β”‚ gNMI:6030 β”‚ β”‚ gNMI:6030 β”‚ β”‚ gNMI:6030 β”‚ β”‚ gNMI:6030 β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ``` -## πŸ”§ Target Fabric +## πŸŽ› Why Kestra? + +We chose [Kestra](https://kestra.io) as the orchestration engine for several reasons: + +| Feature | Benefit | +|---------|---------| +| **Declarative YAML workflows** | Infrastructure-as-Code for orchestration logic | +| **Built-in UI** | Dashboard, logs, metrics, execution history β€” no custom development | +| **Native webhooks** | Direct NetBox integration without custom FastAPI server | +| **Event-driven triggers** | Schedule, webhook, flow triggers out of the box | +| **Python task support** | Run containerized Python scripts with dependencies | +| **DAG support** | Automatic dependency ordering with `io.kestra.core.tasks.flows.Dag` | +| **Retry & error handling** | Built-in retry policies and error notifications | +| **Secrets management** | Native secrets storage for credentials | + +## 🎯 Target Fabric This project is designed for the Arista EVPN-VXLAN ContainerLab topology: @@ -69,12 +92,12 @@ Reference: [arista-evpn-vxlan-clab](https://gitea.arnodo.fr/Damien/arista-evpn-v Progress is tracked via issues. See [all issues](https://gitea.arnodo.fr/Damien/fabric-orchestrator/issues) or filter by phase: -| Phase | Description | Issues | -| ----------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | -| **Phase 1** | YANG Path Discovery - Map EOS 4.35.0F YANG models, validate gNMI | [phase-1-yang-discovery](https://gitea.arnodo.fr/Damien/fabric-orchestrator/issues?type=all&state=all&labels=1) | -| **Phase 2** | Minimal Reconciler - VLANs/VNIs, diff engine, CLI plan/apply | [phase-2-minimal-reconciler](https://gitea.arnodo.fr/Damien/fabric-orchestrator/issues?type=all&state=all&labels=2) | -| **Phase 3** | Full Fabric - BGP, MLAG, VRFs, dependency ordering | [phase-3-full-fabric](https://gitea.arnodo.fr/Damien/fabric-orchestrator/issues?type=all&state=all&labels=3) | -| **Phase 4** | Event-Driven - gNMI Subscribe, drift detection, webhooks | [phase-4-event-driven](https://gitea.arnodo.fr/Damien/fabric-orchestrator/issues?type=all&state=all&labels=4) | +| Phase | Description | Issues | +|-------|-------------|--------| +| **Phase 1** | YANG Path Discovery - Map EOS 4.35.0F YANG models, validate gNMI | [phase-1-yang-discovery](https://gitea.arnodo.fr/Damien/fabric-orchestrator/issues?type=all&state=all&labels=1) | +| **Phase 2** | Core Components - NetBox client, diff engine, gNMI operations | [phase-2-minimal-reconciler](https://gitea.arnodo.fr/Damien/fabric-orchestrator/issues?type=all&state=all&labels=2) | +| **Phase 3** | Full Fabric - BGP, MLAG, VRFs, YANG mappers | [phase-3-full-fabric](https://gitea.arnodo.fr/Damien/fabric-orchestrator/issues?type=all&state=all&labels=3) | +| **Phase 4** | Kestra Integration - Workflows, webhooks, drift detection | [phase-4-kestra](https://gitea.arnodo.fr/Damien/fabric-orchestrator/issues?type=all&state=all&labels=4) | πŸ“Œ **Project Board**: [View Kanban](https://gitea.arnodo.fr/Damien/fabric-orchestrator/projects) @@ -84,56 +107,71 @@ Progress is tracked via issues. See [all issues](https://gitea.arnodo.fr/Damien/ fabric-orchestrator/ β”œβ”€β”€ README.md β”œβ”€β”€ pyproject.toml -β”œβ”€β”€ docker-compose.yml # Redis, API server -β”œβ”€β”€ src/ +β”œβ”€β”€ docker-compose.yml # Kestra + PostgreSQL +β”‚ +β”œβ”€β”€ kestra/ # Kestra workflows +β”‚ └── flows/ +β”‚ β”œβ”€β”€ fabric-reconcile.yml # Main plan/apply workflow +β”‚ β”œβ”€β”€ netbox-webhook.yml # NetBox webhook handler +β”‚ β”œβ”€β”€ drift-detection.yml # Drift monitoring workflow +β”‚ └── device-config.yml # Per-device configuration +β”‚ +β”œβ”€β”€ src/ # Python package (reusable code) β”‚ β”œβ”€β”€ __init__.py -β”‚ β”œβ”€β”€ cli.py # CLI interface (plan, apply, drift) -β”‚ β”œβ”€β”€ api.py # FastAPI server for webhooks -β”‚ β”œβ”€β”€ reconciler/ -β”‚ β”‚ β”œβ”€β”€ engine.py # Core reconciliation logic -β”‚ β”‚ β”œβ”€β”€ diff.py # State comparison -β”‚ β”‚ └── planner.py # Change ordering/dependencies -β”‚ β”œβ”€β”€ yang/ -β”‚ β”‚ β”œβ”€β”€ mapper.py # NetBox intent β†’ YANG paths -β”‚ β”‚ β”œβ”€β”€ paths.py # YANG path definitions -β”‚ β”‚ └── validators.py # Schema validation +β”‚ β”œβ”€β”€ cli.py # CLI for YANG discovery (discover commands) β”‚ β”œβ”€β”€ gnmi/ -β”‚ β”‚ β”œβ”€β”€ client.py # gNMI client wrapper -β”‚ β”‚ └── transactions.py # Atomic operations +β”‚ β”‚ β”œβ”€β”€ __init__.py +β”‚ β”‚ β”œβ”€β”€ client.py # gNMI client wrapper (pygnmi) +β”‚ β”‚ └── README.md β”‚ β”œβ”€β”€ netbox/ -β”‚ β”‚ β”œβ”€β”€ client.py # NetBox API client -β”‚ β”‚ └── models.py # Intent data models -β”‚ └── events/ -β”‚ β”œβ”€β”€ handlers.py # Event handlers -β”‚ └── bus.py # Event bus (Redis) +β”‚ β”‚ β”œβ”€β”€ __init__.py +β”‚ β”‚ β”œβ”€β”€ client.py # NetBox API client (pynetbox) +β”‚ β”‚ └── models.py # Pydantic models for intent validation +β”‚ └── yang/ +β”‚ β”œβ”€β”€ __init__.py +β”‚ β”œβ”€β”€ mapper.py # NetBox intent β†’ YANG paths +β”‚ └── paths.py # YANG path definitions +β”‚ +β”œβ”€β”€ scripts/ # Scripts called by Kestra workflows +β”‚ β”œβ”€β”€ get_fabric_intent.py +β”‚ β”œβ”€β”€ diff_engine.py +β”‚ └── apply_changes.py +β”‚ β”œβ”€β”€ tests/ +β”‚ └── docs/ - β”œβ”€β”€ architecture.md + β”œβ”€β”€ cli-user-guide.md # CLI documentation β”œβ”€β”€ yang-paths.md # Documented YANG paths - └── netbox-schema.md # ConfigContext schema + └── netbox-data-model.md # NetBox schema documentation ``` ## πŸ› οΈ Technology Stack -| Component | Technology | Purpose | -| --------------- | -------------------------- | ------------------------------------ | -| Source of Truth | NetBox | Intent definition via ConfigContexts | -| Transport | gNMI | Configuration and telemetry | -| Data Models | YANG (OpenConfig + Arista) | Structured configuration | -| Orchestrator | Python (asyncio) | Reconciliation engine | -| CLI | Click + Rich | User interface | -| API | FastAPI | Webhook receiver | -| Event Bus | Redis | Async event handling | -| Lab | ContainerLab + cEOS | Development environment | +| Component | Technology | Purpose | +|-----------|------------|---------| +| Source of Truth | NetBox + BGP Plugin | Intent definition via native models | +| Orchestrator | **Kestra** | Declarative workflow orchestration | +| Transport | gNMI | Configuration and telemetry | +| Data Models | YANG (OpenConfig + Arista) | Structured configuration | +| Python Library | pygnmi + pynetbox | gNMI/NetBox interactions | +| CLI | Click + Rich | YANG discovery tools | +| Validation | Pydantic v2 | Intent data validation | +| Lab | ContainerLab + cEOS | Development environment | ## πŸ”— Related Projects - [arista-evpn-vxlan-clab](https://gitea.arnodo.fr/Damien/arista-evpn-vxlan-clab) - Target fabric topology - [projet-vxlan-automation](https://gitea.arnodo.fr/Damien/projet-vxlan-automation) - Previous NetBox RenderConfig work - [Arista YANG Models](https://github.com/aristanetworks/yang/tree/master/EOS-4.35.0F) - EOS 4.35.0F YANG definitions +- [Kestra Documentation](https://kestra.io/docs) - Orchestration platform docs ## πŸ“š References +### Kestra +- [Kestra Documentation](https://kestra.io/docs) +- [Kestra Python Plugin](https://kestra.io/plugins/plugin-script-python) +- [Kestra Webhook Triggers](https://kestra.io/docs/workflow-components/triggers/webhook-trigger) + ### YANG / gNMI - [Arista gNMI Documentation](https://aristanetworks.github.io/openmgmt/configuration/gnmi/) - [OpenConfig Models](https://github.com/openconfig/public) @@ -145,26 +183,95 @@ fabric-orchestrator/ ## πŸš€ Getting Started -*Coming in Phase 1* +### Prerequisites + +- Docker and Docker Compose +- Python 3.12+ +- `uv` package manager +- Access to ContainerLab with cEOS images + +### Quick Start ```bash # Clone the repository git clone https://gitea.arnodo.fr/Damien/fabric-orchestrator.git cd fabric-orchestrator -# Install dependencies +# Start Kestra +docker compose up -d + +# Access Kestra UI +open http://localhost:8080 + +# Install Python dependencies (for CLI tools) uv sync # Verify gNMI connectivity to your fabric -fabric-orch discover --target leaf1:6030 +uv run fabric-orch discover capabilities --target leaf1:6030 -# Generate execution plan -fabric-orch plan +# Explore YANG paths +uv run fabric-orch discover get --target leaf1:6030 \ + --path "/interfaces/interface[name=Ethernet1]/state" +``` -# Apply changes -fabric-orch apply +### Kestra Workflow Example + +```yaml +id: fabric-reconcile +namespace: network.fabric +description: Reconcile fabric state with NetBox intent + +inputs: + - id: device + type: STRING + required: false + - id: auto_apply + type: BOOLEAN + defaults: false + +tasks: + - id: get_intent + type: io.kestra.plugin.scripts.python.Script + containerImage: ghcr.io/damien/fabric-orchestrator:latest + script: | + from kestra import Kestra + from src.netbox import FabricNetBoxClient + + client = FabricNetBoxClient() + intent = client.get_fabric_intent() + Kestra.outputs({"intent": intent.model_dump()}) + + - id: compute_diff + type: io.kestra.plugin.scripts.python.Script + containerImage: ghcr.io/damien/fabric-orchestrator:latest + script: | + from kestra import Kestra + # Compute diff between intent and current state + Kestra.outputs({"changes": changes, "has_changes": len(changes) > 0}) + + - id: apply_changes + type: io.kestra.plugin.scripts.python.Script + runIf: "{{ outputs.compute_diff.vars.has_changes and inputs.auto_apply }}" + containerImage: ghcr.io/damien/fabric-orchestrator:latest + script: | + from src.gnmi import GNMIClient + # Apply changes via gNMI Set + +triggers: + - id: netbox_webhook + type: io.kestra.plugin.core.trigger.Webhook + key: "{{ secret('NETBOX_WEBHOOK_KEY') }}" + + - id: schedule + type: io.kestra.plugin.core.trigger.Schedule + cron: "0 */6 * * *" + +errors: + - id: notify_failure + type: io.kestra.plugin.notifications.slack.SlackExecution + url: "{{ secret('SLACK_WEBHOOK') }}" ``` --- -**Status**: 🚧 Active Development - Phase 1 +**Status**: 🚧 Active Development - Migrating to Kestra orchestration (Phase 4)