diff --git a/configs/grafana/weathermap-dashboard.json b/configs/grafana/weathermap-dashboard.json index 34c9ea7..b809e98 100644 --- a/configs/grafana/weathermap-dashboard.json +++ b/configs/grafana/weathermap-dashboard.json @@ -118,7 +118,7 @@ "label": "campus-spine1", "position": [ 1600, - 80 + 0 ], "isConnection": false, "useConstantSpacing": false, @@ -173,8 +173,8 @@ "id": "dc-spine1", "label": "dc-spine1", "position": [ - 350, - 80 + 100, + 0 ], "isConnection": false, "useConstantSpacing": false, @@ -229,8 +229,8 @@ "id": "dc-spine2", "label": "dc-spine2", "position": [ - 750, - 80 + 280, + 0 ], "isConnection": false, "useConstantSpacing": false, @@ -285,8 +285,8 @@ "id": "campus-spine2", "label": "campus-spine2", "position": [ - 1850, - 80 + 1780, + 0 ], "isConnection": false, "useConstantSpacing": false, @@ -341,8 +341,8 @@ "id": "campus-border-leaf1", "label": "campus-border-leaf1", "position": [ - 1450, - 260 + 1600, + 600 ], "isConnection": false, "useConstantSpacing": false, @@ -397,8 +397,8 @@ "id": "dc-leaf1", "label": "dc-leaf1", "position": [ - 80, - 260 + 100, + 900 ], "isConnection": false, "useConstantSpacing": false, @@ -460,8 +460,8 @@ "id": "campus-border-leaf2", "label": "campus-border-leaf2", "position": [ - 1555, - 260 + 1780, + 600 ], "isConnection": false, "useConstantSpacing": false, @@ -516,8 +516,8 @@ "id": "dc-leaf4", "label": "dc-leaf4", "position": [ - 395, - 260 + 640, + 900 ], "isConnection": false, "useConstantSpacing": false, @@ -572,8 +572,8 @@ "id": "dc-leaf3", "label": "dc-leaf3", "position": [ - 290, - 260 + 460, + 900 ], "isConnection": false, "useConstantSpacing": false, @@ -628,8 +628,8 @@ "id": "dc-leaf5", "label": "dc-leaf5", "position": [ - 500, - 260 + 820, + 900 ], "isConnection": false, "useConstantSpacing": false, @@ -691,8 +691,8 @@ "id": "dc-border-leaf1", "label": "dc-border-leaf1", "position": [ - 920, - 260 + 100, + 600 ], "isConnection": false, "useConstantSpacing": false, @@ -747,8 +747,8 @@ "id": "dc-leaf6", "label": "dc-leaf6", "position": [ - 605, - 260 + 1000, + 900 ], "isConnection": false, "useConstantSpacing": false, @@ -810,8 +810,8 @@ "id": "dc-access1", "label": "dc-access1", "position": [ - 132, - 440 + 100, + 1200 ], "isConnection": false, "useConstantSpacing": false, @@ -866,8 +866,8 @@ "id": "dc-access2", "label": "dc-access2", "position": [ - 342, - 440 + 280, + 1200 ], "isConnection": false, "useConstantSpacing": false, @@ -922,8 +922,8 @@ "id": "dc-leaf8", "label": "dc-leaf8", "position": [ - 815, - 260 + 280, + 1050 ], "isConnection": false, "useConstantSpacing": false, @@ -978,8 +978,8 @@ "id": "dc-leaf7", "label": "dc-leaf7", "position": [ - 710, - 260 + 100, + 1050 ], "isConnection": false, "useConstantSpacing": false, @@ -1034,8 +1034,8 @@ "id": "dc-access4", "label": "dc-access4", "position": [ - 762, - 440 + 640, + 1200 ], "isConnection": false, "useConstantSpacing": false, @@ -1090,8 +1090,8 @@ "id": "dc-access3", "label": "dc-access3", "position": [ - 552, - 440 + 460, + 1200 ], "isConnection": false, "useConstantSpacing": false, @@ -1146,8 +1146,8 @@ "id": "dc-border-leaf2", "label": "dc-border-leaf2", "position": [ - 1025, - 260 + 280, + 600 ], "isConnection": false, "useConstantSpacing": false, @@ -1202,8 +1202,8 @@ "id": "dc-leaf2", "label": "dc-leaf2", "position": [ - 185, - 260 + 280, + 900 ], "isConnection": false, "useConstantSpacing": false, @@ -1265,8 +1265,8 @@ "id": "campus-leaf1", "label": "campus-leaf1", "position": [ - 1660, - 260 + 1600, + 900 ], "isConnection": false, "useConstantSpacing": false, @@ -1328,8 +1328,8 @@ "id": "campus-access1", "label": "campus-access1", "position": [ - 1712, - 440 + 1600, + 1200 ], "isConnection": false, "useConstantSpacing": false, @@ -1384,8 +1384,8 @@ "id": "campus-leaf2", "label": "campus-leaf2", "position": [ - 1765, - 260 + 1780, + 900 ], "isConnection": false, "useConstantSpacing": false, @@ -1447,8 +1447,8 @@ "id": "campus-leaf3", "label": "campus-leaf3", "position": [ - 1870, - 260 + 1960, + 900 ], "isConnection": false, "useConstantSpacing": false, @@ -1510,8 +1510,8 @@ "id": "campus-access2", "label": "campus-access2", "position": [ - 1922, - 440 + 1780, + 1200 ], "isConnection": false, "useConstantSpacing": false, @@ -1566,8 +1566,8 @@ "id": "campus-leaf4", "label": "campus-leaf4", "position": [ - 1975, - 260 + 2140, + 900 ], "isConnection": false, "useConstantSpacing": false, @@ -1629,8 +1629,8 @@ "id": "core1", "label": "core1", "position": [ - 1150, - 260 + 1300, + 300 ], "isConnection": false, "useConstantSpacing": false, @@ -1685,8 +1685,8 @@ "id": "core2", "label": "core2", "position": [ - 1280, - 260 + 1480, + 300 ], "isConnection": false, "useConstantSpacing": false, diff --git a/envrc.sample b/envrc.sample index 1d698b9..52e9916 100644 --- a/envrc.sample +++ b/envrc.sample @@ -1,4 +1,11 @@ export GRAFANA_URL="" export GRAFANA_TOKEN="" +export GRAFANA_DATASOURCE_UID="" export IPFABRIC_URL="" export IPFABRIC_TOKEN="" + +# Optional -- see scripts/README.md for defaults and when to override +# export IPFABRIC_SNAPSHOT="$last" +# export PROMETHEUS_URL="http://172.16.0.71:9090" +# export GRAFANA_DASHBOARD_UID="evpn-vxlan-fabric-weathermap" +# export GRAFANA_WEATHERMAP_PLUGIN_ID="tamirsuliman-weathermap-panel" diff --git a/scripts/README.md b/scripts/README.md index 9554650..b07384b 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -26,32 +26,159 @@ comment thread for the live-validation history (auth quirks, plugin id, a PromQL escaping bug, a panel-crash fix — worth reading before touching the anchor/regex-escaping logic). -## Usage +## How to generate and provision the dashboard, start to finish + +This walks through the whole thing from nothing — no `weathermap-dashboard.json`, +no dashboard in Grafana — to a fully working, live-updating dashboard. Useful +for a first-ever setup, or after a full reset (deleting the generated JSON +and the Grafana dashboard, keeping `dashboard-base.json` — that file is the +hand-authored part of the project and is never deleted/regenerated). + +### 1. Gather credentials + +Four values are required; two more are worth setting so `--provision` works +without extra flags. Each has a fiddly API path and a much simpler UI path — +UI is recommended unless you're scripting this. + +**IPFabric API token** (`IPFABRIC_TOKEN`) +- UI: IPFabric → user menu (top right) → **Settings** → **API Tokens** → **Add token**. Needs Inventory/Snapshots read access. +- No simpler API alternative — tokens can only be created through the UI or by an admin via the same UI. + +**Grafana service account token** (`GRAFANA_TOKEN`) +- UI (recommended — the API path is multiple chained calls: create the service account, then create a token under it, threading the returned service account ID between them): + 1. Grafana → **Administration** → **Users and access** → **Service accounts** → **Add service account**. + 2. Give it a name (e.g. `weathermap-generator`), role **Editor** (needs to create/update dashboards). + 3. Open the new service account → **Add service account token** → copy the token immediately, it's shown once. + +**Prometheus datasource UID** (`GRAFANA_DATASOURCE_UID`) +- UI: Grafana → **Connections** → **Data sources** → click the Prometheus datasource that can see the gnmic metrics (`NetLab` in this lab) → the UID is the last segment of the page URL (`.../datasources/edit/`). +- API alternative: `curl -s -H "Authorization: Bearer $GRAFANA_TOKEN" "$GRAFANA_URL/api/datasources" | jq '.[] | {name, uid}'` +- **This must be a datasource that can actually see the gnmic metrics** — see the note in the Environment variables table below; picking the wrong one renders the panel with no data (happened once, see #48). + +**Weathermap plugin ID** (`GRAFANA_WEATHERMAP_PLUGIN_ID`, optional — only if a different fork than `tamirsuliman-weathermap-panel` is installed) +- UI: Grafana → **Administration** → **Plugins and data** → **Plugins**, search "weathermap", open it — the id is in the page URL (`.../plugins/`). +- API alternative: `curl -s -H "Authorization: Bearer $GRAFANA_TOKEN" "$GRAFANA_URL/api/plugins?panelId=..." ` is more involved than it's worth; the UI is faster here. + +**IPFabric/Grafana URLs** — just the base URLs you already use in a browser (e.g. `https://ipfabric.example.com`, `https://grafana.example.com`). + +Copy `envrc.sample` to `.envrc` (already gitignored) and fill in the values, +or `export` them directly in your shell: ```bash -export IPFABRIC_URL=https:// -export IPFABRIC_TOKEN= -python3 scripts/generate_weathermap.py # writes the merged JSON only - -export GRAFANA_URL=https:// -export GRAFANA_TOKEN= -export GRAFANA_DATASOURCE_UID= -python3 scripts/generate_weathermap.py --provision # also provisions via the Grafana API +cp envrc.sample .envrc # then edit .envrc +source .envrc ``` -Re-run after any topology change (`evpn-lab.clab.yml`) to regenerate the -weathermap panel and re-merge/re-provision. The script always regenerates the -weathermap panel fresh from live IPFabric data (no diffing/incremental -state), so a plain re-run already picks up any topology change — there's no -separate "detect changes" step. The only manual/operational step is -*triggering* that re-run after a change; there's no watcher or CI hook doing -it automatically yet. +### 2. Dry run first (no Grafana changes yet) -If you need to change the BGP/ports/throughput panels, template variables, or -layout, edit `configs/grafana/dashboard-base.json` directly and re-run the -script (or run it with `--provision` to push the edit live) — no topology -data is needed for that path, the weathermap panel just gets regenerated -alongside it. +```bash +python3 scripts/generate_weathermap.py +``` + +This only needs `IPFABRIC_URL`/`IPFABRIC_TOKEN` (and optionally `GRAFANA_URL`/`GRAFANA_TOKEN`, +to preserve any live node positions — see #53). It writes +`configs/grafana/weathermap-dashboard.json` and prints a summary: + +``` +Fetching devices from IPFabric (...)... + 28 devices +Fetching connectivity-matrix... + 61 fabric links after Management-plane filter + dedup (702 raw rows) +Fetching interface speeds... +Fetching live node positions from ... (preserve manual repositioning)... + 0 node(s) with an existing live position <- 0 is expected/correct on a first-ever run +Cross-checking interface names against live exporter (...)... +Loading manual dashboard base (configs/grafana/dashboard-base.json)... +Wrote configs/grafana/weathermap-dashboard.json (28 nodes, 61 links, 4 panels total) +``` + +If any "interface-name / bandwidth mismatch(es)" are printed to stderr, read +them — they mean a link's query won't resolve to data until the underlying +gnmic/IPFabric naming mismatch is fixed (not this script's job to fix, see +"Known gaps" below), but the link itself is still generated, not dropped. + +Open the written JSON and sanity-check it if you like before pushing it live +— it's plain, readable JSON, no need to trust it blindly. + +### 3. Provision it + +```bash +python3 scripts/generate_weathermap.py --provision +``` + +Requires `GRAFANA_URL`, `GRAFANA_TOKEN`, `GRAFANA_DATASOURCE_UID` in addition +to the IPFabric variables. On success: + +``` +Provisioning dashboard 'evpn-vxlan-fabric-weathermap' to https://... + success: /d/evpn-vxlan-fabric-weathermap/evpn-vxlan-fabric-weathermap +``` + +### 4. Verify + +**Via the UI (recommended, and the only way to actually see it render)**: +open `$GRAFANA_URL/d/evpn-vxlan-fabric-weathermap` in a browser. Check: +- The weathermap panel shows all nodes, roughly grouped spine-top/access-bottom (see #53), all colored green (up) with no data gaps. +- The `site`/`device` dropdown variables at the top of the dashboard filter the BGP Sessions and Ports/Interfaces tables when changed. +- BGP Sessions and Ports/Interfaces tables are populated, states colored (green=Up, red=Down). +- Throughput per Site shows a signed graph (in above the axis, out below) with plausible non-zero values. + +This step has no good API substitute — dashboard rendering, panel colors, +and transformations (the ports table's Grafana-side merge, in particular) +only actually execute in a browser. If you don't have one handy, at minimum +confirm the queries resolve (below), then ask someone who does have UI +access to eyeball it. + +**Via the API (queries only, not rendering)** — useful for CI/scripting or a +quick sanity check without opening a browser: + +```bash +# Any panel's query, pulled straight from the generated JSON, e.g.: +curl -s -X POST -H "Authorization: Bearer $GRAFANA_TOKEN" -H "Content-Type: application/json" \ + "$GRAFANA_URL/api/ds/query" -d '{ + "queries": [{"refId":"A","datasource":{"type":"prometheus","uid":"'"$GRAFANA_DATASOURCE_UID"'"}, + "expr":"min by (device) (interfaces_interface_state_oper_status)","instant":true}], + "from":"now-5m","to":"now"}' | python3 -m json.tool +``` + +A much friendlier equivalent that doesn't need hand-built JSON: Grafana's +**Explore** view (left sidebar → **Explore**, pick the Prometheus datasource, +paste a PromQL expression from the panel JSON, run it). This is the manual +alternative for every `/api/ds/query` validation call used during +development of this dashboard (#49–#54) — same result, no `curl`/`jq` +required. + +### 5. Re-running after a topology change + +```bash +python3 scripts/generate_weathermap.py --provision +``` + +Same command — the script always regenerates the weathermap panel fresh from +live IPFabric data (no diffing/incremental state), so a plain re-run already +picks up any topology change. There's no separate "detect changes" step, and +no watcher/CI hook triggering it automatically yet — re-running after a +change is a manual/operational step for now. + +Nodes you've manually repositioned in the Grafana UI keep that position +across the re-run (#53); only genuinely new nodes get the layered default. + +If you only need to change the BGP/ports/throughput panels, template +variables, or panel layout, edit `configs/grafana/dashboard-base.json` +directly and re-run the script (or `--provision` to push it live) — no +topology fetch is skipped today (the script always does the full IPFabric +round-trip regardless), but no topology data is *needed* for that kind of +edit to take effect. + +### Troubleshooting + +| Symptom | Likely cause | Check | +|---|---|---| +| `Refusing to provision: set GRAFANA_DATASOURCE_UID...` | Env var unset or still the placeholder | `echo $GRAFANA_DATASOURCE_UID` | +| Panel loads but shows no data at all | Wrong datasource UID — points at a Prometheus instance that can't see the gnmic metrics | Grafana UI → Explore → run `up` against the datasource you set; confirm gnmic-labeled series come back | +| `Base dashboard has no panel titled '__WEATHERMAP_SLOT__'` | `dashboard-base.json` was edited and the slot panel's `title` got changed/removed | Check the panel with `id: 1` in `dashboard-base.json` still has that exact title | +| A node's manual position keeps resetting to the default layout on every rerun | `GRAFANA_URL`/`GRAFANA_TOKEN` not set for that run, so the live-position fetch was skipped entirely | Check the printed line `Fetching live node positions from ...` appears in the run's output — if it's missing, those two vars weren't set | +| `Fetching live dashboard for position preservation failed: HTTP 401/403` | Bad/expired Grafana token | Regenerate the service account token (UI steps above) | ## Environment variables