Merge pull request 'Telemetry: gnmic + Prometheus + Grafana weathermap dashboard, iperf3 traffic generator' (#59) from feat/telemetry into main

Reviewed-on: #59

assets/weathermap-pipeline.svg

@@ -0,0 +1,72 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 350" font-family="Helvetica, Arial, sans-serif">
+  <defs>
+    <marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
+      <path d="M 0 0 L 10 5 L 0 10 z" fill="#555"/>
+    </marker>
+  </defs>
+
+  <style>
+    .box { stroke-width: 1.5; rx: 6; }
+    .input { fill: #e8f0fe; stroke: #4285f4; }
+    .manual { fill: #fef7e0; stroke: #f9ab00; }
+    .script { fill: #e6f4ea; stroke: #34a853; }
+    .output { fill: #f3e8fd; stroke: #a142f4; }
+    .grafana { fill: #fce8e6; stroke: #ea4335; }
+    .label { font-size: 13px; fill: #202124; }
+    .sublabel { font-size: 10px; fill: #5f6368; }
+    .arrow { stroke: #555; stroke-width: 1.5; fill: none; marker-end: url(#arrow); }
+    .caption { font-size: 11px; fill: #5f6368; font-style: italic; }
+  </style>
+
+  <!-- inputs -->
+  <rect class="box input" x="20" y="30" width="150" height="50"/>
+  <text class="label" x="95" y="50" text-anchor="middle">IPFabric</text>
+  <text class="sublabel" x="95" y="66" text-anchor="middle">topology, inventory</text>
+
+  <rect class="box input" x="20" y="100" width="150" height="50"/>
+  <text class="label" x="95" y="120" text-anchor="middle">Prometheus</text>
+  <text class="sublabel" x="95" y="136" text-anchor="middle">gnmic metrics, live labels</text>
+
+  <!-- generator script -->
+  <rect class="box script" x="220" y="55" width="200" height="70"/>
+  <text class="label" x="320" y="82" text-anchor="middle" font-size="11.5">generate_weathermap.py</text>
+  <text class="sublabel" x="320" y="98" text-anchor="middle">weathermap panel only</text>
+  <text class="sublabel" x="320" y="112" text-anchor="middle">targets + options.weathermap</text>
+
+  <path class="arrow" d="M170,55 L220,80"/>
+  <path class="arrow" d="M170,125 L220,100"/>
+
+  <!-- manual base -->
+  <rect class="box manual" x="220" y="220" width="200" height="70"/>
+  <text class="label" x="320" y="247" text-anchor="middle">dashboard-base.json</text>
+  <text class="sublabel" x="320" y="263" text-anchor="middle">hand-authored, committed</text>
+  <text class="sublabel" x="320" y="277" text-anchor="middle">BGP/ports/throughput/vars</text>
+
+  <!-- merge -->
+  <rect class="box script" x="470" y="130" width="160" height="60"/>
+  <text class="label" x="550" y="155" text-anchor="middle">merge</text>
+  <text class="sublabel" x="550" y="171" text-anchor="middle" font-size="8.5">splices into __WEATHERMAP_SLOT__</text>
+
+  <path class="arrow" d="M420,90 L470,145"/>
+  <path class="arrow" d="M420,255 L470,175"/>
+
+  <!-- output artifact -->
+  <rect class="box output" x="660" y="60" width="210" height="60"/>
+  <text class="label" x="765" y="85" text-anchor="middle" font-size="11.5">weathermap-dashboard.json</text>
+  <text class="sublabel" x="765" y="101" text-anchor="middle">generated build artifact</text>
+
+  <path class="arrow" d="M630,150 L765,120"/>
+
+  <!-- grafana -->
+  <rect class="box grafana" x="660" y="210" width="210" height="60"/>
+  <text class="label" x="765" y="235" text-anchor="middle">Grafana</text>
+  <text class="sublabel" x="765" y="251" text-anchor="middle" font-size="9">--provision (POST /api/dashboards/db)</text>
+
+  <path class="arrow" d="M630,175 L765,210"/>
+  <path class="arrow" d="M765,120 L765,210" stroke-dasharray="3,3"/>
+
+  <!-- read-back for position preservation -->
+  <path class="arrow" d="M765,270 C 620,330 490,300 420,275" stroke-dasharray="3,3"/>
+  <text class="caption" x="545" y="325" text-anchor="middle">live node positions read back before each regeneration (see #53)</text>
+
+</svg>

scripts/README.md

@@ -1,139 +1,67 @@
 # scripts/generate_weathermap.py
 
-Generates a [weathermap-ng](https://github.com/allamiro/grafana-network-weathermap-ng)
+Generates the **weathermap panel only** from live IPFabric + Prometheus
-**panel only** from live IPFabric topology + gnmic/Prometheus metrics, merges
+data, merges it into a hand-authored dashboard base, and provisions the
-it into a manually-authored dashboard base, and writes the merged result to
+result to Grafana.
-`configs/grafana/weathermap-dashboard.json` (committed to Gitea as the build
-artifact, same pattern as the retired Flow Panel YAML). Optionally provisions
-it into Grafana via the HTTP API.
 
-## File structure (see #52)
+![pipeline diagram](../assets/weathermap-pipeline.svg)
 
-| File | What it is |
+## Files
-|---|---|
-| `configs/grafana/dashboard-base.json` | **Manually authored**, committed source of truth for everything except the weathermap panel: BGP sessions table, ports/interfaces table, throughput-per-site graphs, `site`/`device` template variables, panel layout. Edit this directly for anything in those panels. Contains a reserved slot panel titled `__WEATHERMAP_SLOT__` (id `1`, `gridPos` fixed) that the script splices the generated weathermap panel into. |
-| `configs/grafana/weathermap-dashboard.json` | **Generated build artifact** — base + freshly-generated weathermap panel, merged. This is what actually gets provisioned to Grafana. Don't hand-edit; re-run the script. |
 
-The script itself only ever knows about the weathermap panel (`targets` +
+| File                                        | What it is                                                                                                                                                                                      | Edit it?                |
-`options.weathermap`) — it has no knowledge of the other three panels or the
+| ------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- |
-template variables. That split is deliberate: topology-driven content
+| `configs/grafana/dashboard-base.json`       | Hand-authored: BGP table, ports table, throughput graphs, `site`/`device` variables, layout. Has one reserved slot panel (`title: "__WEATHERMAP_SLOT__"`) where the weathermap gets spliced in. | Yes, directly           |
-(nodes/links, generated from live IPFabric+Prometheus data) is separated from
+| `configs/grafana/weathermap-dashboard.json` | Generated build artifact — base + fresh weathermap panel, merged. What actually gets provisioned.                                                                                               | No — regenerate instead |
-hand-tuned dashboard composition (table layout, transformations, thresholds),
-which doesn't need to regenerate on every topology change.
 
-See issue #48 for the panel schema research this is built against, and its
+Why split like this: topology data (nodes/links) changes with the lab and
-comment thread for the live-validation history (auth quirks, plugin id, a
+should regenerate every time; the rest of the dashboard (table layout,
-PromQL escaping bug, a panel-crash fix — worth reading before touching the
+thresholds, transformations) is hand-tuned and shouldn't be touched by a
-anchor/regex-escaping logic).
+topology refresh. See #52.
 
-## How to generate and provision the dashboard, start to finish
+## Quickstart
-
-This walks through the whole thing from nothing — no `weathermap-dashboard.json`,
-no dashboard in Grafana — to a fully working, live-updating dashboard. Useful
-for a first-ever setup, or after a full reset (deleting the generated JSON
-and the Grafana dashboard, keeping `dashboard-base.json` — that file is the
-hand-authored part of the project and is never deleted/regenerated).
-
-### 1. Gather credentials
-
-Four values are required; two more are worth setting so `--provision` works
-without extra flags. Each has a fiddly API path and a much simpler UI path —
-UI is recommended unless you're scripting this.
-
-**IPFabric API token** (`IPFABRIC_TOKEN`)
-- UI: IPFabric → user menu (top right) → **Settings** → **API Tokens** → **Add token**. Needs Inventory/Snapshots read access.
-- No simpler API alternative — tokens can only be created through the UI or by an admin via the same UI.
-
-**Grafana service account token** (`GRAFANA_TOKEN`)
-- UI (recommended — the API path is multiple chained calls: create the service account, then create a token under it, threading the returned service account ID between them):
-  1. Grafana → **Administration** → **Users and access** → **Service accounts** → **Add service account**.
-  2. Give it a name (e.g. `weathermap-generator`), role **Editor** (needs to create/update dashboards).
-  3. Open the new service account → **Add service account token** → copy the token immediately, it's shown once.
-
-**Prometheus datasource UID** (`GRAFANA_DATASOURCE_UID`)
-- UI: Grafana → **Connections** → **Data sources** → click the Prometheus datasource that can see the gnmic metrics (`NetLab` in this lab) → the UID is the last segment of the page URL (`.../datasources/edit/<uid>`).
-- API alternative: `curl -s -H "Authorization: Bearer $GRAFANA_TOKEN" "$GRAFANA_URL/api/datasources" | jq '.[] | {name, uid}'`
-- **This must be a datasource that can actually see the gnmic metrics** — see the note in the Environment variables table below; picking the wrong one renders the panel with no data (happened once, see #48).
-
-**Weathermap plugin ID** (`GRAFANA_WEATHERMAP_PLUGIN_ID`, optional — only if a different fork than `tamirsuliman-weathermap-panel` is installed)
-- UI: Grafana → **Administration** → **Plugins and data** → **Plugins**, search "weathermap", open it — the id is in the page URL (`.../plugins/<id>`).
-- API alternative: `curl -s -H "Authorization: Bearer $GRAFANA_TOKEN" "$GRAFANA_URL/api/plugins?panelId=..." ` is more involved than it's worth; the UI is faster here.
-
-**IPFabric/Grafana URLs** — just the base URLs you already use in a browser (e.g. `https://ipfabric.example.com`, `https://grafana.example.com`).
-
-Copy `envrc.sample` to `.envrc` (already gitignored) and fill in the values,
-or `export` them directly in your shell:
 
 ```bash
-cp envrc.sample .envrc   # then edit .envrc
+cp envrc.sample .envrc   # fill in the values (see "Credentials" below)
 source .envrc
+
+python3 scripts/generate_weathermap.py              # dry run: writes the JSON only
+python3 scripts/generate_weathermap.py --provision   # also pushes it to Grafana
 ```
 
-### 2. Dry run first (no Grafana changes yet)
+Re-run the same `--provision` command any time the lab topology changes —
+the script always rebuilds the weathermap panel from live data, so there's
+nothing to "detect", just re-run it. It's not automated yet (no watcher/CI
+hook), so re-running is a manual step for now.
+
+## Credentials
+
+| Value                         | Env var                                                                 | Get it from the UI                                                                                    | Get it via API                                                                                               |
+| ----------------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ |
+| IPFabric token                | `IPFABRIC_TOKEN`                                                        | IPFabric → user menu → Settings → API Tokens → Add token                                              | not possible, UI-only                                                                                        |
+| Grafana service account token | `GRAFANA_TOKEN`                                                         | Grafana → Administration → Service accounts → Add → give it **Editor** role → Add token               | possible but multi-step (create account, then create token under it) — UI is simpler                         |
+| Prometheus datasource UID     | `GRAFANA_DATASOURCE_UID`                                                | Grafana → Connections → Data sources → open the one with gnmic data → UID is the last part of the URL | `curl -s -H "Authorization: Bearer $GRAFANA_TOKEN" "$GRAFANA_URL/api/datasources" \| jq '.[] \| {name,uid}'` |
+| Weathermap plugin ID          | `GRAFANA_WEATHERMAP_PLUGIN_ID` (optional, only if not the default fork) | Grafana → Administration → Plugins → search "weathermap" → id is in the URL                           | —                                                                                                            |
+
+⚠️ The datasource must be the one that can actually see gnmic metrics —
+picking an unrelated Prometheus instance renders the panel with no data
+(see #48).
+
+## Verifying the result
+
+**In the browser** (the only way to actually see it render — colors,
+tables, and the ports-table merge transformation all happen client-side):
+
+1. Open `$GRAFANA_URL/d/evpn-vxlan-fabric-weathermap`.
+2. Weathermap: nodes grouped spine-top → access-bottom, all green.
+3. `site`/`device` dropdowns filter the two tables below.
+4. BGP Sessions / Ports tables: populated, green=Up / red=Down.
+5. Throughput per Site: signed graph, non-zero values.
+
+**Without a browser**, use Grafana's **Explore** view (paste a PromQL
+expression from the panel JSON and run it) — same result as the UI check
+above minus the rendering, no `curl` needed. Or hit the API directly:
 
 ```bash
-python3 scripts/generate_weathermap.py
-```
-
-This only needs `IPFABRIC_URL`/`IPFABRIC_TOKEN` (and optionally `GRAFANA_URL`/`GRAFANA_TOKEN`,
-to preserve any live node positions — see #53). It writes
-`configs/grafana/weathermap-dashboard.json` and prints a summary:
-
-```
-Fetching devices from IPFabric (...)...
-  28 devices
-Fetching connectivity-matrix...
-  61 fabric links after Management-plane filter + dedup (702 raw rows)
-Fetching interface speeds...
-Fetching live node positions from ... (preserve manual repositioning)...
-  0 node(s) with an existing live position          <- 0 is expected/correct on a first-ever run
-Cross-checking interface names against live exporter (...)...
-Loading manual dashboard base (configs/grafana/dashboard-base.json)...
-Wrote configs/grafana/weathermap-dashboard.json (28 nodes, 61 links, 4 panels total)
-```
-
-If any "interface-name / bandwidth mismatch(es)" are printed to stderr, read
-them — they mean a link's query won't resolve to data until the underlying
-gnmic/IPFabric naming mismatch is fixed (not this script's job to fix, see
-"Known gaps" below), but the link itself is still generated, not dropped.
-
-Open the written JSON and sanity-check it if you like before pushing it live
-— it's plain, readable JSON, no need to trust it blindly.
-
-### 3. Provision it
-
-```bash
-python3 scripts/generate_weathermap.py --provision
-```
-
-Requires `GRAFANA_URL`, `GRAFANA_TOKEN`, `GRAFANA_DATASOURCE_UID` in addition
-to the IPFabric variables. On success:
-
-```
-Provisioning dashboard 'evpn-vxlan-fabric-weathermap' to https://...
-  success: /d/evpn-vxlan-fabric-weathermap/evpn-vxlan-fabric-weathermap
-```
-
-### 4. Verify
-
-**Via the UI (recommended, and the only way to actually see it render)**:
-open `$GRAFANA_URL/d/evpn-vxlan-fabric-weathermap` in a browser. Check:
-- The weathermap panel shows all nodes, roughly grouped spine-top/access-bottom (see #53), all colored green (up) with no data gaps.
-- The `site`/`device` dropdown variables at the top of the dashboard filter the BGP Sessions and Ports/Interfaces tables when changed.
-- BGP Sessions and Ports/Interfaces tables are populated, states colored (green=Up, red=Down).
-- Throughput per Site shows a signed graph (in above the axis, out below) with plausible non-zero values.
-
-This step has no good API substitute — dashboard rendering, panel colors,
-and transformations (the ports table's Grafana-side merge, in particular)
-only actually execute in a browser. If you don't have one handy, at minimum
-confirm the queries resolve (below), then ask someone who does have UI
-access to eyeball it.
-
-**Via the API (queries only, not rendering)** — useful for CI/scripting or a
-quick sanity check without opening a browser:
-
-```bash
-# Any panel's query, pulled straight from the generated JSON, e.g.:
 curl -s -X POST -H "Authorization: Bearer $GRAFANA_TOKEN" -H "Content-Type: application/json" \
   "$GRAFANA_URL/api/ds/query" -d '{
     "queries": [{"refId":"A","datasource":{"type":"prometheus","uid":"'"$GRAFANA_DATASOURCE_UID"'"},
@@ -141,118 +69,99 @@ curl -s -X POST -H "Authorization: Bearer $GRAFANA_TOKEN" -H "Content-Type: appl
     "from":"now-5m","to":"now"}' | python3 -m json.tool
 ```
 
-A much friendlier equivalent that doesn't need hand-built JSON: Grafana's
+## Troubleshooting
-**Explore** view (left sidebar → **Explore**, pick the Prometheus datasource,
-paste a PromQL expression from the panel JSON, run it). This is the manual
-alternative for every `/api/ds/query` validation call used during
-development of this dashboard (#49–#54) — same result, no `curl`/`jq`
-required.
 
-### 5. Re-running after a topology change
+| Symptom                                                    | Cause                                                               | Check                                                              |
-
+| ---------------------------------------------------------- | ------------------------------------------------------------------- | ------------------------------------------------------------------ |
-```bash
+| `Refusing to provision: set GRAFANA_DATASOURCE_UID...`     | Env var unset                                                       | `echo $GRAFANA_DATASOURCE_UID`                                     |
-python3 scripts/generate_weathermap.py --provision
+| Panel renders with no data                                 | Wrong datasource — can't see gnmic metrics                          | Explore → run `up` against it, confirm gnmic series come back      |
-```
+| `Base dashboard has no panel titled '__WEATHERMAP_SLOT__'` | Someone edited `dashboard-base.json` and renamed/removed that panel | Check panel `id: 1` still has that exact title                     |
-
+| Manually-moved node keeps resetting position               | `GRAFANA_URL`/`GRAFANA_TOKEN` weren't set for that run              | Look for `Fetching live node positions from ...` in the run output |
-Same command — the script always regenerates the weathermap panel fresh from
+| `HTTP 401/403` fetching live positions                     | Expired/bad Grafana token                                           | Re-issue the service account token                                 |
-live IPFabric data (no diffing/incremental state), so a plain re-run already
-picks up any topology change. There's no separate "detect changes" step, and
-no watcher/CI hook triggering it automatically yet — re-running after a
-change is a manual/operational step for now.
-
-Nodes you've manually repositioned in the Grafana UI keep that position
-across the re-run (#53); only genuinely new nodes get the layered default.
-
-If you only need to change the BGP/ports/throughput panels, template
-variables, or panel layout, edit `configs/grafana/dashboard-base.json`
-directly and re-run the script (or `--provision` to push it live) — no
-topology fetch is skipped today (the script always does the full IPFabric
-round-trip regardless), but no topology data is *needed* for that kind of
-edit to take effect.
-
-### Troubleshooting
-
-| Symptom | Likely cause | Check |
-|---|---|---|
-| `Refusing to provision: set GRAFANA_DATASOURCE_UID...` | Env var unset or still the placeholder | `echo $GRAFANA_DATASOURCE_UID` |
-| Panel loads but shows no data at all | Wrong datasource UID — points at a Prometheus instance that can't see the gnmic metrics | Grafana UI → Explore → run `up` against the datasource you set; confirm gnmic-labeled series come back |
-| `Base dashboard has no panel titled '__WEATHERMAP_SLOT__'` | `dashboard-base.json` was edited and the slot panel's `title` got changed/removed | Check the panel with `id: 1` in `dashboard-base.json` still has that exact title |
-| A node's manual position keeps resetting to the default layout on every rerun | `GRAFANA_URL`/`GRAFANA_TOKEN` not set for that run, so the live-position fetch was skipped entirely | Check the printed line `Fetching live node positions from ...` appears in the run's output — if it's missing, those two vars weren't set |
-| `Fetching live dashboard for position preservation failed: HTTP 401/403` | Bad/expired Grafana token | Regenerate the service account token (UI steps above) |
 
 ## Environment variables
 
-| Variable | Required | Default | Notes |
+| Variable                       | Required           | Default                               |
-|---|---|---|---|
+| ------------------------------ | ------------------ | ------------------------------------- |
-| `--base` (CLI flag, not env) | no | `configs/grafana/dashboard-base.json` | Manual dashboard base to merge the weathermap panel into. |
+| `IPFABRIC_URL`                 | yes                | —                                     |
-| `IPFABRIC_URL` | yes | — | e.g. `https://ipfabric.example.com` |
+| `IPFABRIC_TOKEN`               | yes                | —                                     |
-| `IPFABRIC_TOKEN` | yes | — | Inventory/Snapshots read access. Sent as `X-API-Token`, not `Authorization: Bearer` — IPFabric's REST API does not use bearer auth. |
+| `IPFABRIC_SNAPSHOT`            | no                 | `$last`                               |
-| `IPFABRIC_SNAPSHOT` | no | `$last` | |
+| `PROMETHEUS_URL`               | no                 | `http://172.16.0.71:9090`             |
-| `PROMETHEUS_URL` | no | `http://172.16.0.71:9090` | The **in-topology** Prometheus (see #45), used at generation time to validate the IPFabric→gnmic interface alias and to discover live VTEP/VLAN pairs. This does not have to be the same instance Grafana queries at render time — see below. |
+| `GRAFANA_URL`                  | no*                | —                                     |
-| `GRAFANA_URL` | no (see #53) | — | Required for `--provision`. Also used, if set, to fetch the currently-provisioned dashboard and preserve any manually drag-and-drop-repositioned node positions — see step 3 below. Without it, every node gets the layered default position, even ones a human previously repositioned in the UI. |
+| `GRAFANA_TOKEN`                | no*                | —                                     |
-| `GRAFANA_TOKEN` | no (see #53) | — | Grafana Service Account token, sent as `Authorization: Bearer`. Same conditions as `GRAFANA_URL` above. |
+| `GRAFANA_DATASOURCE_UID`       | with `--provision` | —                                     |
-| `GRAFANA_DATASOURCE_UID` | with `--provision` | — | UID of the Prometheus datasource in Grafana that will actually back the panel. **This must be a datasource that can see the gnmic metrics** — an unrelated/external Prometheus instance with no gnmic data will make the panel render with no values (this happened once, see #48). |
+| `GRAFANA_DASHBOARD_UID`        | no                 | `evpn-vxlan-fabric-weathermap`        |
-| `GRAFANA_DASHBOARD_UID` | no | `evpn-vxlan-fabric-weathermap` | |
+| `GRAFANA_WEATHERMAP_PLUGIN_ID` | no                 | `tamirsuliman-weathermap-panel`       |
-| `GRAFANA_WEATHERMAP_PLUGIN_ID` | no | `tamirsuliman-weathermap-panel` | The installed weathermap-ng plugin id. Override if a different fork is installed — plugin ids don't always match the upstream repo name (the schema research in #48 was done against the `allamiro` fork's source; what's actually installed here is a different fork with a stricter/different runtime schema — see the `ANCHOR` handling in the script). |
+| `--base` (CLI flag)            | no                 | `configs/grafana/dashboard-base.json` |
 
-## What the script does
+\* `GRAFANA_URL`/`GRAFANA_TOKEN` are required for `--provision`, and
+optional otherwise — if set on a plain (non-`--provision`) run, they're
+used to read back live node positions so manual repositioning survives
+the next regeneration (see #53). Skip them and every node just gets the
+default layout.
 
-1. Fetches device inventory + connectivity-matrix from IPFabric.
+## How it works
-2. Filters the connectivity-matrix to physical Ethernet-to-Ethernet links
+
-   only: drops Management-plane neighbor entries and `.100`/`.200`
+1. Fetch device inventory + connectivity-matrix from IPFabric.
-   subinterface rows (802.1Q tags used for the gold VRF stitching on Core —
+2. Keep only physical Ethernet↔Ethernet links (drop management-plane and
-   they ride the same physical port as their parent interface and gnmic only
+   `.100`/`.200` subinterface rows), dedupe the two directions IPFabric
-   exports physical interface counters), and dedupes the two directions
+   reports per link.
-   IPFabric reports for each physical link into one.
+3. Position each node:
-3. Computes node positions (see #53):
+    - **New node** → layered default: Y-tier by role parsed from the
-   - **Default layered layout**, for nodes with no known position yet: role
+      hostname (`spine → core → border-leaf → leaf → access`), X grouped
-     is parsed from the hostname naming convention (`*-spine*`→spine,
+      by site within the tier. A hostname that matches no role is logged
-     `core*`→core, `*-border-leaf*`→border-leaf, `*-leaf*`→leaf,
+      and dropped into a fallback tier, never silently misplaced (#53).
-     `*-access*`→access — same trust level as the `site` parsing used for
+    - **Known node** → whatever position is currently live in Grafana,
-     the Prometheus relabel, not IPFabric-derived), giving Y-tiers top to
+      unchanged. This is why `GRAFANA_URL`/`GRAFANA_TOKEN` matter even on
-     bottom `spine → core → border-leaf → leaf → access`. X position is
+      a dry run (#53).
-     grouped/columned by site within each tier, so e.g. dc leafs and campus
+4. Build the panel's PromQL `targets`: node status, per-side link tx (each
-     leafs land in the same row but different column bands. A hostname
+   side its own egress counter, so both directions are represented — #57),
-     matching no role pattern is logged (not silently placed in the wrong
+   VXLAN MAC/VNI tooltip — each with an explicit `legendFormat`.
-     tier) and put in a fallback tier below `access`.
+5. Build `nodes[]`/`links[]`, including the plugin's `anchors` tally
-   - **Live-position preservation**: position is edited by drag-and-drop
+   (link count per side) — required by the installed plugin fork even
-     directly in the Grafana UI, not in the git-committed base file — a
+   though the schema doc says it's optional (#48).
-     deliberate exception to the "file is the only source of truth" rule
+6. Cross-check every interface name against the live gnmic exporter
-     from #52, because that's simply not where this field is edited. Before
+   (`Et`→`Ethernet`, `Po`→`Port-Channel`, etc.); mismatches are logged,
-     computing the default layout above, the script fetches the currently
+   not dropped — the link stays, just without data until fixed upstream.
-     provisioned dashboard's weathermap panel (if `GRAFANA_URL`/`GRAFANA_TOKEN`
+7. Load `dashboard-base.json`, substitute the real datasource UID, splice
-     are set) and reuses each existing node's live position as-is. Only
+   the generated panel into `__WEATHERMAP_SLOT__` (keeping that slot's
-     genuinely new nodes (not in that live map — first-ever run, or a fresh
+   `gridPos`/`id` — layout stays manual).
-     device added to the topology) get the layered default. A device removed
+8. Write `weathermap-dashboard.json`; provision it if `--provision`.
-     from the topology simply has no entry in this run's node/link output at
-     all, so no orphaned position or dangling link reference is possible.
-4. Builds the panel's `targets`: one PromQL query per metric family (BGP
-   status, interface tx, interface rx, VXLAN MAC/VNI), each with an explicit
-   `legendFormat` so the resolved display name is predictable.
-5. Builds `nodes[]` and `links[]` referencing those resolved legend strings,
-   including the plugin's `anchors` tally (per-node count of link
-   attachments per side) and numeric anchor enum on each link side —
-   omitting these crashes the panel on load in the installed plugin fork,
-   despite the (fork-specific) schema research saying it's safe to skip.
-6. Cross-checks every IPFabric interface name, aliased to gnmic's naming
-   (`Et`→`Ethernet`, `Po`→`Port-Channel`, `Lo`→`Loopback`, `Vl`→`Vlan`,
-   `Ma`→`Management`), against the live exporter. Any link whose aliased
-   name has no matching series is logged, not silently dropped — the actual
-   fix for a real mismatch belongs in gnmic interface aliasing (#43), not in
-   this script.
-7. Loads `configs/grafana/dashboard-base.json`, substitutes the real
-   Prometheus datasource UID into its `__DATASOURCE_UID__` placeholders,
-   finds the panel titled `__WEATHERMAP_SLOT__` and splices in the generated
-   weathermap panel content (keeping the slot's `gridPos`/`id`, so the manual
-   layout is never repositioned).
-8. Writes the merged dashboard JSON to `configs/grafana/weathermap-dashboard.json`,
-   and provisions it via `POST /api/dashboards/db` if `--provision` is passed.
 
 ## Known gaps
 
-- **VXLAN MAC-per-VNI target**: the `vlan` join key used to correlate
+- **VXLAN MAC-per-VNI tooltip**: the `vlan` join key doesn't match on live
-  VLAN→VNI mapping with FDB entries (query verbatim from #44) doesn't
+  data for any VTEP node, likely an Arista internal-vs-front-panel VLAN
-  actually match on live data for any VTEP node — likely an Arista
+  translation gap. Only affects that one decorative tooltip metric, not
-  internal-VLAN-vs-front-panel-VLAN translation the OpenConfig paths don't
+  node/link status or traffic coloring. Tracked in #44.
-  reconcile. Only affects the decorative per-VTEP tooltip metric, not
+
-  node/link status or traffic coloring. Tracked in #44, not fixed here.
+# scripts/generate_traffic.sh
+
+Generates real DC↔Campus traffic over VRF `gold` using `iperf3` (bundled
+in the `network-multitool` image every host container runs), with a live
+bandwidth dashboard. Without this, host containers sit idle and IPFabric
+ARP/MAC tables, Grafana throughput graphs, and the weathermap panel stay
+empty until someone manually generates traffic (see #55).
+
+## Quickstart
+
+```bash
+./scripts/generate_traffic.sh <duration_seconds>
+```
+
+## How it works
+
+- Starts `iperf3 -s` on the DC gold-VRF servers: `dc-server2`
+  (10.34.34.102), `dc-server4` (10.78.78.104).
+- Runs `iperf3 -c -R` from the paired campus gold-VRF hosts, reversing the
+  stream so the DC server pushes to the campus consumer: `dc-server2` →
+  `campus-host1`, `dc-server4` → `campus-host2` — exercising the full
+  DC→Core→Campus stitched EVPN Type-5 path end to end.
+- Redraws a terminal dashboard every second for the run duration: server
+  list, and live Mbits/sec per client session parsed from `iperf3 -i 1`
+  output.
+- On exit (duration end or Ctrl-C), kills the client processes and stops
+  the `iperf3 -s` processes on the DC servers — no leftover state.
+
+`dc-server1`/`dc-server3` (VLAN 40, VRF default, no gateway) are out of
+scope — this script only exercises the routed gold VRF path.

scripts/generate_traffic.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# Generates DC<->Campus traffic over VRF gold using iperf3 (bundled in the
+# network-multitool image every host container runs), with a live
+# server/client bandwidth dashboard. Refs #55.
+set -euo pipefail
+
+DURATION="${1:?Usage: $0 <duration_seconds>}"
+if ! [[ "$DURATION" =~ ^[0-9]+$ ]] || [[ "$DURATION" -lt 1 ]]; then
+    echo "Duration must be a positive integer (seconds)" >&2
+    exit 1
+fi
+
+LAB_PREFIX="clab-arista-evpn-fabric"
+PORT=5301
+
+# server_name:server_ip:client_name  — gold VRF pairs, stitched EVPN
+# Type-5 path DC -> Core -> Campus (see README Host Addressing table)
+PAIRS=(
+    "dc-server2:10.34.34.102:campus-host1"
+    "dc-server4:10.78.78.104:campus-host2"
+)
+
+WORKDIR="$(mktemp -d)"
+CLIENT_PIDS=()
+
+cleanup() {
+    for pid in "${CLIENT_PIDS[@]:-}"; do
+        kill "$pid" >/dev/null 2>&1 || true
+    done
+    for pair in "${PAIRS[@]}"; do
+        IFS=':' read -r server _ _ <<<"$pair"
+        docker exec "${LAB_PREFIX}-${server}" pkill -f "iperf3 -s -p ${PORT}" >/dev/null 2>&1 || true
+    done
+    rm -rf "$WORKDIR"
+}
+trap cleanup EXIT INT TERM
+
+echo "Starting iperf3 servers..."
+for pair in "${PAIRS[@]}"; do
+    IFS=':' read -r server server_ip _ <<<"$pair"
+    docker exec -d "${LAB_PREFIX}-${server}" iperf3 -s -p "$PORT"
+done
+sleep 1
+
+echo "Starting iperf3 clients for ${DURATION}s..."
+for pair in "${PAIRS[@]}"; do
+    IFS=':' read -r server server_ip client <<<"$pair"
+    logfile="${WORKDIR}/${client}.log"
+    # -R: DC hosts the service, campus is the consumer -- data should flow
+    # server -> client (download), not client -> server, to match how a
+    # real DC-hosted service/campus-consumer pair behaves.
+    docker exec "${LAB_PREFIX}-${client}" iperf3 -c "$server_ip" -p "$PORT" -R \
+        -t "$DURATION" -i 1 --forceflush -f m >"$logfile" 2>&1 &
+    CLIENT_PIDS+=("$!")
+done
+
+for ((elapsed = 0; elapsed <= DURATION; elapsed++)); do
+    clear
+    echo "EVPN/VXLAN lab traffic generator — ${elapsed}/${DURATION}s"
+    echo
+    echo "Servers (iperf3 -s):"
+    for pair in "${PAIRS[@]}"; do
+        IFS=':' read -r server server_ip _ <<<"$pair"
+        echo "  ${server} (${server_ip}:${PORT})"
+    done
+    echo
+    echo "Clients (live bandwidth):"
+    for pair in "${PAIRS[@]}"; do
+        IFS=':' read -r server server_ip client <<<"$pair"
+        logfile="${WORKDIR}/${client}.log"
+        last_line="$(grep -E 'Mbits/sec' "$logfile" 2>/dev/null | tail -1 || true)"
+        bw="$(sed -E 's/.*[[:space:]]([0-9.]+ Mbits\/sec).*/\1/' <<<"$last_line")"
+        [[ -z "$last_line" ]] && bw="waiting..."
+        printf "  %-13s -> %-13s : %s\n" "$server" "$client" "$bw"
+    done
+    sleep 1
+done
+
+wait "${CLIENT_PIDS[@]}" 2>/dev/null || true
+
+echo
+echo "Done. Summary:"
+for pair in "${PAIRS[@]}"; do
+    IFS=':' read -r server _ client <<<"$pair"
+    logfile="${WORKDIR}/${client}.log"
+    summary="$(grep -E 'receiver' "$logfile" 2>/dev/null || true)"
+    echo "  ${server} -> ${client}: ${summary:-no data}"
+done

scripts/generate_weathermap.py

@@ -4,7 +4,7 @@ metrics, merge it into a manually-authored dashboard base, and optionally
 provision the result into Grafana.
 
 Scope (see #52): this script knows only about the weathermap panel --
-targets (node status / link tx / link rx / VXLAN tooltip queries) and
+targets (node status / link tx per side / VXLAN tooltip queries) and
 options.weathermap (nodes/links/scale/settings). It has no knowledge of the
 BGP sessions table, ports/interfaces table, or throughput panels -- those
 live in the manually-authored `configs/grafana/dashboard-base.json` and are
@@ -363,6 +363,12 @@ def build_weathermap(devices, links, interface_speeds, positions, prometheus_url
         link_defs.append({
             "id": f"{link['a_host']}-{a_gnmic_int}--{link['z_host']}-{z_gnmic_int}",
             "nodes": [{"id": link["a_host"]}, {"id": link["z_host"]}],
+            # Each side's query is that side's own tx (egress) counter, not the
+            # far end's rx -- see #57. a_host tx and z_host rx both describe the
+            # *same* A->Z flow measured from opposite ends (the a_host->z_host
+            # direction, counted twice), leaving the Z->A direction never
+            # queried by either side. Using each node's own tx gives two
+            # independent, opposite-direction measurements instead.
             "sides": {
                 "A": {
                     "bandwidth": a_bw,
@@ -371,7 +377,7 @@ def build_weathermap(devices, links, interface_speeds, positions, prometheus_url
                 },
                 "Z": {
                     "bandwidth": z_bw,
-                    "query": f"{link['z_host']} {z_gnmic_int} rx",
+                    "query": f"{link['z_host']} {z_gnmic_int} tx",
                     "labelOffset": 55, "anchor": ANCHOR["Left"], "dashboardLink": "",
                 },
             },
@@ -410,11 +416,6 @@ def build_weathermap(devices, links, interface_speeds, positions, prometheus_url
             "expr": f'rate(interfaces_interface_state_counters_out_octets{{device=~"{host_regex}", interface=~"{interface_regex}"}}[5m]) * 8',
             "legendFormat": "{{device}} {{interface}} tx",
         },
-        {
-            "refId": "C",
-            "expr": f'rate(interfaces_interface_state_counters_in_octets{{device=~"{host_regex}", interface=~"{interface_regex}"}}[5m]) * 8',
-            "legendFormat": "{{device}} {{interface}} rx",
-        },
     ]
     if vtep_hosts:
         vtep_regex = "|".join(promql_escape(h) for h in vtep_hosts)