Merge pull request 'Telemetry: gnmic + Prometheus + Grafana weathermap dashboard, iperf3 traffic generator' (#59) from feat/telemetry into main

Reviewed-on: #59

assets/weathermap-pipeline.svg

@@ -0,0 +1,72 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 350" font-family="Helvetica, Arial, sans-serif">
+  <defs>
+    <marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
+      <path d="M 0 0 L 10 5 L 0 10 z" fill="#555"/>
+    </marker>
+  </defs>
+
+  <style>
+    .box { stroke-width: 1.5; rx: 6; }
+    .input { fill: #e8f0fe; stroke: #4285f4; }
+    .manual { fill: #fef7e0; stroke: #f9ab00; }
+    .script { fill: #e6f4ea; stroke: #34a853; }
+    .output { fill: #f3e8fd; stroke: #a142f4; }
+    .grafana { fill: #fce8e6; stroke: #ea4335; }
+    .label { font-size: 13px; fill: #202124; }
+    .sublabel { font-size: 10px; fill: #5f6368; }
+    .arrow { stroke: #555; stroke-width: 1.5; fill: none; marker-end: url(#arrow); }
+    .caption { font-size: 11px; fill: #5f6368; font-style: italic; }
+  </style>
+
+  <!-- inputs -->
+  <rect class="box input" x="20" y="30" width="150" height="50"/>
+  <text class="label" x="95" y="50" text-anchor="middle">IPFabric</text>
+  <text class="sublabel" x="95" y="66" text-anchor="middle">topology, inventory</text>
+
+  <rect class="box input" x="20" y="100" width="150" height="50"/>
+  <text class="label" x="95" y="120" text-anchor="middle">Prometheus</text>
+  <text class="sublabel" x="95" y="136" text-anchor="middle">gnmic metrics, live labels</text>
+
+  <!-- generator script -->
+  <rect class="box script" x="220" y="55" width="200" height="70"/>
+  <text class="label" x="320" y="82" text-anchor="middle" font-size="11.5">generate_weathermap.py</text>
+  <text class="sublabel" x="320" y="98" text-anchor="middle">weathermap panel only</text>
+  <text class="sublabel" x="320" y="112" text-anchor="middle">targets + options.weathermap</text>
+
+  <path class="arrow" d="M170,55 L220,80"/>
+  <path class="arrow" d="M170,125 L220,100"/>
+
+  <!-- manual base -->
+  <rect class="box manual" x="220" y="220" width="200" height="70"/>
+  <text class="label" x="320" y="247" text-anchor="middle">dashboard-base.json</text>
+  <text class="sublabel" x="320" y="263" text-anchor="middle">hand-authored, committed</text>
+  <text class="sublabel" x="320" y="277" text-anchor="middle">BGP/ports/throughput/vars</text>
+
+  <!-- merge -->
+  <rect class="box script" x="470" y="130" width="160" height="60"/>
+  <text class="label" x="550" y="155" text-anchor="middle">merge</text>
+  <text class="sublabel" x="550" y="171" text-anchor="middle" font-size="8.5">splices into __WEATHERMAP_SLOT__</text>
+
+  <path class="arrow" d="M420,90 L470,145"/>
+  <path class="arrow" d="M420,255 L470,175"/>
+
+  <!-- output artifact -->
+  <rect class="box output" x="660" y="60" width="210" height="60"/>
+  <text class="label" x="765" y="85" text-anchor="middle" font-size="11.5">weathermap-dashboard.json</text>
+  <text class="sublabel" x="765" y="101" text-anchor="middle">generated build artifact</text>
+
+  <path class="arrow" d="M630,150 L765,120"/>
+
+  <!-- grafana -->
+  <rect class="box grafana" x="660" y="210" width="210" height="60"/>
+  <text class="label" x="765" y="235" text-anchor="middle">Grafana</text>
+  <text class="sublabel" x="765" y="251" text-anchor="middle" font-size="9">--provision (POST /api/dashboards/db)</text>
+
+  <path class="arrow" d="M630,175 L765,210"/>
+  <path class="arrow" d="M765,120 L765,210" stroke-dasharray="3,3"/>
+
+  <!-- read-back for position preservation -->
+  <path class="arrow" d="M765,270 C 620,330 490,300 420,275" stroke-dasharray="3,3"/>
+  <text class="caption" x="545" y="325" text-anchor="middle">live node positions read back before each regeneration (see #53)</text>
+
+</svg>

envrc.sample

@@ -1,4 +1,11 @@
 export GRAFANA_URL=""
 export GRAFANA_TOKEN=""
+export GRAFANA_DATASOURCE_UID=""
 export IPFABRIC_URL=""
 export IPFABRIC_TOKEN=""
+
+# Optional -- see scripts/README.md for defaults and when to override
+# export IPFABRIC_SNAPSHOT="$last"
+# export PROMETHEUS_URL="http://172.16.0.71:9090"
+# export GRAFANA_DASHBOARD_UID="evpn-vxlan-fabric-weathermap"
+# export GRAFANA_WEATHERMAP_PLUGIN_ID="tamirsuliman-weathermap-panel"

scripts/README.md

@@ -1,131 +1,167 @@
 # scripts/generate_weathermap.py
 
-Generates a [weathermap-ng](https://github.com/allamiro/grafana-network-weathermap-ng)
+Generates the **weathermap panel only** from live IPFabric + Prometheus
-**panel only** from live IPFabric topology + gnmic/Prometheus metrics, merges
+data, merges it into a hand-authored dashboard base, and provisions the
-it into a manually-authored dashboard base, and writes the merged result to
+result to Grafana.
-`configs/grafana/weathermap-dashboard.json` (committed to Gitea as the build
-artifact, same pattern as the retired Flow Panel YAML). Optionally provisions
-it into Grafana via the HTTP API.
 
-## File structure (see #52)
+![pipeline diagram](../assets/weathermap-pipeline.svg)
 
-| File | What it is |
+## Files
-|---|---|
-| `configs/grafana/dashboard-base.json` | **Manually authored**, committed source of truth for everything except the weathermap panel: BGP sessions table, ports/interfaces table, throughput-per-site graphs, `site`/`device` template variables, panel layout. Edit this directly for anything in those panels. Contains a reserved slot panel titled `__WEATHERMAP_SLOT__` (id `1`, `gridPos` fixed) that the script splices the generated weathermap panel into. |
-| `configs/grafana/weathermap-dashboard.json` | **Generated build artifact** — base + freshly-generated weathermap panel, merged. This is what actually gets provisioned to Grafana. Don't hand-edit; re-run the script. |
 
-The script itself only ever knows about the weathermap panel (`targets` +
+| File                                        | What it is                                                                                                                                                                                      | Edit it?                |
-`options.weathermap`) — it has no knowledge of the other three panels or the
+| ------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- |
-template variables. That split is deliberate: topology-driven content
+| `configs/grafana/dashboard-base.json`       | Hand-authored: BGP table, ports table, throughput graphs, `site`/`device` variables, layout. Has one reserved slot panel (`title: "__WEATHERMAP_SLOT__"`) where the weathermap gets spliced in. | Yes, directly           |
-(nodes/links, generated from live IPFabric+Prometheus data) is separated from
+| `configs/grafana/weathermap-dashboard.json` | Generated build artifact — base + fresh weathermap panel, merged. What actually gets provisioned.                                                                                               | No — regenerate instead |
-hand-tuned dashboard composition (table layout, transformations, thresholds),
-which doesn't need to regenerate on every topology change.
 
-See issue #48 for the panel schema research this is built against, and its
+Why split like this: topology data (nodes/links) changes with the lab and
-comment thread for the live-validation history (auth quirks, plugin id, a
+should regenerate every time; the rest of the dashboard (table layout,
-PromQL escaping bug, a panel-crash fix — worth reading before touching the
+thresholds, transformations) is hand-tuned and shouldn't be touched by a
-anchor/regex-escaping logic).
+topology refresh. See #52.
 
-## Usage
+## Quickstart
 
 ```bash
-export IPFABRIC_URL=https://<ipfabric-instance>
+cp envrc.sample .envrc   # fill in the values (see "Credentials" below)
-export IPFABRIC_TOKEN=<token>
+source .envrc
-python3 scripts/generate_weathermap.py                 # writes the merged JSON only
 
-export GRAFANA_URL=https://<external-grafana-instance>
+python3 scripts/generate_weathermap.py              # dry run: writes the JSON only
-export GRAFANA_TOKEN=<token>
+python3 scripts/generate_weathermap.py --provision   # also pushes it to Grafana
-export GRAFANA_DATASOURCE_UID=<prometheus-datasource-uid-in-grafana>
-python3 scripts/generate_weathermap.py --provision      # also provisions via the Grafana API
 ```
 
-Re-run after any topology change (`evpn-lab.clab.yml`) to regenerate the
+Re-run the same `--provision` command any time the lab topology changes —
-weathermap panel and re-merge/re-provision. The script always regenerates the
+the script always rebuilds the weathermap panel from live data, so there's
-weathermap panel fresh from live IPFabric data (no diffing/incremental
+nothing to "detect", just re-run it. It's not automated yet (no watcher/CI
-state), so a plain re-run already picks up any topology change — there's no
+hook), so re-running is a manual step for now.
-separate "detect changes" step. The only manual/operational step is
-*triggering* that re-run after a change; there's no watcher or CI hook doing
-it automatically yet.
 
-If you need to change the BGP/ports/throughput panels, template variables, or
+## Credentials
-layout, edit `configs/grafana/dashboard-base.json` directly and re-run the
+
-script (or run it with `--provision` to push the edit live) — no topology
+| Value                         | Env var                                                                 | Get it from the UI                                                                                    | Get it via API                                                                                               |
-data is needed for that path, the weathermap panel just gets regenerated
+| ----------------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ |
-alongside it.
+| IPFabric token                | `IPFABRIC_TOKEN`                                                        | IPFabric → user menu → Settings → API Tokens → Add token                                              | not possible, UI-only                                                                                        |
+| Grafana service account token | `GRAFANA_TOKEN`                                                         | Grafana → Administration → Service accounts → Add → give it **Editor** role → Add token               | possible but multi-step (create account, then create token under it) — UI is simpler                         |
+| Prometheus datasource UID     | `GRAFANA_DATASOURCE_UID`                                                | Grafana → Connections → Data sources → open the one with gnmic data → UID is the last part of the URL | `curl -s -H "Authorization: Bearer $GRAFANA_TOKEN" "$GRAFANA_URL/api/datasources" \| jq '.[] \| {name,uid}'` |
+| Weathermap plugin ID          | `GRAFANA_WEATHERMAP_PLUGIN_ID` (optional, only if not the default fork) | Grafana → Administration → Plugins → search "weathermap" → id is in the URL                           | —                                                                                                            |
+
+⚠️ The datasource must be the one that can actually see gnmic metrics —
+picking an unrelated Prometheus instance renders the panel with no data
+(see #48).
+
+## Verifying the result
+
+**In the browser** (the only way to actually see it render — colors,
+tables, and the ports-table merge transformation all happen client-side):
+
+1. Open `$GRAFANA_URL/d/evpn-vxlan-fabric-weathermap`.
+2. Weathermap: nodes grouped spine-top → access-bottom, all green.
+3. `site`/`device` dropdowns filter the two tables below.
+4. BGP Sessions / Ports tables: populated, green=Up / red=Down.
+5. Throughput per Site: signed graph, non-zero values.
+
+**Without a browser**, use Grafana's **Explore** view (paste a PromQL
+expression from the panel JSON and run it) — same result as the UI check
+above minus the rendering, no `curl` needed. Or hit the API directly:
+
+```bash
+curl -s -X POST -H "Authorization: Bearer $GRAFANA_TOKEN" -H "Content-Type: application/json" \
+  "$GRAFANA_URL/api/ds/query" -d '{
+    "queries": [{"refId":"A","datasource":{"type":"prometheus","uid":"'"$GRAFANA_DATASOURCE_UID"'"},
+                 "expr":"min by (device) (interfaces_interface_state_oper_status)","instant":true}],
+    "from":"now-5m","to":"now"}' | python3 -m json.tool
+```
+
+## Troubleshooting
+
+| Symptom                                                    | Cause                                                               | Check                                                              |
+| ---------------------------------------------------------- | ------------------------------------------------------------------- | ------------------------------------------------------------------ |
+| `Refusing to provision: set GRAFANA_DATASOURCE_UID...`     | Env var unset                                                       | `echo $GRAFANA_DATASOURCE_UID`                                     |
+| Panel renders with no data                                 | Wrong datasource — can't see gnmic metrics                          | Explore → run `up` against it, confirm gnmic series come back      |
+| `Base dashboard has no panel titled '__WEATHERMAP_SLOT__'` | Someone edited `dashboard-base.json` and renamed/removed that panel | Check panel `id: 1` still has that exact title                     |
+| Manually-moved node keeps resetting position               | `GRAFANA_URL`/`GRAFANA_TOKEN` weren't set for that run              | Look for `Fetching live node positions from ...` in the run output |
+| `HTTP 401/403` fetching live positions                     | Expired/bad Grafana token                                           | Re-issue the service account token                                 |
 
 ## Environment variables
 
-| Variable | Required | Default | Notes |
+| Variable                       | Required           | Default                               |
-|---|---|---|---|
+| ------------------------------ | ------------------ | ------------------------------------- |
-| `--base` (CLI flag, not env) | no | `configs/grafana/dashboard-base.json` | Manual dashboard base to merge the weathermap panel into. |
+| `IPFABRIC_URL`                 | yes                | —                                     |
-| `IPFABRIC_URL` | yes | — | e.g. `https://ipfabric.example.com` |
+| `IPFABRIC_TOKEN`               | yes                | —                                     |
-| `IPFABRIC_TOKEN` | yes | — | Inventory/Snapshots read access. Sent as `X-API-Token`, not `Authorization: Bearer` — IPFabric's REST API does not use bearer auth. |
+| `IPFABRIC_SNAPSHOT`            | no                 | `$last`                               |
-| `IPFABRIC_SNAPSHOT` | no | `$last` | |
+| `PROMETHEUS_URL`               | no                 | `http://172.16.0.71:9090`             |
-| `PROMETHEUS_URL` | no | `http://172.16.0.71:9090` | The **in-topology** Prometheus (see #45), used at generation time to validate the IPFabric→gnmic interface alias and to discover live VTEP/VLAN pairs. This does not have to be the same instance Grafana queries at render time — see below. |
+| `GRAFANA_URL`                  | no*                | —                                     |
-| `GRAFANA_URL` | no (see #53) | — | Required for `--provision`. Also used, if set, to fetch the currently-provisioned dashboard and preserve any manually drag-and-drop-repositioned node positions — see step 3 below. Without it, every node gets the layered default position, even ones a human previously repositioned in the UI. |
+| `GRAFANA_TOKEN`                | no*                | —                                     |
-| `GRAFANA_TOKEN` | no (see #53) | — | Grafana Service Account token, sent as `Authorization: Bearer`. Same conditions as `GRAFANA_URL` above. |
+| `GRAFANA_DATASOURCE_UID`       | with `--provision` | —                                     |
-| `GRAFANA_DATASOURCE_UID` | with `--provision` | — | UID of the Prometheus datasource in Grafana that will actually back the panel. **This must be a datasource that can see the gnmic metrics** — an unrelated/external Prometheus instance with no gnmic data will make the panel render with no values (this happened once, see #48). |
+| `GRAFANA_DASHBOARD_UID`        | no                 | `evpn-vxlan-fabric-weathermap`        |
-| `GRAFANA_DASHBOARD_UID` | no | `evpn-vxlan-fabric-weathermap` | |
+| `GRAFANA_WEATHERMAP_PLUGIN_ID` | no                 | `tamirsuliman-weathermap-panel`       |
-| `GRAFANA_WEATHERMAP_PLUGIN_ID` | no | `tamirsuliman-weathermap-panel` | The installed weathermap-ng plugin id. Override if a different fork is installed — plugin ids don't always match the upstream repo name (the schema research in #48 was done against the `allamiro` fork's source; what's actually installed here is a different fork with a stricter/different runtime schema — see the `ANCHOR` handling in the script). |
+| `--base` (CLI flag)            | no                 | `configs/grafana/dashboard-base.json` |
 
-## What the script does
+\* `GRAFANA_URL`/`GRAFANA_TOKEN` are required for `--provision`, and
+optional otherwise — if set on a plain (non-`--provision`) run, they're
+used to read back live node positions so manual repositioning survives
+the next regeneration (see #53). Skip them and every node just gets the
+default layout.
 
-1. Fetches device inventory + connectivity-matrix from IPFabric.
+## How it works
-2. Filters the connectivity-matrix to physical Ethernet-to-Ethernet links
+
-   only: drops Management-plane neighbor entries and `.100`/`.200`
+1. Fetch device inventory + connectivity-matrix from IPFabric.
-   subinterface rows (802.1Q tags used for the gold VRF stitching on Core —
+2. Keep only physical Ethernet↔Ethernet links (drop management-plane and
-   they ride the same physical port as their parent interface and gnmic only
+   `.100`/`.200` subinterface rows), dedupe the two directions IPFabric
-   exports physical interface counters), and dedupes the two directions
+   reports per link.
-   IPFabric reports for each physical link into one.
+3. Position each node:
-3. Computes node positions (see #53):
+    - **New node** → layered default: Y-tier by role parsed from the
-   - **Default layered layout**, for nodes with no known position yet: role
+      hostname (`spine → core → border-leaf → leaf → access`), X grouped
-     is parsed from the hostname naming convention (`*-spine*`→spine,
+      by site within the tier. A hostname that matches no role is logged
-     `core*`→core, `*-border-leaf*`→border-leaf, `*-leaf*`→leaf,
+      and dropped into a fallback tier, never silently misplaced (#53).
-     `*-access*`→access — same trust level as the `site` parsing used for
+    - **Known node** → whatever position is currently live in Grafana,
-     the Prometheus relabel, not IPFabric-derived), giving Y-tiers top to
+      unchanged. This is why `GRAFANA_URL`/`GRAFANA_TOKEN` matter even on
-     bottom `spine → core → border-leaf → leaf → access`. X position is
+      a dry run (#53).
-     grouped/columned by site within each tier, so e.g. dc leafs and campus
+4. Build the panel's PromQL `targets`: node status, per-side link tx (each
-     leafs land in the same row but different column bands. A hostname
+   side its own egress counter, so both directions are represented — #57),
-     matching no role pattern is logged (not silently placed in the wrong
+   VXLAN MAC/VNI tooltip — each with an explicit `legendFormat`.
-     tier) and put in a fallback tier below `access`.
+5. Build `nodes[]`/`links[]`, including the plugin's `anchors` tally
-   - **Live-position preservation**: position is edited by drag-and-drop
+   (link count per side) — required by the installed plugin fork even
-     directly in the Grafana UI, not in the git-committed base file — a
+   though the schema doc says it's optional (#48).
-     deliberate exception to the "file is the only source of truth" rule
+6. Cross-check every interface name against the live gnmic exporter
-     from #52, because that's simply not where this field is edited. Before
+   (`Et`→`Ethernet`, `Po`→`Port-Channel`, etc.); mismatches are logged,
-     computing the default layout above, the script fetches the currently
+   not dropped — the link stays, just without data until fixed upstream.
-     provisioned dashboard's weathermap panel (if `GRAFANA_URL`/`GRAFANA_TOKEN`
+7. Load `dashboard-base.json`, substitute the real datasource UID, splice
-     are set) and reuses each existing node's live position as-is. Only
+   the generated panel into `__WEATHERMAP_SLOT__` (keeping that slot's
-     genuinely new nodes (not in that live map — first-ever run, or a fresh
+   `gridPos`/`id` — layout stays manual).
-     device added to the topology) get the layered default. A device removed
+8. Write `weathermap-dashboard.json`; provision it if `--provision`.
-     from the topology simply has no entry in this run's node/link output at
-     all, so no orphaned position or dangling link reference is possible.
-4. Builds the panel's `targets`: one PromQL query per metric family (BGP
-   status, interface tx, interface rx, VXLAN MAC/VNI), each with an explicit
-   `legendFormat` so the resolved display name is predictable.
-5. Builds `nodes[]` and `links[]` referencing those resolved legend strings,
-   including the plugin's `anchors` tally (per-node count of link
-   attachments per side) and numeric anchor enum on each link side —
-   omitting these crashes the panel on load in the installed plugin fork,
-   despite the (fork-specific) schema research saying it's safe to skip.
-6. Cross-checks every IPFabric interface name, aliased to gnmic's naming
-   (`Et`→`Ethernet`, `Po`→`Port-Channel`, `Lo`→`Loopback`, `Vl`→`Vlan`,
-   `Ma`→`Management`), against the live exporter. Any link whose aliased
-   name has no matching series is logged, not silently dropped — the actual
-   fix for a real mismatch belongs in gnmic interface aliasing (#43), not in
-   this script.
-7. Loads `configs/grafana/dashboard-base.json`, substitutes the real
-   Prometheus datasource UID into its `__DATASOURCE_UID__` placeholders,
-   finds the panel titled `__WEATHERMAP_SLOT__` and splices in the generated
-   weathermap panel content (keeping the slot's `gridPos`/`id`, so the manual
-   layout is never repositioned).
-8. Writes the merged dashboard JSON to `configs/grafana/weathermap-dashboard.json`,
-   and provisions it via `POST /api/dashboards/db` if `--provision` is passed.
 
 ## Known gaps
 
-- **VXLAN MAC-per-VNI target**: the `vlan` join key used to correlate
+- **VXLAN MAC-per-VNI tooltip**: the `vlan` join key doesn't match on live
-  VLAN→VNI mapping with FDB entries (query verbatim from #44) doesn't
+  data for any VTEP node, likely an Arista internal-vs-front-panel VLAN
-  actually match on live data for any VTEP node — likely an Arista
+  translation gap. Only affects that one decorative tooltip metric, not
-  internal-VLAN-vs-front-panel-VLAN translation the OpenConfig paths don't
+  node/link status or traffic coloring. Tracked in #44.
-  reconcile. Only affects the decorative per-VTEP tooltip metric, not
+
-  node/link status or traffic coloring. Tracked in #44, not fixed here.
+# scripts/generate_traffic.sh
+
+Generates real DC↔Campus traffic over VRF `gold` using `iperf3` (bundled
+in the `network-multitool` image every host container runs), with a live
+bandwidth dashboard. Without this, host containers sit idle and IPFabric
+ARP/MAC tables, Grafana throughput graphs, and the weathermap panel stay
+empty until someone manually generates traffic (see #55).
+
+## Quickstart
+
+```bash
+./scripts/generate_traffic.sh <duration_seconds>
+```
+
+## How it works
+
+- Starts `iperf3 -s` on the DC gold-VRF servers: `dc-server2`
+  (10.34.34.102), `dc-server4` (10.78.78.104).
+- Runs `iperf3 -c -R` from the paired campus gold-VRF hosts, reversing the
+  stream so the DC server pushes to the campus consumer: `dc-server2` →
+  `campus-host1`, `dc-server4` → `campus-host2` — exercising the full
+  DC→Core→Campus stitched EVPN Type-5 path end to end.
+- Redraws a terminal dashboard every second for the run duration: server
+  list, and live Mbits/sec per client session parsed from `iperf3 -i 1`
+  output.
+- On exit (duration end or Ctrl-C), kills the client processes and stops
+  the `iperf3 -s` processes on the DC servers — no leftover state.
+
+`dc-server1`/`dc-server3` (VLAN 40, VRF default, no gateway) are out of
+scope — this script only exercises the routed gold VRF path.

scripts/generate_traffic.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# Generates DC<->Campus traffic over VRF gold using iperf3 (bundled in the
+# network-multitool image every host container runs), with a live
+# server/client bandwidth dashboard. Refs #55.
+set -euo pipefail
+
+DURATION="${1:?Usage: $0 <duration_seconds>}"
+if ! [[ "$DURATION" =~ ^[0-9]+$ ]] || [[ "$DURATION" -lt 1 ]]; then
+    echo "Duration must be a positive integer (seconds)" >&2
+    exit 1
+fi
+
+LAB_PREFIX="clab-arista-evpn-fabric"
+PORT=5301
+
+# server_name:server_ip:client_name  — gold VRF pairs, stitched EVPN
+# Type-5 path DC -> Core -> Campus (see README Host Addressing table)
+PAIRS=(
+    "dc-server2:10.34.34.102:campus-host1"
+    "dc-server4:10.78.78.104:campus-host2"
+)
+
+WORKDIR="$(mktemp -d)"
+CLIENT_PIDS=()
+
+cleanup() {
+    for pid in "${CLIENT_PIDS[@]:-}"; do
+        kill "$pid" >/dev/null 2>&1 || true
+    done
+    for pair in "${PAIRS[@]}"; do
+        IFS=':' read -r server _ _ <<<"$pair"
+        docker exec "${LAB_PREFIX}-${server}" pkill -f "iperf3 -s -p ${PORT}" >/dev/null 2>&1 || true
+    done
+    rm -rf "$WORKDIR"
+}
+trap cleanup EXIT INT TERM
+
+echo "Starting iperf3 servers..."
+for pair in "${PAIRS[@]}"; do
+    IFS=':' read -r server server_ip _ <<<"$pair"
+    docker exec -d "${LAB_PREFIX}-${server}" iperf3 -s -p "$PORT"
+done
+sleep 1
+
+echo "Starting iperf3 clients for ${DURATION}s..."
+for pair in "${PAIRS[@]}"; do
+    IFS=':' read -r server server_ip client <<<"$pair"
+    logfile="${WORKDIR}/${client}.log"
+    # -R: DC hosts the service, campus is the consumer -- data should flow
+    # server -> client (download), not client -> server, to match how a
+    # real DC-hosted service/campus-consumer pair behaves.
+    docker exec "${LAB_PREFIX}-${client}" iperf3 -c "$server_ip" -p "$PORT" -R \
+        -t "$DURATION" -i 1 --forceflush -f m >"$logfile" 2>&1 &
+    CLIENT_PIDS+=("$!")
+done
+
+for ((elapsed = 0; elapsed <= DURATION; elapsed++)); do
+    clear
+    echo "EVPN/VXLAN lab traffic generator — ${elapsed}/${DURATION}s"
+    echo
+    echo "Servers (iperf3 -s):"
+    for pair in "${PAIRS[@]}"; do
+        IFS=':' read -r server server_ip _ <<<"$pair"
+        echo "  ${server} (${server_ip}:${PORT})"
+    done
+    echo
+    echo "Clients (live bandwidth):"
+    for pair in "${PAIRS[@]}"; do
+        IFS=':' read -r server server_ip client <<<"$pair"
+        logfile="${WORKDIR}/${client}.log"
+        last_line="$(grep -E 'Mbits/sec' "$logfile" 2>/dev/null | tail -1 || true)"
+        bw="$(sed -E 's/.*[[:space:]]([0-9.]+ Mbits\/sec).*/\1/' <<<"$last_line")"
+        [[ -z "$last_line" ]] && bw="waiting..."
+        printf "  %-13s -> %-13s : %s\n" "$server" "$client" "$bw"
+    done
+    sleep 1
+done
+
+wait "${CLIENT_PIDS[@]}" 2>/dev/null || true
+
+echo
+echo "Done. Summary:"
+for pair in "${PAIRS[@]}"; do
+    IFS=':' read -r server _ client <<<"$pair"
+    logfile="${WORKDIR}/${client}.log"
+    summary="$(grep -E 'receiver' "$logfile" 2>/dev/null || true)"
+    echo "  ${server} -> ${client}: ${summary:-no data}"
+done

scripts/generate_weathermap.py

@@ -4,7 +4,7 @@ metrics, merge it into a manually-authored dashboard base, and optionally
 provision the result into Grafana.
 
 Scope (see #52): this script knows only about the weathermap panel --
-targets (node status / link tx / link rx / VXLAN tooltip queries) and
+targets (node status / link tx per side / VXLAN tooltip queries) and
 options.weathermap (nodes/links/scale/settings). It has no knowledge of the
 BGP sessions table, ports/interfaces table, or throughput panels -- those
 live in the manually-authored `configs/grafana/dashboard-base.json` and are
@@ -363,6 +363,12 @@ def build_weathermap(devices, links, interface_speeds, positions, prometheus_url
         link_defs.append({
             "id": f"{link['a_host']}-{a_gnmic_int}--{link['z_host']}-{z_gnmic_int}",
             "nodes": [{"id": link["a_host"]}, {"id": link["z_host"]}],
+            # Each side's query is that side's own tx (egress) counter, not the
+            # far end's rx -- see #57. a_host tx and z_host rx both describe the
+            # *same* A->Z flow measured from opposite ends (the a_host->z_host
+            # direction, counted twice), leaving the Z->A direction never
+            # queried by either side. Using each node's own tx gives two
+            # independent, opposite-direction measurements instead.
             "sides": {
                 "A": {
                     "bandwidth": a_bw,
@@ -371,7 +377,7 @@ def build_weathermap(devices, links, interface_speeds, positions, prometheus_url
                 },
                 "Z": {
                     "bandwidth": z_bw,
-                    "query": f"{link['z_host']} {z_gnmic_int} rx",
+                    "query": f"{link['z_host']} {z_gnmic_int} tx",
                     "labelOffset": 55, "anchor": ANCHOR["Left"], "dashboardLink": "",
                 },
             },
@@ -410,11 +416,6 @@ def build_weathermap(devices, links, interface_speeds, positions, prometheus_url
             "expr": f'rate(interfaces_interface_state_counters_out_octets{{device=~"{host_regex}", interface=~"{interface_regex}"}}[5m]) * 8',
             "legendFormat": "{{device}} {{interface}} tx",
         },
-        {
-            "refId": "C",
-            "expr": f'rate(interfaces_interface_state_counters_in_octets{{device=~"{host_regex}", interface=~"{interface_regex}"}}[5m]) * 8',
-            "legendFormat": "{{device}} {{interface}} rx",
-        },
     ]
     if vtep_hosts:
         vtep_regex = "|".join(promql_escape(h) for h in vtep_hosts)