Merge pull request 'Telemetry: gnmic + Prometheus + Grafana weathermap dashboard, iperf3 traffic generator' (#59) from feat/telemetry into main

Reviewed-on: #59

README.md

@@ -367,6 +367,11 @@ only appears as the sample *value* of
 `(device, interface, vlan)`. Per-VNI MAC counts require a PromQL join on `vlan`
 rather than a native `vni` label.
 
+`site` (`dc`/`campus`/`core`) is also not gnmic-exported. It's added as a
+Prometheus `metric_relabel_configs` rule deriving it from `device` via the
+`<area>-<role><n>` naming convention regex (see #49), rather than repeating
+`label_replace()` in every dashboard query.
+
 - Config: `configs/prometheus/prometheus.yml`
 - This is separate from, and does not replace, the existing external Prometheus
   instance — no cutover yet, both run in parallel pending validation

assets/weathermap-pipeline.svg

@@ -0,0 +1,72 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 350" font-family="Helvetica, Arial, sans-serif">
+  <defs>
+    <marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
+      <path d="M 0 0 L 10 5 L 0 10 z" fill="#555"/>
+    </marker>
+  </defs>
+
+  <style>
+    .box { stroke-width: 1.5; rx: 6; }
+    .input { fill: #e8f0fe; stroke: #4285f4; }
+    .manual { fill: #fef7e0; stroke: #f9ab00; }
+    .script { fill: #e6f4ea; stroke: #34a853; }
+    .output { fill: #f3e8fd; stroke: #a142f4; }
+    .grafana { fill: #fce8e6; stroke: #ea4335; }
+    .label { font-size: 13px; fill: #202124; }
+    .sublabel { font-size: 10px; fill: #5f6368; }
+    .arrow { stroke: #555; stroke-width: 1.5; fill: none; marker-end: url(#arrow); }
+    .caption { font-size: 11px; fill: #5f6368; font-style: italic; }
+  </style>
+
+  <!-- inputs -->
+  <rect class="box input" x="20" y="30" width="150" height="50"/>
+  <text class="label" x="95" y="50" text-anchor="middle">IPFabric</text>
+  <text class="sublabel" x="95" y="66" text-anchor="middle">topology, inventory</text>
+
+  <rect class="box input" x="20" y="100" width="150" height="50"/>
+  <text class="label" x="95" y="120" text-anchor="middle">Prometheus</text>
+  <text class="sublabel" x="95" y="136" text-anchor="middle">gnmic metrics, live labels</text>
+
+  <!-- generator script -->
+  <rect class="box script" x="220" y="55" width="200" height="70"/>
+  <text class="label" x="320" y="82" text-anchor="middle" font-size="11.5">generate_weathermap.py</text>
+  <text class="sublabel" x="320" y="98" text-anchor="middle">weathermap panel only</text>
+  <text class="sublabel" x="320" y="112" text-anchor="middle">targets + options.weathermap</text>
+
+  <path class="arrow" d="M170,55 L220,80"/>
+  <path class="arrow" d="M170,125 L220,100"/>
+
+  <!-- manual base -->
+  <rect class="box manual" x="220" y="220" width="200" height="70"/>
+  <text class="label" x="320" y="247" text-anchor="middle">dashboard-base.json</text>
+  <text class="sublabel" x="320" y="263" text-anchor="middle">hand-authored, committed</text>
+  <text class="sublabel" x="320" y="277" text-anchor="middle">BGP/ports/throughput/vars</text>
+
+  <!-- merge -->
+  <rect class="box script" x="470" y="130" width="160" height="60"/>
+  <text class="label" x="550" y="155" text-anchor="middle">merge</text>
+  <text class="sublabel" x="550" y="171" text-anchor="middle" font-size="8.5">splices into __WEATHERMAP_SLOT__</text>
+
+  <path class="arrow" d="M420,90 L470,145"/>
+  <path class="arrow" d="M420,255 L470,175"/>
+
+  <!-- output artifact -->
+  <rect class="box output" x="660" y="60" width="210" height="60"/>
+  <text class="label" x="765" y="85" text-anchor="middle" font-size="11.5">weathermap-dashboard.json</text>
+  <text class="sublabel" x="765" y="101" text-anchor="middle">generated build artifact</text>
+
+  <path class="arrow" d="M630,150 L765,120"/>
+
+  <!-- grafana -->
+  <rect class="box grafana" x="660" y="210" width="210" height="60"/>
+  <text class="label" x="765" y="235" text-anchor="middle">Grafana</text>
+  <text class="sublabel" x="765" y="251" text-anchor="middle" font-size="9">--provision (POST /api/dashboards/db)</text>
+
+  <path class="arrow" d="M630,175 L765,210"/>
+  <path class="arrow" d="M765,120 L765,210" stroke-dasharray="3,3"/>
+
+  <!-- read-back for position preservation -->
+  <path class="arrow" d="M765,270 C 620,330 490,300 420,275" stroke-dasharray="3,3"/>
+  <text class="caption" x="545" y="325" text-anchor="middle">live node positions read back before each regeneration (see #53)</text>
+
+</svg>

configs/grafana/dashboard-base.json

@@ -0,0 +1,197 @@
+{
+  "uid": "evpn-vxlan-fabric-weathermap",
+  "title": "EVPN/VXLAN Fabric Weathermap",
+  "tags": ["evpn", "vxlan", "weathermap", "auto-generated"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 0,
+  "templating": {
+    "list": [
+      {
+        "name": "site",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "__DATASOURCE_UID__" },
+        "query": { "query": "label_values(interfaces_interface_state_oper_status, site)", "refId": "site" },
+        "refresh": 2,
+        "sort": 1,
+        "multi": true,
+        "includeAll": true,
+        "current": { "text": "All", "value": "$__all" }
+      },
+      {
+        "name": "device",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "__DATASOURCE_UID__" },
+        "query": {
+          "query": "label_values(interfaces_interface_state_oper_status{site=~\"$site\"}, device)",
+          "refId": "device"
+        },
+        "refresh": 2,
+        "sort": 1,
+        "multi": true,
+        "includeAll": true,
+        "current": { "text": "All", "value": "$__all" }
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "title": "__WEATHERMAP_SLOT__",
+      "gridPos": { "h": 24, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "id": 2,
+      "type": "table",
+      "title": "BGP Sessions",
+      "gridPos": { "h": 10, "w": 12, "x": 0, "y": 24 },
+      "datasource": { "type": "prometheus", "uid": "__DATASOURCE_UID__" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "__DATASOURCE_UID__" },
+          "instant": true,
+          "format": "table",
+          "expr": "network_instances_network_instance_protocols_protocol_bgp_neighbors_neighbor_state_session_state{device=~\"$device\"}"
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "__name__": true,
+              "instance": true,
+              "job": true,
+              "protocol_identifier": true,
+              "protocol_name": true,
+              "subscription_name": true
+            },
+            "renameByName": {
+              "device": "Device",
+              "neighbor_address": "Neighbor Address",
+              "network_instance_name": "VRF",
+              "Value": "State"
+            }
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            { "type": "value", "options": { "0": { "text": "Down", "color": "red" }, "1": { "text": "Up", "color": "green" } } }
+          ],
+          "custom": { "cellOptions": { "type": "color-background" } }
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "State" },
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "color-background" } },
+              {
+                "id": "mappings",
+                "value": [
+                  { "type": "value", "options": { "0": { "text": "Down", "color": "red" }, "1": { "text": "Up", "color": "green" } } }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "id": 3,
+      "type": "table",
+      "title": "Ports / Interfaces",
+      "gridPos": { "h": 10, "w": 12, "x": 12, "y": 24 },
+      "datasource": { "type": "prometheus", "uid": "__DATASOURCE_UID__" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "__DATASOURCE_UID__" },
+          "instant": true,
+          "format": "table",
+          "expr": "interfaces_interface_state_oper_status{device=~\"$device\"}"
+        },
+        {
+          "refId": "B",
+          "datasource": { "type": "prometheus", "uid": "__DATASOURCE_UID__" },
+          "instant": true,
+          "format": "table",
+          "expr": "rate(interfaces_interface_state_counters_in_octets{device=~\"$device\"}[5m]) * 8"
+        },
+        {
+          "refId": "C",
+          "datasource": { "type": "prometheus", "uid": "__DATASOURCE_UID__" },
+          "instant": true,
+          "format": "table",
+          "expr": "rate(interfaces_interface_state_counters_out_octets{device=~\"$device\"}[5m]) * 8"
+        }
+      ],
+      "transformations": [
+        { "id": "merge", "options": {} },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "__name__": true,
+              "instance": true,
+              "job": true,
+              "subscription_name": true
+            },
+            "renameByName": {
+              "device": "Device",
+              "interface": "Interface",
+              "Value #A": "Status",
+              "Value #B": "In (bps)",
+              "Value #C": "Out (bps)"
+            }
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "Status" },
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "color-background" } },
+              {
+                "id": "mappings",
+                "value": [
+                  { "type": "value", "options": { "0": { "text": "Down", "color": "red" }, "1": { "text": "Up", "color": "green" } } }
+                ]
+              }
+            ]
+          },
+          { "matcher": { "id": "byName", "options": "In (bps)" }, "properties": [{ "id": "unit", "value": "bps" }] },
+          { "matcher": { "id": "byName", "options": "Out (bps)" }, "properties": [{ "id": "unit", "value": "bps" }] }
+        ]
+      }
+    },
+    {
+      "id": 4,
+      "type": "timeseries",
+      "title": "Throughput per Site",
+      "gridPos": { "h": 10, "w": 24, "x": 0, "y": 34 },
+      "datasource": { "type": "prometheus", "uid": "__DATASOURCE_UID__" },
+      "fieldConfig": { "defaults": { "unit": "bps" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "__DATASOURCE_UID__" },
+          "expr": "sum by (site) (rate(interfaces_interface_state_counters_in_octets{site=~\"$site\", interface!~\"Management.*\"}[5m]) * 8)",
+          "legendFormat": "{{site}} in"
+        },
+        {
+          "refId": "B",
+          "datasource": { "type": "prometheus", "uid": "__DATASOURCE_UID__" },
+          "expr": "-sum by (site) (rate(interfaces_interface_state_counters_out_octets{site=~\"$site\", interface!~\"Management.*\"}[5m]) * 8)",
+          "legendFormat": "{{site}} out"
+        }
+      ]
+    }
+  ]
+}

configs/prometheus/prometheus.yml

@@ -16,6 +16,16 @@ scrape_configs:
       - action: labeldrop
         regex: source
 
+      # site: not exported by gnmic -- derived from the hostname naming
+      # convention (see CLAUDE.md: dc-*/campus-*/core* regex used for
+      # IPFabric site separation) here instead of repeating label_replace()
+      # in every dashboard query, see issue #49
+      - source_labels: [device]
+        target_label: site
+        regex: (campus|core|dc)-?.*
+        replacement: $1
+        action: replace
+
       # interface: raw label is already "interface_name" (OpenConfig key on the
       # "interface" path element) — normalize to "interface" for consistency
       - source_labels: [interface_name]

envrc.sample

@@ -0,0 +1,11 @@
+export GRAFANA_URL=""
+export GRAFANA_TOKEN=""
+export GRAFANA_DATASOURCE_UID=""
+export IPFABRIC_URL=""
+export IPFABRIC_TOKEN=""
+
+# Optional -- see scripts/README.md for defaults and when to override
+# export IPFABRIC_SNAPSHOT="$last"
+# export PROMETHEUS_URL="http://172.16.0.71:9090"
+# export GRAFANA_DASHBOARD_UID="evpn-vxlan-fabric-weathermap"
+# export GRAFANA_WEATHERMAP_PLUGIN_ID="tamirsuliman-weathermap-panel"

scripts/README.md

@@ -1,79 +1,167 @@
 # scripts/generate_weathermap.py
 
-Generates a [weathermap-ng](https://github.com/allamiro/grafana-network-weathermap-ng)
+Generates the **weathermap panel only** from live IPFabric + Prometheus
-panel from live IPFabric topology + gnmic/Prometheus metrics, and writes a full
+data, merges it into a hand-authored dashboard base, and provisions the
-Grafana dashboard-as-code JSON to `configs/grafana/weathermap-dashboard.json`
+result to Grafana.
-(committed to Gitea as the source of truth, same pattern as the retired Flow
-Panel YAML). Optionally provisions it into Grafana via the HTTP API.
 
-See issue #48 for the panel schema research this is built against, and its
+![pipeline diagram](../assets/weathermap-pipeline.svg)
-comment thread for the live-validation history (auth quirks, plugin id, a
-PromQL escaping bug, a panel-crash fix — worth reading before touching the
-anchor/regex-escaping logic).
 
-## Usage
+## Files
+
+| File                                        | What it is                                                                                                                                                                                      | Edit it?                |
+| ------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- |
+| `configs/grafana/dashboard-base.json`       | Hand-authored: BGP table, ports table, throughput graphs, `site`/`device` variables, layout. Has one reserved slot panel (`title: "__WEATHERMAP_SLOT__"`) where the weathermap gets spliced in. | Yes, directly           |
+| `configs/grafana/weathermap-dashboard.json` | Generated build artifact — base + fresh weathermap panel, merged. What actually gets provisioned.                                                                                               | No — regenerate instead |
+
+Why split like this: topology data (nodes/links) changes with the lab and
+should regenerate every time; the rest of the dashboard (table layout,
+thresholds, transformations) is hand-tuned and shouldn't be touched by a
+topology refresh. See #52.
+
+## Quickstart
 
 ```bash
-export IPFABRIC_URL=https://<ipfabric-instance>
+cp envrc.sample .envrc   # fill in the values (see "Credentials" below)
-export IPFABRIC_TOKEN=<token>
+source .envrc
-python3 scripts/generate_weathermap.py                 # writes the JSON only
 
-export GRAFANA_URL=https://<external-grafana-instance>
+python3 scripts/generate_weathermap.py              # dry run: writes the JSON only
-export GRAFANA_TOKEN=<token>
+python3 scripts/generate_weathermap.py --provision   # also pushes it to Grafana
-export GRAFANA_DATASOURCE_UID=<prometheus-datasource-uid-in-grafana>
-python3 scripts/generate_weathermap.py --provision      # also provisions via the Grafana API
 ```
 
-Re-run after any topology change (`evpn-lab.clab.yml`) to regenerate and
+Re-run the same `--provision` command any time the lab topology changes —
-re-provision.
+the script always rebuilds the weathermap panel from live data, so there's
+nothing to "detect", just re-run it. It's not automated yet (no watcher/CI
+hook), so re-running is a manual step for now.
+
+## Credentials
+
+| Value                         | Env var                                                                 | Get it from the UI                                                                                    | Get it via API                                                                                               |
+| ----------------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ |
+| IPFabric token                | `IPFABRIC_TOKEN`                                                        | IPFabric → user menu → Settings → API Tokens → Add token                                              | not possible, UI-only                                                                                        |
+| Grafana service account token | `GRAFANA_TOKEN`                                                         | Grafana → Administration → Service accounts → Add → give it **Editor** role → Add token               | possible but multi-step (create account, then create token under it) — UI is simpler                         |
+| Prometheus datasource UID     | `GRAFANA_DATASOURCE_UID`                                                | Grafana → Connections → Data sources → open the one with gnmic data → UID is the last part of the URL | `curl -s -H "Authorization: Bearer $GRAFANA_TOKEN" "$GRAFANA_URL/api/datasources" \| jq '.[] \| {name,uid}'` |
+| Weathermap plugin ID          | `GRAFANA_WEATHERMAP_PLUGIN_ID` (optional, only if not the default fork) | Grafana → Administration → Plugins → search "weathermap" → id is in the URL                           | —                                                                                                            |
+
+⚠️ The datasource must be the one that can actually see gnmic metrics —
+picking an unrelated Prometheus instance renders the panel with no data
+(see #48).
+
+## Verifying the result
+
+**In the browser** (the only way to actually see it render — colors,
+tables, and the ports-table merge transformation all happen client-side):
+
+1. Open `$GRAFANA_URL/d/evpn-vxlan-fabric-weathermap`.
+2. Weathermap: nodes grouped spine-top → access-bottom, all green.
+3. `site`/`device` dropdowns filter the two tables below.
+4. BGP Sessions / Ports tables: populated, green=Up / red=Down.
+5. Throughput per Site: signed graph, non-zero values.
+
+**Without a browser**, use Grafana's **Explore** view (paste a PromQL
+expression from the panel JSON and run it) — same result as the UI check
+above minus the rendering, no `curl` needed. Or hit the API directly:
+
+```bash
+curl -s -X POST -H "Authorization: Bearer $GRAFANA_TOKEN" -H "Content-Type: application/json" \
+  "$GRAFANA_URL/api/ds/query" -d '{
+    "queries": [{"refId":"A","datasource":{"type":"prometheus","uid":"'"$GRAFANA_DATASOURCE_UID"'"},
+                 "expr":"min by (device) (interfaces_interface_state_oper_status)","instant":true}],
+    "from":"now-5m","to":"now"}' | python3 -m json.tool
+```
+
+## Troubleshooting
+
+| Symptom                                                    | Cause                                                               | Check                                                              |
+| ---------------------------------------------------------- | ------------------------------------------------------------------- | ------------------------------------------------------------------ |
+| `Refusing to provision: set GRAFANA_DATASOURCE_UID...`     | Env var unset                                                       | `echo $GRAFANA_DATASOURCE_UID`                                     |
+| Panel renders with no data                                 | Wrong datasource — can't see gnmic metrics                          | Explore → run `up` against it, confirm gnmic series come back      |
+| `Base dashboard has no panel titled '__WEATHERMAP_SLOT__'` | Someone edited `dashboard-base.json` and renamed/removed that panel | Check panel `id: 1` still has that exact title                     |
+| Manually-moved node keeps resetting position               | `GRAFANA_URL`/`GRAFANA_TOKEN` weren't set for that run              | Look for `Fetching live node positions from ...` in the run output |
+| `HTTP 401/403` fetching live positions                     | Expired/bad Grafana token                                           | Re-issue the service account token                                 |
 
 ## Environment variables
 
-| Variable | Required | Default | Notes |
+| Variable                       | Required           | Default                               |
-|---|---|---|---|
+| ------------------------------ | ------------------ | ------------------------------------- |
-| `IPFABRIC_URL` | yes | — | e.g. `https://ipfabric.example.com` |
+| `IPFABRIC_URL`                 | yes                | —                                     |
-| `IPFABRIC_TOKEN` | yes | — | Inventory/Snapshots read access. Sent as `X-API-Token`, not `Authorization: Bearer` — IPFabric's REST API does not use bearer auth. |
+| `IPFABRIC_TOKEN`               | yes                | —                                     |
-| `IPFABRIC_SNAPSHOT` | no | `$last` | |
+| `IPFABRIC_SNAPSHOT`            | no                 | `$last`                               |
-| `PROMETHEUS_URL` | no | `http://172.16.0.71:9090` | The **in-topology** Prometheus (see #45), used at generation time to validate the IPFabric→gnmic interface alias and to discover live VTEP/VLAN pairs. This does not have to be the same instance Grafana queries at render time — see below. |
+| `PROMETHEUS_URL`               | no                 | `http://172.16.0.71:9090`             |
-| `GRAFANA_URL` | with `--provision` | — | |
+| `GRAFANA_URL`                  | no*                | —                                     |
-| `GRAFANA_TOKEN` | with `--provision` | — | Grafana Service Account token, sent as `Authorization: Bearer`. |
+| `GRAFANA_TOKEN`                | no*                | —                                     |
-| `GRAFANA_DATASOURCE_UID` | with `--provision` | — | UID of the Prometheus datasource in Grafana that will actually back the panel. **This must be a datasource that can see the gnmic metrics** — an unrelated/external Prometheus instance with no gnmic data will make the panel render with no values (this happened once, see #48). |
+| `GRAFANA_DATASOURCE_UID`       | with `--provision` | —                                     |
-| `GRAFANA_DASHBOARD_UID` | no | `evpn-vxlan-fabric-weathermap` | |
+| `GRAFANA_DASHBOARD_UID`        | no                 | `evpn-vxlan-fabric-weathermap`        |
-| `GRAFANA_WEATHERMAP_PLUGIN_ID` | no | `tamirsuliman-weathermap-panel` | The installed weathermap-ng plugin id. Override if a different fork is installed — plugin ids don't always match the upstream repo name (the schema research in #48 was done against the `allamiro` fork's source; what's actually installed here is a different fork with a stricter/different runtime schema — see the `ANCHOR` handling in the script). |
+| `GRAFANA_WEATHERMAP_PLUGIN_ID` | no                 | `tamirsuliman-weathermap-panel`       |
+| `--base` (CLI flag)            | no                 | `configs/grafana/dashboard-base.json` |
 
-## What the script does
+\* `GRAFANA_URL`/`GRAFANA_TOKEN` are required for `--provision`, and
+optional otherwise — if set on a plain (non-`--provision`) run, they're
+used to read back live node positions so manual repositioning survives
+the next regeneration (see #53). Skip them and every node just gets the
+default layout.
 
-1. Fetches device inventory + connectivity-matrix from IPFabric.
+## How it works
-2. Filters the connectivity-matrix to physical Ethernet-to-Ethernet links
+
-   only: drops Management-plane neighbor entries and `.100`/`.200`
+1. Fetch device inventory + connectivity-matrix from IPFabric.
-   subinterface rows (802.1Q tags used for the gold VRF stitching on Core —
+2. Keep only physical Ethernet↔Ethernet links (drop management-plane and
-   they ride the same physical port as their parent interface and gnmic only
+   `.100`/`.200` subinterface rows), dedupe the two directions IPFabric
-   exports physical interface counters), and dedupes the two directions
+   reports per link.
-   IPFabric reports for each physical link into one.
+3. Position each node:
-3. Computes node positions as a simple site-grouped grid (dc/core/campus
+    - **New node** → layered default: Y-tier by role parsed from the
-   bands) — IPFabric has no layout data.
+      hostname (`spine → core → border-leaf → leaf → access`), X grouped
-4. Builds the panel's `targets`: one PromQL query per metric family (BGP
+      by site within the tier. A hostname that matches no role is logged
-   status, interface tx, interface rx, VXLAN MAC/VNI), each with an explicit
+      and dropped into a fallback tier, never silently misplaced (#53).
-   `legendFormat` so the resolved display name is predictable.
+    - **Known node** → whatever position is currently live in Grafana,
-5. Builds `nodes[]` and `links[]` referencing those resolved legend strings,
+      unchanged. This is why `GRAFANA_URL`/`GRAFANA_TOKEN` matter even on
-   including the plugin's `anchors` tally (per-node count of link
+      a dry run (#53).
-   attachments per side) and numeric anchor enum on each link side —
+4. Build the panel's PromQL `targets`: node status, per-side link tx (each
-   omitting these crashes the panel on load in the installed plugin fork,
+   side its own egress counter, so both directions are represented — #57),
-   despite the (fork-specific) schema research saying it's safe to skip.
+   VXLAN MAC/VNI tooltip — each with an explicit `legendFormat`.
-6. Cross-checks every IPFabric interface name, aliased to gnmic's naming
+5. Build `nodes[]`/`links[]`, including the plugin's `anchors` tally
-   (`Et`→`Ethernet`, `Po`→`Port-Channel`, `Lo`→`Loopback`, `Vl`→`Vlan`,
+   (link count per side) — required by the installed plugin fork even
-   `Ma`→`Management`), against the live exporter. Any link whose aliased
+   though the schema doc says it's optional (#48).
-   name has no matching series is logged, not silently dropped — the actual
+6. Cross-check every interface name against the live gnmic exporter
-   fix for a real mismatch belongs in gnmic interface aliasing (#43), not in
+   (`Et`→`Ethernet`, `Po`→`Port-Channel`, etc.); mismatches are logged,
-   this script.
+   not dropped — the link stays, just without data until fixed upstream.
-7. Writes the dashboard JSON, and provisions it via `POST
+7. Load `dashboard-base.json`, substitute the real datasource UID, splice
-   /api/dashboards/db` if `--provision` is passed.
+   the generated panel into `__WEATHERMAP_SLOT__` (keeping that slot's
+   `gridPos`/`id` — layout stays manual).
+8. Write `weathermap-dashboard.json`; provision it if `--provision`.
 
 ## Known gaps
 
-- **VXLAN MAC-per-VNI target**: the `vlan` join key used to correlate
+- **VXLAN MAC-per-VNI tooltip**: the `vlan` join key doesn't match on live
-  VLAN→VNI mapping with FDB entries (query verbatim from #44) doesn't
+  data for any VTEP node, likely an Arista internal-vs-front-panel VLAN
-  actually match on live data for any VTEP node — likely an Arista
+  translation gap. Only affects that one decorative tooltip metric, not
-  internal-VLAN-vs-front-panel-VLAN translation the OpenConfig paths don't
+  node/link status or traffic coloring. Tracked in #44.
-  reconcile. Only affects the decorative per-VTEP tooltip metric, not
+
-  node/link status or traffic coloring. Tracked in #44, not fixed here.
+# scripts/generate_traffic.sh
+
+Generates real DC↔Campus traffic over VRF `gold` using `iperf3` (bundled
+in the `network-multitool` image every host container runs), with a live
+bandwidth dashboard. Without this, host containers sit idle and IPFabric
+ARP/MAC tables, Grafana throughput graphs, and the weathermap panel stay
+empty until someone manually generates traffic (see #55).
+
+## Quickstart
+
+```bash
+./scripts/generate_traffic.sh <duration_seconds>
+```
+
+## How it works
+
+- Starts `iperf3 -s` on the DC gold-VRF servers: `dc-server2`
+  (10.34.34.102), `dc-server4` (10.78.78.104).
+- Runs `iperf3 -c -R` from the paired campus gold-VRF hosts, reversing the
+  stream so the DC server pushes to the campus consumer: `dc-server2` →
+  `campus-host1`, `dc-server4` → `campus-host2` — exercising the full
+  DC→Core→Campus stitched EVPN Type-5 path end to end.
+- Redraws a terminal dashboard every second for the run duration: server
+  list, and live Mbits/sec per client session parsed from `iperf3 -i 1`
+  output.
+- On exit (duration end or Ctrl-C), kills the client processes and stops
+  the `iperf3 -s` processes on the DC servers — no leftover state.
+
+`dc-server1`/`dc-server3` (VLAN 40, VRF default, no gateway) are out of
+scope — this script only exercises the routed gold VRF path.

scripts/generate_traffic.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# Generates DC<->Campus traffic over VRF gold using iperf3 (bundled in the
+# network-multitool image every host container runs), with a live
+# server/client bandwidth dashboard. Refs #55.
+set -euo pipefail
+
+DURATION="${1:?Usage: $0 <duration_seconds>}"
+if ! [[ "$DURATION" =~ ^[0-9]+$ ]] || [[ "$DURATION" -lt 1 ]]; then
+    echo "Duration must be a positive integer (seconds)" >&2
+    exit 1
+fi
+
+LAB_PREFIX="clab-arista-evpn-fabric"
+PORT=5301
+
+# server_name:server_ip:client_name  — gold VRF pairs, stitched EVPN
+# Type-5 path DC -> Core -> Campus (see README Host Addressing table)
+PAIRS=(
+    "dc-server2:10.34.34.102:campus-host1"
+    "dc-server4:10.78.78.104:campus-host2"
+)
+
+WORKDIR="$(mktemp -d)"
+CLIENT_PIDS=()
+
+cleanup() {
+    for pid in "${CLIENT_PIDS[@]:-}"; do
+        kill "$pid" >/dev/null 2>&1 || true
+    done
+    for pair in "${PAIRS[@]}"; do
+        IFS=':' read -r server _ _ <<<"$pair"
+        docker exec "${LAB_PREFIX}-${server}" pkill -f "iperf3 -s -p ${PORT}" >/dev/null 2>&1 || true
+    done
+    rm -rf "$WORKDIR"
+}
+trap cleanup EXIT INT TERM
+
+echo "Starting iperf3 servers..."
+for pair in "${PAIRS[@]}"; do
+    IFS=':' read -r server server_ip _ <<<"$pair"
+    docker exec -d "${LAB_PREFIX}-${server}" iperf3 -s -p "$PORT"
+done
+sleep 1
+
+echo "Starting iperf3 clients for ${DURATION}s..."
+for pair in "${PAIRS[@]}"; do
+    IFS=':' read -r server server_ip client <<<"$pair"
+    logfile="${WORKDIR}/${client}.log"
+    # -R: DC hosts the service, campus is the consumer -- data should flow
+    # server -> client (download), not client -> server, to match how a
+    # real DC-hosted service/campus-consumer pair behaves.
+    docker exec "${LAB_PREFIX}-${client}" iperf3 -c "$server_ip" -p "$PORT" -R \
+        -t "$DURATION" -i 1 --forceflush -f m >"$logfile" 2>&1 &
+    CLIENT_PIDS+=("$!")
+done
+
+for ((elapsed = 0; elapsed <= DURATION; elapsed++)); do
+    clear
+    echo "EVPN/VXLAN lab traffic generator — ${elapsed}/${DURATION}s"
+    echo
+    echo "Servers (iperf3 -s):"
+    for pair in "${PAIRS[@]}"; do
+        IFS=':' read -r server server_ip _ <<<"$pair"
+        echo "  ${server} (${server_ip}:${PORT})"
+    done
+    echo
+    echo "Clients (live bandwidth):"
+    for pair in "${PAIRS[@]}"; do
+        IFS=':' read -r server server_ip client <<<"$pair"
+        logfile="${WORKDIR}/${client}.log"
+        last_line="$(grep -E 'Mbits/sec' "$logfile" 2>/dev/null | tail -1 || true)"
+        bw="$(sed -E 's/.*[[:space:]]([0-9.]+ Mbits\/sec).*/\1/' <<<"$last_line")"
+        [[ -z "$last_line" ]] && bw="waiting..."
+        printf "  %-13s -> %-13s : %s\n" "$server" "$client" "$bw"
+    done
+    sleep 1
+done
+
+wait "${CLIENT_PIDS[@]}" 2>/dev/null || true
+
+echo
+echo "Done. Summary:"
+for pair in "${PAIRS[@]}"; do
+    IFS=':' read -r server _ client <<<"$pair"
+    logfile="${WORKDIR}/${client}.log"
+    summary="$(grep -E 'receiver' "$logfile" 2>/dev/null || true)"
+    echo "  ${server} -> ${client}: ${summary:-no data}"
+done

scripts/generate_weathermap.py

@@ -1,6 +1,14 @@
 #!/usr/bin/env python3
-"""Generate a weathermap-ng panel config from IPFabric topology + gnmic/Prometheus
+"""Generate a weathermap-ng PANEL (only) from IPFabric topology + gnmic/Prometheus
-metrics, and optionally provision it into a Grafana dashboard.
+metrics, merge it into a manually-authored dashboard base, and optionally
+provision the result into Grafana.
+
+Scope (see #52): this script knows only about the weathermap panel --
+targets (node status / link tx per side / VXLAN tooltip queries) and
+options.weathermap (nodes/links/scale/settings). It has no knowledge of the
+BGP sessions table, ports/interfaces table, or throughput panels -- those
+live in the manually-authored `configs/grafana/dashboard-base.json` and are
+not touched by this script.
 
 Data sources (see gitea issues #44, #47, #48):
   - IPFabric REST API: device inventory + connectivity-matrix (topology)
@@ -8,8 +16,13 @@ Data sources (see gitea issues #44, #47, #48):
     values, used both to validate the IPFabric->gnmic interface-name mapping
     and to resolve which VTEPs/VLANs actually have VXLAN data.
 
+Merge: the weathermap panel is generated fresh from live data on every run
+(no diffing), then spliced into the base dashboard's reserved slot (the panel
+titled "__WEATHERMAP_SLOT__"), keeping that slot's gridPos so the manually
+authored layout is never repositioned by the generator. See #52.
+
 Usage:
-    python3 scripts/generate_weathermap.py [--output PATH] [--provision]
+    python3 scripts/generate_weathermap.py [--base PATH] [--output PATH] [--provision]
 
 Environment:
     IPFABRIC_URL        e.g. https://ipfabric.example.com
@@ -48,7 +61,25 @@ SITE_ORDER = ["dc", "core", "campus"]
 GRID_COLUMNS = 6
 GRID_SPACING_X = 180
 GRID_SPACING_Y = 150
-SITE_BAND_Y = {"dc": 0, "core": 450, "campus": 750}
+
+# Layered default layout (see #53): device role is parsed from the hostname
+# naming convention, same trust level as the `site` parsing already in place
+# for the Prometheus relabel (#49) -- not IPFabric-derived, not configurable.
+# Checked in this order so e.g. "campus-border-leaf1" matches border-leaf
+# before the more general leaf pattern.
+ROLE_PATTERNS = [
+    ("spine", re.compile(r"-spine")),
+    ("core", re.compile(r"^core")),
+    ("border-leaf", re.compile(r"-border-leaf")),
+    ("leaf", re.compile(r"-leaf")),
+    ("access", re.compile(r"-access")),
+]
+ROLE_TIER_ORDER = ["spine", "core", "border-leaf", "leaf", "access"]
+TIER_SPACING_Y = 300
+# Per-site X band start, wide enough that the largest role/site group (8 dc
+# leafs) doesn't spill into the next site's band at GRID_COLUMNS=6.
+SITE_X_OFFSET = {"dc": 100, "core": 1300, "campus": 1600}
+UNMATCHED_ROLE_TIER_Y = len(ROLE_TIER_ORDER) * TIER_SPACING_Y + 300
 
 # tamirsuliman-weathermap-panel's numeric anchor enum (Center=0, Top=1,
 # Bottom=2, Left=3, Right=4) -- reverse-engineered from module.js, since
@@ -168,27 +199,49 @@ def promql_escape(s):
 # Topology processing
 # --------------------------------------------------------------------------
 
-def build_layout(devices):
+def parse_role(hostname):
-    by_site = {}
+    """Device role from the hostname naming convention -- see ROLE_PATTERNS.
+    Returns None if nothing matches (logged by the caller, not silently
+    defaulted into the wrong tier)."""
+    for role, pattern in ROLE_PATTERNS:
+        if pattern.search(hostname):
+            return role
+    return None
+
+
+def build_layout(devices, mismatches):
+    """Default layered layout for nodes with no live (Grafana-drag-and-drop)
+    position yet -- see #53. Y-tier by role (spine top, access bottom), X
+    grouped/columned by site within each tier so same-role devices from
+    different sites don't overlap."""
+    groups = {}  # (role, site) -> [hostname, ...]
+    unmatched = []
     for dev in devices:
-        by_site.setdefault(dev["siteName"], []).append(dev["hostname"])
+        host, site = dev["hostname"], dev["siteName"]
+        role = parse_role(host)
+        if role is None:
+            unmatched.append(host)
+            continue
+        groups.setdefault((role, site), []).append(host)
 
     positions = {}
-    for site in SITE_ORDER:
+    for role_idx, role in enumerate(ROLE_TIER_ORDER):
-        band_y = SITE_BAND_Y.get(site, 0)
+        tier_y = role_idx * TIER_SPACING_Y
-        for idx, host in enumerate(sorted(by_site.get(site, []))):
+        for site in SITE_ORDER:
-            col, row = idx % GRID_COLUMNS, idx // GRID_COLUMNS
+            x_offset = SITE_X_OFFSET.get(site, max(SITE_X_OFFSET.values()) + 300)
-            positions[host] = [100 + col * GRID_SPACING_X, band_y + row * GRID_SPACING_Y]
+            for idx, host in enumerate(sorted(groups.get((role, site), []))):
+                col, row = idx % GRID_COLUMNS, idx // GRID_COLUMNS
+                positions[host] = [x_offset + col * GRID_SPACING_X, tier_y + row * GRID_SPACING_Y]
 
-    # Any site not in SITE_ORDER (shouldn't happen given the naming convention,
+    if unmatched:
-    # but don't silently drop nodes if it does) gets stacked below everything else.
+        mismatches.append(
-    extra_band_y = max(SITE_BAND_Y.values()) + 300
+            f"{len(unmatched)} hostname(s) matched no role pattern (spine/core/border-leaf/leaf/access), "
-    for site, hosts in by_site.items():
+            f"placed in a fallback tier instead of guessing: {', '.join(sorted(unmatched))}"
-        if site in SITE_ORDER:
+        )
-            continue
+        for idx, host in enumerate(sorted(unmatched)):
-        for idx, host in enumerate(sorted(hosts)):
             col, row = idx % GRID_COLUMNS, idx // GRID_COLUMNS
-            positions[host] = [100 + col * GRID_SPACING_X, extra_band_y + row * GRID_SPACING_Y]
+            positions[host] = [100 + col * GRID_SPACING_X, UNMATCHED_ROLE_TIER_Y + row * GRID_SPACING_Y]
+
     return positions
 
 
@@ -261,11 +314,25 @@ def build_weathermap(devices, links, interface_speeds, positions, prometheus_url
             "position": positions[host],
             "isConnection": False,
             "useConstantSpacing": False,
-            "compactVerticalLinks": False,
+            # True (not the plugin default) so rendered node HEIGHT is
+            # constant, decoupled from per-node link count -- see #54. Ground
+            # truth confirmed against the installed plugin's real module.js
+            # (not the #48 schema doc, which doesn't cover this): height is
+            # `fontSize + 2*padding.vertical` when compactVerticalLinks is
+            # true, unconditionally; when false, it's the larger of that or
+            # a term proportional to max(anchors[Left].numLinks,
+            # anchors[Right].numLinks) -- exactly why campus-leaf1 (3 links)
+            # and campus-leaf2 (4 links) rendered at different heights.
+            # Width is unaffected either way: it's always recomputed from
+            # the label text at render time (no override field exists), and
+            # useConstantSpacing only pulls in Top/Bottom anchor link count,
+            # which this generator never uses (links always attach
+            # Right/Left -- see anchor_counts above).
+            "compactVerticalLinks": True,
             "padding": {"horizontal": 12, "vertical": 6},
             "colors": dict(NODE_COLORS),
             "nodeIcon": None,
-            "statusQuery": f"BGP {host}",
+            "statusQuery": f"STATUS {host}",
             "nodeStatusColorTarget": "border",
             "statusValueMappings": [dict(m) for m in STATUS_VALUE_MAPPINGS],
             "anchors": {a: {"numLinks": anchor_counts[host][a], "numFilledLinks": 0} for a in ANCHOR.values()},
@@ -296,6 +363,12 @@ def build_weathermap(devices, links, interface_speeds, positions, prometheus_url
         link_defs.append({
             "id": f"{link['a_host']}-{a_gnmic_int}--{link['z_host']}-{z_gnmic_int}",
             "nodes": [{"id": link["a_host"]}, {"id": link["z_host"]}],
+            # Each side's query is that side's own tx (egress) counter, not the
+            # far end's rx -- see #57. a_host tx and z_host rx both describe the
+            # *same* A->Z flow measured from opposite ends (the a_host->z_host
+            # direction, counted twice), leaving the Z->A direction never
+            # queried by either side. Using each node's own tx gives two
+            # independent, opposite-direction measurements instead.
             "sides": {
                 "A": {
                     "bandwidth": a_bw,
@@ -304,7 +377,7 @@ def build_weathermap(devices, links, interface_speeds, positions, prometheus_url
                 },
                 "Z": {
                     "bandwidth": z_bw,
-                    "query": f"{link['z_host']} {z_gnmic_int} rx",
+                    "query": f"{link['z_host']} {z_gnmic_int} tx",
                     "labelOffset": 55, "anchor": ANCHOR["Left"], "dashboardLink": "",
                 },
             },
@@ -317,32 +390,48 @@ def build_weathermap(devices, links, interface_speeds, positions, prometheus_url
     interface_regex = "|".join(sorted(interface_regex_parts))
 
     # -- targets (one query per metric family, per task spec) --
+    #
+    # Node status (refId A): a device is "up" only if at least one interface
+    # is up AND every BGP session is up, if it has any -- see #49. The old
+    # "BGP {{device}}" query sourced raw per-neighbor session-state series,
+    # so a device with multiple neighbors collided on one legend string and
+    # only one arbitrarily won. `min by (device)` picks the *worst* session
+    # (0 beats 1), and devices with zero BGP sessions (access switches) are
+    # not penalized: the `or` fallback substitutes a constant 1 for any
+    # device present in device-up but absent from the BGP series entirely.
+    device_up = f'min by (device) (interfaces_interface_state_oper_status{{device=~"{host_regex}"}})'
+    bgp_worst = (
+        f'min by (device) (network_instances_network_instance_protocols_protocol_bgp_neighbors_neighbor_state_session_state'
+        f'{{network_instance_name="default", device=~"{host_regex}"}})'
+    )
+    bgp_ok_or_not_applicable = f'({bgp_worst} or ({device_up} * 0 + 1))'
     targets = [
         {
             "refId": "A",
-            "expr": f'min by (device) (network_instances_network_instance_protocols_protocol_bgp_neighbors_neighbor_state_session_state{{network_instance_name="default", device=~"{host_regex}"}})',
+            "expr": f'{device_up} * {bgp_ok_or_not_applicable}',
-            "legendFormat": "BGP {{device}}",
+            "legendFormat": "STATUS {{device}}",
         },
         {
             "refId": "B",
             "expr": f'rate(interfaces_interface_state_counters_out_octets{{device=~"{host_regex}", interface=~"{interface_regex}"}}[5m]) * 8',
             "legendFormat": "{{device}} {{interface}} tx",
         },
-        {
-            "refId": "C",
-            "expr": f'rate(interfaces_interface_state_counters_in_octets{{device=~"{host_regex}", interface=~"{interface_regex}"}}[5m]) * 8',
-            "legendFormat": "{{device}} {{interface}} rx",
-        },
     ]
     if vtep_hosts:
         vtep_regex = "|".join(promql_escape(h) for h in vtep_hosts)
         # Verbatim join from #44 "VXLAN (MAC count per VNI)", scoped to VTEP nodes.
+        # RHS is wrapped in max by (device, vlan) so the join key is always
+        # unique -- a Prometheus restart or relabel change otherwise leaves
+        # the pre-restart (frozen) and post-restart series briefly coexisting
+        # within the 5m staleness window, both matching the same (device,
+        # vlan) group, which trips PromQL's "many-to-many matching not
+        # allowed" error. See #50.
         targets.append({
             "refId": "D",
             "expr": (
                 f'count by (device, vlan) (network_instances_network_instance_fdb_mac_table_entries_entry_vlan{{device=~"{vtep_regex}"}})\n'
                 f'* on(device, vlan) group_left(vlan_to_vni_state_vni)\n'
-                f'interfaces_interface_arista_vxlan_vlan_to_vnis_vlan_to_vni_state_vni{{device=~"{vtep_regex}"}}'
+                f'max by (device, vlan) (interfaces_interface_arista_vxlan_vlan_to_vnis_vlan_to_vni_state_vni{{device=~"{vtep_regex}"}})'
             ),
             "legendFormat": "VNI {{device}} {{vlan}}",
         })
@@ -375,34 +464,102 @@ def build_weathermap(devices, links, interface_speeds, positions, prometheus_url
 
 
 # --------------------------------------------------------------------------
-# Grafana provisioning
+# Weathermap panel (only) -- see #52, scope narrowed from full-dashboard
 # --------------------------------------------------------------------------
 
-def build_dashboard(weathermap, targets, dashboard_uid, datasource_uid, plugin_id):
+WEATHERMAP_SLOT_TITLE = "__WEATHERMAP_SLOT__"
+WEATHERMAP_PANEL_TITLE = "Fabric Weathermap"  # what the slot is renamed to on merge
+
+
+def build_weathermap_panel(weathermap, targets, datasource_uid, plugin_id):
+    """The weathermap panel's own content: type/datasource/targets/options.
+    Deliberately has no gridPos/id/title of its own -- those come from
+    whatever slot it's merged into (see merge_weathermap_into_base)."""
     return {
-        "dashboard": {
+        "type": plugin_id,
-            "uid": dashboard_uid,
+        "datasource": {"type": "prometheus", "uid": datasource_uid},
-            "title": "EVPN/VXLAN Fabric Weathermap",
+        "targets": [dict(t, datasource={"type": "prometheus", "uid": datasource_uid}) for t in targets],
-            "tags": ["evpn", "vxlan", "weathermap", "auto-generated"],
+        "options": {"weathermap": weathermap},
-            "timezone": "browser",
-            "schemaVersion": 39,
-            "version": 0,
-            "panels": [
-                {
-                    "id": 1,
-                    "type": plugin_id,
-                    "title": "Fabric Weathermap",
-                    "gridPos": {"h": 24, "w": 24, "x": 0, "y": 0},
-                    "datasource": {"type": "prometheus", "uid": datasource_uid},
-                    "targets": [dict(t, datasource={"type": "prometheus", "uid": datasource_uid}) for t in targets],
-                    "options": {"weathermap": weathermap},
-                }
-            ],
-        },
-        "overwrite": True,
     }
 
 
+# --------------------------------------------------------------------------
+# Manual base dashboard + merge (see #52)
+# --------------------------------------------------------------------------
+
+def load_base_dashboard(path):
+    with open(path) as f:
+        return json.load(f)
+
+
+def substitute_placeholders(obj, replacements):
+    """Recursively replace exact-match string placeholders (e.g. the
+    datasource UID token) anywhere in the manually-authored base JSON."""
+    if isinstance(obj, dict):
+        return {k: substitute_placeholders(v, replacements) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [substitute_placeholders(v, replacements) for v in obj]
+    if isinstance(obj, str) and obj in replacements:
+        return replacements[obj]
+    return obj
+
+
+def merge_weathermap_into_base(base_dashboard, weathermap_panel, dashboard_uid):
+    """Splice the freshly generated weathermap panel into the base
+    dashboard's reserved slot (matched by title), keeping the slot's
+    gridPos/id -- layout stays manual, the generator never repositions it."""
+    dashboard = dict(base_dashboard)
+    dashboard["uid"] = dashboard_uid
+    panels = []
+    found = False
+    for panel in dashboard.get("panels", []):
+        if panel.get("title") == WEATHERMAP_SLOT_TITLE:
+            found = True
+            merged = dict(panel)
+            merged.update(weathermap_panel)
+            merged["title"] = WEATHERMAP_PANEL_TITLE
+            panels.append(merged)
+        else:
+            panels.append(panel)
+    if not found:
+        sys.exit(f"Base dashboard has no panel titled {WEATHERMAP_SLOT_TITLE!r} -- nowhere to merge the weathermap panel")
+    dashboard["panels"] = panels
+    return dashboard
+
+
+def fetch_live_node_positions(grafana_url, api_token, dashboard_uid):
+    """Node positions as currently provisioned in Grafana, keyed by node id
+    -- see #53. Position is edited live via drag-and-drop in the Grafana UI,
+    not in the git-committed base file, so it's the one weathermap field that
+    must be sourced from live Grafana state rather than regenerated or read
+    from the manual base -- a manual repositioning must survive every rerun.
+
+    Returns {} if the dashboard doesn't exist yet (first-ever run) or has no
+    weathermap panel yet -- everything falls back to the default layered
+    layout in that case. Any other HTTP error is treated as a real
+    misconfiguration (e.g. a bad token) and raised, rather than silently
+    treated as "no dashboard" -- that would risk quietly discarding every
+    manual position on a run that should have failed loudly instead."""
+    req = urllib.request.Request(
+        f"{grafana_url.rstrip('/')}/api/dashboards/uid/{dashboard_uid}",
+        headers={"Authorization": f"Bearer {api_token}"},
+        method="GET",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            payload = json.load(resp)
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return {}
+        sys.exit(f"Fetching live dashboard for position preservation failed: HTTP {e.code} {e.read().decode()}")
+
+    for panel in payload.get("dashboard", {}).get("panels", []):
+        if panel.get("title") == WEATHERMAP_PANEL_TITLE:
+            nodes = panel.get("options", {}).get("weathermap", {}).get("nodes", [])
+            return {n["id"]: n["position"] for n in nodes if "position" in n}
+    return {}
+
+
 def provision_to_grafana(grafana_url, api_token, dashboard_payload):
     req = urllib.request.Request(
         f"{grafana_url.rstrip('/')}/api/dashboards/db",
@@ -421,16 +578,23 @@ def provision_to_grafana(grafana_url, api_token, dashboard_payload):
 
 def main():
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--base", default="configs/grafana/dashboard-base.json",
+                         help="manually-authored dashboard base JSON (everything but the weathermap panel)")
     parser.add_argument("--output", default="configs/grafana/weathermap-dashboard.json",
-                         help="where to write the generated dashboard-as-code JSON")
+                         help="where to write the merged dashboard-as-code JSON (build artifact)")
     parser.add_argument("--provision", action="store_true",
-                         help="also POST the dashboard to the Grafana API (requires GRAFANA_* env vars)")
+                         help="also POST the merged dashboard to the Grafana API (requires GRAFANA_* env vars)")
     args = parser.parse_args()
 
     ipfabric_url = env("IPFABRIC_URL", required=True)
     ipfabric_token = env("IPFABRIC_TOKEN", required=True)
     snapshot = env("IPFABRIC_SNAPSHOT", "$last")
     prometheus_url = env("PROMETHEUS_URL", "http://172.16.0.71:9090")
+    dashboard_uid = env("GRAFANA_DASHBOARD_UID", "evpn-vxlan-fabric-weathermap")
+    datasource_uid = env("GRAFANA_DATASOURCE_UID", "PROMETHEUS_DATASOURCE_UID_PLACEHOLDER")
+    plugin_id = env("GRAFANA_WEATHERMAP_PLUGIN_ID", "tamirsuliman-weathermap-panel")
+    grafana_url = env("GRAFANA_URL")
+    grafana_token = env("GRAFANA_TOKEN")
 
     print(f"Fetching devices from IPFabric ({ipfabric_url}, snapshot={snapshot})...")
     devices = fetch_devices(ipfabric_url, ipfabric_token, snapshot)
@@ -444,32 +608,51 @@ def main():
     print("Fetching interface speeds...")
     interface_speeds = fetch_interface_speeds(ipfabric_url, ipfabric_token, snapshot)
 
-    positions = build_layout(devices)
-
     mismatches = []
+    default_positions = build_layout(devices, mismatches)
+
+    # Position specifically is edited live in the Grafana UI (drag-and-drop),
+    # not in the git-committed base file -- see #53. Reuse whatever's live
+    # for nodes that already exist there; only brand-new nodes get the
+    # layered default. Iterating over default_positions (this run's device
+    # set) rather than the live map means a removed device's stale live
+    # position is simply never looked up again, no orphaned entry survives.
+    if grafana_url and grafana_token:
+        print(f"Fetching live node positions from {grafana_url} (preserve manual repositioning)...")
+        live_positions = fetch_live_node_positions(grafana_url, grafana_token, dashboard_uid)
+        print(f"  {len(live_positions)} node(s) with an existing live position")
+    else:
+        print("GRAFANA_URL/GRAFANA_TOKEN not set -- skipping live position fetch, using layered default for all nodes", file=sys.stderr)
+        live_positions = {}
+    positions = {host: live_positions.get(host, default) for host, default in default_positions.items()}
+
     print(f"Cross-checking interface names against live exporter ({prometheus_url})...")
     weathermap, targets = build_weathermap(devices, links, interface_speeds, positions, prometheus_url, mismatches)
 
     if mismatches:
-        print(f"\n{len(mismatches)} interface-name / bandwidth mismatch(es) found (link kept, not dropped):", file=sys.stderr)
+        print(f"\n{len(mismatches)} mismatch(es) found (link/position kept, not dropped):", file=sys.stderr)
         for m in mismatches:
             print(f"  - {m}", file=sys.stderr)
         print(file=sys.stderr)
 
-    dashboard_uid = env("GRAFANA_DASHBOARD_UID", "evpn-vxlan-fabric-weathermap")
+    weathermap_panel = build_weathermap_panel(weathermap, targets, datasource_uid, plugin_id)
-    datasource_uid = env("GRAFANA_DATASOURCE_UID", "PROMETHEUS_DATASOURCE_UID_PLACEHOLDER")
+
-    plugin_id = env("GRAFANA_WEATHERMAP_PLUGIN_ID", "tamirsuliman-weathermap-panel")
+    print(f"Loading manual dashboard base ({args.base})...")
-    dashboard_payload = build_dashboard(weathermap, targets, dashboard_uid, datasource_uid, plugin_id)
+    base_dashboard = load_base_dashboard(args.base)
+    base_dashboard = substitute_placeholders(base_dashboard, {"__DATASOURCE_UID__": datasource_uid})
+    dashboard = merge_weathermap_into_base(base_dashboard, weathermap_panel, dashboard_uid)
+    dashboard_payload = {"dashboard": dashboard, "overwrite": True}
 
     os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
     with open(args.output, "w") as f:
         json.dump(dashboard_payload, f, indent=2)
         f.write("\n")
-    print(f"Wrote {args.output} ({len(weathermap['nodes'])} nodes, {len(weathermap['links'])} links)")
+    print(f"Wrote {args.output} ({len(weathermap['nodes'])} nodes, {len(weathermap['links'])} links, "
+          f"{len(dashboard['panels'])} panels total)")
 
     if args.provision:
-        grafana_url = env("GRAFANA_URL", required=True)
+        if not (grafana_url and grafana_token):
-        grafana_token = env("GRAFANA_TOKEN", required=True)
+            sys.exit("Refusing to provision: GRAFANA_URL and GRAFANA_TOKEN must both be set")
         if datasource_uid == "PROMETHEUS_DATASOURCE_UID_PLACEHOLDER":
             sys.exit("Refusing to provision: set GRAFANA_DATASOURCE_UID to the real Prometheus datasource UID first")
         print(f"Provisioning dashboard '{dashboard_uid}' to {grafana_url}...")