Layered default layout by hostname role + preserve live node positions

Default node position for spine/core/border-leaf/leaf/access is now a
Y-tier by role (parsed from hostname, same trust level as the site
parsing from #49) with X grouped by site within each tier, replacing
the flat site-grid heuristic.

Position specifically is now read from the live Grafana dashboard before
computing anything, so a node manually drag-and-dropped in the UI keeps
its position across regeneration -- only nodes with no live position yet
(new devices, or first-ever run) get the layered default. Devices removed
from the topology are simply absent from this run's output, so no
orphaned position/link reference is possible.

Refs #53

configs/grafana/weathermap-dashboard.json

@@ -117,8 +117,8 @@
                 "id": "campus-spine1",
                 "label": "campus-spine1",
                 "position": [
-                  460,
+                  2595,
-                  900
+                  93
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -173,8 +173,8 @@
                 "id": "dc-spine1",
                 "label": "dc-spine1",
                 "position": [
-                  460,
+                  -793,
-                  300
+                  31
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -229,8 +229,8 @@
                 "id": "dc-spine2",
                 "label": "dc-spine2",
                 "position": [
-                  640,
+                  -275,
-                  300
+                  27
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -285,8 +285,8 @@
                 "id": "campus-spine2",
                 "label": "campus-spine2",
                 "position": [
-                  640,
+                  2147,
-                  900
+                  58
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -341,8 +341,8 @@
                 "id": "campus-border-leaf1",
                 "label": "campus-border-leaf1",
                 "position": [
-                  460,
+                  1744,
-                  750
+                  -152
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -398,7 +398,7 @@
                 "label": "dc-leaf1",
                 "position": [
                   100,
-                  150
+                  900
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -460,8 +460,8 @@
                 "id": "campus-border-leaf2",
                 "label": "campus-border-leaf2",
                 "position": [
-                  640,
+                  1407,
-                  750
+                  -163
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -516,8 +516,8 @@
                 "id": "dc-leaf4",
                 "label": "dc-leaf4",
                 "position": [
-                  640,
+                  -687,
-                  150
+                  538
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -572,8 +572,8 @@
                 "id": "dc-leaf3",
                 "label": "dc-leaf3",
                 "position": [
-                  460,
+                  -920,
-                  150
+                  540
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -628,8 +628,8 @@
                 "id": "dc-leaf5",
                 "label": "dc-leaf5",
                 "position": [
-                  820,
+                  -226,
-                  150
+                  525
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -691,8 +691,8 @@
                 "id": "dc-border-leaf1",
                 "label": "dc-border-leaf1",
                 "position": [
-                  820,
+                  215,
-                  0
+                  -169
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -747,8 +747,8 @@
                 "id": "dc-leaf6",
                 "label": "dc-leaf6",
                 "position": [
-                  1000,
+                  56,
-                  150
+                  532
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -810,8 +810,8 @@
                 "id": "dc-access1",
                 "label": "dc-access1",
                 "position": [
-                  100,
+                  -1432,
-                  0
+                  773
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -866,8 +866,8 @@
                 "id": "dc-access2",
                 "label": "dc-access2",
                 "position": [
-                  280,
+                  -803,
-                  0
+                  719
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -922,8 +922,8 @@
                 "id": "dc-leaf8",
                 "label": "dc-leaf8",
                 "position": [
-                  280,
+                  816,
-                  300
+                  553
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -978,8 +978,8 @@
                 "id": "dc-leaf7",
                 "label": "dc-leaf7",
                 "position": [
-                  100,
+                  430,
-                  300
+                  530
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -1034,8 +1034,8 @@
                 "id": "dc-access4",
                 "label": "dc-access4",
                 "position": [
-                  640,
+                  644,
-                  0
+                  707
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -1090,8 +1090,8 @@
                 "id": "dc-access3",
                 "label": "dc-access3",
                 "position": [
-                  460,
+                  -87,
-                  0
+                  704
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -1146,8 +1146,8 @@
                 "id": "dc-border-leaf2",
                 "label": "dc-border-leaf2",
                 "position": [
-                  1000,
+                  595,
-                  0
+                  -172
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -1202,8 +1202,8 @@
                 "id": "dc-leaf2",
                 "label": "dc-leaf2",
                 "position": [
-                  280,
+                  -1289,
-                  150
+                  532
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -1265,8 +1265,8 @@
                 "id": "campus-leaf1",
                 "label": "campus-leaf1",
                 "position": [
-                  820,
+                  1925,
-                  750
+                  520
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -1328,8 +1328,8 @@
                 "id": "campus-access1",
                 "label": "campus-access1",
                 "position": [
-                  100,
+                  2061,
-                  750
+                  720
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -1384,8 +1384,8 @@
                 "id": "campus-leaf2",
                 "label": "campus-leaf2",
                 "position": [
-                  1000,
+                  2161,
-                  750
+                  520
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -1447,8 +1447,8 @@
                 "id": "campus-leaf3",
                 "label": "campus-leaf3",
                 "position": [
-                  100,
+                  2585,
-                  900
+                  510
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -1510,8 +1510,8 @@
                 "id": "campus-access2",
                 "label": "campus-access2",
                 "position": [
-                  280,
+                  2955,
-                  750
+                  521
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -1566,8 +1566,8 @@
                 "id": "campus-leaf4",
                 "label": "campus-leaf4",
                 "position": [
-                  280,
+                  2812,
-                  900
+                  731
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -1629,8 +1629,8 @@
                 "id": "core1",
                 "label": "core1",
                 "position": [
-                  100,
+                  912,
-                  450
+                  71
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,
@@ -1685,8 +1685,8 @@
                 "id": "core2",
                 "label": "core2",
                 "position": [
-                  280,
+                  1251,
-                  450
+                  90
                 ],
                 "isConnection": false,
                 "useConstantSpacing": false,

scripts/README.md

@@ -62,8 +62,8 @@ alongside it.
 | `IPFABRIC_TOKEN` | yes | — | Inventory/Snapshots read access. Sent as `X-API-Token`, not `Authorization: Bearer` — IPFabric's REST API does not use bearer auth. |
 | `IPFABRIC_SNAPSHOT` | no | `$last` | |
 | `PROMETHEUS_URL` | no | `http://172.16.0.71:9090` | The **in-topology** Prometheus (see #45), used at generation time to validate the IPFabric→gnmic interface alias and to discover live VTEP/VLAN pairs. This does not have to be the same instance Grafana queries at render time — see below. |
-| `GRAFANA_URL` | with `--provision` | — | |
+| `GRAFANA_URL` | no (see #53) | — | Required for `--provision`. Also used, if set, to fetch the currently-provisioned dashboard and preserve any manually drag-and-drop-repositioned node positions — see step 3 below. Without it, every node gets the layered default position, even ones a human previously repositioned in the UI. |
-| `GRAFANA_TOKEN` | with `--provision` | — | Grafana Service Account token, sent as `Authorization: Bearer`. |
+| `GRAFANA_TOKEN` | no (see #53) | — | Grafana Service Account token, sent as `Authorization: Bearer`. Same conditions as `GRAFANA_URL` above. |
 | `GRAFANA_DATASOURCE_UID` | with `--provision` | — | UID of the Prometheus datasource in Grafana that will actually back the panel. **This must be a datasource that can see the gnmic metrics** — an unrelated/external Prometheus instance with no gnmic data will make the panel render with no values (this happened once, see #48). |
 | `GRAFANA_DASHBOARD_UID` | no | `evpn-vxlan-fabric-weathermap` | |
 | `GRAFANA_WEATHERMAP_PLUGIN_ID` | no | `tamirsuliman-weathermap-panel` | The installed weathermap-ng plugin id. Override if a different fork is installed — plugin ids don't always match the upstream repo name (the schema research in #48 was done against the `allamiro` fork's source; what's actually installed here is a different fork with a stricter/different runtime schema — see the `ANCHOR` handling in the script). |
@@ -77,8 +77,28 @@ alongside it.
    they ride the same physical port as their parent interface and gnmic only
    exports physical interface counters), and dedupes the two directions
    IPFabric reports for each physical link into one.
-3. Computes node positions as a simple site-grouped grid (dc/core/campus
+3. Computes node positions (see #53):
-   bands) — IPFabric has no layout data.
+   - **Default layered layout**, for nodes with no known position yet: role
+     is parsed from the hostname naming convention (`*-spine*`→spine,
+     `core*`→core, `*-border-leaf*`→border-leaf, `*-leaf*`→leaf,
+     `*-access*`→access — same trust level as the `site` parsing used for
+     the Prometheus relabel, not IPFabric-derived), giving Y-tiers top to
+     bottom `spine → core → border-leaf → leaf → access`. X position is
+     grouped/columned by site within each tier, so e.g. dc leafs and campus
+     leafs land in the same row but different column bands. A hostname
+     matching no role pattern is logged (not silently placed in the wrong
+     tier) and put in a fallback tier below `access`.
+   - **Live-position preservation**: position is edited by drag-and-drop
+     directly in the Grafana UI, not in the git-committed base file — a
+     deliberate exception to the "file is the only source of truth" rule
+     from #52, because that's simply not where this field is edited. Before
+     computing the default layout above, the script fetches the currently
+     provisioned dashboard's weathermap panel (if `GRAFANA_URL`/`GRAFANA_TOKEN`
+     are set) and reuses each existing node's live position as-is. Only
+     genuinely new nodes (not in that live map — first-ever run, or a fresh
+     device added to the topology) get the layered default. A device removed
+     from the topology simply has no entry in this run's node/link output at
+     all, so no orphaned position or dangling link reference is possible.
 4. Builds the panel's `targets`: one PromQL query per metric family (BGP
    status, interface tx, interface rx, VXLAN MAC/VNI), each with an explicit
    `legendFormat` so the resolved display name is predictable.

scripts/generate_weathermap.py

@@ -61,7 +61,25 @@ SITE_ORDER = ["dc", "core", "campus"]
 GRID_COLUMNS = 6
 GRID_SPACING_X = 180
 GRID_SPACING_Y = 150
-SITE_BAND_Y = {"dc": 0, "core": 450, "campus": 750}
+
+# Layered default layout (see #53): device role is parsed from the hostname
+# naming convention, same trust level as the `site` parsing already in place
+# for the Prometheus relabel (#49) -- not IPFabric-derived, not configurable.
+# Checked in this order so e.g. "campus-border-leaf1" matches border-leaf
+# before the more general leaf pattern.
+ROLE_PATTERNS = [
+    ("spine", re.compile(r"-spine")),
+    ("core", re.compile(r"^core")),
+    ("border-leaf", re.compile(r"-border-leaf")),
+    ("leaf", re.compile(r"-leaf")),
+    ("access", re.compile(r"-access")),
+]
+ROLE_TIER_ORDER = ["spine", "core", "border-leaf", "leaf", "access"]
+TIER_SPACING_Y = 300
+# Per-site X band start, wide enough that the largest role/site group (8 dc
+# leafs) doesn't spill into the next site's band at GRID_COLUMNS=6.
+SITE_X_OFFSET = {"dc": 100, "core": 1300, "campus": 1600}
+UNMATCHED_ROLE_TIER_Y = len(ROLE_TIER_ORDER) * TIER_SPACING_Y + 300
 
 # tamirsuliman-weathermap-panel's numeric anchor enum (Center=0, Top=1,
 # Bottom=2, Left=3, Right=4) -- reverse-engineered from module.js, since
@@ -181,27 +199,49 @@ def promql_escape(s):
 # Topology processing
 # --------------------------------------------------------------------------
 
-def build_layout(devices):
+def parse_role(hostname):
-    by_site = {}
+    """Device role from the hostname naming convention -- see ROLE_PATTERNS.
+    Returns None if nothing matches (logged by the caller, not silently
+    defaulted into the wrong tier)."""
+    for role, pattern in ROLE_PATTERNS:
+        if pattern.search(hostname):
+            return role
+    return None
+
+
+def build_layout(devices, mismatches):
+    """Default layered layout for nodes with no live (Grafana-drag-and-drop)
+    position yet -- see #53. Y-tier by role (spine top, access bottom), X
+    grouped/columned by site within each tier so same-role devices from
+    different sites don't overlap."""
+    groups = {}  # (role, site) -> [hostname, ...]
+    unmatched = []
     for dev in devices:
-        by_site.setdefault(dev["siteName"], []).append(dev["hostname"])
+        host, site = dev["hostname"], dev["siteName"]
+        role = parse_role(host)
+        if role is None:
+            unmatched.append(host)
+            continue
+        groups.setdefault((role, site), []).append(host)
 
     positions = {}
-    for site in SITE_ORDER:
+    for role_idx, role in enumerate(ROLE_TIER_ORDER):
-        band_y = SITE_BAND_Y.get(site, 0)
+        tier_y = role_idx * TIER_SPACING_Y
-        for idx, host in enumerate(sorted(by_site.get(site, []))):
+        for site in SITE_ORDER:
-            col, row = idx % GRID_COLUMNS, idx // GRID_COLUMNS
+            x_offset = SITE_X_OFFSET.get(site, max(SITE_X_OFFSET.values()) + 300)
-            positions[host] = [100 + col * GRID_SPACING_X, band_y + row * GRID_SPACING_Y]
+            for idx, host in enumerate(sorted(groups.get((role, site), []))):
+                col, row = idx % GRID_COLUMNS, idx // GRID_COLUMNS
+                positions[host] = [x_offset + col * GRID_SPACING_X, tier_y + row * GRID_SPACING_Y]
 
-    # Any site not in SITE_ORDER (shouldn't happen given the naming convention,
+    if unmatched:
-    # but don't silently drop nodes if it does) gets stacked below everything else.
+        mismatches.append(
-    extra_band_y = max(SITE_BAND_Y.values()) + 300
+            f"{len(unmatched)} hostname(s) matched no role pattern (spine/core/border-leaf/leaf/access), "
-    for site, hosts in by_site.items():
+            f"placed in a fallback tier instead of guessing: {', '.join(sorted(unmatched))}"
-        if site in SITE_ORDER:
+        )
-            continue
+        for idx, host in enumerate(sorted(unmatched)):
-        for idx, host in enumerate(sorted(hosts)):
             col, row = idx % GRID_COLUMNS, idx // GRID_COLUMNS
-            positions[host] = [100 + col * GRID_SPACING_X, extra_band_y + row * GRID_SPACING_Y]
+            positions[host] = [100 + col * GRID_SPACING_X, UNMATCHED_ROLE_TIER_Y + row * GRID_SPACING_Y]
+
     return positions
 
 
@@ -413,6 +453,7 @@ def build_weathermap(devices, links, interface_speeds, positions, prometheus_url
 # --------------------------------------------------------------------------
 
 WEATHERMAP_SLOT_TITLE = "__WEATHERMAP_SLOT__"
+WEATHERMAP_PANEL_TITLE = "Fabric Weathermap"  # what the slot is renamed to on merge
 
 
 def build_weathermap_panel(weathermap, targets, datasource_uid, plugin_id):
@@ -461,7 +502,7 @@ def merge_weathermap_into_base(base_dashboard, weathermap_panel, dashboard_uid):
             found = True
             merged = dict(panel)
             merged.update(weathermap_panel)
-            merged["title"] = "Fabric Weathermap"
+            merged["title"] = WEATHERMAP_PANEL_TITLE
             panels.append(merged)
         else:
             panels.append(panel)
@@ -471,6 +512,39 @@ def merge_weathermap_into_base(base_dashboard, weathermap_panel, dashboard_uid):
     return dashboard
 
 
+def fetch_live_node_positions(grafana_url, api_token, dashboard_uid):
+    """Node positions as currently provisioned in Grafana, keyed by node id
+    -- see #53. Position is edited live via drag-and-drop in the Grafana UI,
+    not in the git-committed base file, so it's the one weathermap field that
+    must be sourced from live Grafana state rather than regenerated or read
+    from the manual base -- a manual repositioning must survive every rerun.
+
+    Returns {} if the dashboard doesn't exist yet (first-ever run) or has no
+    weathermap panel yet -- everything falls back to the default layered
+    layout in that case. Any other HTTP error is treated as a real
+    misconfiguration (e.g. a bad token) and raised, rather than silently
+    treated as "no dashboard" -- that would risk quietly discarding every
+    manual position on a run that should have failed loudly instead."""
+    req = urllib.request.Request(
+        f"{grafana_url.rstrip('/')}/api/dashboards/uid/{dashboard_uid}",
+        headers={"Authorization": f"Bearer {api_token}"},
+        method="GET",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            payload = json.load(resp)
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return {}
+        sys.exit(f"Fetching live dashboard for position preservation failed: HTTP {e.code} {e.read().decode()}")
+
+    for panel in payload.get("dashboard", {}).get("panels", []):
+        if panel.get("title") == WEATHERMAP_PANEL_TITLE:
+            nodes = panel.get("options", {}).get("weathermap", {}).get("nodes", [])
+            return {n["id"]: n["position"] for n in nodes if "position" in n}
+    return {}
+
+
 def provision_to_grafana(grafana_url, api_token, dashboard_payload):
     req = urllib.request.Request(
         f"{grafana_url.rstrip('/')}/api/dashboards/db",
@@ -501,6 +575,11 @@ def main():
     ipfabric_token = env("IPFABRIC_TOKEN", required=True)
     snapshot = env("IPFABRIC_SNAPSHOT", "$last")
     prometheus_url = env("PROMETHEUS_URL", "http://172.16.0.71:9090")
+    dashboard_uid = env("GRAFANA_DASHBOARD_UID", "evpn-vxlan-fabric-weathermap")
+    datasource_uid = env("GRAFANA_DATASOURCE_UID", "PROMETHEUS_DATASOURCE_UID_PLACEHOLDER")
+    plugin_id = env("GRAFANA_WEATHERMAP_PLUGIN_ID", "tamirsuliman-weathermap-panel")
+    grafana_url = env("GRAFANA_URL")
+    grafana_token = env("GRAFANA_TOKEN")
 
     print(f"Fetching devices from IPFabric ({ipfabric_url}, snapshot={snapshot})...")
     devices = fetch_devices(ipfabric_url, ipfabric_token, snapshot)
@@ -514,21 +593,33 @@ def main():
     print("Fetching interface speeds...")
     interface_speeds = fetch_interface_speeds(ipfabric_url, ipfabric_token, snapshot)
 
-    positions = build_layout(devices)
-
     mismatches = []
+    default_positions = build_layout(devices, mismatches)
+
+    # Position specifically is edited live in the Grafana UI (drag-and-drop),
+    # not in the git-committed base file -- see #53. Reuse whatever's live
+    # for nodes that already exist there; only brand-new nodes get the
+    # layered default. Iterating over default_positions (this run's device
+    # set) rather than the live map means a removed device's stale live
+    # position is simply never looked up again, no orphaned entry survives.
+    if grafana_url and grafana_token:
+        print(f"Fetching live node positions from {grafana_url} (preserve manual repositioning)...")
+        live_positions = fetch_live_node_positions(grafana_url, grafana_token, dashboard_uid)
+        print(f"  {len(live_positions)} node(s) with an existing live position")
+    else:
+        print("GRAFANA_URL/GRAFANA_TOKEN not set -- skipping live position fetch, using layered default for all nodes", file=sys.stderr)
+        live_positions = {}
+    positions = {host: live_positions.get(host, default) for host, default in default_positions.items()}
+
     print(f"Cross-checking interface names against live exporter ({prometheus_url})...")
     weathermap, targets = build_weathermap(devices, links, interface_speeds, positions, prometheus_url, mismatches)
 
     if mismatches:
-        print(f"\n{len(mismatches)} interface-name / bandwidth mismatch(es) found (link kept, not dropped):", file=sys.stderr)
+        print(f"\n{len(mismatches)} mismatch(es) found (link/position kept, not dropped):", file=sys.stderr)
         for m in mismatches:
             print(f"  - {m}", file=sys.stderr)
         print(file=sys.stderr)
 
-    dashboard_uid = env("GRAFANA_DASHBOARD_UID", "evpn-vxlan-fabric-weathermap")
-    datasource_uid = env("GRAFANA_DATASOURCE_UID", "PROMETHEUS_DATASOURCE_UID_PLACEHOLDER")
-    plugin_id = env("GRAFANA_WEATHERMAP_PLUGIN_ID", "tamirsuliman-weathermap-panel")
     weathermap_panel = build_weathermap_panel(weathermap, targets, datasource_uid, plugin_id)
 
     print(f"Loading manual dashboard base ({args.base})...")
@@ -545,8 +636,8 @@ def main():
           f"{len(dashboard['panels'])} panels total)")
 
     if args.provision:
-        grafana_url = env("GRAFANA_URL", required=True)
+        if not (grafana_url and grafana_token):
-        grafana_token = env("GRAFANA_TOKEN", required=True)
+            sys.exit("Refusing to provision: GRAFANA_URL and GRAFANA_TOKEN must both be set")
         if datasource_uid == "PROMETHEUS_DATASOURCE_UID_PLACEHOLDER":
             sys.exit("Refusing to provision: set GRAFANA_DATASOURCE_UID to the real Prometheus datasource UID first")
         print(f"Provisioning dashboard '{dashboard_uid}' to {grafana_url}...")