Source code for squadds.ml.capacitance.benchmark

"""Scoring of capacitance models with the paper's metric rules.

Per layout and entry kind: the median over its entries of ``|p - y| / max(|y|, 0.1 fF)`` in %, the MAE and the RMSE in fF; then
the mean over layouts (the layout mean), so a 68-conductor chip and a two-pad capacitor count once each. Kinds: ``mutual``
(every off-diagonal pair, the headline), ``signal-signal``, ``signal-ground``, ``signal_self``, ``ground_self``. Unknown
(NaN) label entries are dropped and counted. Every report carries the wall time, machine and thread count.

Validation scoring is open to everyone. Test scoring (``split="test"``) is maintainer-only: once per declared model, recorded
in a ledger, and only for models whose training keys are disjoint from the scored layouts (models trained on the TRAIN split).
"""

from __future__ import annotations

import json
import os
import time
from dataclasses import dataclass, field
from pathlib import Path

import numpy as np

FLOOR_FF = 0.1
KINDS = ("mutual", "signal-signal", "signal-ground", "signal_self", "ground_self")


# ---------------------------------------------------------------------------------------------------------------- models
[docs] def checkpoint_paths(model) -> list[Path]: """Checkpoint files of a model given as a CapacitanceModel, a run dir / .pt path, or a list of them.""" from .models import CapacitanceModel if isinstance(model, CapacitanceModel): if model.entry.get("learned_prior") or model.entry.get("stack") is None: raise ValueError( f"{model.name}: array scoring needs a prior-based model (not a learned-prior / solver-free model)" ) return [Path(model.root) / c["path"] for c in model.entry["checkpoints"]] if isinstance(model, (str, os.PathLike)): p = Path(model) return [p / "model.pt" if p.is_dir() else p] if isinstance(model, (list, tuple)): return [q for m in model for q in checkpoint_paths(m)] raise TypeError(f"cannot read checkpoints from {type(model).__name__}")
[docs] def load_checkpoints(model, threads: int = 2) -> list[tuple[dict, object]]: """[(checkpoint dict, network in eval mode)] of a model (CapacitanceModel, path(s), run dir(s), or checkpoint dict(s)).""" import torch from .network import build_model torch.set_num_threads(int(threads)) items = model if isinstance(model, (list, tuple)) and model and isinstance(model[0], dict) else None if isinstance(model, dict): items = [model] if items is None: items = [torch.load(p, map_location="cpu", weights_only=False) for p in checkpoint_paths(model)] out = [] for c in items: s = c["scales"] net = build_model(len(s["X"]["median"]), len(s["E"]["median"]), len(s["G"]["median"]), c["rounds"]) net.load_state_dict(c["state"]) net.eval() out.append((c, net)) if len({c["arm"] for c, _ in out}) != 1: raise ValueError("an ensemble must share one arm") return out
def _cols(src: list, dst: list) -> list[int]: missing = [k for k in dst if k not in src] if missing: raise KeyError(f"features {missing[:5]} of the checkpoint are not in the data") return [src.index(k) for k in dst]
[docs] def predict_arrays(models, d: dict, schema: dict, rows=None, chunk: int = 64) -> np.ndarray: """Ensemble-mean Maxwell matrices (fF) of shard rows ``rows`` of one shard dict ``d`` (all rows by default): the checkpoint's own standardisation, intercepts and arm, as ``infer.Predictor.from_features`` does per layout.""" import torch rows = np.arange(len(d["key"])) if rows is None else np.asarray(rows) pairs = d["pairs"] out = np.zeros((len(rows), d["sig"].shape[1], d["sig"].shape[1])) for c, net in models: cs = c.get("schema", schema) nk, ek, gk = cs["nodekeys"], cs["edgekeys"], cs.get("globkeys", schema["globkeys"]) xi, ei_, gi = _cols(schema["nodekeys"], nk), _cols(schema["edgekeys"], ek), _cols(schema["globkeys"], gk) arm = c["arm"] direct = arm == "direct_gnn" eg = ek.index("is_ground_pair") ni = nk.index("role_ground") for s in range(0, len(rows), chunk): b = rows[s : s + chunk] X, E, G = d["X"][b][:, :, xi].copy(), d["E"][b][:, :, ei_].copy(), d["G"][b][:, gi].copy() kinds = E[:, :, eg].astype(int) if direct: di = c["direct_intercepts"] base = np.where(kinds == 0, di["0"], di["1"]).astype(np.float32) else: lp = np.log(d["pri"][b].astype(np.float32)) base = (2 * lp[:, :, 1] - lp[:, :, 0]).astype(np.float32) E = np.concatenate([E, (lp[:, :, 1] - lp[:, :, 0])[:, :, None]], -1).astype(np.float32) if arm in ("pair_mlp", "mesh", "lofo_pair"): X, G = X[:, :, [ni]], np.zeros((len(b), 1), np.float32) zs = [] for name, z in (("X", X), ("E", E), ("G", G)): med, iqr = np.array(c["scales"][name]["median"]), np.array(c["scales"][name]["iqr"]) zs.append(np.clip((np.where(np.isfinite(z), z, med) - med) / iqr, -10, 10).astype(np.float32)) cl = c["closure_intercepts"] bbase = np.where(d["sig"][b], cl.get("1", cl.get("True")), cl.get("0", cl.get("False"))).astype(np.float32) T = torch.from_numpy M = torch.ones(d["sig"][b].shape, dtype=torch.bool) PM = torch.ones(base.shape, dtype=torch.bool) with torch.inference_mode(): C, _ = net(T(zs[0]), T(zs[1]), T(zs[2]), M, PM, T(pairs), T(base), T(bbase), direct) out[s : s + len(b)] += C.numpy().astype(np.float64) out /= len(models) if not np.isfinite(out).all(): raise FloatingPointError("non-finite prediction") return out
# ---------------------------------------------------------------------------------------------------------------- metrics
[docs] def entry_kinds(sig: np.ndarray, pairs: np.ndarray) -> np.ndarray: i, j = pairs.T return np.where(sig[i] & sig[j], "signal-signal", "signal-ground")
[docs] def layout_rows(y: np.ndarray, p: np.ndarray, sig: np.ndarray, pairs: np.ndarray, key: str, family: str) -> list[dict]: """Per-kind metrics of one layout (y, p: Maxwell matrices in fF).""" i, j = pairs.T yk, pk = -y[i, j], -p[i, j] kk = entry_kinds(sig, pairs) dy, dp = np.diag(y), np.diag(p) sets = { "mutual": (yk, pk), "signal-signal": (yk[kk == "signal-signal"], pk[kk == "signal-signal"]), "signal-ground": (yk[kk == "signal-ground"], pk[kk == "signal-ground"]), "signal_self": (dy[sig], dp[sig]), "ground_self": (dy[~sig], dp[~sig]), } out = [] for kind, (a, b) in sets.items(): m = np.isfinite(a) if not m.any(): continue a, b = a[m].astype(float), b[m].astype(float) err = np.abs(b - a) out.append( dict( key=key, family=family, kind=kind, n=int(m.sum()), dropped=int((~m).sum()), median_pct=float(np.median(err / np.maximum(np.abs(a), FLOOR_FF)) * 100), mae_fF=float(err.mean()), rmse_fF=float(np.sqrt(np.mean(err**2))), ) ) return out
[docs] def summarize(layouts) -> tuple: """(per family x kind layout means, overall per kind layout means) of a layout-metric frame.""" import pandas as pd L = layouts if isinstance(layouts, pd.DataFrame) else pd.DataFrame(layouts) agg = dict( layouts=("key", "size"), entries=("n", "sum"), median_pct=("median_pct", "mean"), mae_fF=("mae_fF", "mean") ) agg["rmse_fF"] = ("rmse_fF", "mean") fam = L.groupby(["family", "kind"]).agg(**agg).reset_index() tot = L.groupby("kind").agg(**agg).reset_index() return fam, tot
[docs] @dataclass class ScoreReport: layouts: object by_family: object overall: object timing: dict = field(default_factory=dict) split: str = "val" model: str = ""
[docs] def headline(self, kind: str = "mutual") -> dict: r = self.overall[self.overall.kind == kind].iloc[0] return dict( kind=kind, layouts=int(r.layouts), median_pct=float(r.median_pct), mae_fF=float(r.mae_fF), rmse_fF=float(r.rmse_fF), )
def __str__(self) -> str: h = self.headline() t = self.timing return ( f"{self.model} on {self.split}: {h['layouts']} layouts, layout-mean median {h['median_pct']:.4f} % " f"(MAE {h['mae_fF']:.4f} fF, RMSE {h['rmse_fF']:.4f} fF); {t.get('seconds', 0):.1f} s, " f"{t.get('threads')} threads, {t.get('machine')}" )
[docs] def to_dict(self) -> dict: return dict( model=self.model, split=self.split, headline=self.headline(), overall=self.overall.to_dict("records"), by_family=self.by_family.to_dict("records"), timing=self.timing, )
# ---------------------------------------------------------------------------------------------------------------- guards def _guard(split, maintainer, keys, models, ledger_dir, model_id): if split != "test": return None if not maintainer: raise PermissionError( "test scoring is maintainer-only (once per declared model, TRAIN-split models only); score on split='val'" ) if ledger_dir is None: raise ValueError("maintainer test scoring needs ledger_dir (the once-per-model ledger)") if models is not None: trained = set().union(*[set(map(str, c.get("train_keys", []))) for c, _ in models]) overlap = trained & set(map(str, keys)) if overlap: raise PermissionError( f"{len(overlap)} scored test layouts are training layouts of this model (e.g. {sorted(overlap)[:3]})" ) lp = Path(ledger_dir) / f"test__{model_id}.scored" lp.parent.mkdir(parents=True, exist_ok=True) try: fd = os.open(lp, os.O_CREAT | os.O_EXCL | os.O_WRONLY) except FileExistsError: raise PermissionError( f"model {model_id} was already scored on test ({lp.read_text(encoding='utf-8').strip()})" ) from None with os.fdopen(fd, "w") as f: f.write(json.dumps(dict(time=time.strftime("%Y-%m-%dT%H:%M:%S%z"), layouts=len(keys)))) return lp def _model_id(models) -> str: import hashlib h = hashlib.sha256() for _, net in models: for k, v in sorted(net.state_dict().items()): h.update(k.encode()) h.update(v.numpy().tobytes()) return h.hexdigest()[:16] # ---------------------------------------------------------------------------------------------------------------- score
[docs] def score( model, data, labels=None, split: str = "val", maintainer: bool = False, threads: int = 2, families=None, keys=None, ledger_dir=None, name: str | None = None, **predict_kwargs, ) -> ScoreReport: """Score a model on the ``split`` rows of a shard dir, or on layouts with labels. model: ``"QCap"`` / ``"QCap-Fast"`` (released), a CapacitanceModel, a run dir / .pt path (or a list = ensemble), or checkpoint dict(s). data: a shard dir (array route: the stored embedding and priors) or a list of layouts (GDS paths / PreparedLayouts, predicted end to end) with ``labels`` = one (names, Maxwell matrix fF) or ``cap_matrix.csv`` path per layout. ``families`` / ``keys`` restrict the shard rows.""" from ._runtime import machine from .training_data import read_q3d_csv, read_schema, read_shards if split == "test" and not maintainer: _guard(split, maintainer, [], None, ledger_dir, "") t0 = time.perf_counter() rows = [] if isinstance(data, (str, os.PathLike)) and Path(data).is_dir(): from .models import is_model_name, load_model if is_model_name(model): model = load_model(model) models = load_checkpoints(model, threads) schema = read_schema(data) sel = [] for d in read_shards(data): m = d["split"] == split if families is not None: m &= np.isin(d["family"], list(families)) if keys is not None: m &= np.isin(d["key"], list(keys)) if m.any(): sel.append((d, np.flatnonzero(m))) all_keys = [str(k) for d, ix in sel for k in d["key"][ix]] if not all_keys: raise ValueError(f"no {split!r} rows in {data}") mid = _model_id(models) _guard(split, maintainer, all_keys, models, ledger_dir, mid) tn = 0.0 for d, ix in sel: t = time.perf_counter() P = predict_arrays(models, d, schema, ix) tn += time.perf_counter() - t for b, p in zip(ix, P): rows += layout_rows( d["C"][b].astype(float), p, d["sig"][b], d["pairs"], str(d["key"][b]), str(d["family"][b]) ) timing = dict( network_s=tn, per_layout_network_s=tn / len(all_keys), route="shard arrays (stored embedding + priors)" ) label = name or f"{models[0][0]['arm']} x{len(models)} ({mid})" else: from .models import CapacitanceModel, load_model from .predictor import CapacitancePredictor items = list(data) if isinstance(data, (list, tuple)) else [data] if labels is None or len(labels) != len(items): raise ValueError("layout scoring needs one label per layout") if split == "test": raise PermissionError("maintainer test scoring runs on shard dirs (train keys are checked there)") if isinstance(model, str): model = load_model(model) if not isinstance(model, CapacitanceModel): from .train import trained_model model = trained_model(checkpoint_paths(model)) pred = CapacitancePredictor(model, threads=threads) times = [] for k, (lay, lab) in enumerate(zip(items, labels)): names, C = (read_q3d_csv(lab)[:2]) if isinstance(lab, (str, os.PathLike)) else lab r = pred.predict(lay, **predict_kwargs) times.append(r.timing_s["total_s"]) if sorted(r.names) != sorted(names): raise ValueError(f"layout {k}: predicted nets {r.names} differ from the label nets {names}") idx = [list(names).index(x) for x in r.names] y = np.asarray(C, float)[np.ix_(idx, idx)] sig = np.array([x != "G" for x in r.names]) rows += layout_rows( y, r.matrix_fF, sig, np.array([(i, j) for i in range(len(sig)) for j in range(i + 1, len(sig))]), str(getattr(lay, "source", lay)), "layouts", ) timing = dict( total_s=float(np.sum(times)), per_layout_median_s=float(np.median(times)), route="end to end from the layout", ) label = name or model.name import pandas as pd L = pd.DataFrame(rows) fam, tot = summarize(L) timing.update(seconds=time.perf_counter() - t0, threads=int(threads), machine=machine()) return ScoreReport(L, fam, tot, timing, split, label)
[docs] def paired_difference(a, b, kind: str = "mutual", n_boot: int = 2000, seed: int = 0) -> dict: """Layout-paired difference a - b of the layout means (median % and MAE fF) with a layout-bootstrap 95% interval. a, b: ScoreReports or layout frames scored on the same layouts.""" La = (a.layouts if hasattr(a, "layouts") else a).query("kind == @kind").set_index("key") Lb = (b.layouts if hasattr(b, "layouts") else b).query("kind == @kind").set_index("key") common = La.index.intersection(Lb.index) if len(common) != len(La) or len(common) != len(Lb): raise ValueError("the two scores are not on the same layouts") rng = np.random.default_rng(seed) out = {"layouts": len(common)} for col in ("median_pct", "mae_fF", "rmse_fF"): d = (La.loc[common, col] - Lb.loc[common, col]).to_numpy() boot = d[rng.integers(0, len(d), (n_boot, len(d)))].mean(1) if len(d) > 1 else np.array([d.mean()] * 2) out[col] = dict(diff=float(d.mean()), lo=float(np.percentile(boot, 2.5)), hi=float(np.percentile(boot, 97.5))) return out