Source code for squadds.ml.capacitance.dataset

"""Query the ``SQuADDS/SQuADDS_PlanarCapacitance`` dataset (raw and processed tables) with pandas / pyarrow only.

No ``[ml]`` dependency: this module needs pandas, pyarrow and huggingface_hub (all core SQuADDS dependencies), so
``from squadds.ml.capacitance.dataset import PlanarCapacitanceDB`` works on a plain ``pip install squadds``.

    db = PlanarCapacitanceDB()                                  # HF repo
    db = PlanarCapacitanceDB(local_dir="/path/to/SQuADDS_PlanarCapacitance")   # or env SQUADDS_PLANAR_CAP_DATA
    ids = db.designs(family="interdigital", split="val").design_id[:5]
    db.labels(ids)        # {design_id: N x N Maxwell matrix (pandas DataFrame, fF, ground first)}

Tables are read file by file with pyarrow column / row filters, so a query never loads more of the dataset than it asks for.
"""

from __future__ import annotations

import fnmatch
import json
import os
from pathlib import Path

import numpy as np
import pandas as pd

from squadds.datasets.planar_capacitance_schema import (
    GLOBAL_FEATURES,
    NODE_FEATURES,
    REPO_ID,
    TABLES,
)

ENV_VAR = "SQUADDS_PLANAR_CAP_DATA"
# pinned HF commit of REPO_ID (2026-11-14: coarse prior table renamed physics_prior); revision="main" for the latest
DATASET_REVISION = "c79953313874e9c116ca2f39a414d92025d18484"

TABLE_GLOBS = {
    "designs": "data/designs/*.parquet",
    "layouts": "data/layouts/*.parquet",
    "simulations": "data/simulations/*.parquet",
    "labels": "data/labels/*.parquet",
    "label_passes": "data/label_passes/*.parquet",
    "geometric_embedding_nodes": "data/geometric_embedding/nodes-*.parquet",
    "geometric_embedding_edges": "data/geometric_embedding/edges-*.parquet",
    "geometric_embedding_globals": "data/geometric_embedding/globals-*.parquet",
    "physics_prior": "data/physics_prior/*.parquet",
    "optimized_physics_prior": "data/optimized_physics_prior/*.parquet",
    "model_input": "data/model_input/*.parquet",
}
DESIGN_COLUMNS = [c for c, _ in TABLES["designs"] if c != "params"]
NEAREST_FEATURES = (
    [*GLOBAL_FEATURES, "log_n_conductors"] + [f"mean_{f}" for f in NODE_FEATURES] + [f"max_{f}" for f in NODE_FEATURES]
)


[docs] def long_to_matrix( df: pd.DataFrame, value: str, i: str = "a_idx", j: str = "b_idx", a: str = "net_a", b: str = "net_b" ): """Symmetric N x N DataFrame (index = columns = net names, in index order) from long-format entries ``i <= j``. A table that stores mutual entries only (no diagonal) gets the Maxwell diagonal from closure: ``C_ii = -sum_{j != i} C_ij`` (the ground net is one of the nets, so this is exact for a closed system). """ n = int(max(df[i].max(), df[j].max())) + 1 names = [None] * n for ii, jj, na, nb in zip(df[i], df[j], df[a], df[b]): names[int(ii)], names[int(jj)] = na, nb C = np.full((n, n), np.nan) ii, jj, v = df[i].to_numpy(int), df[j].to_numpy(int), df[value].to_numpy(float) C[ii, jj] = v C[jj, ii] = v diag = np.isnan(np.diag(C)) if diag.any(): off = np.where(np.eye(n, dtype=bool), 0.0, np.nan_to_num(C)) C[diag, diag] = -off.sum(1)[diag] return pd.DataFrame(C, index=names, columns=names)
[docs] class PlanarCapacitanceDB: """Read access to the planar capacitance dataset. Args: revision: HF revision (commit sha, tag or branch) of ``repo_id``; default ``DATASET_REVISION`` (the pinned commit), ``"main"`` for the latest; ignored with a local directory. local_dir: a local copy of the repo (same layout as on HF). Default: env ``SQUADDS_PLANAR_CAP_DATA``, else HF. repo_id: HF dataset repo (default ``SQuADDS/SQuADDS_PlanarCapacitance``). token: HF token (default: the logged-in token, if any). """ def __init__(self, revision: str | None = None, local_dir=None, repo_id: str = REPO_ID, token: str | None = None): local_dir = local_dir or os.environ.get(ENV_VAR) self.local_dir = Path(local_dir).expanduser() if local_dir else None if self.local_dir is not None and not self.local_dir.is_dir(): raise FileNotFoundError(f"planar capacitance data directory not found: {self.local_dir}") self.repo_id, self.revision, self.token = repo_id, revision or DATASET_REVISION, token self._repo_files: list[str] | None = None self._nearest_cache: dict = {} def __repr__(self) -> str: where = str(self.local_dir) if self.local_dir else f"hf://datasets/{self.repo_id}@{self.revision}" return f"PlanarCapacitanceDB({where})" # ------------------------------------------------------------------------------------------------ files def _list_repo_files(self) -> list[str]: if self._repo_files is None: from huggingface_hub import HfApi self._repo_files = HfApi(token=self.token).list_repo_files( self.repo_id, repo_type="dataset", revision=self.revision ) return self._repo_files def _download(self, filename: str) -> Path: from huggingface_hub import hf_hub_download return Path( hf_hub_download(self.repo_id, filename, repo_type="dataset", revision=self.revision, token=self.token) )
[docs] def tables(self) -> dict[str, list[str]]: """Available tables -> their files (repo-relative paths); tables with no file are left out.""" if self.local_dir is not None: rel = [p.relative_to(self.local_dir).as_posix() for p in self.local_dir.glob("data/**/*.parquet")] else: rel = self._list_repo_files() out = {t: sorted(f for f in rel if fnmatch.fnmatch(f, g)) for t, g in TABLE_GLOBS.items()} return {t: f for t, f in out.items() if f}
[docs] def files(self, table: str) -> list[Path]: if table not in TABLE_GLOBS: raise KeyError(f"unknown table {table!r}; tables: {sorted(TABLE_GLOBS)}") if self.local_dir is not None: return sorted(self.local_dir.glob(TABLE_GLOBS[table])) return [self._download(f) for f in self.tables().get(table, [])]
[docs] def read(self, table: str, columns=None, design_ids=None, filters=None) -> pd.DataFrame: """Rows of one table (all its split files) with optional column selection and pyarrow row filters (``[("split", "=", "val")]``); ``design_ids`` adds ``design_id in design_ids``.""" import pyarrow.parquet as pq flt = list(filters or []) if design_ids is not None: ids = list(dict.fromkeys(design_ids)) if not ids: return pd.DataFrame(columns=columns or [c for c, _ in TABLES.get(table, ())]) flt.append(("design_id", "in", ids)) parts = [] for f in self.files(table): names = set(pq.read_schema(f).names) if any(c not in names for c, _, _ in flt): continue cols = [c for c in columns if c in names] if columns else None t = pq.read_table(f, columns=cols, filters=flt or None) if t.num_rows: parts.append(t.to_pandas()) if not parts: return pd.DataFrame(columns=columns or [c for c, _ in TABLES.get(table, ())]) return pd.concat(parts, ignore_index=True)
# ------------------------------------------------------------------------------------------------ raw tables
[docs] def designs( self, family=None, split=None, n_conductors=None, collection=None, config=None, columns=None, design_ids=None, ) -> pd.DataFrame: """Design rows (without the ``params`` JSON unless asked for in ``columns``). ``family`` matches ``family``, ``paper_family`` or ``model_family``; ``config`` is the SQuADDS config name; each filter takes a value or a list.""" def as_list(v): return [v] if isinstance(v, (str, int, np.integer)) else list(v) flt = [] for col, v in (("split", split), ("collection", collection), ("squadds_config", config)): if v is not None: flt.append((col, "in", as_list(v))) df = self.read("designs", columns=columns or DESIGN_COLUMNS, design_ids=design_ids, filters=flt) if family is not None: fam = set(as_list(family)) mask = np.zeros(len(df), bool) for col in ("family", "paper_family", "model_family"): if col in df: mask |= df[col].isin(fam).to_numpy() df = df[mask] if n_conductors is not None: df = df[df["n_conductors"].isin([float(x) for x in as_list(n_conductors)])] return df.reset_index(drop=True)
[docs] def params(self, design_ids) -> dict[str, dict]: """Generator / component options of each design (parsed ``params`` JSON).""" df = self.read("designs", columns=["design_id", "params"], design_ids=design_ids) return {d: json.loads(p) if p else {} for d, p in zip(df.design_id, df.params)}
[docs] def simulations(self, design_ids=None, role: str | None = None, columns=None) -> pd.DataFrame: flt = [("role", "=", role)] if role else None return self.read("simulations", columns=columns, design_ids=design_ids, filters=flt)
[docs] def label_passes(self, design_ids, columns=None) -> pd.DataFrame: return self.read("label_passes", columns=columns, design_ids=design_ids)
[docs] def labels(self, design_ids, role: str | None = "reference") -> dict[str, pd.DataFrame]: """Final Maxwell matrices (fF, ground first) keyed by design_id (``role`` = simulation role, default the reference Q3D solve); ``role=None`` returns every simulation keyed by simulation_id. Designs without a public label (test split) are absent.""" sims = self.simulations(design_ids, role=role, columns=["simulation_id", "design_id", "role"]) if sims.empty: return {} lab = self.read( "labels", columns=["simulation_id", "design_id", "net_a", "net_b", "a_idx", "b_idx", "C_fF"], filters=[("simulation_id", "in", sims.simulation_id.tolist())], ) key = "design_id" if role else "simulation_id" return {k: long_to_matrix(g, "C_fF") for k, g in lab.groupby(key, sort=False)}
[docs] def gds_path(self, design_id: str) -> Path: """Local path of the design's GDS (downloaded from the HF repo when needed).""" d = self.designs(design_ids=[design_id], columns=["design_id", "layout_id"]) if d.empty: raise KeyError(f"unknown design_id {design_id!r}") lay = self.read("layouts", columns=["layout_id", "path"], filters=[("layout_id", "=", d.layout_id.iloc[0])]) if lay.empty: raise KeyError(f"no layout row for {design_id!r} ({d.layout_id.iloc[0]})") rel = lay.path.iloc[0] return self.local_dir / rel if self.local_dir is not None else self._download(rel)
# ------------------------------------------------------------------------------------------------ processed tables def _prior(self, table: str, design_ids, value: str) -> dict[str, pd.DataFrame]: df = self.read(table, design_ids=design_ids) return {k: long_to_matrix(g, value, "i", "j") for k, g in df.groupby("design_id", sort=False)}
[docs] def physics_prior(self, design_ids, value: str = "C_fF") -> dict[str, pd.DataFrame]: """Single thin-film MoM solve at p = 0.375 as N x N Maxwell matrices (fF, ground first).""" return self._prior("physics_prior", design_ids, value)
[docs] def optimized_physics_prior(self, design_ids, value: str = "C_R_fF") -> dict[str, pd.DataFrame]: """C^R = R(1.5, 3) on the corrected stack as N x N Maxwell matrices (``value="mutual_1p5_fF"`` / ``"mutual_3_fF"`` for the two solves, diagonal from closure).""" return self._prior("optimized_physics_prior", design_ids, value)
[docs] def model_input(self, design_ids) -> pd.DataFrame: return self.read("model_input", design_ids=design_ids)
[docs] def embedding(self, design_ids) -> dict[str, dict]: """``planar-capacitance-graph-v1`` per design: ``nodes`` (one row per conductor, ground = node 0), ``edges`` (pairs i < j) and ``globals`` (dict).""" nodes = self.read("geometric_embedding_nodes", design_ids=design_ids) edges = self.read("geometric_embedding_edges", design_ids=design_ids) glob = self.read("geometric_embedding_globals", design_ids=design_ids) out = {} for d, g in glob.groupby("design_id", sort=False): out[d] = { "nodes": nodes[nodes.design_id == d].sort_values("node_idx").reset_index(drop=True), "edges": edges[edges.design_id == d].sort_values(["i", "j"]).reset_index(drop=True), "globals": g.iloc[0].to_dict(), } return out
# ------------------------------------------------------------------------------------------------ nearest neighbours
[docs] def pooled_embedding(self, design_ids=None, family=None, split=None) -> pd.DataFrame: """One vector per design (index design_id): the 4 global features, log N, and the mean and the max of each of the 61 node features over the signal conductors (ground node excluded); 127 values, not standardized.""" if design_ids is None and (family is not None or split is not None): design_ids = self.designs( family=family, split=split, columns=["design_id", "family", "paper_family", "model_family", "split"] ).design_id cols = ["design_id", "node_idx", *NODE_FEATURES] nodes = self.read("geometric_embedding_nodes", columns=cols, design_ids=design_ids) glob = ( self.read( "geometric_embedding_globals", columns=["design_id", "n_conductors", *GLOBAL_FEATURES], design_ids=design_ids, ) .drop_duplicates("design_id") .set_index("design_id") ) sig = nodes[nodes.node_idx > 0].drop(columns="node_idx").groupby("design_id") mean = sig.mean().add_prefix("mean_") mx = sig.max().add_prefix("max_") g = glob[list(GLOBAL_FEATURES)].astype(float) g["log_n_conductors"] = np.log(glob["n_conductors"].astype(float)) out = g.join(mean, how="inner").join(mx, how="inner") return out[NEAREST_FEATURES]
[docs] def nearest(self, query, k: int = 5, family=None, split=None) -> pd.DataFrame: """The k designs closest to ``query`` (a design_id, or a 127-value vector in ``pooled_embedding`` order) in the pooled embedding space, standardized per feature over the candidate set (Euclidean distance). Candidates: designs of ``family`` (default: the query design's ``model_family``; all designs when the query is a vector and no family is given) and ``split``. This is a simple geometric similarity, not a capacitance distance. """ qvec = None if isinstance(query, str): if family is None: d = self.designs(design_ids=[query], columns=["design_id", "model_family"]) if d.empty: raise KeyError(f"unknown design_id {query!r}") family = d.model_family.iloc[0] else: qvec = np.asarray(query, float).ravel() if qvec.size != len(NEAREST_FEATURES): raise ValueError(f"query vector must have {len(NEAREST_FEATURES)} values (pooled_embedding order)") key = (repr(family), repr(split)) if key not in self._nearest_cache: self._nearest_cache[key] = self.pooled_embedding(family=family, split=split) P = self._nearest_cache[key] if isinstance(query, str): if query not in P.index: P = pd.concat([P, self.pooled_embedding(design_ids=[query])]) qvec = P.loc[query].to_numpy(float) X = P.to_numpy(float) mu, sd = np.nanmean(X, 0), np.nanstd(X, 0) sd[~(sd > 0)] = 1.0 dist = np.sqrt(np.nansum(((X - mu) / sd - (qvec - mu) / sd) ** 2, 1)) res = pd.DataFrame({"design_id": P.index, "distance": dist}) if isinstance(query, str): res = res[res.design_id != query] res = res.nsmallest(k, "distance").reset_index(drop=True) meta = self.designs(design_ids=res.design_id, columns=["design_id", "collection", "model_family", "split"]) return res.merge(meta, on="design_id", how="left")