"""Training data of the planar-capacitance networks: the P2d shard layout, label reading and QC, and shard builders.
A shard directory holds one ``n0NN.npz`` per conductor count N (ground included) and ``schema.json``. Arrays per file
(L layouts, P = N(N-1)/2 pairs ``i < j``):
key (L,) str split (L,) str family (L,) str
X (L, N, 61) f32 node vector (60 features + missing_log_min_halfwidth), node 0 = ground G
E (L, P, 42) f32 pair vector G (L, 4) f32 global vector
C (L, N, N) f32 Maxwell matrix in fF, ground first (NaN = unknown entry, masked in training)
sig (L, N) bool signal mask pri (L, P, 2) f32 positive prior mutuals (fF) at 1.5 and 3 panels per gap
pairs (P, 2) int
``build_shards`` turns layouts (GDS) and their labels (a Maxwell matrix with net names, or a Q3D ``cap_matrix.csv``) into
shards with the same embedding and the ``optimized_physics_prior`` solves the released models read. Net names are matched by
name and never merged; every label passes QC (symmetry, signs, closure, positive definiteness) or is refused.
"""
from __future__ import annotations
import csv
import hashlib
import json
import os
import platform
import re
import time
from dataclasses import dataclass, field
from pathlib import Path
import numpy as np
GROUND = "G"
PRIOR_LEVELS = (1.5, 3.0)
TO_FF = {"F": 1e15, "nF": 1e6, "pF": 1e3, "fF": 1.0, "aF": 1e-3}
ARRAYS = ("key", "split", "family", "X", "E", "G", "C", "sig", "pri", "pairs")
[docs]
class LabelQCError(ValueError):
"""A label matrix failed QC (symmetry, Maxwell signs, closure, positive definiteness, net names)."""
[docs]
def base_schema() -> dict:
"""Feature names and order of the networks' input (nodekeys 61, edgekeys 42, globkeys 4) and the prior's physics."""
from ._core import paper as PA
from .models import MANIFEST
s = json.loads((Path(__file__).with_name("_core") / "schema_p2d.json").read_text(encoding="utf-8"))
st = MANIFEST["prior_stacks"]["corrected"]
return {
"nodekeys": list(s["nodekeys"]),
"edgekeys": list(s["edgekeys"]),
"globkeys": list(s["globkeys"]),
"physics": {
"prior_method": "local_gap",
"alpha": PA.ALPHA,
"ensemble": "arithmetic mean of complete predicted Maxwell matrices",
"prior_version": st["version"],
"prior_stack": "corrected",
"levels_ppg": list(PRIOR_LEVELS),
},
}
[docs]
def pairs_of(n: int) -> np.ndarray:
return np.array([(i, j) for i in range(n) for j in range(i + 1, n)], dtype=np.int64).reshape(-1, 2)
[docs]
def sha256_file(p) -> str:
h = hashlib.sha256()
with open(p, "rb") as f:
for b in iter(lambda: f.read(1 << 20), b""):
h.update(b)
return h.hexdigest()
# ---------------------------------------------------------------------------------------------------------------- shards
[docs]
def read_schema(shard_dir) -> dict:
return json.loads((Path(shard_dir) / "schema.json").read_text(encoding="utf-8"))
[docs]
def shard_files(shard_dir) -> list[Path]:
files = sorted(Path(shard_dir).glob("n*.npz"))
if not files:
raise FileNotFoundError(f"no n*.npz shards in {shard_dir}")
return files
[docs]
def read_shards(shard_dir) -> list[dict]:
"""Every shard file as a dict of arrays (sorted by N), plus ``n`` and ``file``."""
out = []
for p in shard_files(shard_dir):
with np.load(p, allow_pickle=False) as z:
d = {k: z[k] for k in z.files}
d["n"], d["file"] = int(d["sig"].shape[1]), p.name
out.append(d)
return out
[docs]
def read_names(shard_dir) -> dict:
"""{key: net names (G first)} of a shard dir built here (``names.json``); {} for shards without it."""
p = Path(shard_dir) / "names.json"
return json.loads(p.read_text(encoding="utf-8")) if p.exists() else {}
[docs]
def shard_summary(shard_dir) -> dict:
"""Layout counts per split and family, conductor-count range and the prior version of a shard dir."""
from collections import Counter
c, ns = Counter(), set()
for d in read_shards(shard_dir):
c.update(zip(d["split"].tolist(), d["family"].tolist()))
ns.add(d["n"])
s = read_schema(shard_dir)
return {
"layouts": sum(c.values()),
"by_split": {sp: sum(v for (a, _), v in c.items() if a == sp) for sp in sorted({a for a, _ in c})},
"by_family": {f"{a}/{b}": v for (a, b), v in sorted(c.items())},
"n_range": [min(ns), max(ns)],
"prior_version": s.get("physics", {}).get("prior_version"),
}
[docs]
def write_shards(rows: list[dict], out_dir, schema: dict, extra: dict | None = None) -> Path:
"""Write per-layout rows (keys of ARRAYS minus pairs, plus ``names``) as a shard dir; refuses duplicate keys."""
out = Path(out_dir)
out.mkdir(parents=True, exist_ok=False)
keys = [r["key"] for r in rows]
dup = sorted({k for k in keys if keys.count(k) > 1})
if dup:
raise ValueError(f"duplicate layout keys: {dup[:5]}")
by_n: dict[int, list[dict]] = {}
for r in rows:
by_n.setdefault(len(r["sig"]), []).append(r)
for n, rs in sorted(by_n.items()):
z = {
"key": np.array([r["key"] for r in rs]),
"split": np.array([r["split"] for r in rs]),
"family": np.array([r["family"] for r in rs]),
"X": np.stack([r["X"] for r in rs]).astype(np.float32),
"E": np.stack([r["E"] for r in rs]).astype(np.float32),
"G": np.stack([r["G"] for r in rs]).astype(np.float32),
"C": np.stack([r["C"] for r in rs]).astype(np.float32),
"sig": np.stack([r["sig"] for r in rs]).astype(bool),
"pri": np.stack([r["pri"] for r in rs]).astype(np.float32),
"pairs": pairs_of(n),
}
np.savez_compressed(out / f"n{n:03d}.npz", **z)
names = {r["key"]: list(r["names"]) for r in rows if r.get("names") is not None}
if names:
(out / "names.json").write_text(json.dumps(names), encoding="utf-8")
s = dict(schema)
s["shard_sha256"] = {p.name: sha256_file(p) for p in sorted(out.glob("n*.npz"))}
if extra:
s.update(extra)
(out / "schema.json").write_text(json.dumps(s, indent=1), encoding="utf-8")
return out
[docs]
def subset_shards(src, out_dir, train_keys=None, val_keys=None, keep=None) -> Path:
"""Copy of a shard dir restricted to ``train_keys`` (split set to train) and ``val_keys`` (split set to val), or to rows for
which ``keep(shard_dict) -> bool mask`` is True (splits unchanged)."""
src, out = Path(src), Path(out_dir)
out.mkdir(parents=True, exist_ok=False)
tr, va = set(map(str, train_keys or ())), set(map(str, val_keys or ()))
if tr & va:
raise ValueError(f"keys both train and val: {sorted(tr & va)[:5]}")
found = set()
for d in read_shards(src):
k = d["key"].astype(str)
if keep is not None:
m = np.asarray(keep(d), bool)
split = d["split"]
else:
t, v = np.isin(k, list(tr)), np.isin(k, list(va))
m = t | v
split = np.where(t, "train", "val")
found.update(k[m].tolist())
if m.any():
z = {a: (d[a] if a == "pairs" else d[a][m]) for a in d if a not in ("n", "file")}
if keep is None:
z["split"] = split[m]
np.savez_compressed(out / d["file"], **z)
if keep is None and found != tr | va:
raise KeyError(f"keys not found in {src}: {sorted((tr | va) - found)[:5]}")
s = read_schema(src)
s["shard_sha256"] = {p.name: sha256_file(p) for p in sorted(out.glob("n*.npz"))}
s["subset_of"] = str(src)
(out / "schema.json").write_text(json.dumps(s, indent=1), encoding="utf-8")
names = read_names(src)
if names:
(out / "names.json").write_text(json.dumps({k: v for k, v in names.items() if k in found}), encoding="utf-8")
return out
[docs]
def merge_shards(
base_dir,
user_dir,
out_dir,
user_split: str | None = None,
all_split: str | None = None,
allow_prior_mismatch: bool = False,
) -> Path:
"""Base shards + user shards -> one shard dir for collective retraining. Feature schemas and the prior version must agree
(a user prior computed under another stack is refused); duplicate keys are refused. Base arrays the user rows lack (e.g.
QCap's ``origin`` / ``group``) are filled with ``user`` / the row's family. ``user_split`` overrides the user rows'
split, ``all_split`` every row's (``"train"`` for collective training)."""
bs, us = read_schema(base_dir), read_schema(user_dir)
for k in ("nodekeys", "edgekeys", "globkeys"):
if list(bs[k]) != list(us[k]):
raise ValueError(f"feature schema {k} differs between {base_dir} and {user_dir}")
bp, up = bs.get("physics", {}), us.get("physics", {})
if float(bp.get("alpha", np.nan)) != float(up.get("alpha", np.nan)):
raise ValueError(f"prior alpha differs: base {bp.get('alpha')}, user {up.get('alpha')}")
if bp.get("prior_version") != up.get("prior_version") and not allow_prior_mismatch:
raise ValueError(
f"prior version differs: base {bp.get('prior_version')!r}, user {up.get('prior_version')!r}. Build the user "
"shards under the base's prior stack (or pass allow_prior_mismatch=True knowingly)"
)
base = {d["n"]: d for d in read_shards(base_dir)}
user = {d["n"]: d for d in read_shards(user_dir)}
out = Path(out_dir)
out.mkdir(parents=True, exist_ok=False)
seen: set = set()
for n in sorted(set(base) | set(user)):
b, u = base.get(n), user.get(n)
if u is not None:
u = dict(u)
if user_split is not None:
u["split"] = np.full(len(u["key"]), user_split)
if b is not None and u is not None:
for k in set(b) - set(u) - {"n", "file"}:
if b[k].dtype.kind != "U":
raise ValueError(f"base array {k!r} has no user counterpart and is not a string array")
u[k] = u["family"].copy() if k == "group" else np.full(len(u["key"]), "user")
extra = set(u) - set(b) - {"n", "file"}
if extra:
raise ValueError(f"user arrays {sorted(extra)} are not in the base shards")
z = {k: (b[k] if k == "pairs" else np.concatenate([b[k], u[k]])) for k in b if k not in ("n", "file")}
else:
z = {k: v for k, v in (b or u).items() if k not in ("n", "file")}
if all_split is not None:
z["split"] = np.full(len(z["key"]), all_split)
for k in z["key"].astype(str):
if k in seen:
raise ValueError(f"duplicate layout key {k!r} (base and user shards overlap)")
seen.add(k)
np.savez_compressed(out / f"n{n:03d}.npz", **z)
s = dict(bs)
s["shard_sha256"] = {p.name: sha256_file(p) for p in sorted(out.glob("n*.npz"))}
s["merged"] = {
"base": str(base_dir),
"user": str(user_dir),
"base_schema_sha256": sha256_file(Path(base_dir) / "schema.json"),
"user_schema_sha256": sha256_file(Path(user_dir) / "schema.json"),
"user_split": user_split,
"all_split": all_split,
}
(out / "schema.json").write_text(json.dumps(s, indent=1), encoding="utf-8")
names = {**read_names(base_dir), **read_names(user_dir)}
if names:
(out / "names.json").write_text(json.dumps(names), encoding="utf-8")
return out
# ---------------------------------------------------------------------------------------------------------------- labels
[docs]
def read_q3d_csv(path) -> tuple[list[str], np.ndarray, dict]:
"""Q3D ``cap_matrix.csv`` (one row per adaptive pass, columns ``C(a,b) [unit]``, also the ``C(a,b)*1e15 [pF]`` re-export)
-> (net names, Maxwell matrix in fF of the last complete pass, info). Names: G first, then the file's order of first
appearance."""
rows = [r for r in csv.reader(open(path, encoding="utf-8")) if r]
hdr, data = rows[0], rows[1:]
cols, order = {}, []
for j, h in enumerate(hdr):
m = re.match(r"\s*C\(([^,]+),([^)]+)\)(\*1e15)?\s*\[(\w+)\]", h)
if not m:
continue
a, b = m.group(1).strip(), m.group(2).strip()
if m.group(4) not in TO_FF:
raise ValueError(f"{path}: unknown unit {m.group(4)!r} in column {h!r}")
cols[(a, b)] = (j, TO_FF[m.group(4)] * (1e-15 if m.group(3) else 1.0))
for x in (a, b):
if x not in order:
order.append(x)
if not cols:
raise ValueError(f"{path}: no C(a,b) columns")
names = ([GROUND] if GROUND in order else []) + [x for x in order if x != GROUND]
n = len(names)
missing = [(a, b) for a in names for b in names if (a, b) not in cols and (b, a) not in cols]
if missing:
raise ValueError(f"{path}: matrix entries missing from the export: {missing[:5]}")
passes = []
for r in data:
C = np.full((n, n), np.nan)
for ia, a in enumerate(names):
for ib, b in enumerate(names):
j, s = cols.get((a, b)) or cols[(b, a)]
try:
C[ia, ib] = float(r[j]) * s
except (ValueError, IndexError):
pass
passes.append(C)
complete = [k for k, C in enumerate(passes) if np.isfinite(C).all()]
if not complete:
raise LabelQCError(f"{path}: no complete adaptive pass")
last = complete[-1]
info = {"passes": len(passes), "label_pass_index": last, "complete_passes": len(complete)}
if len(complete) >= 2:
a, b = passes[complete[-2]], passes[last]
big = np.abs(b) >= 0.1
info["last_pass_change_max_pct"] = (
float(np.max(np.abs(b - a)[big] / np.abs(b)[big]) * 100) if big.any() else 0.0
)
return names, passes[last], info
[docs]
def label_qc(C, names, sym_rtol: float | None = None, sign_atol: float = 1e-6, closure_rtol: float = 1e-4) -> dict:
"""Label QC of a Maxwell matrix (fF, names G first). Returns the checks; raises LabelQCError on any failure."""
from .qc import SYM_RTOL
sym_rtol = SYM_RTOL if sym_rtol is None else sym_rtol
C = np.asarray(C, float)
n = len(names)
fail = []
if C.shape != (n, n):
raise LabelQCError(f"matrix shape {C.shape} does not match {n} net names")
if len(set(names)) != n:
raise LabelQCError(f"duplicate net names {names}")
if names[0] != GROUND:
raise LabelQCError(f"ground net {GROUND!r} must be first (names {names})")
if not np.isfinite(C).all():
raise LabelQCError("non-finite entries in the label matrix")
scale = float(np.abs(C).max())
off = ~np.eye(n, dtype=bool)
sym = float(np.abs(C - C.T).max() / max(scale, 1e-30))
chk = {"symmetry_rel": sym}
if sym > sym_rtol:
fail.append(f"asymmetric (max |C - C^T| / max |C| = {sym:.3g} > {sym_rtol:g})")
C = (C + C.T) / 2
if not np.all(np.diag(C) > 0):
fail.append("non-positive diagonal entries")
chk["offdiag_max_fF"] = float(C[off].max()) if n > 1 else None
if n > 1 and C[off].max() > sign_atol:
fail.append(f"positive off-diagonal entry ({C[off].max():.3g} fF): not Maxwell signs")
sig = np.array([x != GROUND for x in names])
rows = C.sum(1)
chk["closure_min_rel"] = float(np.min(rows[sig] / np.diag(C)[sig])) if sig.any() else None
if sig.any() and np.min(rows[sig] / np.diag(C)[sig]) < -closure_rtol:
fail.append(f"closure: a signal row sums below zero (min row sum / self = {chk['closure_min_rel']:.3g})")
eig = float(np.linalg.eigvalsh(C[np.ix_(sig, sig)]).min()) if sig.any() else None
chk["signal_block_min_eig_fF"] = eig
if eig is not None and eig <= 0:
fail.append(f"signal block not positive definite (min eigenvalue {eig:.3g} fF)")
if fail:
raise LabelQCError("; ".join(fail))
chk["passed"] = True
return chk
# ---------------------------------------------------------------------------------------------------------------- designs
[docs]
@dataclass
class Design:
"""One layout and its label.
gds: GDS path. labels: a ``cap_matrix.csv`` path (last complete pass), or a Maxwell matrix in fF with ``label_names``
(any net order; ground ``G``), or None (inference-only rows are not allowed in training shards).
names: how the layout's conductors are named, matched to the label names: ``"auto"`` (port-stub layouts, i.e. a (2, 0)
marker: N / S fixed by the stubs and the pads named by the sibling design JSON's Q3D polygon map, the angle-slot or the
file-order convention, whichever matches the label (``resolve_dataset_names``); else C1..Cn in file order),
``"dataset"`` (angle slots P<k>), ``"dataset_file_order"`` (N, S, P1, ... in file order), ``"design_json"``,
``"file_order"`` (C1..Cn), or an explicit list of signal names in file order.
"""
gds: str
labels: object = None
label_names: list | None = None
key: str | None = None
family: str = "user"
split: str = "train"
names: object = "auto"
ground_layers: tuple = ((1, 0),)
conductor_layers: tuple = ((1, 10),)
meta: dict = field(default_factory=dict)
[docs]
def resolved_key(self) -> str:
if self.key:
return str(self.key)
p = Path(self.gds)
return f"{p.parent.name}/{p.stem}"
[docs]
def as_design(x) -> Design:
if isinstance(x, Design):
return x
if isinstance(x, dict):
return Design(**x)
raise TypeError(f"cannot read a design from {type(x).__name__}: pass a Design or a dict of its fields")
def _has_marker(path, layer=(2, 0)) -> bool:
import gdstk
lib = gdstk.read_gds(str(path))
cell = lib.top_level()[0]
return any((p.layer, p.datatype) == layer for p in cell.polygons)
[docs]
def design_json_names(gds) -> list[str] | None:
"""Signal net names in GDS file order from the design's sibling ``<stem>.json`` (the Q3D polygon map
``ansys_gds.q3d_signal.mapping_polygons`` + ``nets``), or None when there is no such map."""
p = Path(gds).with_suffix(".json")
if not p.is_file():
return None
try:
q = json.loads(p.read_text(encoding="utf-8"))["ansys_gds"]["q3d_signal"]
a2n = {v["assignment"]: k for k, v in q["nets"].items()}
order = sorted(q["mapping_polygons"], key=lambda k: int(k.rsplit("_", 1)[-1]))
names = [a2n[q["mapping_polygons"][k]] for k in order]
except (KeyError, ValueError, TypeError, AttributeError):
return None
return [x for x in names if x != GROUND]
[docs]
def resolve_dataset_names(signals, markers, label_names, json_names=None) -> tuple[list[str], str]:
"""Signal names of a port-stub layout (dataset convention) that match the label nets, and which naming gave them.
N and S are fixed geometrically (N = body at the north stub (2, 0), S = body at the south stub (3, 0) or the origin).
Candidates, each kept only if its names are unique, equal the label's signal nets and put N and S on those bodies:
``design_json`` (the Q3D polygon map of the sibling design JSON: exp13-15, quantum-metal exports), ``dataset`` (pads
P<k> by angle slot: the multi-pad star, exp12) and ``dataset_file_order`` (N, S, P1, P2, ... in file order). The design
JSON wins when valid; otherwise one valid candidate (or two that agree) is used, and anything else is refused."""
from ._core._tlsq.uge.scene import tlsq_namer
want = sorted(x for x in label_names if x != GROUND)
geo = tlsq_namer(signals, markers)
n_i, s_i = geo.index("N"), geo.index("S")
cands = {}
if json_names is not None:
cands["design_json"] = list(json_names)
cands["dataset"] = geo
cands["dataset_file_order"] = ["N", "S"] + [f"P{k}" for k in range(1, len(signals) - 1)]
def ok(c):
return (
len(c) == len(signals)
and sorted(c) == want
and len(set(c)) == len(c)
and "N" in c
and "S" in c
and c.index("N") == n_i
and c.index("S") == s_i
)
valid = {k: c for k, c in cands.items() if ok(c)}
if "design_json" in valid:
return valid["design_json"], "design_json"
if not valid:
raise LabelQCError(
f"no naming of the layout bodies matches the label nets {want} ({cands}); pass names= (a list in file order); "
"nets are never merged"
)
if len(valid) == 2 and valid["dataset"] != valid["dataset_file_order"]:
raise LabelQCError(
f"ambiguous naming: {valid} both match the label nets {want}; pass names='dataset', 'dataset_file_order' "
"or a list in file order"
)
k = "dataset" if "dataset" in valid else "dataset_file_order"
return valid[k], k
[docs]
def scene_of(d: Design, label_names=None):
"""(Scene, naming used) for a design: the training geometry (no crop), ground = net 0. ``label_names`` lets
``names="auto"`` pick the dataset naming that matches the label (see ``resolve_dataset_names``)."""
from ._core._tlsq.uge.scene import scene_from_gds, tlsq_namer
mode = d.names
if mode == "auto":
mode = "dataset" if _has_marker(d.gds) else "file_order"
if mode == "dataset" and label_names is not None:
used = {}
jn = design_json_names(d.gds)
def auto_namer(S, m):
names, used["k"] = resolve_dataset_names(S, m, label_names, jn)
return names
sc = scene_from_gds(d.gds, namer=auto_namer)
return sc, f"auto:{used['k']}"
if mode == "dataset":
return scene_from_gds(d.gds, namer=tlsq_namer), "dataset"
if mode in ("dataset_file_order", "design_json"):
if mode == "design_json":
sig = design_json_names(d.gds)
if sig is None:
raise LabelQCError(f"{d.gds}: no design JSON with a Q3D polygon map next to the GDS")
else:
sig = None
def ordered(S, m):
ns = sig if sig is not None else ["N", "S"] + [f"P{k}" for k in range(1, len(S) - 1)]
if len(S) != len(ns):
raise LabelQCError(f"{d.gds}: {len(S)} signal polygons but {len(ns)} names ({mode})")
return list(ns)
return scene_from_gds(d.gds, namer=ordered), mode
if isinstance(mode, (list, tuple)):
sig = [str(x) for x in mode if x != GROUND]
def namer(S, m):
if len(S) != len(sig):
raise LabelQCError(f"{d.gds}: {len(S)} signal polygons but {len(sig)} names given")
return sig
return scene_from_gds(d.gds, namer=namer), "explicit"
if mode != "file_order":
raise ValueError(
f"names={mode!r}: use 'auto', 'dataset', 'dataset_file_order', 'design_json', 'file_order' or a list of names"
)
from ._core import api as A
from ._runtime import prepare_layout
lay = prepare_layout(d.gds, ground_layers=d.ground_layers, conductor_layers=d.conductor_layers)
return A.scene_from_prepared(lay), "file_order"
[docs]
def labels_of(d: Design) -> tuple[list[str], np.ndarray, dict]:
if d.labels is None:
raise LabelQCError(f"{d.resolved_key()}: no labels (training rows need a Maxwell matrix)")
if isinstance(d.labels, (str, os.PathLike)):
names, C, info = read_q3d_csv(d.labels)
return names, C, {"source": str(d.labels), **info}
if d.label_names is None:
raise LabelQCError(f"{d.resolved_key()}: a label matrix needs label_names")
return [str(x) for x in d.label_names], np.asarray(d.labels, float), {"source": "matrix"}
def _stack_registry():
from .models import registry_view
return registry_view({}, ".", "squadds-training-data")
[docs]
def embed_design(d: Design, threads: int = 2, prior: str = "optimized", schema: dict | None = None) -> dict:
"""Shard row of one design: embedding (exact, no far-sample decimation), prior solves at 1.5 and 3 panels per gap under
the corrected stack, and the QC-checked label reordered to the scene's net order (G first)."""
from . import _runtime as RT
from ._core import api as A
from ._core import paper as PA
from ._core._tlsq.uge import blocks as B
from .predictor import quiet
if prior != "optimized":
raise NotImplementedError(
f"prior={prior!r}: training shards carry the optimized_physics_prior solves (1.5, 3); other priors are TODO"
)
schema = schema or base_schema()
key = d.resolved_key()
lnames, Cl, linfo = labels_of(d)
if GROUND not in lnames:
raise LabelQCError(f"{key}: the label has no ground net {GROUND!r} (names {lnames})")
lnames_g = [GROUND] + [x for x in lnames if x != GROUND]
perm = [lnames.index(x) for x in lnames_g]
try:
qc = label_qc(Cl[np.ix_(perm, perm)], lnames_g)
except LabelQCError as e:
raise LabelQCError(f"{key}: label QC failed: {e}") from e
t0 = time.perf_counter()
sc, naming = scene_of(d, lnames)
names = list(sc.names()) if callable(getattr(sc, "names", None)) else [c.name for c in sc.conductors]
if sorted(names) != sorted(lnames):
raise LabelQCError(
f"{key}: layout nets {names} do not match the label nets {lnames} (naming {naming!r}); nets are never merged"
)
old = dict(B.FAR_DECIMATION)
B.FAR_DECIMATION.update(k=0, d_far=100.0)
try:
from ._core._tlsq.uge.graph import scene_graph
row = PA.lt.add_tokens(sc, scene_graph(sc))
finally:
B.FAR_DECIMATION.clear()
B.FAR_DECIMATION.update(old)
t1 = time.perf_counter()
if list(row["names"]) != names:
raise RuntimeError(f"{key}: embedding names {row['names']} differ from the scene {names}")
with RT.LOCK, quiet():
threads = RT.configure(threads)
PA.apply_stack(_stack_registry(), {"stack": "corrected"}, threads)
P, tl = PA.solve_levels(sc, list(PRIOR_LEVELS), threads)
stack = {"name": "corrected", "patches": list(A._STATE.get("patches", [])), "mode": A._STATE.get("mode")}
t2 = time.perf_counter()
n = len(names)
pr = pairs_of(n)
nk, ek, gk = schema["nodekeys"], schema["edgekeys"], schema["globkeys"]
assert nk[-1] == "missing_log_min_halfwidth"
wi = nk.index("log_min_halfwidth")
X = np.array([[row["nodes"][a][k] for k in nk[:-1]] for a in range(n)], np.float32)
X = np.concatenate([X, (~np.isfinite(X[:, wi])).astype(np.float32)[:, None]], -1)
E = np.array([[row["edges"][(int(i), int(j))][k] for k in ek] for i, j in pr], np.float32).reshape(len(pr), len(ek))
G = np.array([row["global"][k] for k in gk], np.float32)
pri = np.stack([np.asarray(P[p], float)[pr[:, 0], pr[:, 1]] for p in PRIOR_LEVELS], -1).astype(np.float32)
if not (np.isfinite(E).all() and np.isfinite(G).all() and np.isfinite(np.delete(X, wi, axis=-1)).all()):
raise ValueError(f"{key}: non-finite embedding features")
if not (np.isfinite(pri).all() and (pri > 0).all()):
raise ValueError(f"{key}: prior mutuals not finite and positive")
idx = [lnames.index(x) for x in names]
C = np.asarray(Cl, float)[np.ix_(idx, idx)]
C = (C + C.T) / 2
return {
"key": key,
"split": d.split,
"family": d.family,
"names": names,
"X": X,
"E": E,
"G": G,
"C": C.astype(np.float32),
"sig": np.array([x != GROUND for x in names]),
"pri": pri,
"info": {
"gds": str(d.gds),
"naming": naming,
"label": linfo,
"qc": qc,
"stack": stack,
"timing_s": {"embedding": t1 - t0, "prior": t2 - t1, "prior_levels": tl},
**({"meta": d.meta} if d.meta else {}),
},
}
def _embed_worker(args):
d, threads, prior, on_error = args
try:
return embed_design(d, threads=threads, prior=prior), None
except (LabelQCError, ValueError) as e:
if on_error == "raise":
raise
return None, {"key": d.resolved_key(), "gds": str(d.gds), "reason": str(e)}
[docs]
def build_shards(
designs, out_dir, prior: str = "optimized", threads: int = 2, workers: int = 1, on_error: str = "raise"
) -> dict:
"""User designs -> a P2d shard dir (``n0NN.npz`` + ``schema.json`` + ``names.json`` + ``build.json``).
designs: Design objects or dicts of their fields. Labels are QC-checked and matched to the layout nets by name; a design
that fails is refused (``on_error="raise"``, default) or skipped and listed in ``build.json`` (``"skip"``). Priors are
computed on this machine (macOS is not bit-identical to the released x86 priors: ~2.4e-3 relative at most); ``threads``
per design, ``workers`` designs at a time (spawned processes)."""
from . import _runtime as RT
if on_error not in ("raise", "skip"):
raise ValueError("on_error must be 'raise' or 'skip'")
ds = [as_design(x) for x in designs]
if not ds:
raise ValueError("no designs")
schema = base_schema()
t = time.perf_counter()
jobs = [(d, threads, prior, on_error) for d in ds]
if workers > 1:
from concurrent.futures import ProcessPoolExecutor
from multiprocessing import get_context
with ProcessPoolExecutor(min(workers, len(ds)), mp_context=get_context("spawn")) as ex:
res = list(ex.map(_embed_worker, jobs))
else:
res = [_embed_worker(j) for j in jobs]
rows = [r for r, e in res if r is not None]
refused = [e for r, e in res if e is not None]
if not rows:
raise LabelQCError(f"every design was refused: {refused[:3]}")
from squadds import __version__
build = {
"created": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
"machine": RT.machine(),
"platform": platform.platform(),
"threads": threads,
"workers": workers,
"package": f"squadds {__version__}",
"code_sha256": sha256_file(__file__),
"layouts": len(rows),
"refused": refused,
"seconds": time.perf_counter() - t,
"note": "priors computed on this machine; released shards use HPC x86 priors",
}
out = write_shards(rows, out_dir, schema, extra={"build": {k: v for k, v in build.items() if k != "refused"}})
build["designs"] = {r["key"]: r["info"] for r in rows}
(out / "build.json").write_text(json.dumps(build, indent=1, default=str), encoding="utf-8")
return {"out_dir": str(out), "layouts": len(rows), "refused": refused, "seconds": build["seconds"]}
# ---------------------------------------------------------------------------------------------------------------- dataset