"""Contributing planar capacitance data and models to ``SQuADDS/SQuADDS_PlanarCapacitance`` (Hugging Face pull requests).
Data: a contribution directory in the template layout (``squadds/database/templates/planar_capacitance_contribution``)::
manifest.yaml contributor, licence, solver, stack, setup, layer map, units, generator, family
designs.csv design, gds, family, params_json, net_order (G first, then conductors in GDS file order), notes
gds/<design>.gds
sims/<design>/<sim_tag>/cap_matrix.csv Q3D export as is (one row per pass) or matrix.csv / matrix.parquet
sims/<design>/<sim_tag>/convergence.txt the Q3D .conv file (optional)
sims/<design>/<sim_tag>/run.json solver version, cores, solve_s, wall_s, peak_mem_GB, machine (optional)
nets/<design>.json {net: [conductor-layer polygon indices]} (optional explicit net map)
``validate_capacitance_contribution(dir)`` runs the dataset validator (schema, GDS readback, net names, label QC, duplicates,
stack, licence, privacy) and ``contribute_capacitance(dir)`` opens a pull request adding ``contrib/<org>-<batch>/`` and its
``validation_report.md``. Maintainers merge, compute the priors on HPC and ingest the rows as ``source="community"``,
``split="none"``.
Models: ``contribute_model(repo_id, revision)`` validates a model repository (checkpoints load, sha256, required metadata,
validation score with median % and MAE / RMSE in fF plus time, machine and threads) and opens a pull request adding its entry
to ``model_zoo.yaml``. Test-set scores are never self-reported: they are run by the maintainers, once per declared model, only
for models trained on the TRAIN split.
"""
from __future__ import annotations
import csv
import io
import json
import os
import re
import time
from pathlib import Path
import numpy as np
from squadds.datasets.planar_capacitance_schema import REPO_ID
from . import qc as Q
LICENCES = {"cc-by-4.0": "CC-BY-4.0", "cc0-1.0": "CC0-1.0", "cc0": "CC0-1.0", "cc-by 4.0": "CC-BY-4.0"}
MANIFEST_REQUIRED = {
"contributor": dict,
"licence": str,
"solver": dict,
"stack": dict,
"setup": dict,
"layer_map": dict,
"ground_net_name": str,
"units": dict,
"generator": dict,
"family": str,
"description": str,
}
DESIGN_COLUMNS = ("design", "gds", "family", "params_json", "net_order", "notes")
MATRIX_FILES = ("cap_matrix.csv", "matrix.csv", "matrix.parquet")
CONV_FILES = ("convergence.txt",)
UPLOAD_DIRS = ("gds", "sims", "nets")
ZOO_FILE = "model_zoo.yaml"
MODEL_META_FILES = ("squadds_model.yaml", "squadds_model.json")
MODEL_REQUIRED = ("name", "title", "licence", "training_data", "val_score", "contributor")
VAL_REQUIRED = ("median_pct", "mae_fF", "rmse_fF", "layouts", "machine", "threads")
VAL_TIME = ("seconds", "seconds_per_layout")
_ABS = re.compile(r"^(/|~/|[A-Za-z]:[\\/]|\\\\)")
def _slug(s: str) -> str:
return re.sub(r"[^a-z0-9._-]+", "-", str(s).lower()).strip("-.")
def _yaml():
try:
import yaml
except ImportError as e:
raise ImportError("contributions need pyyaml: pip install pyyaml (or 'squadds[ml]')") from e
return yaml
def _api(api=None, token=None):
if api is not None:
return api
from huggingface_hub import HfApi
from squadds.database.contributor_env import load_contributor_environment
load_contributor_environment()
return HfApi(token=token or os.getenv("HUGGINGFACE_API_KEY") or None)
[docs]
def submitter_record() -> dict:
"""The contributor block of the repo-root ``.env`` (GROUP_NAME, PI_NAME, INSTITUTION, USER_NAME, CONTRIB_MISC)."""
from squadds.database.contributor_env import build_contributor_record
return build_contributor_record()
# ---------------------------------------------------------------------------------------------------------------- privacy
def _scrub(o, path: str, fixes: list):
"""Absolute local paths -> file names (recorded in ``fixes``)."""
if isinstance(o, dict):
return {k: _scrub(v, f"{path}.{k}", fixes) for k, v in o.items()}
if isinstance(o, list):
return [_scrub(v, f"{path}[{i}]", fixes) for i, v in enumerate(o)]
if isinstance(o, str) and _ABS.match(o.strip()) and len(o.strip()) > 1:
fixes.append(f"{path}: absolute path replaced by its file name")
return re.split(r"[\\/]", o.strip().rstrip("/\\"))[-1]
return o
[docs]
def sanitized_manifest(manifest: dict) -> tuple[dict, list[str]]:
"""The manifest as it is uploaded: absolute paths stripped, the e-mail dropped unless ``contributor.email_public``."""
fixes: list[str] = []
m = _scrub(manifest, "manifest", fixes)
c = dict(m.get("contributor") or {})
if c.get("email") and not c.get("email_public"):
c.pop("email")
fixes.append("manifest.contributor.email removed (email_public is not true)")
m["contributor"] = c
return m, fixes
def _sanitized_designs_csv(rows: list[dict], fixes: list[str]) -> str:
out = io.StringIO()
cols = list(dict.fromkeys([*DESIGN_COLUMNS, *(k for r in rows for k in r)]))
w = csv.DictWriter(out, fieldnames=cols)
w.writeheader()
for r in rows:
w.writerow({k: _scrub(r.get(k, ""), f"designs.csv[{r.get('design')}].{k}", fixes) for k in cols})
return out.getvalue()
# ---------------------------------------------------------------------------------------------------------------- reading
[docs]
def load_manifest(path) -> dict:
p = Path(path) / "manifest.yaml"
if not p.exists():
raise FileNotFoundError(f"{p} not found (copy the template manifest.yaml)")
return _yaml().safe_load(p.read_text(encoding="utf-8")) or {}
[docs]
def read_designs(path) -> list[dict]:
p = Path(path) / "designs.csv"
if not p.exists():
raise FileNotFoundError(f"{p} not found (copy the template designs.csv)")
with open(p, newline="", encoding="utf-8") as f:
return [
{k.strip(): (v or "").strip() for k, v in r.items() if k}
for r in csv.DictReader(line for line in f if not line.lstrip().startswith("#"))
]
[docs]
def parse_net_order(s: str) -> list[str]:
s = (s or "").strip()
if s.startswith("["):
return [str(x) for x in json.loads(s)]
return [x.strip() for x in re.split(r"[;|,\s]+", s) if x.strip()]
[docs]
def read_sim(sim_dir, unit: str = "fF") -> dict:
"""One simulation directory -> {"names", "C" (label, given unit), "passes", "unit", "convergence", "run"}."""
d = Path(sim_dir)
out: dict = {"convergence": None, "run": None}
if (d / "cap_matrix.csv").exists():
m = Q.read_cap_matrix(d / "cap_matrix.csv")
complete = [(p, C) for p, C in m["passes"] if np.isfinite(C).all()]
out.update(names=m["names"], passes=m["passes"], unit="fF", units_found=m["units"], freq_GHz=m["freq_GHz"])
out["C"] = complete[-1][1] if complete else np.full((len(m["names"]),) * 2, np.nan)
else:
f = next((d / n for n in ("matrix.parquet", "matrix.csv") if (d / n).exists()), None)
if f is None:
raise FileNotFoundError(f"{d}: no {' / '.join(MATRIX_FILES)}")
import pandas as pd
df = pd.read_parquet(f) if f.suffix == ".parquet" else pd.read_csv(f)
val = "C_fF" if "C_fF" in df else "C"
if val not in df or not {"net_a", "net_b"} <= set(df.columns):
raise ValueError(f"{f}: needs columns net_a, net_b and C_fF (or C in the manifest unit)")
u = "fF" if val == "C_fF" else unit
order = list(dict.fromkeys([*map(str, df.net_a), *map(str, df.net_b)]))
names = ([Q.GROUND] if Q.GROUND in order else []) + [x for x in order if x != Q.GROUND]
ix = {n: i for i, n in enumerate(names)}
def mat(sub):
C = np.full((len(names), len(names)), np.nan)
for a, b, v in zip(sub.net_a, sub.net_b, sub[val]):
C[ix[str(a)], ix[str(b)]] = C[ix[str(b)], ix[str(a)]] = float(v)
return C
if "pass" in df:
passes = [(int(p), mat(g)) for p, g in df.groupby("pass")]
out.update(passes=passes, C=passes[-1][1])
else:
out.update(passes=None, C=mat(df))
out.update(names=names, unit=u)
conv = next((p for p in [*(d / n for n in CONV_FILES), *sorted(d.glob("*.conv"))] if p.exists()), None)
if conv is not None:
out["convergence"] = Q.read_conv(conv)
if (d / "run.json").exists():
out["run"] = json.loads((d / "run.json").read_text(encoding="utf-8"))
return out
# ---------------------------------------------------------------------------------------------------------------- validator
def _manifest_qc(m: dict) -> Q.QCReport:
r = Q.QCReport("manifest")
for k, t in MANIFEST_REQUIRED.items():
if k not in m or m[k] in (None, "", {}):
r.error(f"manifest field {k!r} missing")
elif not isinstance(m[k], t):
r.error(f"manifest field {k!r} must be a {t.__name__}")
c = m.get("contributor") or {}
if isinstance(c, dict) and not (c.get("name") and c.get("affiliation")):
r.error("contributor needs name and affiliation")
lic = LICENCES.get(str(m.get("licence", "")).strip().lower())
if lic is None:
r.error(f"licence {m.get('licence')!r} not accepted: data contributions must be CC-BY-4.0 or CC0-1.0")
r.info["licence"] = lic
solver = m.get("solver") or {}
if isinstance(solver, dict) and not solver.get("name"):
r.error("solver.name missing")
units = m.get("units") or {}
cap = str(units.get("capacitance", "")).strip() if isinstance(units, dict) else ""
try:
Q.to_fF(1.0, cap)
except ValueError:
r.error(f"units.capacitance {cap!r} not one of {sorted(Q.TO_FF)} (or 'pF*1e15')")
lm = m.get("layer_map") or {}
if isinstance(lm, dict) and not (lm.get("ground") and lm.get("conductor")):
r.error("layer_map needs ground and conductor layer lists, e.g. {ground: [[1, 0]], conductor: [[1, 10]]}")
return r
[docs]
def validate_capacitance_contribution(
path,
reference="auto",
check_prior: bool = False,
threads: int = 2,
report_path=None,
revision: str | None = None,
) -> Q.QCReport:
"""Validate a contribution directory (``PLAN_DATASET_HF.md`` §4.2). Returns a ``QCReport`` with one child per design.
reference: the dataset's designs (DataFrame with ``design_id``, ``geom_hash``, ``split``) for the duplicate check;
``"auto"`` reads them with ``PlanarCapacitanceDB`` (a warning when the dataset is not reachable), None skips it.
check_prior: also compare every label with ``optimized_physics_prior`` (needs ``[ml]``; seconds per design; report only).
report_path: also write the markdown report there."""
root = Path(path)
rep = Q.QCReport(f"contribution {root.name}")
try:
m = load_manifest(root)
rows = read_designs(root)
except (FileNotFoundError, ValueError) as e:
rep.error(str(e))
return _finish(rep, report_path)
except Exception as e: # noqa: BLE001 (YAML / CSV syntax)
rep.error(f"cannot parse the manifest or designs.csv: {e}")
return _finish(rep, report_path)
rep.extend(_manifest_qc(m), "manifest")
_, fixes = sanitized_manifest(m)
rep.fixes += fixes
_sanitized_designs_csv(rows, rep.fixes)
st = m.get("stack") if isinstance(m.get("stack"), dict) else {}
freq = (m.get("setup") or {}).get("freq_GHz") if isinstance(m.get("setup"), dict) else None
rep.extend(Q.stack_qc(st, freq), "stack")
if not rows:
rep.error("designs.csv has no rows")
missing_cols = [c for c in ("design", "gds", "net_order") if rows and c not in rows[0]]
if missing_cols:
rep.error(f"designs.csv misses columns {missing_cols} (template columns: {', '.join(DESIGN_COLUMNS)})")
return _finish(rep, report_path)
names = [r["design"] for r in rows]
dup_names = sorted({n for n in names if names.count(n) > 1})
if dup_names:
rep.error(f"design names repeated in designs.csv: {dup_names[:5]}")
unit = str((m.get("units") or {}).get("capacitance", "fF"))
layer_map = m.get("layer_map") or Q.DEFAULT_LAYERS
gname = str(m.get("ground_net_name") or Q.GROUND)
hashes = {}
for row in rows:
child = _validate_design(root, row, unit, layer_map, gname, check_prior, threads)
rep.children.append(child)
hashes[row["design"]] = child.info.get("geom_hash")
if isinstance(reference, str) and reference == "auto":
try:
reference = Q.reference_hashes(revision=revision)
except Exception as e: # noqa: BLE001
rep.warn(f"dataset not reachable for the duplicate check ({type(e).__name__}: {e}); maintainers rerun it")
reference = None
rep.extend(Q.duplicate_qc(hashes, reference), "duplicates")
rep.info.update(
designs=len(rows),
designs_ok=sum(c.ok for c in rep.children),
simulations=sum(c.info.get("simulations", 0) for c in rep.children),
family=m.get("family"),
licence=rep.info.get("licence"),
)
return _finish(rep, report_path)
def _finish(rep: Q.QCReport, report_path) -> Q.QCReport:
if report_path is not None:
Path(report_path).write_text(rep.to_markdown("Validation report"), encoding="utf-8")
return rep
def _validate_design(root: Path, row: dict, unit, layer_map, gname, check_prior, threads) -> Q.QCReport:
d = row["design"]
r = Q.QCReport(d)
if not re.fullmatch(r"[A-Za-z0-9._-]+", d or ""):
r.error(f"design name {d!r}: use letters, digits, '.', '_' or '-'")
return r
order = parse_net_order(row.get("net_order", ""))
order = [Q.GROUND if x == gname else x for x in order]
if not order:
r.error("net_order is empty")
return r
gds = row.get("gds") or f"gds/{d}.gds"
gp = root / gds if not _ABS.match(gds) else root / "gds" / Path(gds).name
if not gp.exists():
r.error(f"GDS {gds} not found")
return r
nets = None
if (root / "nets" / f"{d}.json").exists():
nets = json.loads((root / "nets" / f"{d}.json").read_text(encoding="utf-8"))
g = Q.gds_qc(gp, layer_map, nets, subject="gds")
r.extend(g, "gds")
if "n_conductors" in g.info:
r.extend(Q.conductor_count_qc(g.info["n_conductors"], order), "nets")
if nets and [Q.GROUND, *nets] != order:
r.error(f"net_order {order} differs from the nets map order {[Q.GROUND, *nets]}")
sims = sorted(p for p in (root / "sims" / d).glob("*") if p.is_dir()) if (root / "sims" / d).exists() else []
r.info["simulations"] = len(sims)
if not sims and str(row.get("unlabelled", "")).lower() not in ("1", "true", "yes"):
r.error(
f"no simulation under sims/{d}/<sim_tag>/ (declare the design unlabelled=true to contribute geometry only)"
)
for s in sims:
try:
sim = read_sim(s, unit)
except Exception as e: # noqa: BLE001
r.error(f"{s.name}: cannot read the matrix ({e})")
continue
sim_names = [Q.GROUND if x == gname else x for x in sim["names"]]
lq = Q.label_qc(
sim["C"],
sim_names,
net_order=order,
unit=sim["unit"],
passes=sim["passes"],
convergence=sim["convergence"],
flagged_zero=_flagged(row),
subject=s.name,
)
mat = lq.info.pop("matrix_fF", None)
r.extend(lq, f"sim {s.name}")
if sim.get("run"):
fx: list[str] = []
_scrub(sim["run"], f"sims/{d}/{s.name}/run.json", fx)
r.fixes += fx
if check_prior and lq.ok and mat is not None:
r.extend(Q.prior_sanity(mat, order, gp, layer_map, threads=threads), f"sim {s.name}")
return r
def _flagged(row: dict) -> list[tuple[str, str]]:
s = row.get("zero_entries", "")
return [tuple(p.split("-", 1)) for p in re.split(r"[;\s]+", s) if "-" in p]
# ---------------------------------------------------------------------------------------------------------------- data PR
def _upload_files(root: Path) -> list[Path]:
files = []
for top in UPLOAD_DIRS:
if (root / top).exists():
files += [p for p in sorted((root / top).rglob("*")) if p.is_file() and not p.name.startswith(".")]
return files
[docs]
def contribute_capacitance(
path,
repo_id: str = REPO_ID,
dry_run: bool = False,
org: str | None = None,
batch: str | None = None,
reference="auto",
check_prior: bool = False,
api=None,
token: str | None = None,
) -> dict:
"""Validate ``path`` and open a pull request on the dataset ``repo_id`` adding ``contrib/<org>-<batch>/`` (manifest,
designs, GDS, simulations, ``validation_report.md``, ``submission.json`` with the ``.env`` contributor block).
A contribution with errors is refused (``ValueError``). dry_run: validate and return the planned files without any
network call. Returns ``{"ok", "pr_url", "folder", "files", "report"}``."""
from huggingface_hub import CommitOperationAdd
root = Path(path)
rep = validate_capacitance_contribution(root, reference=reference, check_prior=check_prior)
if not rep.ok:
raise ValueError("the contribution failed validation:\n- " + "\n- ".join(rep.all_errors()[:20]))
m = load_manifest(root)
org = _slug(org or m.get("org") or (m.get("contributor") or {}).get("affiliation") or "community")
batch = _slug(batch or m.get("batch") or time.strftime("%Y%m%d"))
folder = f"contrib/{org}-{batch}"
man, _ = sanitized_manifest(m)
fx: list[str] = []
designs_csv = _sanitized_designs_csv(read_designs(root), fx)
try:
sub = submitter_record()
except Exception as e: # noqa: BLE001
sub = {"note": f".env contributor block not read: {e}"}
md = rep.to_markdown(f"Validation report: {folder}")
md += "\n## Submitted by (.env)\n\n" + "\n".join(f"- {k}: {v}" for k, v in sub.items() if v) + "\n"
yaml = _yaml()
blobs = {
"manifest.yaml": yaml.safe_dump(man, sort_keys=False).encode(),
"designs.csv": designs_csv.encode(),
"validation_report.md": md.encode(),
"submission.json": json.dumps(
{"submitted_by": sub, "created": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "validator": _version()}, indent=1
).encode(),
}
ops = [CommitOperationAdd(path_in_repo=f"{folder}/{k}", path_or_fileobj=v) for k, v in blobs.items()]
for p in _upload_files(root):
rel = p.relative_to(root).as_posix()
if p.name == "run.json":
fxr: list[str] = []
data = json.dumps(_scrub(json.loads(p.read_text(encoding="utf-8")), rel, fxr), indent=1).encode()
ops.append(CommitOperationAdd(path_in_repo=f"{folder}/{rel}", path_or_fileobj=data))
else:
ops.append(CommitOperationAdd(path_in_repo=f"{folder}/{rel}", path_or_fileobj=str(p)))
files = [o.path_in_repo for o in ops]
out = {"ok": True, "folder": folder, "files": files, "report": md, "pr_url": None, "repo_id": repo_id}
if dry_run:
return out
info = _api(api, token).create_commit(
repo_id=repo_id,
repo_type="dataset",
operations=ops,
commit_message=f"Contribution {org}-{batch}: {rep.info.get('designs')} designs ({m.get('family')})",
commit_description=md[:60000],
create_pr=True,
)
out["pr_url"] = getattr(info, "pr_url", None) or str(info)
return out
def _version() -> str:
try:
from squadds import __version__
return f"squadds {__version__}"
except Exception: # noqa: BLE001
return "squadds"
# ---------------------------------------------------------------------------------------------------------------- models
def _model_meta(root: Path) -> dict:
for n in MODEL_META_FILES:
p = root / n
if p.exists():
return (
_yaml().safe_load(p.read_text(encoding="utf-8"))
if p.suffix == ".yaml"
else json.loads(p.read_text(encoding="utf-8"))
)
return {}
def _load_contributed(root: Path, meta: dict, paths: list[Path]):
base = meta.get("base") or {}
from .models import is_model_name, load_model
if is_model_name(base.get("name")):
return load_model(base["name"], path=root)
from .train import trained_model
return trained_model(paths, name=_slug(meta.get("name", "contributed")) or "contributed", title=meta.get("title"))
[docs]
def validate_model(
model_repo_id: str,
revision: str | None = None,
local_dir=None,
metadata: dict | None = None,
val_data=None,
threads: int = 2,
api=None,
token: str | None = None,
loader=None,
) -> tuple[Q.QCReport, dict]:
"""Validate a model repository for the model zoo. Returns (report, zoo entry).
The repository holds ``*.pt`` checkpoints (``squadds-ml train`` run dirs or ``model.pt`` files) and ``squadds_model.yaml``
with ``name``, ``title``, ``licence``, ``contributor``, ``training_data`` (``sha256``, ``split_used``, ``description``) and
``val_score`` (``median_pct``, ``mae_fF``, ``rmse_fF``, ``layouts``, ``seconds`` or ``seconds_per_layout``, ``machine``,
``threads``); ``metadata``
overrides the file. val_data: a validation shard dir: the score is then computed here with ``benchmark.score`` and replaces
the self-reported one. local_dir: a local copy of the repository (no download)."""
from .models import sha256_file
rep = Q.QCReport(f"model {model_repo_id}@{revision or 'main'}")
hf = None
if local_dir is None:
hf = _api(api, token)
sha = hf.repo_info(model_repo_id, revision=revision).sha
local_dir = hf.snapshot_download(repo_id=model_repo_id, revision=sha)
revision = sha
elif not (revision and re.fullmatch(r"[0-9a-f]{40}", str(revision))):
rep.warn(f"revision {revision!r} is not a full commit sha: the zoo entry must pin one")
root = Path(local_dir)
meta = {**_model_meta(root), **(metadata or {})}
for k in MODEL_REQUIRED:
if not meta.get(k):
rep.error(f"model metadata field {k!r} missing (squadds_model.yaml)")
lic = str(meta.get("licence", "")).strip()
if lic and not re.fullmatch(r"[A-Za-z0-9.+-]+", lic):
rep.error(f"licence {lic!r} is not an SPDX identifier")
td = meta.get("training_data") or {}
if not td.get("sha256"):
rep.error("training_data.sha256 missing (sha256 of the training shards / model_input used)")
split_used = str(td.get("split_used", "")).lower()
if not split_used:
rep.error("training_data.split_used missing (train | train+val | all | custom)")
test_eligible = split_used == "train"
if "test" in split_used or split_used == "all":
rep.warn(
"trained on test layouts: the model is listed without any test score (not eligible for maintainer scoring)"
)
paths = sorted(p for p in root.rglob("*.pt") if not any(x.startswith(".") for x in p.relative_to(root).parts))
if not paths:
rep.error("no *.pt checkpoint in the repository")
files = [{"path": p.relative_to(root).as_posix(), "sha256": sha256_file(p)} for p in paths]
model = None
if paths:
try:
model = (loader or _load_contributed)(root, meta, paths)
rep.info["arm"] = (getattr(model, "entry", {}) or {}).get("arm")
except Exception as e: # noqa: BLE001
rep.error(f"checkpoints do not load: {type(e).__name__}: {e}")
val = dict(meta.get("val_score") or {})
if val_data is not None and model is not None:
from .benchmark import score
s = score(model, val_data, split="val", threads=threads)
h = s.headline()
computed = {
"median_pct": h["median_pct"],
"mae_fF": h["mae_fF"],
"rmse_fF": h["rmse_fF"],
"layouts": h["layouts"],
"seconds": s.timing.get("seconds"),
"machine": s.timing.get("machine"),
"threads": s.timing.get("threads"),
"split": "val",
"source": "computed by contribute_model",
}
if val and val.get("median_pct") is not None and abs(float(val["median_pct"]) - h["median_pct"]) > 1e-3:
rep.warn(
f"self-reported val median {val['median_pct']} % differs from the computed {h['median_pct']:.4f} %"
)
val = computed
elif val:
val.setdefault("source", "self-reported")
rep.warn("validation score self-reported (pass val_data to compute it here)")
if str(val.get("split", "val")).lower() == "test":
rep.error("val_score is a test score: test scores are maintainer-run only, report the validation split")
miss = [k for k in VAL_REQUIRED if val.get(k) in (None, "")]
if all(val.get(k) in (None, "") for k in VAL_TIME):
miss.append(" or ".join(VAL_TIME))
if meta.get("val_score") is not None or val_data is not None:
if miss:
rep.error(f"val_score misses {miss} (median % with MAE and RMSE in fF, plus time, machine and threads)")
rep.info.update(files=len(files), val_median_pct=val.get("median_pct"), val_mae_fF=val.get("mae_fF"))
entry = {
"id": _slug(meta.get("name", model_repo_id.split("/")[-1])),
"repo_id": model_repo_id,
"revision": revision,
"title": meta.get("title"),
"arm": rep.info.get("arm"),
"base": meta.get("base"),
"licence": lic or None,
"contributor": meta.get("contributor"),
"files": files,
"training_data": td,
"val_score": val,
"test_eligible": test_eligible,
"validated": {"date": time.strftime("%Y-%m-%d"), "by": _version()},
}
return rep, entry
[docs]
def contribute_model(
model_repo_id: str,
revision: str | None = None,
registry_repo: str = REPO_ID,
metadata: dict | None = None,
val_data=None,
dry_run: bool = False,
local_dir=None,
threads: int = 2,
api=None,
token: str | None = None,
loader=None,
) -> dict:
"""Validate a model repository (``validate_model``) and open a pull request on ``registry_repo`` adding its entry to
``model_zoo.yaml`` plus ``model_zoo/<id>/validation_report.md``. A model with errors is refused (``ValueError``).
Returns ``{"ok", "pr_url", "entry", "report", "files"}``."""
from huggingface_hub import CommitOperationAdd
hf = None if (dry_run and local_dir is not None and api is None) else _api(api, token)
rep, entry = validate_model(
model_repo_id, revision, local_dir, metadata, val_data, threads, api=hf, token=token, loader=loader
)
if not rep.ok:
raise ValueError("the model failed validation:\n- " + "\n- ".join(rep.all_errors()[:20]))
zoo = {"version": 1, "models": []}
if hf is not None:
try:
p = hf.hf_hub_download(repo_id=registry_repo, repo_type="dataset", filename=ZOO_FILE)
zoo = _yaml().safe_load(Path(p).read_text(encoding="utf-8")) or zoo
except Exception as e: # noqa: BLE001
if "404" not in str(e) and "EntryNotFound" not in type(e).__name__ and "not found" not in str(e).lower():
raise
zoo.setdefault("models", [])
if any(x.get("repo_id") == entry["repo_id"] and x.get("revision") == entry["revision"] for x in zoo["models"]):
raise ValueError(f"{entry['repo_id']}@{entry['revision']} is already in {ZOO_FILE}")
zoo["models"].append(entry)
md = rep.to_markdown(f"Model validation report: {model_repo_id}@{entry['revision']}")
ops = [
CommitOperationAdd(path_in_repo=ZOO_FILE, path_or_fileobj=_yaml().safe_dump(zoo, sort_keys=False).encode()),
CommitOperationAdd(path_in_repo=f"model_zoo/{entry['id']}/validation_report.md", path_or_fileobj=md.encode()),
]
out = {"ok": True, "entry": entry, "report": md, "files": [o.path_in_repo for o in ops], "pr_url": None}
if dry_run:
return out
info = hf.create_commit(
repo_id=registry_repo,
repo_type="dataset",
operations=ops,
commit_message=f"Model zoo: add {entry['id']} ({model_repo_id}@{str(entry['revision'])[:8]})",
commit_description=md[:60000],
create_pr=True,
)
out["pr_url"] = getattr(info, "pr_url", None) or str(info)
return out