#!/usr/bin/env python3 """Bench 03 — repeatability. Ask for the same thing three times; how different is what you get? python3 scripts/bench-repeatability.py Reads the model-selection run in scripts/bench02/ and writes apps/site/public/benchmarks/repeatability/run.json plus the src/data copy. These are the same frames the model-selection run is built from: eight briefs, three models, three runs each, every request carrying the production system prompt and the requested aspect ratio. What is measured, per (brief, model) triple: variance mean |Δ| over the three pairwise combinations (0 = identical) compositionIoU IoU of edge maps over the same pairs (1 = same framing) Frames are resampled to a common long edge before comparison so resolution is not a factor. Aspect ratios still differ between models, so the CROSS-MODEL aggregate is indicative rather than exact — recorded in the output as such, not buried in a footnote. """ import itertools import json from pathlib import Path import numpy as np from PIL import Image ROOT = Path(__file__).resolve().parent.parent SRC = ROOT / "scripts/bench02" RAW = SRC / "raw.json" OUT = ROOT / "apps/site/public/benchmarks/repeatability" SRC_DATA = ROOT / "apps/site/src/data/benchmarks" LONG_EDGE = 640 # common grid, so a 1408px frame and a 1024px frame compare fairly def load(name): for ext in ("png", "jpg", "webp"): p = SRC / f"{name}.{ext}" if p.exists(): im = Image.open(p).convert("RGB") w, h = im.size s = LONG_EDGE / max(w, h) im = im.resize((max(1, round(w * s)), max(1, round(h * s))), Image.LANCZOS) return np.asarray(im, dtype=np.float64) return None def edge_map(a, pct=90): g = a.mean(axis=2) gx = g[1:-1, 2:] - g[1:-1, :-2] gy = g[2:, 1:-1] - g[:-2, 1:-1] mag = np.hypot(gx, gy) return mag > np.percentile(mag, pct) def iou(a, b): u = np.logical_or(a, b).sum() return float(np.logical_and(a, b).sum() / u) if u else 0.0 def main(): raw = json.loads(RAW.read_text()) briefs = {b["id"]: b for b in raw["briefs"]} groups = {} for r in raw["records"]: if r.get("error"): continue groups.setdefault((r["briefId"], r["model"]), []).append(r["name"]) per_group = [] for (brief_id, model), names in sorted(groups.items()): frames = [(n, load(n)) for n in sorted(names)] frames = [(n, f) for n, f in frames if f is not None] if len(frames) < 2: continue variances, ious, pairs = [], [], [] for (na, a), (nb, b) in itertools.combinations(frames, 2): # Frames within a group share a ratio, but rounding can differ by a # pixel — align before differencing. if a.shape != b.shape: h = min(a.shape[0], b.shape[0]) w = min(a.shape[1], b.shape[1]) a, b = a[:h, :w], b[:h, :w] v = float(np.abs(a - b).mean()) i = iou(edge_map(a), edge_map(b)) variances.append(v) ious.append(i) pairs.append({"a": na, "b": nb, "variance": round(v, 2), "compositionIoU": round(i, 4)}) per_group.append( { "briefId": brief_id, "briefLabel": briefs[brief_id]["label"], "mode": briefs[brief_id]["mode"], "constrained": brief_id.endswith("-constrained"), "model": model, "runs": len(frames), "frames": [n for n, _ in frames], "variance": round(float(np.mean(variances)), 2), "compositionIoU": round(float(np.mean(ious)), 4), "pairs": pairs, } ) def agg(rows, key): out = {} for r in rows: out.setdefault(r[key], []).append(r) return [ { key: k, "groups": len(v), "variance": round(float(np.mean([x["variance"] for x in v])), 2), "compositionIoU": round(float(np.mean([x["compositionIoU"] for x in v])), 4), } for k, v in sorted(out.items(), key=lambda kv: np.mean([x["variance"] for x in kv[1]])) ] constrained = [r for r in per_group if r["constrained"]] openbrief = [r for r in per_group if not r["constrained"]] result = { "runDate": raw["runDate"], "runsPerBrief": raw["runsPerBrief"], "harness": "scripts/model-selection-test.mjs", "metricsScript": "scripts/bench-repeatability.py", "commonLongEdgePx": LONG_EDGE, # Models still deliver different pixel grids, so the cross-model # aggregate is indicative; within a model it is exact. "crossModelComparisonIsIndicative": True, "briefs": raw["briefs"], "byModel": agg(per_group, "model"), "byMode": agg(per_group, "mode"), "byBriefStyle": [ { "style": "Open brief", "variance": round(float(np.mean([x["variance"] for x in openbrief])), 2), "compositionIoU": round(float(np.mean([x["compositionIoU"] for x in openbrief])), 4), }, { "style": "Brief with constraints", "variance": round(float(np.mean([x["variance"] for x in constrained])), 2), "compositionIoU": round(float(np.mean([x["compositionIoU"] for x in constrained])), 4), }, ], "groups": per_group, } payload = json.dumps(result, indent=2, ensure_ascii=False) + "\n" OUT.mkdir(parents=True, exist_ok=True) (OUT / "run.json").write_text(payload) SRC_DATA.mkdir(parents=True, exist_ok=True) (SRC_DATA / "repeatability.json").write_text(payload) print(f"wrote {OUT/'run.json'}") print(f"wrote {SRC_DATA/'repeatability.json'}") print("\nby model:") for r in result["byModel"]: print(f" {r['model']:<8} variance {r['variance']:>6} compositionIoU {r['compositionIoU']:.3f}") print("by mode:") for r in result["byMode"]: print(f" {r['mode']:<12} variance {r['variance']:>6} compositionIoU {r['compositionIoU']:.3f}") print("by brief style:") for r in result["byBriefStyle"]: print(f" {r['style']:<22} variance {r['variance']:>6} compositionIoU {r['compositionIoU']:.3f}") if __name__ == "__main__": main()