#!/usr/bin/env python3 """Turn Bench 02's raw run into the numbers the page renders. node --env-file=scripts/.env scripts/model-selection-test.mjs # produces raw.json python3 scripts/bench-model-metrics.py # produces run.json Reads scripts/bench02/raw.json + the generated PNGs, writes apps/site/public/benchmarks/model-selection/run.json and the src/data copy, the same two-destination pattern as bench-metrics.py. WHAT IS MEASURED HERE, AND WHAT IS NOT. Measured, per model: latencyMs median / p90 wall clock per image aspectAccuracy delivered aspect / requested aspect (1.0 = exact) failureRate slots that returned nothing sharpness Laplacian variance — descriptive, NOT a quality score contrast std-dev of luminance colourfulness Hasler-Susstrunk metric meanTone mean R/G/B NOT measured here, on purpose: whether the image is any good, and whether it followed the brief. Those are scored by people against the rubric in docs/bench-02-model-selection-plan.md, entered into scores.json, and merged in below. The published page must label them as human judgement — they are the only numbers on it that are not computed. The three models exist for DIFFERENT JOBS, not as a ranking. This script therefore emits no total, no composite score and no ordering. If a future edit adds one, it is contradicting the product's own framing. """ import json from pathlib import Path import numpy as np from PIL import Image ROOT = Path(__file__).resolve().parent.parent RAW = ROOT / "scripts/bench02/raw.json" IMG_DIR = ROOT / "scripts/bench02" SCORES = ROOT / "scripts/bench02/scores.json" # optional human scoring OUT = ROOT / "apps/site/public/benchmarks/model-selection" SRC_DATA = ROOT / "apps/site/src/data/benchmarks" def laplacian_var(a): g = a.mean(axis=2) h, w = g.shape k = np.array([[0, 1, 0], [1, -4, 1], [0, 1, 0]], dtype=np.float64) out = np.zeros((h - 2, w - 2)) for dy in range(3): for dx in range(3): if k[dy, dx]: out += k[dy, dx] * g[dy : dy + h - 2, dx : dx + w - 2] return float(out.var()) def colourfulness(a): """Hasler-Susstrunk. Separates 'F is more saturated' from 'F is sharper'.""" r, g, b = a[:, :, 0], a[:, :, 1], a[:, :, 2] rg = r - g yb = 0.5 * (r + g) - b return float( np.sqrt(rg.std() ** 2 + yb.std() ** 2) + 0.3 * np.sqrt(rg.mean() ** 2 + yb.mean() ** 2) ) def pct(values, p): if not values: return None s = sorted(values) i = min(len(s) - 1, int(round((p / 100) * (len(s) - 1)))) return s[i] def main(): if not RAW.exists(): raise SystemExit( f"{RAW} not found — run scripts/model-selection-test.mjs first." ) raw = json.loads(RAW.read_text()) human = json.loads(SCORES.read_text()) if SCORES.exists() else {} per_image = [] for rec in raw["records"]: row = dict(rec) if rec.get("error"): per_image.append(row) continue # The models do not agree on a container — the lite model returns JPEG, # the others PNG — so the extension is whatever was delivered. p = next( (q for ext in ("png", "jpg", "webp") if (q := IMG_DIR / f"{rec['name']}.{ext}").exists()), None, ) if p is not None: a = np.asarray(Image.open(p).convert("RGB"), dtype=np.float64) row["sharpness"] = round(laplacian_var(a), 2) row["contrast"] = round(float(a.mean(axis=2).std()), 2) row["colourfulness"] = round(colourfulness(a), 2) row["meanTone"] = [round(float(a[:, :, c].mean()), 2) for c in range(3)] # Internal cost fields are not part of the published run. for k in ("gatewayCostUsd", "gatewayBilledUsd", "creditsCharged"): row.pop(k, None) # Human scores, if any, keyed by image name. Kept clearly separate from # everything computed above. if rec["name"] in human: row["human"] = human[rec["name"]] per_image.append(row) models = [] for m in raw["models"]: mine = [r for r in per_image if r.get("model") == m["label"]] ok = [r for r in mine if not r.get("error")] lat = [r["latencyMs"] for r in ok] def avg(field, digits=2): vals = [r[field] for r in ok if r.get(field) is not None] return round(sum(vals) / len(vals), digits) if vals else None scored = [r["human"] for r in ok if "human" in r] m = {k: v for k, v in m.items() if k != "creditsPerImage"} models.append( { **m, "images": len(mine), "succeeded": len(ok), "failureRatePct": round(100 * (len(mine) - len(ok)) / len(mine), 1) if mine else None, "latencyMedianMs": pct(lat, 50), "latencyP90Ms": pct(lat, 90), # 4 decimals: Flux only accepts preset sizes, so its drift from # the requested ratio is small but real — rounding to 2 would # print a flat 1.0 and hide exactly what this measures. "aspectAccuracyMean": avg("aspectAccuracy", 4), "sharpnessMean": avg("sharpness"), "contrastMean": avg("contrast"), "colourfulnessMean": avg("colourfulness"), # Human judgement, surfaced separately and labelled as such. "briefAdherenceMean": ( round(sum(s.get("briefAdherence", 0) for s in scored) / len(scored), 2) if scored else None ), "briefAdherenceScoredImages": len(scored), } ) result = { "runDate": raw["runDate"], "runsPerBrief": raw["runsPerBrief"], # Disclosed, not hidden: when true the run was assembled in more than # one pass (--append), reusing images already on disk instead of # re-shooting them. Same code and same session, but a reader deserves # to know it was not one uninterrupted sweep. "assembledInMultiplePasses": bool(raw.get("appendedToPriorRun")), "harness": "scripts/model-selection-test.mjs", "metricsScript": "scripts/bench-model-metrics.py", "briefs": raw["briefs"], "models": models, "images": per_image, "note": ( "The three models serve different jobs; this run deliberately emits no " "composite score and no ranking. Brief adherence is human judgement " "against a published rubric, everything else is computed." ), } payload = json.dumps(result, indent=2) + "\n" OUT.mkdir(parents=True, exist_ok=True) (OUT / "run.json").write_text(payload) SRC_DATA.mkdir(parents=True, exist_ok=True) (SRC_DATA / "model-selection.json").write_text(payload) print(f"wrote {OUT / 'run.json'}") print(f"wrote {SRC_DATA / 'model-selection.json'}") for m in models: print( f"\n{m['label']} n={m['succeeded']}/{m['images']}" f" median {m['latencyMedianMs']}ms p90 {m['latencyP90Ms']}ms" ) print( f" aspect {m['aspectAccuracyMean']} sharp {m['sharpnessMean']}" f" colour {m['colourfulnessMean']}" ) if m["briefAdherenceScoredImages"] == 0: print(" brief adherence: NOT SCORED — see scores.json in the plan doc") if __name__ == "__main__": main()