Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

feat(vibe-tests): add Gemini design judge via [redacted] (#1085)

Ernestb05cdf5a8d5e
diff --git a/internal/vibe-tests/src/design-judge-gemini.py b/internal/vibe-tests/src/design-judge-gemini.py
new file mode 100644
index 0000000000..8da3035ff3
--- /dev/null
+++ b/internal/vibe-tests/src/design-judge-gemini.py
@@ -0,0 +1,337 @@
+#!/usr/bin/env python3
+"""
+Night Watch Design Judge — Gemini 2.5 Pro Preview via Meta InternalProxy (mTLS).
+
+Runs on Meta devvm/devgpu. No external API key needed — authenticates using
+the machine's mTLS identity cert at /var/facebook/x509_identities/server.pem.
+
+Usage:
+ python3 design-judge-gemini.py \
+ --ideals internal/vibe-tests/ideals \
+ --screenshots /tmp/vibe-screenshots-<iteration_id> \
+ --iteration <iteration_id> \
+ --output /tmp/design-scores-gemini.json
+
+ # Resume a partial run:
+ python3 design-judge-gemini.py ... --resume
+
+ # Dry run (validate inputs, don't call API):
+ python3 design-judge-gemini.py ... --dry-run
+"""
+import argparse
+import base64
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+
+CERT = "/var/facebook/x509_identities/server.pem"
+CA = "/var/facebook/rootcanal/ca.pem"
+MODEL = "gemini-2.5-pro-preview"
+BASE = "https://internal-proxy.example.net/v1beta/models"
+API_KEY = "sk-internal-proxy-dummy-1234567890"
+TARGETS = ["xds", "baseline", "html"]
+
+# Prompt registry — maps prompt ID to human-readable description for the judge
+PROMPTS = {
+ "dd-1": "Data table with sortable columns",
+ "dd-2": "Transaction list with amount, date, status",
+ "dd-3": "Analytics dashboard with charts",
+ "fwc-1": "Login form with validation",
+ "fwc-2": "Multi-step form wizard",
+ "fwc-3": "Search with autocomplete",
+ "fwc-4": "Confirmation delete dialog",
+ "io-1": "Empty state with call to action",
+ "io-2": "Loading skeleton screens",
+ "io-3": "Toast notification stack",
+ "io-4": "Progress indicator",
+ "ps-1": "Product grid with filters",
+ "ps-2": "Product card with quick actions",
+ "ps-3": "Shopping cart summary",
+ "ps-4": "Product detail with breadcrumbs",
+ "rc-1": "Responsive navigation menu",
+ "rc-2": "Sidebar to bottom sheet on mobile",
+ "rc-3": "Responsive data table to cards",
+ "sd-1": "Button with loading state",
+ "sd-2": "Loading to success button states",
+ "tc-1": "Color scheme switcher",
+ "tc-3": "Typography scale demo",
+ "tc-6": "Settings page with theme controls",
+ "ty-1": "Article with rich typography",
+ "ty-3": "Metrics dashboard card",
+ "wd-1": "E-commerce checkout flow",
+ "wd-2": "User registration wizard",
+ "wd-3": "Onboarding flow (4 steps)",
+ "cwm-1": "Rich text editor toolbar",
+ "cwm-2": "Kanban board with drag handles",
+ "cwm-3": "Notion page header with icon/cover picker",
+}
+
+# Note: {{ and }} are escaped braces so .format() works with {prompt_text}
+JUDGE_PROMPT = """You are a professional UI design reviewer. Compare IMAGE 1 (ideal reference) to IMAGE 2 (generated output) for the prompt: "{prompt_text}"
+
+Score IMAGE 2 against IMAGE 1 on 5 dimensions (0-100 each). Use the full scale — avoid clustering at 0/50/100. Most UIs score 20-90.
+
+- layout (weight 25%): page structure, regions, stacking order, grid
+- hierarchy (weight 25%): visual weight, heading/body size, primary action prominence
+- spacing (weight 20%): margins, padding, gaps proportionally similar
+- components (weight 15%): interactive element affordances, border-radius, shadows
+- color (weight 15%): palette match, primary/accent/surface/text colors
+
+Scoring guidance:
+- Blank/error screenshot: 0-5
+- Completely different UI: 5-20
+- Right concept, looks different: 30-60
+- Close with noticeable differences: 60-80
+- Very close, minor differences: 80-95
+- Near-identical: 95-100
+
+Compute overall = layout*0.25 + hierarchy*0.25 + spacing*0.20 + components*0.15 + color*0.15
+
+Respond with ONLY a JSON object (no explanation, no markdown):
+{{"layout": <int>, "hierarchy": <int>, "spacing": <int>, "components": <int>, "color": <int>, "overall": <float>, "notes": "<1-2 sentences>"}}"""
+
+
+def parse_args():
+ p = argparse.ArgumentParser(description="Night Watch Design Judge (Gemini via InternalProxy)")
+ p.add_argument("--ideals", required=True, help="Path to directory containing ideal PNG files")
+ p.add_argument("--screenshots", required=True, help="Path to directory containing screenshot PNGs")
+ p.add_argument("--iteration", required=True, help="Iteration ID (e.g. 7e7514ec)")
+ p.add_argument("--output", default="/tmp/design-scores-gemini.json", help="Output JSON path")
+ p.add_argument("--prompts", help="Comma-separated list of prompt IDs to score (default: all with ideals)")
+ p.add_argument("--resume", action="store_true", help="Resume from existing output file")
+ p.add_argument("--dry-run", action="store_true", help="Validate inputs without calling API")
+ p.add_argument("--model", default=MODEL, help=f"Gemini model (default: {MODEL})")
+ return p.parse_args()
+
+
+def check_devvm():
+ if not os.path.exists(CERT):
+ print(f"ERROR: mTLS cert not found at {CERT}", file=sys.stderr)
+ print("This script requires a Meta devvm or devgpu with machine identity certs.", file=sys.stderr)
+ sys.exit(1)
+
+
+def encode_image(path):
+ with open(path, "rb") as f:
+ return base64.b64encode(f.read()).decode("utf-8")
+
+
+def call_gemini(ideal_path, screenshot_path, prompt_text, model=MODEL, dry_run=False):
+ if dry_run:
+ return {"layout": 0, "hierarchy": 0, "spacing": 0, "components": 0, "color": 0,
+ "overall": 0.0, "notes": "DRY RUN"}
+
+ ideal_b64 = encode_image(ideal_path)
+ ss_b64 = encode_image(screenshot_path)
+
+ payload = {
+ "contents": [{"role": "user", "parts": [
+ {"inlineData": {"mimeType": "image/png", "data": ideal_b64}},
+ {"inlineData": {"mimeType": "image/png", "data": ss_b64}},
+ {"text": JUDGE_PROMPT.format(prompt_text=prompt_text)}
+ ]}],
+ "generationConfig": {
+ "temperature": 0.1,
+ "maxOutputTokens": 4096,
+ "responseMimeType": "application/json",
+ },
+ }
+
+ with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
+ json.dump(payload, f)
+ tmpfile = f.name
+
+ try:
+ result = subprocess.run(
+ [
+ "curl", "-s", "-w", "\n%{http_code}",
+ "--cert", CERT, "--cacert", CA, "--noproxy", "example.net",
+ "-H", "Content-Type: application/json",
+ "-H", f"x-goog-api-key: {API_KEY}",
+ "-d", f"@{tmpfile}",
+ f"{BASE}/{model}:generateContent",
+ ],
+ capture_output=True,
+ text=True,
+ timeout=120,
+ )
+
+ lines = result.stdout.strip().split("\n")
+ http_code = lines[-1]
+ body = "\n".join(lines[:-1])
+
+ if http_code != "200":
+ raise Exception(f"HTTP {http_code}: {body[:300]}")
+
+ resp = json.loads(body)
+ if "error" in resp:
+ raise Exception(f"API error: {resp['error']}")
+
+ text = resp["candidates"][0]["content"]["parts"][0]["text"].strip()
+ # Strip markdown fences (Gemini wraps in ```json even with responseMimeType=json)
+ text = re.sub(r"```json\s*", "", text)
+ text = re.sub(r"```\s*", "", text).strip()
+
+ return json.loads(text)
+ finally:
+ os.unlink(tmpfile)
+
+
+def find_ideal(ideals_dir, prompt_id):
+ """Find the most specific ideal image for a prompt ID."""
+ candidates = [
+ f"{prompt_id}__desktop__light.png",
+ f"{prompt_id}__desktop.png",
+ f"{prompt_id}.png",
+ ]
+ for name in candidates:
+ path = os.path.join(ideals_dir, name)
+ if os.path.exists(path):
+ return path
+ return None
+
+
+def find_screenshot(screenshots_dir, prompt_id, target):
+ """Find the desktop-light screenshot for a prompt/target."""
+ name = f"{prompt_id}-{target}-desktop-light.png"
+ path = os.path.join(screenshots_dir, name)
+ if os.path.exists(path) and os.path.getsize(path) >= 1000:
+ return path
+ return None
+
+
+def compute_averages(results, targets):
+ keys = ["layout", "hierarchy", "spacing", "components", "color", "overall"]
+ avgs = {t: {k: [] for k in keys} for t in targets}
+ for pid, pd in results.items():
+ for t in targets:
+ if t in pd:
+ for k in keys:
+ avgs[t][k].append(pd[t][k])
+ return {
+ t: {k: round(sum(v) / len(v), 1) for k, v in avgs[t].items()}
+ for t in targets
+ if avgs[t]["overall"]
+ }
+
+
+def print_summary(results, avg_out, prompts_to_score, targets):
+ print(f"\n{'='*70}")
+ print(f"{'Prompt':<8} {'XDS':>6} {'Base':>6} {'HTML':>6}")
+ print("-" * 30)
+ for pid in prompts_to_score:
+ if pid in results:
+ x = results[pid].get("xds", {}).get("overall", "—")
+ b = results[pid].get("baseline", {}).get("overall", "—")
+ h = results[pid].get("html", {}).get("overall", "—")
+ print(f"{pid:<8} {str(x):>6} {str(b):>6} {str(h):>6}")
+ print(f"\n{'Target':<10} {'Overall':>8}")
+ print("-" * 20)
+ for t in targets:
+ if t in avg_out:
+ print(f"{t:<10} {avg_out[t]['overall']:>8}")
+
+
+def main():
+ args = parse_args()
+ check_devvm()
+
+ model = args.model
+ outfile = args.output
+
+ # Determine which prompts to score
+ if args.prompts:
+ prompt_ids = [p.strip() for p in args.prompts.split(",")]
+ else:
+ # Auto-discover: all prompts that have an ideal image
+ prompt_ids = sorted(
+ pid for pid in PROMPTS
+ if find_ideal(args.ideals, pid) is not None
+ )
+
+ print(f"Judge: {model}")
+ print(f"Iteration: {args.iteration}")
+ print(f"Prompts: {len(prompt_ids)}")
+ print(f"Output: {outfile}")
+ if args.dry_run:
+ print("DRY RUN — no API calls will be made")
+ print()
+
+ # Load partial results if resuming
+ results = {}
+ if args.resume and os.path.exists(outfile):
+ with open(outfile) as f:
+ old = json.load(f)
+ results = old.get("results", {})
+ cached = sum(len(v) for v in results.values())
+ if cached > 0:
+ print(f"Resuming from {cached} existing scores\n")
+
+ for pid in prompt_ids:
+ prompt_text = PROMPTS.get(pid, pid)
+ ideal = find_ideal(args.ideals, pid)
+ if ideal is None:
+ print(f" SKIP {pid} — no ideal image")
+ continue
+ if pid not in results:
+ results[pid] = {}
+
+ for t in TARGETS:
+ if t in results[pid] and results[pid][t].get("overall", -1) >= 0:
+ print(f" {pid}/{t} ... CACHED {results[pid][t]['overall']}")
+ continue
+
+ ss = find_screenshot(args.screenshots, pid, t)
+ if ss is None:
+ print(f" SKIP {pid}/{t} — no screenshot")
+ continue
+
+ print(f" {pid}/{t} ...", end=" ", flush=True)
+ try:
+ j = call_gemini(ideal, ss, prompt_text, model=model, dry_run=args.dry_run)
+ results[pid][t] = j
+ print(
+ f"{j['overall']:5.1f} "
+ f"L:{j['layout']:2d} H:{j['hierarchy']:2d} S:{j['spacing']:2d} "
+ f"C:{j['components']:2d} Co:{j['color']:2d} "
+ f"{j.get('notes', '')[:70]}"
+ )
+ except Exception as e:
+ print(f"FAIL: {e}")
+
+ # Save incrementally after every score
+ partial = {
+ "iterationId": args.iteration,
+ "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
+ "judge": f"{model}-internal-proxy",
+ "model": model,
+ "results": results,
+ }
+ with open(outfile, "w") as f:
+ json.dump(partial, f, indent=2)
+
+ if not args.dry_run:
+ time.sleep(2)
+
+ avg_out = compute_averages(results, TARGETS)
+ out = {
+ "iterationId": args.iteration,
+ "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
+ "judge": f"{model}-internal-proxy",
+ "model": model,
+ "results": results,
+ "averages": avg_out,
+ }
+ with open(outfile, "w") as f:
+ json.dump(out, f, indent=2)
+
+ print_summary(results, avg_out, prompt_ids, TARGETS)
+ print(f"\nSaved to {outfile}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/internal/vibe-tests/src/post-design-results.py b/internal/vibe-tests/src/post-design-results.py
new file mode 100644
index 0000000000..8eb56ad3bb
--- /dev/null
+++ b/internal/vibe-tests/src/post-design-results.py
@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""
+Post Night Watch design judge results as a GitHub issue comment.
+
+Usage:
+ python3 post-design-results.py \
+ --scores /tmp/design-scores-gemini.json \
+ --release-tag untagged-927529363adeb232eb8b \
+ --issue 1041 \
+ --repo facebookexperimental/xds \
+ --token $GITHUB_TOKEN
+"""
+import argparse
+import json
+import sys
+import urllib.request
+import urllib.error
+
+
+EMOJI = {"xds": "🟣", "baseline": "⚪", "html": "🟠"}
+PROMPT_LABELS = {
+ "dd-1": "Data table with sortable columns",
+ "dd-2": "Transaction list with amount, date, status",
+ "dd-3": "Analytics dashboard with charts",
+ "fwc-1": "Login form with validation",
+ "fwc-2": "Multi-step form wizard",
+ "fwc-3": "Search with autocomplete",
+ "fwc-4": "Confirmation delete dialog",
+ "io-1": "Empty state with call to action",
+ "io-2": "Loading skeleton screens",
+ "io-3": "Toast notification stack",
+ "io-4": "Progress indicator",
+ "ps-1": "Product grid with filters",
+ "ps-2": "Product card with quick actions",
+ "ps-3": "Shopping cart summary",
+ "ps-4": "Product detail with breadcrumbs",
+ "rc-1": "Responsive navigation menu",
+ "rc-2": "Sidebar to bottom sheet on mobile",
+ "rc-3": "Responsive data table to cards",
+ "sd-1": "Button with loading state",
+ "sd-2": "Loading to success button states",
+ "tc-1": "Color scheme switcher",
+ "tc-3": "Typography scale demo",
+ "tc-6": "Settings page with theme controls",
+ "ty-1": "Article with rich typography",
+ "ty-3": "Metrics dashboard card",
+ "wd-1": "E-commerce checkout flow",
+ "wd-2": "User registration wizard",
+ "wd-3": "Onboarding flow (4 steps)",
+ "cwm-1": "Rich text editor toolbar",
+ "cwm-2": "Kanban board with drag handles",
+ "cwm-3": "Notion page header with icon/cover picker",
+}
+
+
+def img_url(release_tag, filename):
+ return f"https://github.com/facebookexperimental/xds/releases/download/{release_tag}/{filename}"
+
+
+def score_emoji(score):
+ if score is None or score == "—":
+ return "—"
+ if score >= 80:
+ return f"✅ **{score}**"
+ if score >= 50:
+ return f"🟡 **{score}**"
+ return f"🔴 **{score}**"
+
+
+def build_comment(data, release_tag):
+ results = data["results"]
+ averages = data.get("averages", {})
+ iteration_id = data["iterationId"]
+ model = data["model"]
+ targets = ["xds", "baseline", "html"]
+
+ lines = []
+ lines.append(f"## 🔬 Night Watch Design Judge — Gemini Vision")
+ lines.append("")
+ lines.append(f"**Judge:** `{model}` via Meta InternalProxy | **Iteration:** `{iteration_id}` | **Method:** mTLS (no external API key)")
+ lines.append("")
+
+ # Overall averages table
+ lines.append("### Overall Averages")
+ lines.append("")
+ lines.append("| Target | Layout | Hierarchy | Spacing | Components | Color | **Overall** |")
+ lines.append("|--------|--------|-----------|---------|------------|-------|-------------|")
+ for t in targets:
+ if t in averages:
+ a = averages[t]
+ lines.append(
+ f"| {t.upper()} | {a['layout']} | {a['hierarchy']} | {a['spacing']} | "
+ f"{a['components']} | {a['color']} | **{a['overall']}** |"
+ )
+ lines.append("")
+ lines.append("---")
+ lines.append("")
+
+ # Per-prompt sections
+ for pid, pd in sorted(results.items()):
+ label = PROMPT_LABELS.get(pid, pid)
+ scores = {t: pd.get(t, {}).get("overall") for t in targets}
+ valid = [s for s in scores.values() if s is not None]
+ best = max(valid) if valid else None
+
+ # Section header with best scores
+ score_parts = " | ".join(
+ f"{t.upper()}: {s if s is not None else 'n/a'}"
+ for t, s in scores.items()
+ )
+ status = "✅" if best and best >= 70 else ("🟡" if best and best >= 40 else "🔴")
+ lines.append(f"#### {status} {pid} — {label}")
+ lines.append(f"**{score_parts}**")
+ lines.append("")
+
+ # Score table
+ lines.append("| Target | Layout | Hier | Space | Comp | Color | Overall | Notes |")
+ lines.append("|--------|--------|------|-------|------|-------|---------|-------|")
+ for t in targets:
+ if t in pd:
+ j = pd[t]
+ notes = j.get("notes", "")[:80]
+ overall = j.get("overall", "—")
+ lines.append(
+ f"| {t.upper()} | {j.get('layout','—')} | {j.get('hierarchy','—')} | "
+ f"{j.get('spacing','—')} | {j.get('components','—')} | {j.get('color','—')} | "
+ f"**{overall}** | {notes} |"
+ )
+ else:
+ lines.append(f"| {t.upper()} | — | — | — | — | — | **n/a** | No screenshot |")
+ lines.append("")
+
+ # Screenshot images
+ has_baseline = "baseline" in pd
+ if has_baseline:
+ lines.append("**Ideal** | **XDS** | **Baseline** | **HTML**")
+ lines.append(":--: | :--: | :--: | :--:")
+ lines.append(
+ f'<img src="{img_url(release_tag, f"{pid}.png")}" width="220"> | '
+ f'<img src="{img_url(release_tag, f"{pid}-xds-desktop-light.png")}" width="220"> | '
+ f'<img src="{img_url(release_tag, f"{pid}-baseline-desktop-light.png")}" width="220"> | '
+ f'<img src="{img_url(release_tag, f"{pid}-html-desktop-light.png")}" width="220">'
+ )
+ else:
+ lines.append("**Ideal** | **XDS** | **HTML**")
+ lines.append(":--: | :--: | :--:")
+ lines.append(
+ f'<img src="{img_url(release_tag, f"{pid}.png")}" width="220"> | '
+ f'<img src="{img_url(release_tag, f"{pid}-xds-desktop-light.png")}" width="220"> | '
+ f'<img src="{img_url(release_tag, f"{pid}-html-desktop-light.png")}" width="220">'
+ )
+ lines.append("")
+ lines.append("---")
+ lines.append("")
+
+ lines.append("— via Navi on behalf of Ernest Tien")
+ return "\n".join(lines)
+
+
+def post_comment(body, repo, issue_number, token):
+ url = f"https://api.github.com/repos/{repo}/issues/{issue_number}/comments"
+ data = json.dumps({"body": body}).encode()
+ req = urllib.request.Request(
+ url,
+ data=data,
+ headers={
+ "Authorization": f"token {token}",
+ "Content-Type": "application/json",
+ "Accept": "application/vnd.github.v3+json",
+ },
+ method="POST",
+ )
+ try:
+ with urllib.request.urlopen(req) as resp:
+ result = json.load(resp)
+ print(f"Posted: {result['html_url']}")
+ return result["html_url"]
+ except urllib.error.HTTPError as e:
+ print(f"HTTP error {e.code}: {e.read().decode()}", file=sys.stderr)
+ sys.exit(1)
+
+
+def main():
+ p = argparse.ArgumentParser(description="Post design judge results to GitHub issue")
+ p.add_argument("--scores", required=True, help="Path to design-scores-gemini.json")
+ p.add_argument("--release-tag", required=True, help="GitHub release tag for image hosting")
+ p.add_argument("--issue", required=True, type=int, help="GitHub issue number")
+ p.add_argument("--repo", default="facebookexperimental/xds", help="GitHub repo")
+ p.add_argument("--token", required=True, help="GitHub token")
+ p.add_argument("--dry-run", action="store_true", help="Print comment without posting")
+ args = p.parse_args()
+
+ with open(args.scores) as f:
+ data = json.load(f)
+
+ body = build_comment(data, args.release_tag)
+
+ if args.dry_run:
+ print(body)
+ return
+
+ post_comment(body, args.repo, args.issue, args.token)
+
+
+if __name__ == "__main__":
+ main()