Skip to content

The Open Standard for Generative UI

Add the model comparison board to the benchmark hero chart (#1079)

rahuldoval94b79176701d
diff --git a/docs/app/(home)/benchmarks/BenchmarkDetailPage.tsx b/docs/app/(home)/benchmarks/BenchmarkDetailPage.tsx
new file mode 100644
index 000000000..ddc9fb846
--- /dev/null
+++ b/docs/app/(home)/benchmarks/BenchmarkDetailPage.tsx
@@ -0,0 +1,101 @@
+import { Footer } from "@/app/(home)/sections/Footer/Footer";
+import type { ReactNode } from "react";
+import { PillLink } from "../components/Button/Button";
+
+type DetailLink = {
+ href: string;
+ label: string;
+ external?: boolean;
+};
+
+export function BenchmarkDetailPage({
+ eyebrow,
+ title,
+ description,
+ sections,
+ sibling,
+ downloads,
+ children,
+}: {
+ eyebrow: string;
+ title: string;
+ description: string;
+ sections: ReadonlyArray<{ id: string; label: string }>;
+ sibling: DetailLink;
+ downloads: ReadonlyArray<DetailLink>;
+ children: ReactNode;
+}) {
+ return (
+ <>
+ <div className="min-h-screen bg-[linear-gradient(to_bottom,var(--openui-foreground),var(--openui-background)_10rem,var(--openui-background)_calc(100%_-_10rem),var(--openui-foreground))] [[data-theme=dark]_&]:bg-[linear-gradient(to_bottom,#000,var(--swatch-neutral-950)_28rem,var(--swatch-neutral-950)_calc(100%_-_28rem),#000)]">
+ <main className="mx-auto flex w-full max-w-[var(--home-container-wide)] gap-4 px-4 pt-16 pb-40 lg:gap-28 lg:pr-8 min-[1249px]:pl-0 min-[1024px]:max-[1248px]:pl-8">
+ <aside className="hidden w-56 shrink-0 lg:block">
+ <div className="sticky top-24">
+ <p className="mb-3 text-sm font-medium text-fd-foreground">On this page</p>
+ <ul className="space-y-2 text-sm text-[color:var(--openui-text-neutral-tertiary)]">
+ {sections.map((section) => (
+ <li key={section.id}>
+ <a href={`#${section.id}`}>{section.label}</a>
+ </li>
+ ))}
+ </ul>
+ </div>
+ </aside>
+
+ <div className="min-w-0 flex-1">
+ <p className="mb-3 text-sm font-medium text-[color:var(--openui-text-neutral-secondary)]">
+ {eyebrow}
+ </p>
+ <h1 className="mb-3 max-w-4xl text-[length:var(--home-title-size)] font-[family-name:var(--home-font-display)] font-medium leading-[var(--home-title-leading)] tracking-[var(--home-title-tracking)] text-[color:var(--openui-text-neutral-primary)]">
+ {title}
+ </h1>
+ <p className="mb-5 max-w-3xl text-[length:var(--home-lead-size)] font-[family-name:var(--home-font-text)] leading-[var(--home-lead-leading)] text-[color:var(--openui-text-neutral-secondary)]">
+ {description}
+ </p>
+ <nav
+ aria-label="Benchmark pages"
+ className="-ml-3 flex flex-wrap items-center gap-1 border-b border-[color:var(--home-hairline)] pb-6"
+ >
+ <PillLink href="/benchmarks" variant="ghost">
+ Benchmark overview
+ </PillLink>
+ <PillLink href={sibling.href} variant="ghost" external={sibling.external}>
+ {sibling.label}
+ </PillLink>
+ <PillLink href="/benchmarks/methodology" variant="ghost">
+ Methodology
+ </PillLink>
+ </nav>
+
+ <article className="prose mt-8 min-w-0 max-w-none">{children}</article>
+
+ <section
+ id="downloads"
+ aria-labelledby="downloads-heading"
+ className="mt-12 border-t border-[color:var(--home-hairline)] pt-8"
+ >
+ <h2 id="downloads-heading" className="mb-4 text-xl font-medium">
+ Data for agents and analysis
+ </h2>
+ <ul className="flex flex-wrap gap-x-6 gap-y-2 text-sm">
+ {downloads.map((link) => (
+ <li key={link.href}>
+ <a
+ href={link.href}
+ target={link.external ? "_blank" : undefined}
+ rel={link.external ? "noopener noreferrer" : undefined}
+ className="underline decoration-[color:var(--home-hairline)] underline-offset-4"
+ >
+ {link.label}
+ </a>
+ </li>
+ ))}
+ </ul>
+ </section>
+ </div>
+ </main>
+ </div>
+ <Footer />
+ </>
+ );
+}
diff --git a/docs/app/(home)/benchmarks/BenchmarksContent.tsx b/docs/app/(home)/benchmarks/BenchmarksContent.tsx
index 441141289..1141a7e53 100644
--- a/docs/app/(home)/benchmarks/BenchmarksContent.tsx
+++ b/docs/app/(home)/benchmarks/BenchmarksContent.tsx
@@ -9,18 +9,19 @@ import { RenderSplit } from "@/components/charts/render-split";
import { RepairFunnelFlow } from "@/components/charts/repair-funnel-flow";
import { SpeedTokens } from "@/components/charts/speed-tokens";
import {
+ blankScreens,
BRIEFS,
+ completionByDensity,
+ completionOver,
COST_MODELS,
+ costPerPass,
FORMAT_ORDER,
LINKS,
+ MODEL_BOARD_SIZE,
MODELS,
- RUNS_PER_FORMAT,
- blankScreens,
- completionByDensity,
- completionOver,
- costPerPass,
production,
repairedShare,
+ RUNS_PER_FORMAT,
} from "@/lib/benchmark-data";
import { ChartLineUp, CurrencyDollarSimple, ShieldCheck, Wrench } from "@phosphor-icons/react";
import { ArrowUpRight } from "lucide-react";
@@ -124,12 +125,13 @@ function Section({
export function BenchmarksContent() {
return (
- <main className={s.page}>
+ <main id="benchmark-content" className={s.page}>
<VizSkin vivid>
<header className={s.hero}>
<div className={s.heroLockup}>
<p className={s.heroAside}>
- 46 real screens, 6 models, 3 formats: 1,104 scored runs each
+ {MODEL_BOARD_SIZE} models on OpenUI, plus a 6-model comparison across 3 generative UI
+ formats
</p>
<div className={s.heroCopy}>
<p className={s.heroEyebrow}>
@@ -137,8 +139,7 @@ export function BenchmarksContent() {
<span className={s.heroTag}>Benchmarks</span>
</p>
<h1 className={s.heroTitle}>
- <span>Generative UI</span>
- <span>Benchmark</span>
+ <span>Generative UI</span> <span>Benchmark</span>
</h1>
</div>
<div className={s.heroActions}>
@@ -169,7 +170,10 @@ export function BenchmarksContent() {
<ReliabilityByModel models={ALL_MODELS} formats={FORMAT_ORDER} />
</section>
- <div className={s.featureBand} aria-label="Headline results">
+ <div className={s.featureBand} aria-labelledby="headline-results-heading">
+ <h2 id="headline-results-heading" className={s.srOnly}>
+ Headline results
+ </h2>
<FeatureGridSection
features={HEADLINE_FEATURES}
showHeader={false}
diff --git a/docs/app/(home)/benchmarks/agent.md/route.ts b/docs/app/(home)/benchmarks/agent.md/route.ts
new file mode 100644
index 000000000..b72bf15f1
--- /dev/null
+++ b/docs/app/(home)/benchmarks/agent.md/route.ts
@@ -0,0 +1,14 @@
+import { benchmarkAgentMarkdown } from "@/lib/benchmark-agent-data";
+
+export const dynamic = "force-static";
+export const revalidate = false;
+
+export function GET() {
+ return new Response(benchmarkAgentMarkdown(), {
+ headers: {
+ "Content-Type": "text/markdown; charset=utf-8",
+ "Cache-Control": "public, max-age=0, s-maxage=86400, stale-while-revalidate=604800",
+ "Content-Disposition": 'inline; filename="openui-generative-ui-benchmark.md"',
+ },
+ });
+}
diff --git a/docs/app/(home)/benchmarks/benchmarks.module.css b/docs/app/(home)/benchmarks/benchmarks.module.css
index 07b6802e6..7b5ff94b8 100644
--- a/docs/app/(home)/benchmarks/benchmarks.module.css
+++ b/docs/app/(home)/benchmarks/benchmarks.module.css
@@ -6,6 +6,17 @@
font-family: var(--home-font-text);
}
+.srOnly {
+ position: absolute;
+ width: 1px;
+ height: 1px;
+ padding: 0;
+ overflow: hidden;
+ clip: rect(0 0 0 0);
+ clip-path: inset(50%);
+ white-space: nowrap;
+}
+
.hero,
.heroGrid,
.column {
@@ -161,6 +172,10 @@
.heroGrid > figure,
.sectionBody > figure {
+ /* the page lays the cards out edge to edge, so there is no card padding for
+ a full-bleed table to escape */
+ --card-pad: 0px;
+
width: 100%;
min-width: 0;
margin: 0;
diff --git a/docs/app/(home)/benchmarks/data.csv/route.ts b/docs/app/(home)/benchmarks/data.csv/route.ts
new file mode 100644
index 000000000..074a4ca9c
--- /dev/null
+++ b/docs/app/(home)/benchmarks/data.csv/route.ts
@@ -0,0 +1,14 @@
+import { modelBoardCsv } from "@/lib/benchmark-agent-data";
+
+export const dynamic = "force-static";
+export const revalidate = false;
+
+export function GET() {
+ return new Response(`${modelBoardCsv()}\n`, {
+ headers: {
+ "Content-Type": "text/csv; charset=utf-8",
+ "Cache-Control": "public, max-age=0, s-maxage=86400, stale-while-revalidate=604800",
+ "Content-Disposition": 'inline; filename="openui-model-board.csv"',
+ },
+ });
+}
diff --git a/docs/app/(home)/benchmarks/data.json/route.ts b/docs/app/(home)/benchmarks/data.json/route.ts
new file mode 100644
index 000000000..956f70f60
--- /dev/null
+++ b/docs/app/(home)/benchmarks/data.json/route.ts
@@ -0,0 +1,14 @@
+import { benchmarkAgentDataset } from "@/lib/benchmark-agent-data";
+
+export const dynamic = "force-static";
+export const revalidate = false;
+
+export function GET() {
+ return Response.json(benchmarkAgentDataset, {
+ headers: {
+ "Cache-Control": "public, max-age=0, s-maxage=86400, stale-while-revalidate=604800",
+ "Content-Disposition": 'inline; filename="openui-generative-ui-benchmark.json"',
+ Link: `<${benchmarkAgentDataset.$schema}>; rel="describedby"`,
+ },
+ });
+}
diff --git a/docs/app/(home)/benchmarks/data.schema.json/route.ts b/docs/app/(home)/benchmarks/data.schema.json/route.ts
new file mode 100644
index 000000000..d20ec81f5
--- /dev/null
+++ b/docs/app/(home)/benchmarks/data.schema.json/route.ts
@@ -0,0 +1,14 @@
+import { benchmarkDataSchema } from "@/lib/benchmark-data-schema";
+
+export const dynamic = "force-static";
+export const revalidate = false;
+
+export function GET() {
+ return Response.json(benchmarkDataSchema, {
+ headers: {
+ "Content-Type": "application/schema+json; charset=utf-8",
+ "Cache-Control": "public, max-age=0, s-maxage=86400, stale-while-revalidate=604800",
+ "Content-Disposition": 'inline; filename="openui-generative-ui-benchmark.schema.json"',
+ },
+ });
+}
diff --git a/docs/app/(home)/benchmarks/framework/agent.md/route.ts b/docs/app/(home)/benchmarks/framework/agent.md/route.ts
new file mode 100644
index 000000000..2ab33fbf3
--- /dev/null
+++ b/docs/app/(home)/benchmarks/framework/agent.md/route.ts
@@ -0,0 +1,14 @@
+import { frameworkBenchmarkMarkdown } from "@/lib/benchmark-agent-data";
+
+export const dynamic = "force-static";
+export const revalidate = false;
+
+export async function GET() {
+ return new Response(frameworkBenchmarkMarkdown(), {
+ headers: {
+ "Content-Type": "text/markdown; charset=utf-8",
+ "Cache-Control": "public, max-age=0, must-revalidate",
+ "Content-Disposition": 'inline; filename="generative-ui-framework-benchmark.md"',
+ },
+ });
+}
diff --git a/docs/app/(home)/benchmarks/framework/data.csv/route.ts b/docs/app/(home)/benchmarks/framework/data.csv/route.ts
new file mode 100644
index 000000000..3f4e3ff1a
--- /dev/null
+++ b/docs/app/(home)/benchmarks/framework/data.csv/route.ts
@@ -0,0 +1,14 @@
+import { frameworkComparisonCsv } from "@/lib/benchmark-agent-data";
+
+export const dynamic = "force-static";
+export const revalidate = false;
+
+export async function GET() {
+ return new Response(frameworkComparisonCsv(), {
+ headers: {
+ "Content-Type": "text/csv; charset=utf-8",
+ "Cache-Control": "public, max-age=0, must-revalidate",
+ "Content-Disposition": 'inline; filename="generative-ui-framework-benchmark.csv"',
+ },
+ });
+}
diff --git a/docs/app/(home)/benchmarks/framework/data.json/route.ts b/docs/app/(home)/benchmarks/framework/data.json/route.ts
new file mode 100644
index 000000000..d7b882ce5
--- /dev/null
+++ b/docs/app/(home)/benchmarks/framework/data.json/route.ts
@@ -0,0 +1,13 @@
+import { frameworkBenchmarkDataset } from "@/lib/benchmark-agent-data";
+
+export const dynamic = "force-static";
+export const revalidate = false;
+
+export async function GET() {
+ return Response.json(frameworkBenchmarkDataset, {
+ headers: {
+ "Cache-Control": "public, max-age=0, must-revalidate",
+ "Content-Disposition": 'inline; filename="generative-ui-framework-benchmark.json"',
+ },
+ });
+}
diff --git a/docs/app/(home)/benchmarks/framework/page.tsx b/docs/app/(home)/benchmarks/framework/page.tsx
new file mode 100644
index 000000000..f33fbfa99
--- /dev/null
+++ b/docs/app/(home)/benchmarks/framework/page.tsx
@@ -0,0 +1,241 @@
+import {
+ BENCHMARK_CANONICAL_URL,
+ BENCHMARK_UPDATED_ISO,
+ FRAMEWORK_BENCHMARK_URL,
+ formatComparisonRows,
+ formatSummaryRows,
+} from "@/lib/benchmark-agent-data";
+import { BRIEFS, LINKS, MODELS, RUNS_PER_FORMAT, RUNS_TOTAL } from "@/lib/benchmark-data";
+import type { Metadata } from "next";
+import { BenchmarkDetailPage } from "../BenchmarkDetailPage";
+
+export const metadata: Metadata = {
+ title: "Generative UI framework benchmark | OpenUI",
+ description:
+ "Compare OpenUI, Google A2UI, and Vercel json-render across six models and 46 interface briefs, with exact server-rendered results and downloads.",
+ alternates: {
+ canonical: "/benchmarks/framework",
+ types: {
+ "application/json": "/benchmarks/framework/data.json",
+ "text/csv": "/benchmarks/framework/data.csv",
+ "text/markdown": "/benchmarks/framework/agent.md",
+ },
+ },
+ openGraph: {
+ type: "article",
+ url: "/benchmarks/framework",
+ title: "Generative UI framework benchmark",
+ description: "OpenUI versus Google A2UI versus Vercel json-render under one condition.",
+ modifiedTime: BENCHMARK_UPDATED_ISO,
+ },
+};
+
+const structuredData = {
+ "@context": "https://schema.org",
+ "@type": "Dataset",
+ "@id": `${FRAMEWORK_BENCHMARK_URL}#dataset`,
+ name: "Generative UI framework benchmark",
+ description:
+ "OpenUI, Google A2UI, and Vercel json-render compared across six models and 46 interface briefs.",
+ url: FRAMEWORK_BENCHMARK_URL,
+ dateModified: BENCHMARK_UPDATED_ISO,
+ isPartOf: { "@id": `${BENCHMARK_CANONICAL_URL}#dataset` },
+ isBasedOn: LINKS.rawData,
+ measurementTechnique: `${BENCHMARK_CANONICAL_URL}/methodology`,
+ variableMeasured: [
+ "Structural validity percentage",
+ "Render success percentage",
+ "Cost per 46-screen pass in USD",
+ ],
+ distribution: [
+ {
+ "@type": "DataDownload",
+ encodingFormat: "application/json",
+ contentUrl: `${FRAMEWORK_BENCHMARK_URL}/data.json`,
+ },
+ {
+ "@type": "DataDownload",
+ encodingFormat: "text/csv",
+ contentUrl: `${FRAMEWORK_BENCHMARK_URL}/data.csv`,
+ },
+ {
+ "@type": "DataDownload",
+ encodingFormat: "text/markdown",
+ contentUrl: `${FRAMEWORK_BENCHMARK_URL}/agent.md`,
+ },
+ ],
+};
+
+export default function FrameworkBenchmarkPage() {
+ return (
+ <>
+ <script
+ type="application/ld+json"
+ dangerouslySetInnerHTML={{
+ __html: JSON.stringify(structuredData).replaceAll("<", "\\u003c"),
+ }}
+ />
+ <BenchmarkDetailPage
+ eyebrow="Generative UI benchmark · Frameworks"
+ title="Generative UI framework benchmark"
+ description="How do OpenUI, Google A2UI, and Vercel json-render compare under the same briefs and generation condition?"
+ sections={[
+ { id: "answer", label: "What this answers" },
+ { id: "summary", label: "Format summary" },
+ { id: "results", label: "Results by model" },
+ { id: "condition", label: "Controlled condition" },
+ { id: "downloads", label: "Agent-readable data" },
+ ]}
+ sibling={{ href: "/benchmarks/language", label: "Language/model benchmark" }}
+ downloads={[
+ { href: "/benchmarks/framework/agent.md", label: "Agent Markdown" },
+ { href: "/benchmarks/framework/data.json", label: "JSON" },
+ { href: "/benchmarks/framework/data.csv", label: "CSV" },
+ { href: LINKS.rawData, label: "Published raw results", external: true },
+ { href: "/benchmarks/data.json", label: "Combined dataset" },
+ ]}
+ >
+ <section id="answer" aria-labelledby="answer-heading">
+ <h2 id="answer-heading">What this benchmark answers</h2>
+ <p>
+ This is a controlled comparison of three generative UI formats: OpenUI, Google A2UI, and
+ Vercel json-render. It covers {MODELS.length} models, {BRIEFS} briefs, and four
+ generations per brief: {RUNS_PER_FORMAT.toLocaleString("en-US")} runs per format and{" "}
+ {RUNS_TOTAL.toLocaleString("en-US")} scored runs in total.
+ </p>
+ <p>
+ Structural validity and render success are separate. A generation can put something on
+ screen while still failing the complete structural checks.
+ </p>
+ </section>
+
+ <section id="summary" aria-labelledby="summary-heading">
+ <h2 id="summary-heading">Format summary</h2>
+ <div className="not-prose overflow-x-auto">
+ <table className="w-full min-w-[680px] border-collapse text-left text-sm">
+ <caption className="sr-only">Summary results for each generative UI format</caption>
+ <thead>
+ <tr className="border-b border-[color:var(--home-hairline)]">
+ <th scope="col" className="py-3 pr-5 font-medium">
+ Format
+ </th>
+ <th scope="col" className="py-3 pr-5 text-right font-medium">
+ Structural validity
+ </th>
+ <th scope="col" className="py-3 pr-5 text-right font-medium">
+ Render success
+ </th>
+ <th scope="col" className="py-3 pr-5 text-right font-medium">
+ Blank screens
+ </th>
+ <th scope="col" className="py-3 text-right font-medium">
+ Runs
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ {formatSummaryRows.map((row) => (
+ <tr key={row.format_id} className="border-b border-[color:var(--home-hairline)]">
+ <th scope="row" className="py-3 pr-5 font-medium">
+ {row.format_name}
+ </th>
+ <td className="py-3 pr-5 text-right tabular-nums">
+ {row.validity_score_percent.toFixed(1)}%
+ </td>
+ <td className="py-3 pr-5 text-right tabular-nums">
+ {row.render_rate_percent.toFixed(1)}%
+ </td>
+ <td className="py-3 pr-5 text-right tabular-nums">{row.blank_screens}</td>
+ <td className="py-3 text-right tabular-nums">
+ {row.runs.toLocaleString("en-US")}
+ </td>
+ </tr>
+ ))}
+ </tbody>
+ </table>
+ </div>
+ </section>
+
+ <section id="results" aria-labelledby="results-heading">
+ <h2 id="results-heading">Results by model and format</h2>
+ <div className="not-prose overflow-x-auto">
+ <table className="w-full min-w-[800px] border-collapse text-left text-sm">
+ <caption className="sr-only">
+ Structural validity, render success, and cost by model and format
+ </caption>
+ <thead>
+ <tr className="border-b border-[color:var(--home-hairline)]">
+ <th scope="col" className="py-3 pr-5 font-medium">
+ Model
+ </th>
+ <th scope="col" className="py-3 pr-5 font-medium">
+ Format
+ </th>
+ <th scope="col" className="py-3 pr-5 text-right font-medium">
+ Valid runs
+ </th>
+ <th scope="col" className="py-3 pr-5 text-right font-medium">
+ Validity
+ </th>
+ <th scope="col" className="py-3 pr-5 text-right font-medium">
+ Render success
+ </th>
+ <th scope="col" className="py-3 text-right font-medium">
+ Cost / 46 screens
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ {formatComparisonRows.map((row) => (
+ <tr
+ key={`${row.model_id}-${row.format_id}`}
+ className="border-b border-[color:var(--home-hairline)]"
+ >
+ <th scope="row" className="py-3 pr-5 font-medium">
+ {row.model_name}
+ </th>
+ <td className="py-3 pr-5">{row.format_name}</td>
+ <td className="py-3 pr-5 text-right tabular-nums">
+ {row.valid_runs}/{row.total_runs}
+ </td>
+ <td className="py-3 pr-5 text-right tabular-nums">
+ {row.validity_score_percent.toFixed(1)}%
+ </td>
+ <td className="py-3 pr-5 text-right tabular-nums">
+ {row.render_rate_percent.toFixed(1)}%
+ </td>
+ <td className="py-3 text-right tabular-nums">
+ {row.cost_per_46_screen_pass_usd === null
+ ? "Not available"
+ : `$${row.cost_per_46_screen_pass_usd.toFixed(2)}`}
+ </td>
+ </tr>
+ ))}
+ </tbody>
+ </table>
+ </div>
+ </section>
+
+ <section id="condition" aria-labelledby="condition-heading">
+ <h2 id="condition-heading">Controlled condition</h2>
+ <ul>
+ <li>One shared catalog of 70 equivalent components.</li>
+ <li>Each format uses the prompt generated by its own SDK.</li>
+ <li>The same two worked examples are carried by all three prompts.</li>
+ <li>Four generations per brief and a 16,384-token output ceiling.</li>
+ <li>Temperature 0.7 where supported, with minimal or no reasoning.</li>
+ <li>
+ Each SDK&rsquo;s shipped validation plus one identical shared completeness layer and
+ component-count floor.
+ </li>
+ </ul>
+ <p>
+ Every row is scored by one build of the shipped OpenUI parser, lang-core 0.2.16. See the
+ {` `}
+ <a href="/benchmarks/methodology">complete methodology and reproduction steps</a>.
+ </p>
+ </section>
+ </BenchmarkDetailPage>
+ </>
+ );
+}
diff --git a/docs/app/(home)/benchmarks/language/agent.md/route.ts b/docs/app/(home)/benchmarks/language/agent.md/route.ts
new file mode 100644
index 000000000..e342af7bc
--- /dev/null
+++ b/docs/app/(home)/benchmarks/language/agent.md/route.ts
@@ -0,0 +1,14 @@
+import { languageBenchmarkMarkdown } from "@/lib/benchmark-agent-data";
+
+export const dynamic = "force-static";
+export const revalidate = false;
+
+export async function GET() {
+ return new Response(languageBenchmarkMarkdown(), {
+ headers: {
+ "Content-Type": "text/markdown; charset=utf-8",
+ "Cache-Control": "public, max-age=0, must-revalidate",
+ "Content-Disposition": 'inline; filename="openui-language-model-benchmark.md"',
+ },
+ });
+}
diff --git a/docs/app/(home)/benchmarks/language/data.csv/route.ts b/docs/app/(home)/benchmarks/language/data.csv/route.ts
new file mode 100644
index 000000000..7ee789339
--- /dev/null
+++ b/docs/app/(home)/benchmarks/language/data.csv/route.ts
@@ -0,0 +1,14 @@
+import { modelBoardCsv } from "@/lib/benchmark-agent-data";
+
+export const dynamic = "force-static";
+export const revalidate = false;
+
+export async function GET() {
+ return new Response(modelBoardCsv(), {
+ headers: {
+ "Content-Type": "text/csv; charset=utf-8",
+ "Cache-Control": "public, max-age=0, must-revalidate",
+ "Content-Disposition": 'inline; filename="openui-language-model-benchmark.csv"',
+ },
+ });
+}
diff --git a/docs/app/(home)/benchmarks/language/data.json/route.ts b/docs/app/(home)/benchmarks/language/data.json/route.ts
new file mode 100644
index 000000000..2b169859f
--- /dev/null
+++ b/docs/app/(home)/benchmarks/language/data.json/route.ts
@@ -0,0 +1,13 @@
+import { languageBenchmarkDataset } from "@/lib/benchmark-agent-data";
+
+export const dynamic = "force-static";
+export const revalidate = false;
+
+export async function GET() {
+ return Response.json(languageBenchmarkDataset, {
+ headers: {
+ "Cache-Control": "public, max-age=0, must-revalidate",
+ "Content-Disposition": 'inline; filename="openui-language-model-benchmark.json"',
+ },
+ });
+}
diff --git a/docs/app/(home)/benchmarks/language/page.tsx b/docs/app/(home)/benchmarks/language/page.tsx
new file mode 100644
index 000000000..869c89691
--- /dev/null
+++ b/docs/app/(home)/benchmarks/language/page.tsx
@@ -0,0 +1,192 @@
+import {
+ BENCHMARK_CANONICAL_URL,
+ LANGUAGE_BENCHMARK_URL,
+ MODEL_BOARD_UPDATED_ISO,
+ modelBoardRows,
+} from "@/lib/benchmark-agent-data";
+import { BRIEFS, MODEL_BOARD_SIZE } from "@/lib/benchmark-data";
+import type { Metadata } from "next";
+import { BenchmarkDetailPage } from "../BenchmarkDetailPage";
+
+export const metadata: Metadata = {
+ title: "OpenUI language and model benchmark | OpenUI",
+ description: `Compare structural validity and measured cost across ${MODEL_BOARD_SIZE} models generating OpenUI, with exact server-rendered data and machine-readable downloads.`,
+ alternates: {
+ canonical: "/benchmarks/language",
+ types: {
+ "application/json": "/benchmarks/language/data.json",
+ "text/csv": "/benchmarks/language/data.csv",
+ "text/markdown": "/benchmarks/language/agent.md",
+ },
+ },
+ openGraph: {
+ type: "article",
+ url: "/benchmarks/language",
+ title: "OpenUI language and model benchmark",
+ description: `Structural validity and cost across ${MODEL_BOARD_SIZE} models generating OpenUI.`,
+ modifiedTime: MODEL_BOARD_UPDATED_ISO,
+ },
+};
+
+const structuredData = {
+ "@context": "https://schema.org",
+ "@type": "Dataset",
+ "@id": `${LANGUAGE_BENCHMARK_URL}#dataset`,
+ name: "OpenUI language and model benchmark",
+ description: `Structural validity and measured cost across ${MODEL_BOARD_SIZE} models generating OpenUI.`,
+ url: LANGUAGE_BENCHMARK_URL,
+ dateModified: MODEL_BOARD_UPDATED_ISO,
+ isPartOf: { "@id": `${BENCHMARK_CANONICAL_URL}#dataset` },
+ measurementTechnique: `${BENCHMARK_CANONICAL_URL}/methodology`,
+ variableMeasured: ["Structural validity percentage", "Cost per task in USD"],
+ distribution: [
+ {
+ "@type": "DataDownload",
+ encodingFormat: "application/json",
+ contentUrl: `${LANGUAGE_BENCHMARK_URL}/data.json`,
+ },
+ {
+ "@type": "DataDownload",
+ encodingFormat: "text/csv",
+ contentUrl: `${LANGUAGE_BENCHMARK_URL}/data.csv`,
+ },
+ {
+ "@type": "DataDownload",
+ encodingFormat: "text/markdown",
+ contentUrl: `${LANGUAGE_BENCHMARK_URL}/agent.md`,
+ },
+ ],
+};
+
+const costLabel = (row: (typeof modelBoardRows)[number]) =>
+ row.cost_per_task_usd === null
+ ? "Not comparable"
+ : row.cost_per_task_usd === 0
+ ? "$0 / free"
+ : `$${row.cost_per_task_usd.toFixed(4)}`;
+
+export default function LanguageBenchmarkPage() {
+ return (
+ <>
+ <script
+ type="application/ld+json"
+ dangerouslySetInnerHTML={{
+ __html: JSON.stringify(structuredData).replaceAll("<", "\\u003c"),
+ }}
+ />
+ <BenchmarkDetailPage
+ eyebrow="OpenUI benchmark · Models"
+ title="OpenUI language and model benchmark"
+ description="How reliably and economically do different models generate structurally valid OpenUI?"
+ sections={[
+ { id: "answer", label: "What this answers" },
+ { id: "results", label: "All model results" },
+ { id: "interpretation", label: "How to interpret it" },
+ { id: "downloads", label: "Agent-readable data" },
+ ]}
+ sibling={{ href: "/benchmarks/framework", label: "Framework benchmark" }}
+ downloads={[
+ { href: "/benchmarks/language/agent.md", label: "Agent Markdown" },
+ { href: "/benchmarks/language/data.json", label: "JSON" },
+ { href: "/benchmarks/language/data.csv", label: "CSV" },
+ { href: "/benchmarks/data.json", label: "Combined dataset" },
+ ]}
+ >
+ <section id="answer" aria-labelledby="answer-heading">
+ <h2 id="answer-heading">What this benchmark answers</h2>
+ <p>
+ This benchmark compares {modelBoardRows.length} models under the same OpenUI task:{" "}
+ {BRIEFS} interface briefs, four generations per brief, scored for structural validity.
+ The chart pairs that score with measured list-price cost per generated task where a
+ comparable API price exists.
+ </p>
+ <p>
+ A high score means the generated component graph parsed, produced a root, resolved its
+ references, used valid required and enum props, avoided truncation, and met the shared
+ component-count floor. It is not a general intelligence score.
+ </p>
+ </section>
+
+ <section id="results" aria-labelledby="results-heading">
+ <h2 id="results-heading">All model results</h2>
+ <p>
+ All {modelBoardRows.length} rows are present below, including the five compact or local
+ models deselected by default in the visual chart, which opens on models at or above 70%.
+ </p>
+ <div className="not-prose overflow-x-auto">
+ <table className="w-full min-w-[760px] border-collapse text-left text-sm">
+ <caption className="sr-only">
+ OpenUI structural validity and measured cost across models
+ </caption>
+ <thead>
+ <tr className="border-b border-[color:var(--home-hairline)]">
+ <th scope="col" className="py-3 pr-5 font-medium">
+ Model
+ </th>
+ <th scope="col" className="py-3 pr-5 font-medium">
+ Provider
+ </th>
+ <th scope="col" className="py-3 pr-5 text-right font-medium">
+ Valid
+ </th>
+ <th scope="col" className="py-3 pr-5 text-right font-medium">
+ Cost / task
+ </th>
+ <th scope="col" className="py-3 pr-5 font-medium">
+ Pricing
+ </th>
+ <th scope="col" className="py-3 pr-5 font-medium">
+ Pareto
+ </th>
+ <th scope="col" className="py-3 font-medium">
+ Default view
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ {modelBoardRows.map((row) => (
+ <tr key={row.model_id} className="border-b border-[color:var(--home-hairline)]">
+ <th scope="row" className="py-3 pr-5 font-medium">
+ {row.model_name}
+ </th>
+ <td className="py-3 pr-5">{row.provider}</td>
+ <td className="py-3 pr-5 text-right tabular-nums">
+ {row.validity_score_percent.toFixed(1)}%
+ </td>
+ <td className="py-3 pr-5 text-right tabular-nums">{costLabel(row)}</td>
+ <td className="py-3 pr-5">{row.cost_type}</td>
+ <td className="py-3 pr-5">{row.pareto_frontier ? "Yes" : "No"}</td>
+ <td className="py-3">{row.default_selected ? "Shown" : "Hidden"}</td>
+ </tr>
+ ))}
+ </tbody>
+ </table>
+ </div>
+ </section>
+
+ <section id="interpretation" aria-labelledby="interpretation-heading">
+ <h2 id="interpretation-heading">How to interpret it</h2>
+ <ul>
+ <li>Provider color identifies a company; it is not a quality highlight.</li>
+ <li>
+ A Pareto point is not dominated on both measured cost and structural validity. It is
+ not a general model ranking.
+ </li>
+ <li>
+ Self-hosted cost is recorded as null because there is no comparable API list price.
+ Null does not mean free.
+ </li>
+ <li>
+ Every row is scored by one build of the shipped OpenUI parser, lang-core 0.2.16, whose
+ parser validates enum and scalar prop values.
+ </li>
+ </ul>
+ <p>
+ The exact generation and scoring condition is documented on the{` `}
+ <a href="/benchmarks/methodology">methodology page</a>.
+ </p>
+ </section>
+ </BenchmarkDetailPage>
+ </>
+ );
+}
diff --git a/docs/app/(home)/benchmarks/methodology/page.tsx b/docs/app/(home)/benchmarks/methodology/page.tsx
index d1c705de5..1b1b54d78 100644
--- a/docs/app/(home)/benchmarks/methodology/page.tsx
+++ b/docs/app/(home)/benchmarks/methodology/page.tsx
@@ -1,25 +1,63 @@
-import { GitHubButton } from "@/app/(home)/components/GitHubButton/GitHubButton";
import { Footer } from "@/app/(home)/sections/Footer/Footer";
+import { BENCHMARK_CANONICAL_URL } from "@/lib/benchmark-agent-data";
+import {
+ BRIEFS,
+ CATALOG_COMPONENTS,
+ FORMATS,
+ LINKS,
+ MODELS,
+ REPO_ROOT,
+ RUNS_PER_FORMAT,
+} from "@/lib/benchmark-data";
+import { ArrowRight, ArrowUpRight } from "lucide-react";
import type { Metadata } from "next";
+import { BevelButton } from "../../components/Button/BevelButton";
import { PillLink } from "../../components/Button/Button";
+import benchmarkStyles from "../benchmarks.module.css";
export const metadata: Metadata = {
title: "Benchmark methodology | OpenUI",
description:
- "How the OpenUI generative UI benchmark is run and scored: the models, the component catalog, each SDK's own prompts and parsers, and the two rubrics.",
+ "How the OpenUI generative UI benchmark is run and scored: 46 briefs, 70 equivalent components, controlled generation settings, and SDK-native validation.",
+ alternates: { canonical: "/benchmarks/methodology" },
+ openGraph: {
+ type: "article",
+ url: "/benchmarks/methodology",
+ title: "Generative UI benchmark methodology | OpenUI",
+ description: "A concise explanation of how the benchmark is generated and scored.",
+ modifiedTime: "2026-08-26",
+ },
};
const SECTIONS = [
- { id: "how-we-ran-it", label: "How we ran it" },
- { id: "rubrics", label: "Rubrics" },
-];
+ { id: "setup", label: "Benchmark setup" },
+ { id: "scoring", label: "What counts as valid" },
+ { id: "reproduce", label: "Reproduce the results" },
+ { id: "notes", label: "Important notes" },
+] as const;
+
+const structuredData = {
+ "@context": "https://schema.org",
+ "@type": "TechArticle",
+ "@id": `${BENCHMARK_CANONICAL_URL}/methodology#article`,
+ headline: "Generative UI benchmark methodology",
+ description: "How the OpenUI generative UI benchmark is generated, scored, and reproduced.",
+ url: `${BENCHMARK_CANONICAL_URL}/methodology`,
+ dateModified: "2026-08-26",
+ author: { "@type": "Organization", name: "OpenUI by Thesys" },
+ isPartOf: { "@id": `${BENCHMARK_CANONICAL_URL}#dataset` },
+ codeRepository: REPO_ROOT,
+};
-/* Same layout as a blog post (app/blog/[slug]/page.tsx) — gradient band, TOC
- rail, prose column, footer — with the post's Methodology section as the
- body. The text is the blog's, not a second version of it. */
export default function BenchmarkMethodologyPage() {
return (
<>
+ <script
+ type="application/ld+json"
+ dangerouslySetInnerHTML={{
+ __html: JSON.stringify(structuredData).replaceAll("<", "\\u003c"),
+ }}
+ />
<div className="min-h-screen bg-[linear-gradient(to_bottom,var(--openui-foreground),var(--openui-background)_10rem,var(--openui-background)_calc(100%_-_10rem),var(--openui-foreground))] [[data-theme=dark]_&]:bg-[linear-gradient(to_bottom,#000,var(--swatch-neutral-950)_28rem,var(--swatch-neutral-950)_calc(100%_-_28rem),#000)]">
<main className="mx-auto flex w-full max-w-[var(--home-container-wide)] gap-4 px-4 pt-16 pb-40 lg:gap-28 lg:pr-8 min-[1249px]:pl-0 min-[1024px]:max-[1248px]:pl-8">
<aside className="hidden w-56 shrink-0 lg:block">
@@ -40,60 +78,137 @@ export default function BenchmarkMethodologyPage() {
Methodology
</h1>
<p className="mb-4 text-[length:var(--home-lead-size)] font-[family-name:var(--home-font-text)] leading-[var(--home-lead-leading)] text-[color:var(--openui-text-neutral-secondary)]">
- How the generative UI benchmark is run and scored.
+ One catalog, the same 46 interface briefs, and a shared scoring standard.
</p>
- <div className="-ml-3 flex flex-wrap items-center gap-1 border-b border-[color:var(--home-hairline)] pb-6">
- <PillLink href="/benchmarks" variant="ghost">
- Benchmarks
- </PillLink>
- <PillLink href="/blog/generative-ui-benchmark" variant="ghost">
- Read the full write-up
+ <nav
+ aria-label="Methodology actions"
+ className="flex flex-wrap items-center gap-2 border-b border-[color:var(--home-hairline)] pb-6"
+ >
+ <BevelButton
+ href="/benchmarks"
+ label="Benchmark"
+ variant="secondary"
+ className={benchmarkStyles.heroButton}
+ badge={
+ <ArrowRight
+ className={benchmarkStyles.actionIcon}
+ strokeWidth={2.25}
+ aria-hidden="true"
+ />
+ }
+ />
+ <PillLink
+ href={REPO_ROOT}
+ variant="ghost"
+ external
+ arrow={<ArrowUpRight size={16} strokeWidth={2.25} aria-hidden="true" />}
+ >
+ Source
</PillLink>
- </div>
+ </nav>
<article className="prose mt-8 min-w-0">
- <h2 id="how-we-ran-it">How we ran it</h2>
- <p>
- We chose 6 frontier models from various model families to test the 3 different
- Generative UI formats, OpenUI, Google A2UI and Vercel json-render.
- </p>
- <p>
- For all of these combinations we used the same catalog of 70 components, built
- entirely from OpenUI&rsquo;s public component library: the open-source chat set, six
- chat blocks from the same library, and twelve components from the public shadcn-chat
- example. The exact catalog is committed in the repo.
- </p>
- <p>
- For each format we used that SDK&rsquo;s own official prompt generator and parser,
- and gave all three the same two worked examples, so no format was hand-tuned against
- the others.
- </p>
- <p>Every model ran every prompt 4 times, giving 1,104 runs per format.</p>
- <p>
- In total 46 different prompts were used that mirror real life requests. None of them
- name a component or a layout to mimic real life user queries.
- </p>
+ <section id="setup" aria-labelledby="setup-heading">
+ <h2 id="setup-heading">Benchmark setup</h2>
+ <p>
+ We test how reliably models generate working interfaces, without telling them
+ which components or layouts to use.
+ </p>
+ <ul>
+ <li>
+ <strong>{BRIEFS} briefs</strong> across five complexity bands, from simple
+ screens to dense workspaces.
+ </li>
+ <li>
+ <strong>{CATALOG_COMPONENTS} equivalent components</strong> available to every
+ format.
+ </li>
+ <li>
+ <strong>Four generations per brief</strong>, with a 16,384-token output limit.
+ </li>
+ <li>
+ <strong>Temperature 0.7</strong> where supported, with minimal or no reasoning.
+ </li>
+ </ul>
+ <p>
+ For the framework comparison, the same briefs run across {MODELS.length} models
+ and {FORMATS.length} formats: OpenUI, Google A2UI, and Vercel json-render. That is{" "}
+ {RUNS_PER_FORMAT.toLocaleString("en-US")} generations per format.
+ </p>
+ <p>
+ Each format uses the prompt generated by its own SDK. All three receive the same
+ two worked examples, and their component catalogs are checked against one shared
+ reference catalog.
+ </p>
+ </section>
- <h2 id="rubrics">Rubrics</h2>
- <p>For evaluation we used the following rubrics:</p>
- <ul>
- <li>
- <strong>Structural validity:</strong> a run passes when it parses, has a root,
- every reference resolves, nothing is orphaned or invented, required props and
- enums are valid, and the output is not truncated. It also needs at least as many
- components as the brief has requirements; that is a count floor, not a check that
- each requirement was addressed. Higher is better.
- </li>
- <li>
- <strong>Render success:</strong> whether anything reached the screen at all,
- measured separately from structural validity. Higher is better.
- </li>
- </ul>
- </article>
+ <section id="scoring" aria-labelledby="scoring-heading">
+ <h2 id="scoring-heading">What counts as valid</h2>
+ <p>
+ Each format is parsed and validated by its own shipped SDK, followed by the same
+ completeness checks for all three formats.
+ </p>
+ <p>A generation passes structural validity when it:</p>
+ <ul>
+ <li>parses and produces a root;</li>
+ <li>uses known components and valid required or enum props;</li>
+ <li>resolves every reference, with nothing orphaned;</li>
+ <li>does not end because it hit the output limit; and</li>
+ <li>
+ contains at least as many reachable components as the brief has requirements.
+ </li>
+ </ul>
+ <p>
+ <strong>Render success is separate.</strong> It records whether a non-blank screen
+ appeared at all. A screen can render and still fail structural validity.
+ </p>
+ </section>
- <div className="mt-12 border-t border-[color:var(--home-hairline)] pt-8">
- <GitHubButton variant="desktopGlow" />
- </div>
+ <section id="reproduce" aria-labelledby="reproduce-heading">
+ <h2 id="reproduce-heading">Reproduce the results</h2>
+ <p>
+ Raw model outputs and scored verdicts are committed to the benchmark repository.
+ The scorer runs offline, so the published outputs can be rescored without model
+ API keys. Recreating the committed result files with no diff is the integrity
+ check.
+ </p>
+ <p>
+ The repository README contains the exact Node and A2UI Python setup, plus commands
+ for rescoring one model or the full benchmark.
+ </p>
+ <p>
+ <a href={LINKS.latestReadme}>Read the reproduction instructions</a> or inspect the{" "}
+ <a href={LINKS.rawOutputs}>published raw outputs</a>.
+ </p>
+ </section>
+
+ <section id="notes" aria-labelledby="notes-heading">
+ <h2 id="notes-heading">Important notes</h2>
+ <div className="not-prose my-6 border-l-2 border-[color:var(--openui-text-neutral-primary)] pl-5 text-sm leading-6 text-[color:var(--openui-text-neutral-secondary)]">
+ Every row is scored by one build of the shipped OpenUI parser,{` `}
+ <strong>lang-core 0.2.16</strong>, whose parser validates enum and scalar prop
+ values. Raw outputs and per-run verdicts are committed for independent rescoring.
+ </div>
+ <ul>
+ <li>
+ The component-count floor prevents tiny valid outputs from passing, but it does
+ not judge visual quality or prove every requirement was understood.
+ </li>
+ <li>
+ Cost uses measured tokens and list prices. Hidden reasoning tokens can make the
+ actual bill higher.
+ </li>
+ <li>
+ Self-hosted model costs are marked as not comparable rather than treated as
+ free.
+ </li>
+ <li>
+ OpenUI built this benchmark. The method and outputs are published so the results
+ can be inspected and challenged.
+ </li>
+ </ul>
+ </section>
+ </article>
</div>
</main>
</div>
diff --git a/docs/app/(home)/benchmarks/page.tsx b/docs/app/(home)/benchmarks/page.tsx
index d64e29979..b7053e064 100644
--- a/docs/app/(home)/benchmarks/page.tsx
+++ b/docs/app/(home)/benchmarks/page.tsx
@@ -1,3 +1,12 @@
+import {
+ BENCHMARK_CANONICAL_URL,
+ BENCHMARK_DATASET_NAME,
+ BENCHMARK_UPDATED_ISO,
+ FRAMEWORK_BENCHMARK_URL,
+ LANGUAGE_BENCHMARK_URL,
+ MODEL_BOARD_UPDATED_ISO,
+} from "@/lib/benchmark-agent-data";
+import { BENCHMARK_VERSION, LINKS, MODEL_BOARD_SIZE } from "@/lib/benchmark-data";
import type { Metadata } from "next";
import { Footer } from "../sections/Footer/Footer";
import { BenchmarksContent } from "./BenchmarksContent";
@@ -5,11 +14,97 @@ import { BenchmarksContent } from "./BenchmarksContent";
export const metadata: Metadata = {
title: "Benchmarks | OpenUI",
description: `We generated the same 46 interfaces with 6 models and 3 UI formats. OpenUI produced the fewest blank screens, used the fewest tokens, and cost the least on every priced model.`,
+ alternates: {
+ canonical: "/benchmarks",
+ types: {
+ "application/json": "/benchmarks/data.json",
+ "application/schema+json": "/benchmarks/data.schema.json",
+ "text/csv": "/benchmarks/data.csv",
+ "text/markdown": "/benchmarks/agent.md",
+ },
+ },
+ openGraph: {
+ type: "article",
+ url: "/benchmarks",
+ title: "Generative UI Benchmark | OpenUI",
+ description: `OpenUI results across ${MODEL_BOARD_SIZE} models, plus a six-model comparison across OpenUI, A2UI and json-render.`,
+ modifiedTime: MODEL_BOARD_UPDATED_ISO,
+ },
+ twitter: {
+ card: "summary_large_image",
+ title: "Generative UI Benchmark | OpenUI",
+ description: `OpenUI results across ${MODEL_BOARD_SIZE} models, plus a six-model comparison across OpenUI, A2UI and json-render.`,
+ },
+};
+
+const structuredData = {
+ "@context": "https://schema.org",
+ "@graph": [
+ {
+ "@type": "Dataset",
+ "@id": `${BENCHMARK_CANONICAL_URL}#dataset`,
+ name: BENCHMARK_DATASET_NAME,
+ description: `OpenUI benchmark measurements across ${MODEL_BOARD_SIZE} models, plus a controlled six-model comparison of OpenUI, A2UI and json-render over 46 interface briefs.`,
+ url: BENCHMARK_CANONICAL_URL,
+ version: BENCHMARK_VERSION,
+ dateModified: MODEL_BOARD_UPDATED_ISO,
+ creator: { "@type": "Organization", name: "OpenUI by Thesys", url: "https://www.openui.com" },
+ isBasedOn: LINKS.rawData,
+ measurementTechnique: `${BENCHMARK_CANONICAL_URL}/methodology`,
+ variableMeasured: [
+ "Structural validity percentage",
+ "Render success percentage",
+ "Cost per task in USD",
+ "Token consumption",
+ "Estimated streaming time",
+ ],
+ distribution: [
+ {
+ "@type": "DataDownload",
+ encodingFormat: "application/json",
+ contentUrl: `${BENCHMARK_CANONICAL_URL}/data.json`,
+ },
+ {
+ "@type": "DataDownload",
+ encodingFormat: "text/csv",
+ contentUrl: `${BENCHMARK_CANONICAL_URL}/data.csv`,
+ },
+ {
+ "@type": "DataDownload",
+ encodingFormat: "text/markdown",
+ contentUrl: `${BENCHMARK_CANONICAL_URL}/agent.md`,
+ },
+ ],
+ subjectOf: `${BENCHMARK_CANONICAL_URL}/data.schema.json`,
+ hasPart: [
+ { "@id": `${LANGUAGE_BENCHMARK_URL}#dataset` },
+ { "@id": `${FRAMEWORK_BENCHMARK_URL}#dataset` },
+ ],
+ },
+ {
+ "@type": "TechArticle",
+ "@id": `${BENCHMARK_CANONICAL_URL}#article`,
+ headline: "Generative UI Benchmark",
+ description: `A controlled comparison of generative UI formats and an OpenUI model board covering ${MODEL_BOARD_SIZE} models.`,
+ mainEntityOfPage: BENCHMARK_CANONICAL_URL,
+ datePublished: BENCHMARK_UPDATED_ISO,
+ dateModified: MODEL_BOARD_UPDATED_ISO,
+ author: { "@type": "Organization", name: "OpenUI by Thesys" },
+ about: { "@id": `${BENCHMARK_CANONICAL_URL}#dataset` },
+ mentions: [LANGUAGE_BENCHMARK_URL, FRAMEWORK_BENCHMARK_URL],
+ },
+ ],
};
export default function BenchmarksPage() {
return (
<>
+ <script
+ type="application/ld+json"
+ dangerouslySetInnerHTML={{
+ __html: JSON.stringify(structuredData).replaceAll("<", "\\u003c"),
+ }}
+ />
<BenchmarksContent />
<Footer />
</>
diff --git a/docs/app/llms.txt/route.ts b/docs/app/llms.txt/route.ts
index 1682b23f6..e21bc03c4 100644
--- a/docs/app/llms.txt/route.ts
+++ b/docs/app/llms.txt/route.ts
@@ -29,6 +29,21 @@ export async function GET() {
"",
...pages,
"",
+ "## Benchmarks",
+ "",
+ `- [Generative UI Benchmark](${new URL("/benchmarks", BASE_URL)}): Visual benchmark page with server-rendered summaries and semantic data tables.`,
+ `- [OpenUI language and model benchmark](${new URL("/benchmarks/language", BASE_URL)}): Focused, canonical comparison of structural validity and cost across 30 models generating OpenUI.`,
+ `- [Language/model benchmark agent document](${new URL("/benchmarks/language/agent.md", BASE_URL)}): Plain Markdown with all model-board results and interpretation notes.`,
+ `- [Language/model benchmark JSON](${new URL("/benchmarks/language/data.json", BASE_URL)}): Focused machine-readable model-board dataset.`,
+ `- [Generative UI framework benchmark](${new URL("/benchmarks/framework", BASE_URL)}): Focused, canonical comparison of OpenUI, Google A2UI, and Vercel json-render.`,
+ `- [Framework benchmark agent document](${new URL("/benchmarks/framework/agent.md", BASE_URL)}): Plain Markdown with format summaries and all model-format rows.`,
+ `- [Framework benchmark JSON](${new URL("/benchmarks/framework/data.json", BASE_URL)}): Focused machine-readable framework-comparison dataset.`,
+ `- [Benchmark agent document](${new URL("/benchmarks/agent.md", BASE_URL)}): Plain Markdown definitions, headline results, model board, provenance and distribution links.`,
+ `- [Benchmark JSON dataset](${new URL("/benchmarks/data.json", BASE_URL)}): Versioned machine-readable benchmark dataset.`,
+ `- [Benchmark JSON Schema](${new URL("/benchmarks/data.schema.json", BASE_URL)}): Field definitions and validation contract for the benchmark JSON dataset.`,
+ `- [OpenUI model-board CSV](${new URL("/benchmarks/data.csv", BASE_URL)}): One row per model with provider, family, score, pricing type and frontier membership.`,
+ `- [Benchmark methodology](${new URL("/benchmarks/methodology", BASE_URL)}): How the benchmark is run and scored.`,
+ "",
"## Optional",
"",
`- [Complete OpenUI documentation](${new URL("/llms-full.txt", BASE_URL)}): Full documentation in one plain-text response; it may exceed an LLM context window.`,
diff --git a/docs/app/sitemap.ts b/docs/app/sitemap.ts
index bfa0c74f6..be1d8a040 100644
--- a/docs/app/sitemap.ts
+++ b/docs/app/sitemap.ts
@@ -10,6 +10,20 @@ const STATIC_PATHS = [
"/integrations",
"/chat",
"/blog",
+ "/benchmarks",
+ "/benchmarks/language",
+ "/benchmarks/language/agent.md",
+ "/benchmarks/language/data.json",
+ "/benchmarks/language/data.csv",
+ "/benchmarks/framework",
+ "/benchmarks/framework/agent.md",
+ "/benchmarks/framework/data.json",
+ "/benchmarks/framework/data.csv",
+ "/benchmarks/methodology",
+ "/benchmarks/agent.md",
+ "/benchmarks/data.json",
+ "/benchmarks/data.schema.json",
+ "/benchmarks/data.csv",
];
export default async function sitemap() {
diff --git a/docs/components/charts/benchmark-charts.tsx b/docs/components/charts/benchmark-charts.tsx
index 717ae3455..6e6e00f62 100644
--- a/docs/components/charts/benchmark-charts.tsx
+++ b/docs/components/charts/benchmark-charts.tsx
@@ -21,7 +21,16 @@ import {
tokens,
} from "@/lib/benchmark-data";
import type { ReactNode } from "react";
-import { Chart, Chip, Row, styles as s, slotClass, useVizSkin } from "./primitives";
+import {
+ Chart,
+ ChartDataDisclosure,
+ Chip,
+ DataTable,
+ Row,
+ styles as s,
+ slotClass,
+ useVizSkin,
+} from "./primitives";
const slotOf = (id: FormatId) => FORMATS.find((f) => f.id === id)!.series;
const markOf = (id: FormatId) => FORMATS.find((f) => f.id === id)!.mark;
@@ -70,8 +79,12 @@ export function CompletionByModel({
>
<MarketingTable
compact
+ edgeToEdgeMobile
className={`${s.benchmarkTable} ${s.tableWide} ${flatTable ? s.tableFlat : ""}`}
>
+ <caption className={s.tableCaption}>
+ Structural validity percentage for each model and generative UI format
+ </caption>
<thead>
<tr>
<th scope="col">Model</th>
@@ -388,6 +401,37 @@ export function TokenMatrix({
);
})}
</div>
+ <ChartDataDisclosure label="View token data">
+ <DataTable>
+ <caption>System prompt and mean output tokens per screen by format</caption>
+ <thead>
+ <tr>
+ <th scope="col">Format</th>
+ <th scope="col" data-numeric="true">
+ System prompt
+ </th>
+ <th scope="col" data-numeric="true">
+ Mean output
+ </th>
+ <th scope="col" data-numeric="true">
+ Combined
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ {shown.map((format) => (
+ <tr key={`data-${format}`}>
+ <th scope="row">{formatLabel(format)}</th>
+ <td data-numeric="true">{tokens.systemPrompt[format].toLocaleString()}</td>
+ <td data-numeric="true">{tokens.outputPerScreen[format].toLocaleString()}</td>
+ <td data-numeric="true">
+ {(tokens.systemPrompt[format] + tokens.outputPerScreen[format]).toLocaleString()}
+ </td>
+ </tr>
+ ))}
+ </tbody>
+ </DataTable>
+ </ChartDataDisclosure>
</Chart>
);
}
@@ -435,7 +479,14 @@ export function CostPerPass({
a priced model to the filter.
</p>
) : (
- <MarketingTable compact className={`${s.benchmarkTable} ${flatTable ? s.tableFlat : ""}`}>
+ <MarketingTable
+ compact
+ edgeToEdgeMobile
+ className={`${s.benchmarkTable} ${flatTable ? s.tableFlat : ""}`}
+ >
+ <caption className={s.tableCaption}>
+ Cost in USD for one 46-screen benchmark pass by model and generative UI format
+ </caption>
<thead>
<tr>
<th scope="col">Model</th>
diff --git a/docs/components/charts/blank-screens-panels.tsx b/docs/components/charts/blank-screens-panels.tsx
index aed96ec7a..0e62ce12f 100644
--- a/docs/components/charts/blank-screens-panels.tsx
+++ b/docs/components/charts/blank-screens-panels.tsx
@@ -10,7 +10,15 @@ import {
formatLabel,
runsFor,
} from "@/lib/benchmark-data";
-import { Chart, Chip, Mark, styles as s, slotClass } from "./primitives";
+import {
+ Chart,
+ ChartDataDisclosure,
+ Chip,
+ DataTable,
+ Mark,
+ styles as s,
+ slotClass,
+} from "./primitives";
/* Render rate per model, one panel per format. Framed as the share of runs
that put a screen in front of the user, so OpenUI's near-perfect
@@ -133,6 +141,43 @@ export function BlankScreensPanels({
</span>
))}
</div>
+ <ChartDataDisclosure label="View render data">
+ <DataTable>
+ <caption>Rendered and blank runs for every model and generative UI format</caption>
+ <thead>
+ <tr>
+ <th scope="col">Model</th>
+ <th scope="col">Format</th>
+ <th scope="col" data-numeric="true">
+ Rendered
+ </th>
+ <th scope="col" data-numeric="true">
+ Blank
+ </th>
+ <th scope="col" data-numeric="true">
+ Render rate
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ {shownFormats.flatMap((format) =>
+ MODELS.filter((model) => selected.includes(model.id)).map((model) => {
+ const blanks = blanksOver(format, [model.id]);
+ const runs = runsFor(model.id);
+ return (
+ <tr key={`data-${format}-${model.id}`}>
+ <th scope="row">{fullName(model)}</th>
+ <td>{formatLabel(format)}</td>
+ <td data-numeric="true">{runs - blanks}</td>
+ <td data-numeric="true">{blanks}</td>
+ <td data-numeric="true">{rate(format, model.id).toFixed(1)}%</td>
+ </tr>
+ );
+ }),
+ )}
+ </tbody>
+ </DataTable>
+ </ChartDataDisclosure>
</Chart>
);
}
diff --git a/docs/components/charts/brand-marks.ts b/docs/components/charts/brand-marks.ts
index b40082a63..58462ab6a 100644
--- a/docs/components/charts/brand-marks.ts
+++ b/docs/components/charts/brand-marks.ts
@@ -28,3 +28,8 @@ export const BRAND_VIEWBOX: Record<string, string> = {
};
export const markViewBox = (id?: string) => (id && BRAND_VIEWBOX[id]) || "0 0 24 24";
+
+/* Brand colours for the model marks. Hexes are simple-icons' published values
+ except Anthropic, which uses its clay brand colour rather than the black
+ wordmark. Marks that are monochrome by design (OpenAI, Moonshot) take the
+ theme's ink so they stay visible in both light and dark. */
diff --git a/docs/components/charts/completion-by-density.tsx b/docs/components/charts/completion-by-density.tsx
index 2e33f452b..7e98df108 100644
--- a/docs/components/charts/completion-by-density.tsx
+++ b/docs/components/charts/completion-by-density.tsx
@@ -10,7 +10,14 @@ import {
completionByDensityFor,
} from "@/lib/benchmark-data";
import { useEffect, useRef, useState } from "react";
-import { Chart, styles as s, slotClass, useVizSkin } from "./primitives";
+import {
+ Chart,
+ ChartDataDisclosure,
+ DataTable,
+ styles as s,
+ slotClass,
+ useVizSkin,
+} from "./primitives";
/* Drawn at the container's real pixel width. the SVG never scales, so text
is always its true size and the plot always fills the card. */
@@ -254,6 +261,37 @@ export function CompletionByDensity({
</span>
))}
</div>
+ <ChartDataDisclosure label="View complexity data">
+ <DataTable>
+ <caption>Structural validity by brief requirement count and generative UI format</caption>
+ <thead>
+ <tr>
+ <th scope="col">Requirements per screen</th>
+ <th scope="col" data-numeric="true">
+ Runs
+ </th>
+ {shownFormats.map((format) => (
+ <th key={format} scope="col" data-numeric="true">
+ {FORMATS.find((item) => item.id === format)!.label}
+ </th>
+ ))}
+ </tr>
+ </thead>
+ <tbody>
+ {completionByDensity.map((row) => (
+ <tr key={`data-${row.band}`}>
+ <th scope="row">{row.band}</th>
+ <td data-numeric="true">{row.runs}</td>
+ {shownFormats.map((format) => (
+ <td key={format} data-numeric="true">
+ {row[format].toFixed(1)}%
+ </td>
+ ))}
+ </tr>
+ ))}
+ </tbody>
+ </DataTable>
+ </ChartDataDisclosure>
</Chart>
);
}
diff --git a/docs/components/charts/primitives.tsx b/docs/components/charts/primitives.tsx
index c22873835..0376b1fb7 100644
--- a/docs/components/charts/primitives.tsx
+++ b/docs/components/charts/primitives.tsx
@@ -1,6 +1,6 @@
"use client";
-import { createContext, useContext, useLayoutEffect, useRef, type ReactNode } from "react";
+import { createContext, useContext, useId, useLayoutEffect, useRef, type ReactNode } from "react";
import { BRAND_MARKS, markViewBox } from "./brand-marks";
import { MASCOT_SHAPES, MASCOT_VIEWBOX } from "./openui-mascot";
import s from "./viz.module.css";
@@ -116,11 +116,24 @@ export function Chart({
}) {
const { vivid } = useVizSkin();
const revealRef = useReveal(vivid);
+ const headingId = useId();
+ const descriptionId = useId();
return (
- <figure ref={revealRef} className={`${s.viz} ${tight ? s.tight : ""} ${vivid ? s.vivid : ""}`}>
+ <figure
+ ref={revealRef}
+ className={`${s.viz} ${tight ? s.tight : ""} ${vivid ? s.vivid : ""}`}
+ aria-labelledby={headingId}
+ aria-describedby={sub ? descriptionId : undefined}
+ >
<figcaption className={s.head}>
- <p className={s.title}>{title}</p>
- {sub ? <p className={s.sub}>{sub}</p> : null}
+ <p id={headingId} className={s.title}>
+ {title}
+ </p>
+ {sub ? (
+ <p id={descriptionId} className={s.sub}>
+ {sub}
+ </p>
+ ) : null}
</figcaption>
{children}
{legend ? (
@@ -138,6 +151,40 @@ export function Chart({
);
}
+/**
+ * A disclosure table together with its own horizontal scroller. The scroller
+ * wraps the table and nothing else, so a wide table slides sideways while the
+ * heading and note above it stay where they are, and two tables inside one
+ * disclosure scroll independently instead of moving as one sheet.
+ */
+export function DataTable({ children }: { children: ReactNode }) {
+ return (
+ <div className={s.dataScroller}>
+ <table className={s.dataTable}>{children}</table>
+ </div>
+ );
+}
+
+/**
+ * A visible-on-demand semantic equivalent for a chart. It stays in the
+ * server-rendered document, so crawlers and assistive technology can recover
+ * exact values without executing hover interactions or interpreting pixels.
+ */
+export function ChartDataDisclosure({
+ label = "View data",
+ children,
+}: {
+ label?: string;
+ children: ReactNode;
+}) {
+ return (
+ <details className={s.dataDisclosure}>
+ <summary>{label}</summary>
+ <div className={s.dataDisclosureBody}>{children}</div>
+ </details>
+ );
+}
+
/** One 32px row: logo chip + name on the left, bar and value in the body. */
export function Row({
label,
diff --git a/docs/components/charts/reliability-by-model.tsx b/docs/components/charts/reliability-by-model.tsx
index 473b271c0..0515650e5 100644
--- a/docs/components/charts/reliability-by-model.tsx
+++ b/docs/components/charts/reliability-by-model.tsx
@@ -4,47 +4,119 @@ import {
COST_MODELS,
FORMATS,
FORMAT_ORDER,
- type FormatId,
MODELS,
- type ModelId,
+ OPENUI_MODEL_BOARD,
completionByModel,
costPerPass,
formatLabel,
+ modelBoardCostPerTask,
+ modelBoardDefaultSelected,
+ modelBoardFamilyFor,
+ modelBoardFrontier,
+ type FormatId,
+ type ModelBoardProvider,
+ type ModelId,
} from "@/lib/benchmark-data";
-import { useEffect, useRef, useState } from "react";
+import { useEffect, useRef, useState, type CSSProperties } from "react";
import { BRAND_MARKS, markViewBox } from "./brand-marks";
-import { Chart, Mark, styles as s } from "./primitives";
+import { Chart, ChartDataDisclosure, DataTable, Mark, styles as s } from "./primitives";
-/* The hero chart: a cost-versus-reliability frontier, one line per format.
- Cost runs right-to-left toward $0 (the CursorBench convention), so the best
- place on the chart is the top right: reliable and cheap. One point per
- model, all six priced at list prices. */
-const H = 440;
-const PAD = { top: 56, bottom: 64 };
-const PH = H - PAD.top - PAD.bottom;
+/* Both tabs share one plot box so switching views does not resize the hero.
+ The box is measured inside the component because its height depends on the
+ available width: a 720px plot on a phone is a canyon, not a chart. */
+const PAD = { top: 44, bottom: 68 };
+type View = "board" | "formats";
-/* chart-local palette (see .frontier in viz.module.css): this is the one
- chart where all three formats get a real colour */
-const hue: Record<FormatId, string> = {
+const FORMAT_HUE: Record<FormatId, string> = {
openui: "var(--pO)",
- /* the competitors keep the same two greys they wear in the bar charts */
a2ui: "var(--c2)",
jsonRender: "var(--c3)",
};
-
+const PROVIDER_HUE: Record<ModelBoardProvider, string> = {
+ xAI: "#5f6470",
+ OpenAI: "#0f9f79",
+ Anthropic: "#d26a3a",
+ Google: "#3978e6",
+ Moonshot: "#7655d6",
+ Meta: "#6059cf",
+ Alibaba: "#b88308",
+ Zhipu: "#0d9488",
+ DeepSeek: "#0891b2",
+ "Thinking Machines": "#c34a91",
+ Microsoft: "#2774c8",
+ Mistral: "#dc5a4f",
+ IBM: "#4d6fb8",
+ Liquid: "#c04f79",
+ InclusionAI: "#7c6f64",
+};
const MODEL_NAME: Record<ModelId, string> = {
sol: "GPT-5.6 Sol",
opus: "Claude Opus 4.8",
kimi: "Kimi K3",
- gemini: "Gemini 3.6 Flash",
+ gemini: "Gemini 3.7 Flash",
qwen: "Qwen3.8 2.4T",
muse: "Muse Spark 1.2",
};
+const PROVIDERS = Array.from(new Set(OPENUI_MODEL_BOARD.map((p) => p.provider)));
+/* The page's established geometry puts cheaper models on the right, so the
+ best trade-off is consistently top-right. Log spacing keeps sub-cent API
+ prices distinguishable. Free models use a tiny display floor; their exact
+ zero price remains available in the tooltip. */
+/* Half-decade steps (1 and 3 per decade) keep the ticks essentially evenly
+ spaced on a log axis while staying round enough to read. A constant ratio is
+ what makes spacing even; mixed ratios are what made the old axis lurch. */
+const decadeFloor = (value: number) => 10 ** Math.floor(Math.log10(value));
+const decadeCeil = (value: number) => 10 ** Math.ceil(Math.log10(value));
+const boardTicksBetween = (min: number, max: number) => {
+ const decades = Math.log10(max) - Math.log10(min);
+ /* half-decade steps over a short span, whole decades once the axis is wide,
+ so the tick count stays around five however much is selected */
+ const step = decades > 2.5 ? 1 : 0.5;
+ const ticks: number[] = [];
+ for (let exponent = Math.log10(max); exponent >= Math.log10(min) - 1e-9; exponent -= step) {
+ const value = 10 ** exponent;
+ /* snap the half-decade to its round neighbour: 0.0316 reads as 0.03 */
+ ticks.push(Number(value.toPrecision(1)));
+ }
+ return ticks;
+};
+
+/* enough decimals for the value, never a rounded-to-nothing "$0.0000" */
+const boardCostLabel = (value: number) => {
+ const decimals = value >= 0.01 ? 2 : value >= 0.001 ? 3 : value >= 0.0001 ? 4 : 5;
+ return `$${value.toFixed(decimals)}`;
+};
+/* The score axis frames whatever is on screen, the same way the cost axis
+ does: floor to a round value just below the lowest visible score, then pick
+ a step that lands 5-6 ticks. Hiding the weak tail lifts the floor and the
+ remaining models spread out instead of crowding the top. */
+const boardScoreAxis = (scores: number[]) => {
+ if (!scores.length) return { min: 0, ticks: [0, 20, 40, 60, 80, 100] };
+ const floor = Math.max(0, Math.floor((Math.min(...scores) - 3) / 10) * 10);
+ const span = 100 - floor;
+ const step = span <= 10 ? 2 : span <= 20 ? 5 : span <= 50 ? 10 : 20;
+ const ticks: number[] = [];
+ for (let value = floor; value <= 100 + 1e-9; value += step) ticks.push(Number(value.toFixed(0)));
+ return { min: floor, ticks };
+};
+const BOARD_REFERENCE_LABELS = [
+ "gpt-5-6-sol",
+ "claude-opus-4-8",
+ "gemini-3-7-flash",
+ "kimi-k3",
+ "muse-spark-1-2",
+ "diffusion-gemma-26b-a4b",
+ "ling-3-tiny",
+ "lfm-2-5-2-6b",
+] as const;
+const BOARD_PARETO = modelBoardFrontier();
+const DEFAULT_BOARD_IDS = OPENUI_MODEL_BOARD.filter((point) =>
+ modelBoardDefaultSelected(point.id),
+).map((point) => point.id);
+const DEFAULT_BOARD_ID_SET = new Set<string>(DEFAULT_BOARD_IDS);
-const C_MAX = 7.4;
-const Y_MIN = 66;
-const X_TICKS = [6, 4, 2, 0];
-const Y_TICKS = [70, 80, 90, 100];
+const sameIds = (left: ReadonlySet<string>, right: ReadonlySet<string>) =>
+ left.size === right.size && [...left].every((id) => right.has(id));
export function ReliabilityByModel({
models,
@@ -54,275 +126,874 @@ export function ReliabilityByModel({
formats?: readonly FormatId[];
}) {
const holder = useRef<HTMLDivElement>(null);
- const [W, setW] = useState(1080);
- const [hover, setHover] = useState<{ m: ModelId; f: FormatId } | null>(null);
+ const [width, setWidth] = useState(1080);
+ const [view, setView] = useState<View>("board");
+ const [selectedBoardIds, setSelectedBoardIds] = useState<Set<string>>(
+ () => new Set(DEFAULT_BOARD_IDS),
+ );
+ const [boardHover, setBoardHover] = useState<string | null>(null);
+ const [formatHover, setFormatHover] = useState<{ m: ModelId; f: FormatId } | null>(null);
+ const [urlReady, setUrlReady] = useState(false);
+ /* How much page there is to the left of the plot. The hanging-label layout
+ spends this space, so it has to be measured rather than assumed: at tablet
+ widths the page padding shrinks well before the chart does, and labels
+ that hang 72px into a 40px gutter end up off the side of the window. */
+ const [pageGutter, setPageGutter] = useState(999);
useEffect(() => {
- const el = holder.current;
- if (!el) return;
- const ro = new ResizeObserver(([entry]) => {
- setW(Math.max(320, Math.round(entry.contentRect.width)));
- });
- ro.observe(el);
- return () => ro.disconnect();
+ const element = holder.current;
+ if (!element) return;
+ const measure = (entry: ResizeObserverEntry) => {
+ setWidth(Math.max(240, Math.round(entry.contentRect.width)));
+ setPageGutter(Math.round(entry.target.getBoundingClientRect().left));
+ };
+ const observer = new ResizeObserver(([entry]) => measure(entry));
+ observer.observe(element);
+ return () => observer.disconnect();
}, []);
- const activeFormats = FORMAT_ORDER.filter((f) => (formats ?? FORMAT_ORDER).includes(f));
- const pricedModels = COST_MODELS.filter((m) => (models ?? MODELS.map((x) => x.id)).includes(m));
+ useEffect(() => {
+ const syncFromUrl = () => {
+ const params = new URLSearchParams(window.location.search);
+ setView(params.get("view") === "formats" ? "formats" : "board");
+ const requestedModels = params.get("models");
+ if (requestedModels !== null) {
+ const validIds = new Set(OPENUI_MODEL_BOARD.map((point) => point.id));
+ setSelectedBoardIds(
+ new Set(requestedModels.split(",").filter((id) => validIds.has(id as never))),
+ );
+ } else {
+ setSelectedBoardIds(new Set(DEFAULT_BOARD_IDS));
+ }
+ setUrlReady(true);
+ };
+ queueMicrotask(syncFromUrl);
+ window.addEventListener("popstate", syncFromUrl);
+ return () => window.removeEventListener("popstate", syncFromUrl);
+ }, []);
- const narrow = W < 560;
- /* narrow screens keep the plot flush with the section edges */
- const PL = narrow ? 44 : 56;
- const PR = narrow ? 12 : 40;
- const PW = W - PL - PR;
- /* reversed: expensive on the left, $0 on the right */
- const x = (cost: number) => PL + ((C_MAX - cost) / C_MAX) * PW;
- const y = (v: number) => PAD.top + (1 - (v - Y_MIN) / (100 - Y_MIN)) * PH;
+ useEffect(() => {
+ if (!urlReady) return;
+ const url = new URL(window.location.href);
+ if (view === "formats") url.searchParams.set("view", "formats");
+ else url.searchParams.delete("view");
+ if (sameIds(selectedBoardIds, DEFAULT_BOARD_ID_SET)) url.searchParams.delete("models");
+ else url.searchParams.set("models", [...selectedBoardIds].join(","));
+ window.history.replaceState(null, "", `${url.pathname}${url.search}${url.hash}`);
+ }, [selectedBoardIds, urlReady, view]);
- if (pricedModels.length === 0) return null;
+ const narrow = width < 620;
+ /* Wide: one tall plot, because there is room to separate 20 models.
+ Narrow: shorter, or the same points stretch into a vertical smear. */
+ const plotHeight = narrow ? 440 : 720;
+ const boardPlotHeight = plotHeight;
+ const PH = plotHeight;
+ const H = plotHeight + PAD.top + PAD.bottom;
+ /* Two ways to place the vertical axis, chosen by whether the page can pay
+ for the first one:
+ hanging — the grid stays flush with the control row (PL = 0) and the
+ labels hang into the page gutter. The viewBox is extended left by
+ GUTTER and the svg pulled back by the same amount, so x = 0 still lands
+ on the container's left edge.
+ inset — the labels move inside the canvas. Used whenever the gutter is
+ too thin to hang into, which is every phone and also the tablet range,
+ where page padding shrinks long before the chart does. */
+ const hanging = pageGutter >= 76;
+ const PL = hanging ? 0 : 52;
+ const PR = 0;
+ const GUTTER = hanging ? 72 : 0;
+ const PW = width - PL - PR;
+ const boardTop = PAD.top;
+ const boardBottom = boardTop + boardPlotHeight;
+ const boardHeight = boardBottom + PAD.bottom;
+ const visibleBoardPoints = OPENUI_MODEL_BOARD.filter((point) => selectedBoardIds.has(point.id));
+ const { min: boardYMin, ticks: boardYTicks } = boardScoreAxis(
+ visibleBoardPoints.map((point) => point.score),
+ );
+ /* The cost axis focuses on what is actually shown, so hiding the cheap tail
+ spreads the remaining models out instead of leaving dead space. Free and
+ self-hosted models pin to the floor tick, which is labelled $0. */
+ const visiblePriced = visibleBoardPoints.filter(
+ (point) => !("unpriced" in point) && modelBoardCostPerTask(point) > 0,
+ );
+ const visibleCosts = visiblePriced.map((point) => modelBoardCostPerTask(point));
+ const hasFreeVisible = visibleBoardPoints.some(
+ (point) => "unpriced" in point || modelBoardCostPerTask(point) === 0,
+ );
+ const boardMaxCost = visibleCosts.length ? decadeCeil(Math.max(...visibleCosts)) : 0.1;
+ const pricedFloor = visibleCosts.length ? decadeFloor(Math.min(...visibleCosts)) : 0.001;
+ /* a free model needs a decade of its own below the cheapest priced one */
+ const boardMinCost = hasFreeVisible ? pricedFloor / 10 : pricedFloor;
+ const boardXTicks = boardTicksBetween(boardMinCost, boardMaxCost);
+ const boardX = (cost: number) => {
+ const displayedCost = Math.max(cost || boardMinCost, boardMinCost);
+ const proportion =
+ (Math.log(displayedCost) - Math.log(boardMinCost)) /
+ (Math.log(boardMaxCost) - Math.log(boardMinCost));
+ return PL + (1 - proportion) * PW;
+ };
+ const boardY = (score: number) =>
+ boardTop + (1 - (score - boardYMin) / (100 - boardYMin)) * boardPlotHeight;
+ /* A tick centred on the first or last gridline hangs half its width off the
+ canvas. On a wide screen that spills into whitespace; on a phone it is cut
+ off. Both edges align inward instead. */
+ const edgeAnchor = (x: number): "start" | "middle" | "end" =>
+ x <= PL + 1 ? "start" : x >= PL + PW - 1 ? "end" : "middle";
+ const formatX = (cost: number) => PL + ((9 - cost) / 9) * PW;
+ const formatY = (score: number) => PAD.top + (1 - (score - 60) / 40) * PH;
+ const visibleProviders = PROVIDERS.filter((provider) =>
+ visibleBoardPoints.some((point) => point.provider === provider),
+ );
- /* one line per format: its six priced points, dearest first */
- const series = activeFormats.map((f) => ({
- f,
- pts: pricedModels
- .map((m) => ({ m, cost: costPerPass[m]![f], score: completionByModel[m][f] }))
+ const pricedModels = COST_MODELS.filter((m) => (models ?? MODELS.map((x) => x.id)).includes(m));
+ const activeFormats = FORMAT_ORDER.filter((f) => (formats ?? FORMAT_ORDER).includes(f));
+ const formatSeries = activeFormats.map((format) => ({
+ format,
+ points: pricedModels
+ .map((model) => ({
+ model,
+ cost: costPerPass[model]![format],
+ score: completionByModel[model][format],
+ }))
.sort((a, b) => b.cost - a.cost),
}));
-
- const hovered = hover
- ? { cost: costPerPass[hover.m]![hover.f], score: completionByModel[hover.m][hover.f] }
+ const hoveredBoard = OPENUI_MODEL_BOARD.find((p) => p.id === boardHover);
+ const hoveredFormat = formatHover
+ ? {
+ cost: costPerPass[formatHover.m]![formatHover.f],
+ score: completionByModel[formatHover.m][formatHover.f],
+ }
: null;
+ /* Keep the default view editorial: label the competitive frontier, a few
+ major reference models, and the local/free extremes. Every other model is
+ still fully named in its hover/focus card and accessible label. */
+ const visiblePareto = visibleBoardPoints
+ .filter(
+ (point) =>
+ !("unpriced" in point) &&
+ !visibleBoardPoints.some(
+ (other) =>
+ other !== point &&
+ !("unpriced" in other) &&
+ modelBoardCostPerTask(other) <= modelBoardCostPerTask(point) &&
+ other.score >= point.score &&
+ (modelBoardCostPerTask(other) < modelBoardCostPerTask(point) ||
+ other.score > point.score),
+ ),
+ )
+ .sort((a, b) => modelBoardCostPerTask(b) - modelBoardCostPerTask(a));
+ const competitivePareto = visiblePareto.filter((point) => point.score >= 80);
+ const competitiveParetoIds = new Set(competitivePareto.map((point) => point.id));
+ /* Wide: every visible point is named, as on the Artificial Analysis
+ reference. The collision pass below shifts and re-anchors labels rather
+ than hiding them, so no model is silently unlabelled.
+ Narrow: 20 names cannot fit beside 20 dots on a 390px screen without
+ running off the edge, so only the frontier is named. Every other model
+ keeps its name in its tap card, its accessible label and the table. */
+ const labelledBoardIds = new Set<string>(
+ (narrow ? visiblePareto : visibleBoardPoints).map((point) => point.id),
+ );
+ const labelFontSize = narrow ? 10 : 11.5;
+ /* Labels sit beside their dot on the same centre line, as on the Artificial
+ Analysis reference. Candidates are tried nearest-first: right of the dot,
+ left of it, then progressively larger vertical offsets on either side. The
+ search widens until a free slot exists, so a label is never dropped on top
+ of another one; anything pushed more than a few pixels off its dot gets a
+ leader line so the pairing stays obvious. */
+ const boardLabels = (() => {
+ /* the axis tick labels occupy the gutter and the plot floor, so they are
+ seeded as obstacles: a model label must not land on "100%" either */
+ const placed: Array<{ left: number; right: number; top: number; bottom: number }> = [
+ ...boardYTicks.map((tick) => ({
+ left: PL - GUTTER,
+ right: PL - 6,
+ top: boardY(tick) - labelFontSize,
+ bottom: boardY(tick) + 8,
+ })),
+ /* and every plotted dot: a label must never sit on top of a data point,
+ its own or anyone else's */
+ ...visibleBoardPoints.map((point) => {
+ const cx = boardX(modelBoardCostPerTask(point));
+ const cy = boardY(point.score);
+ return { left: cx - 7, right: cx + 7, top: cy - 7, bottom: cy + 7 };
+ }),
+ ];
+ const gap = narrow ? 9 : 11;
+ const dotRadius = 4;
+ const overlaps = (box: { left: number; right: number; top: number; bottom: number }) =>
+ placed.some(
+ (other) =>
+ box.left < other.right &&
+ box.right > other.left &&
+ box.top < other.bottom &&
+ box.bottom > other.top,
+ );
+
+ /* nearest offsets first: 0, then alternating down/up in 13px steps */
+ const offsets = [0];
+ for (let step = 1; step <= 26; step++) offsets.push(step * 13, -step * 13);
+
+ return visibleBoardPoints
+ .filter((point) => labelledBoardIds.has(point.id))
+ .sort((a, b) => b.score - a.score || modelBoardCostPerTask(a) - modelBoardCostPerTask(b))
+ .map((point) => {
+ const px = boardX(modelBoardCostPerTask(point));
+ const py = boardY(point.score);
+ const labelWidth = Math.max(36, point.label.length * (narrow ? 5.4 : 6.35));
+ /* keep labels inside the plot: hug the side with more room */
+ const preferred: "start" | "end" = px < PL + PW * 0.55 ? "start" : "end";
+ const sides: Array<"start" | "end"> = [preferred, preferred === "start" ? "end" : "start"];
+
+ for (const offset of offsets) {
+ for (const anchor of sides) {
+ const lx = px + (anchor === "start" ? gap + dotRadius : -(gap + dotRadius));
+ const left = anchor === "start" ? lx : lx - labelWidth;
+ const right = anchor === "start" ? lx + labelWidth : lx;
+ if (left < PL - GUTTER || right > PL + PW) continue;
+ /* baseline that centres the text on the dot, then the offset */
+ const ly = py + labelFontSize * 0.35 + offset;
+ if (ly - labelFontSize < boardTop || ly > boardBottom) continue;
+ const box = {
+ left: left - 4,
+ right: right + 4,
+ top: ly - labelFontSize - 3,
+ bottom: ly + 4,
+ };
+ if (overlaps(box)) continue;
+ placed.push(box);
+ return { point, lx, ly, anchor, offset };
+ }
+ }
+
+ /* Every slot in range was taken. Climb from the plot floor looking for
+ one last gap, measuring the box on the side the text actually runs. */
+ const anchor = preferred;
+ const lx = px + (anchor === "start" ? gap + dotRadius : -(gap + dotRadius));
+ const boxAt = (y: number) => ({
+ left: (anchor === "start" ? lx : lx - labelWidth) - 4,
+ right: (anchor === "start" ? lx + labelWidth : lx) + 4,
+ top: y - labelFontSize - 3,
+ bottom: y + 4,
+ });
+ let ly = boardBottom - 4;
+ while (overlaps(boxAt(ly)) && ly > boardTop + labelFontSize) ly -= 13;
+ /* Still nowhere to put it: leave this one unlabelled rather than print
+ it over a neighbour. It keeps its name in the tap card, its
+ accessible label and the table. Only reachable on the narrowest
+ phones, where the plot is a few hundred pixels wide. */
+ if (overlaps(boxAt(ly))) return null;
+ placed.push(boxAt(ly));
+ return { point, lx, ly, anchor, offset: ly - py };
+ })
+ .filter((entry): entry is NonNullable<typeof entry> => entry !== null);
+ })();
+
+ /* Model marks on the format plot, placed the way the point labels are: the
+ preferred spot first, then stepping away until nothing overlaps. Fixed
+ offsets were fine while the marks were spread out, but at the cheap end
+ of the axis several models land within a few pixels of each other and
+ their logos stacked into an unreadable pile — most visibly on a phone,
+ where the whole plot is a third of the width. A mark that cannot be
+ placed near its own point is dropped rather than dragged somewhere it
+ would read as belonging to a different one. */
+ const formatMarks = (() => {
+ const size = narrow ? 12 : 16;
+ const boxes: Array<{ left: number; right: number; top: number; bottom: number }> = [];
+ /* the plotted squares come first: a logo must never cover a data point */
+ for (const { points } of formatSeries) {
+ for (const point of points) {
+ const cx = formatX(point.cost);
+ const cy = formatY(point.score);
+ boxes.push({ left: cx - 7, right: cx + 7, top: cy - 7, bottom: cy + 7 });
+ }
+ }
+ const overlaps = (box: { left: number; right: number; top: number; bottom: number }) =>
+ boxes.some(
+ (other) =>
+ box.left < other.right &&
+ box.right > other.left &&
+ box.top < other.bottom &&
+ box.bottom > other.top,
+ );
+ const stride = size + 3;
+ const steps = [0];
+ for (let i = 1; i <= 3; i++) steps.push(-i * stride, i * stride);
+
+ return formatSeries
+ .flatMap(({ format, points }) =>
+ points.map((point) => {
+ const model = MODELS.find((item) => item.id === point.model)!;
+ const cx = formatX(point.cost);
+ const py = formatY(point.score);
+ const above = py - (narrow ? 21 : 27);
+ const base = format === "jsonRender" || above < PAD.top ? py + 12 : above;
+ for (const step of steps) {
+ const gy = base + step;
+ if (gy < PAD.top || gy + size > PAD.top + PH) continue;
+ const box = {
+ left: cx - size / 2 - 1,
+ right: cx + size / 2 + 1,
+ top: gy - 1,
+ bottom: gy + size + 1,
+ };
+ if (overlaps(box)) continue;
+ boxes.push(box);
+ return {
+ key: `${format}-${point.model}-mark`,
+ d: BRAND_MARKS[model.mark],
+ scale: size / Number(markViewBox(model.mark).split(" ")[2]),
+ x: cx - size / 2,
+ y: gy,
+ };
+ }
+ return null;
+ }),
+ )
+ .filter((mark): mark is NonNullable<typeof mark> => mark !== null);
+ })();
+
+ const toggleBoardModel = (id: string) => {
+ setSelectedBoardIds((current) => {
+ const next = new Set(current);
+ if (next.has(id)) next.delete(id);
+ else next.add(id);
+ return next;
+ });
+ setBoardHover(null);
+ };
+
return (
<Chart
- title="Reliability vs cost"
- sub="One line per format across all six models at list prices. Up and to the right is better: higher structural validity, less money."
+ title={view === "board" ? "Model comparison" : "Format comparison"}
+ sub={
+ view === "board"
+ ? "Provider-coloured dots with one Pareto frontier. Key references are labeled; hover any point for its name and values."
+ : "The original six-model comparison, one line per format. Higher and further right is better."
+ }
>
- <div ref={holder} className={`${s.svgHolder} ${s.frontier}`} style={{ position: "relative" }}>
- <svg
- className={s.svg}
- width={W}
- height={H}
- viewBox={`0 0 ${W} ${H}`}
- role="img"
- aria-label="Cost versus reliability, one line per format; OpenUI sits highest and furthest toward zero cost"
+ <div className={s.chartControlRow}>
+ <div
+ className={s.axisToggle}
+ role="tablist"
+ aria-label="Chart view"
+ onKeyDown={(event) => {
+ if (event.key !== "ArrowLeft" && event.key !== "ArrowRight") return;
+ event.preventDefault();
+ const nextView = view === "board" ? "formats" : "board";
+ setView(nextView);
+ setBoardHover(null);
+ setFormatHover(null);
+ requestAnimationFrame(() =>
+ document
+ .getElementById(
+ nextView === "board" ? "benchmark-models-tab" : "benchmark-formats-tab",
+ )
+ ?.focus(),
+ );
+ }}
>
- {Y_TICKS.map((t) => (
- <g key={t}>
- <line x1={PL} x2={PL + PW} y1={y(t)} y2={y(t)} stroke="var(--rule)" />
- {t !== 100 ? (
+ <button
+ id="benchmark-models-tab"
+ type="button"
+ role="tab"
+ aria-selected={view === "board"}
+ aria-controls="benchmark-model-chart-panel"
+ aria-describedby="benchmark-model-view-description"
+ tabIndex={view === "board" ? 0 : -1}
+ className={`${s.axisBtn} ${view === "board" ? s.axisBtnOn : ""}`}
+ onClick={() => {
+ setView("board");
+ setFormatHover(null);
+ }}
+ >
+ Model comparison
+ </button>
+ <button
+ id="benchmark-formats-tab"
+ type="button"
+ role="tab"
+ aria-selected={view === "formats"}
+ aria-controls="benchmark-format-chart-panel"
+ aria-describedby="benchmark-format-view-description"
+ tabIndex={view === "formats" ? 0 : -1}
+ className={`${s.axisBtn} ${view === "formats" ? s.axisBtnOn : ""}`}
+ onClick={() => {
+ setView("formats");
+ setBoardHover(null);
+ }}
+ >
+ Format comparison
+ </button>
+ <span id="benchmark-model-view-description" className={s.tableCaption}>
+ Compares OpenUI structural validity and cost across {OPENUI_MODEL_BOARD.length} models
+ using provider-coloured dots and one Pareto frontier.
+ </span>
+ <span id="benchmark-format-view-description" className={s.tableCaption}>
+ Compares validity and cost across OpenUI, A2UI, and json-render for six models.
+ </span>
+ </div>
+
+ {view === "board" ? (
+ <details className={s.modelSelect}>
+ <summary
+ aria-label={`Filter models, ${selectedBoardIds.size} of ${OPENUI_MODEL_BOARD.length} selected`}
+ >
+ <span>Models</span>
+ <strong>
+ {selectedBoardIds.size === OPENUI_MODEL_BOARD.length
+ ? `All ${OPENUI_MODEL_BOARD.length}`
+ : `${selectedBoardIds.size} / ${OPENUI_MODEL_BOARD.length}`}
+ </strong>
+ <span className={s.modelSelectChevron} aria-hidden>
+ ↓
+ </span>
+ </summary>
+ <div className={s.modelSelectMenu} role="group" aria-label="Models shown on chart">
+ <span className={s.tableCaption} aria-live="polite">
+ {selectedBoardIds.size} of {OPENUI_MODEL_BOARD.length} models selected
+ </span>
+ <div className={s.modelSelectActions}>
+ <span>{selectedBoardIds.size} selected</span>
+ <button
+ type="button"
+ onClick={() =>
+ setSelectedBoardIds(new Set(OPENUI_MODEL_BOARD.map((point) => point.id)))
+ }
+ >
+ All
+ </button>
+ <button type="button" onClick={() => setSelectedBoardIds(new Set())}>
+ None
+ </button>
+ </div>
+ <div className={s.modelSelectList}>
+ {PROVIDERS.map((provider) => {
+ const companyModels = OPENUI_MODEL_BOARD.filter(
+ (point) => point.provider === provider,
+ );
+ return (
+ <section
+ key={provider}
+ className={s.modelSelectGroup}
+ role="group"
+ aria-labelledby={`model-provider-${PROVIDERS.indexOf(provider)}`}
+ >
+ <div
+ id={`model-provider-${PROVIDERS.indexOf(provider)}`}
+ className={s.modelSelectGroupTitle}
+ >
+ <span
+ className={`${s.modelSelectDot} ${s.providerHue}`}
+ style={{ background: PROVIDER_HUE[provider] }}
+ aria-hidden
+ />
+ <strong>{provider}</strong>
+ <span>{companyModels.length}</span>
+ </div>
+ {companyModels.map((point) => (
+ <label key={point.id}>
+ <input
+ type="checkbox"
+ checked={selectedBoardIds.has(point.id)}
+ onChange={() => toggleBoardModel(point.id)}
+ />
+ <span>{point.label}</span>
+ </label>
+ ))}
+ </section>
+ );
+ })}
+ </div>
+ </div>
+ </details>
+ ) : null}
+ </div>
+
+ <div
+ id={view === "board" ? "benchmark-model-chart-panel" : "benchmark-format-chart-panel"}
+ ref={holder}
+ role="tabpanel"
+ aria-labelledby={view === "board" ? "benchmark-models-tab" : "benchmark-formats-tab"}
+ className={`${s.svgHolder} ${s.frontier} ${s.edgeAligned}`}
+ style={{ position: "relative" }}
+ >
+ {view === "board" ? (
+ <svg
+ className={s.svg}
+ width={width + GUTTER}
+ height={boardHeight}
+ viewBox={`${-GUTTER} 0 ${width + GUTTER} ${boardHeight}`}
+ style={{ marginLeft: -GUTTER }}
+ role="img"
+ aria-label={`OpenUI structural validity versus cost per task for ${visibleBoardPoints.length} selected models; vertical axis ranges from ${boardYMin} to 100 percent`}
+ >
+ {boardYTicks.map((tick) => (
+ <g key={tick}>
+ <line
+ x1={PL}
+ x2={PL + PW}
+ y1={boardY(tick)}
+ y2={boardY(tick)}
+ stroke="var(--rule)"
+ />
<text
x={PL - 12}
- y={y(t) + 4}
+ y={boardY(tick) + 4}
textAnchor="end"
- fontSize="13"
+ fontSize="12.5"
fill="var(--ink-muted)"
>
- {t}%
+ {tick}%
</text>
- ) : null}
- </g>
- ))}
-
- {X_TICKS.map((t) => (
- <line
- key={`gx-${t}`}
- x1={x(t)}
- x2={x(t)}
- y1={y(100)}
- y2={PAD.top + PH}
- stroke="var(--rule)"
- />
- ))}
-
- {/* the chart header, set on the top axis line like the reference */}
- <text x={PL - 12} y={y(100) - 14} textAnchor="end" fontSize="13" fill="var(--ink-muted)">
- 100%
- </text>
- <text x={PL + 4} y={y(100) - 14} fontSize="14" fill="var(--ink)">
- Reliability vs cost
- </text>
+ </g>
+ ))}
+ {boardXTicks.map((tick) => (
+ <g key={tick}>
+ <line
+ x1={boardX(tick)}
+ x2={boardX(tick)}
+ y1={boardTop}
+ y2={boardBottom}
+ stroke="var(--rule)"
+ />
+ <text
+ x={boardX(tick)}
+ y={boardBottom + 26}
+ textAnchor={edgeAnchor(boardX(tick))}
+ fontSize="12.5"
+ fill="var(--ink-muted)"
+ >
+ {hasFreeVisible && tick === boardMinCost ? "$0" : boardCostLabel(tick)}
+ </text>
+ </g>
+ ))}
+ <text x={PL + 4} y={boardTop - 15} fontSize="13.5" fill="var(--ink)">
+ {narrow ? "Structural validity vs cost" : "Structural validity vs cost, by model"}
+ </text>
+ {hanging ? (
+ <text
+ className={s.axisTitle}
+ transform={`translate(${PL - GUTTER + 16}, ${boardTop + boardPlotHeight / 2}) rotate(-90)`}
+ textAnchor="middle"
+ >
+ Structural validity
+ </text>
+ ) : null}
+ <text x={PL + PW} y={boardTop - 15} textAnchor="end" fontSize="12.5" fill="var(--ink)">
+ better ↗
+ </text>
+ <text className={s.axisTitle} x={PL + PW / 2} y={boardHeight - 10} textAnchor="middle">
+ Cost per task · USD
+ </text>
+ {visiblePareto.length > 1 ? (
+ <path
+ className={s.fadeLate}
+ d={visiblePareto
+ .map(
+ (point, index) =>
+ `${index === 0 ? "M" : "L"} ${boardX(modelBoardCostPerTask(point))} ${boardY(point.score)}`,
+ )
+ .join(" ")}
+ fill="none"
+ stroke="var(--ink)"
+ strokeWidth={1.5}
+ strokeOpacity={0.62}
+ strokeLinejoin="round"
+ strokeLinecap="round"
+ aria-hidden="true"
+ />
+ ) : null}
- {X_TICKS.map((t) => (
+ {visibleBoardPoints.map((point, index) => {
+ const radius = 4.5;
+ const px = boardX(modelBoardCostPerTask(point));
+ const py = boardY(point.score);
+ return (
+ <g key={point.id}>
+ <circle
+ className={`${s.pt} ${s.providerHue}`}
+ cx={px}
+ cy={py}
+ r={radius}
+ fill={PROVIDER_HUE[point.provider]}
+ stroke="var(--surface)"
+ strokeWidth={2}
+ aria-label={`${point.label}, ${point.provider}: ${point.score.toFixed(1)}% valid, ${"unpriced" in point ? "self-hosted, cost not comparable" : `$${modelBoardCostPerTask(point).toFixed(4)} per task`}`}
+ style={{ cursor: "default", "--d": `${300 + index * 18}ms` } as CSSProperties}
+ onMouseEnter={() => setBoardHover(point.id)}
+ onMouseLeave={() => setBoardHover(null)}
+ />
+ </g>
+ );
+ })}
+ {boardLabels.map(({ point, lx, ly, anchor, offset }) => (
+ <g key={`label-${point.id}`} className={s.fadeLate}>
+ {Math.abs(offset) > 6 ? (
+ <line
+ x1={boardX(modelBoardCostPerTask(point))}
+ y1={boardY(point.score)}
+ x2={anchor === "start" ? lx - 3 : lx + 3}
+ y2={ly - labelFontSize * 0.35}
+ stroke="var(--rule)"
+ strokeWidth={1}
+ />
+ ) : null}
+ <text
+ x={lx}
+ y={ly}
+ textAnchor={anchor}
+ fontSize={labelFontSize}
+ fontWeight={450}
+ fill="var(--ink-muted)"
+ stroke="var(--surface)"
+ strokeWidth={4}
+ paintOrder="stroke"
+ >
+ {point.label}
+ </text>
+ </g>
+ ))}
+ </svg>
+ ) : (
+ <svg
+ className={s.svg}
+ width={width + GUTTER}
+ height={H}
+ viewBox={`${-GUTTER} 0 ${width + GUTTER} ${H}`}
+ style={{ marginLeft: -GUTTER }}
+ role="img"
+ aria-label="Structural validity versus cost, one line per generative UI format across six models"
+ >
+ {[60, 70, 80, 90, 100].map((tick) => (
+ <g key={tick}>
+ <line
+ x1={PL}
+ x2={PL + PW}
+ y1={formatY(tick)}
+ y2={formatY(tick)}
+ stroke="var(--rule)"
+ />
+ <text
+ x={PL - 12}
+ y={formatY(tick) + 4}
+ textAnchor="end"
+ fontSize="12.5"
+ fill="var(--ink-muted)"
+ >
+ {tick}%
+ </text>
+ </g>
+ ))}
+ {[9, 6, 3, 0].map((tick) => (
+ <g key={tick}>
+ <line
+ x1={formatX(tick)}
+ x2={formatX(tick)}
+ y1={formatY(100)}
+ y2={PAD.top + PH}
+ stroke="var(--rule)"
+ />
+ <text
+ x={formatX(tick)}
+ y={PAD.top + PH + 28}
+ textAnchor={edgeAnchor(formatX(tick))}
+ fontSize="12.5"
+ fill="var(--ink-muted)"
+ >
+ ${tick}
+ </text>
+ </g>
+ ))}
+ {/* The board's bottom and right edges land on ticks (20% and
+ free/local); this view's floor is 66% with no tick there, so the
+ frame is drawn explicitly and the two tabs match. */}
+ <line x1={PL} x2={PL + PW} y1={PAD.top + PH} y2={PAD.top + PH} stroke="var(--rule)" />
+ <line x1={PL + PW} x2={PL + PW} y1={PAD.top} y2={PAD.top + PH} stroke="var(--rule)" />
+ <text x={PL + 4} y={formatY(100) - 14} fontSize="13.5" fill="var(--ink)">
+ {narrow ? "Structural validity vs cost" : "Structural validity vs cost, by format"}
+ </text>
+ {hanging ? (
+ <text
+ className={s.axisTitle}
+ transform={`translate(${PL - GUTTER + 16}, ${PAD.top + PH / 2}) rotate(-90)`}
+ textAnchor="middle"
+ >
+ Structural validity
+ </text>
+ ) : null}
<text
- key={t}
- x={x(t)}
- y={PAD.top + PH + 28}
- textAnchor="middle"
- fontSize="13"
- fill="var(--ink-muted)"
+ x={PL + PW}
+ y={formatY(100) - 14}
+ textAnchor="end"
+ fontSize="12.5"
+ fill="var(--ink)"
>
- ${t}
+ better ↗
</text>
- ))}
- <text
- x={PL + PW / 2}
- y={PAD.top + PH + 52}
- textAnchor="middle"
- fontSize="13"
- fill="var(--ink-muted)"
- >
- Cost of one benchmark pass, 46 screens
- </text>
- <text x={PL + PW} y={y(100) - 14} textAnchor="end" fontSize="13" fill="var(--ink)">
- better ↗
- </text>
-
- {/* draw ours last so the purple line sits on top */}
- {[...series].reverse().map(({ f, pts }) => {
- const stroke = hue[f];
- const ours = f === "openui";
- const first = pts[0];
- return (
- <g key={f}>
- <polyline
- className={s.line}
- pathLength={1}
- points={pts.map((p) => `${x(p.cost)},${y(p.score)}`).join(" ")}
- fill="none"
- stroke={stroke}
- strokeWidth={ours ? 4.5 : 2.75}
- strokeLinejoin="round"
- strokeLinecap="round"
- style={{ "--d": `${activeFormats.indexOf(f) * 120}ms` } as React.CSSProperties}
- />
- {pts.map((p, i) => {
- const isHover = hover?.m === p.m && hover?.f === f;
- const r = 5 + (isHover ? 1.5 : 0);
- return (
- <rect
- key={p.m}
- className={s.pt}
- x={x(p.cost) - r}
- y={y(p.score) - r}
- width={r * 2}
- height={r * 2}
- fill={ours ? "var(--surface)" : stroke}
- stroke={stroke}
- strokeWidth={ours ? 2.5 : 0}
- style={
- { cursor: "default", "--d": `${350 + i * 45}ms` } as React.CSSProperties
- }
- onMouseEnter={() => setHover({ m: p.m, f })}
- onMouseLeave={() => setHover(null)}
- />
- );
- })}
- {narrow ? null : ours ? (
- (() => {
- /* ours gets a filled tag so the winning line reads first */
- const label = formatLabel(f);
- const tagW = 18 + label.length * 8.2;
- const tx = x(first.cost) - 16;
+ <text className={s.axisTitle} x={PL + PW / 2} y={H - 10} textAnchor="middle">
+ {narrow ? "Cost per pass · 46 screens" : "Cost of one benchmark pass · 46 screens"}
+ </text>
+ {[...formatSeries].reverse().map(({ format, points }) => {
+ const stroke = FORMAT_HUE[format];
+ const ours = format === "openui";
+ const first = points[0];
+ return (
+ <g key={format}>
+ <polyline
+ className={s.line}
+ pathLength={1}
+ points={points.map((p) => `${formatX(p.cost)},${formatY(p.score)}`).join(" ")}
+ fill="none"
+ stroke={stroke}
+ strokeWidth={ours ? 1.75 : 1.4}
+ strokeLinejoin="round"
+ strokeLinecap="round"
+ style={{ "--d": `${activeFormats.indexOf(format) * 120}ms` } as CSSProperties}
+ />
+ {points.map((point, index) => {
+ const radius =
+ 5 + (formatHover?.m === point.model && formatHover?.f === format ? 1.5 : 0);
return (
- <g className={s.fadeLate}>
- <rect
- x={tx - tagW}
- y={y(first.score) - 12}
- width={tagW}
- height={24}
- fill="var(--pO)"
- />
- <text
- x={tx - tagW / 2}
- y={y(first.score) + 4.5}
- textAnchor="middle"
- fontSize="13.5"
- fontWeight="700"
- fill="#fff"
- >
- {label}
- </text>
- </g>
+ <rect
+ key={point.model}
+ className={s.pt}
+ x={formatX(point.cost) - radius}
+ y={formatY(point.score) - radius}
+ width={radius * 2}
+ height={radius * 2}
+ fill={ours ? "var(--surface)" : stroke}
+ stroke={stroke}
+ strokeWidth={ours ? 2.5 : 0}
+ aria-label={`${formatLabel(format)}, ${MODEL_NAME[point.model]}: ${point.score.toFixed(1)}% valid, $${point.cost.toFixed(2)} per pass`}
+ style={
+ { cursor: "default", "--d": `${350 + index * 45}ms` } as CSSProperties
+ }
+ onMouseEnter={() => setFormatHover({ m: point.model, f: format })}
+ onMouseLeave={() => setFormatHover(null)}
+ />
);
- })()
- ) : (
- <text
- className={s.fadeLate}
- x={x(first.cost) - 14}
- y={y(first.score) + 5}
- textAnchor="end"
- fontSize="15"
- fontWeight={600}
- fill={stroke}
- stroke="var(--surface)"
- strokeWidth={4}
- paintOrder="stroke"
- >
- {formatLabel(f)}
- </text>
- )}
- </g>
- );
- })}
-
- {/* every point carries its model's mark; the legend below decodes them */}
- {narrow
- ? null
- : (() => {
- const glyphs = series.flatMap(({ f, pts }) =>
- pts.map((p) => {
- const above = y(p.score) - 30;
- /* the chart header sits on the top gridline, so a mark that
- would rise into it drops under its point instead */
- const collidesWithHeader = above < PAD.top;
- return {
- key: `${f}-${p.m}`,
- mark: MODELS.find((m) => m.id === p.m)!.mark,
- lx: x(p.cost),
- ly: f === "jsonRender" || collidesWithHeader ? y(p.score) + 12 : above,
- };
- }),
- );
- glyphs.sort((a, b) => a.ly - b.ly);
- for (let i = 0; i < glyphs.length; i++) {
- for (let j = 0; j < i; j++) {
- const a = glyphs[j];
- const b = glyphs[i];
- if (Math.abs(a.lx - b.lx) < 20 && Math.abs(a.ly - b.ly) < 18) b.ly = a.ly + 18;
- }
- }
- return glyphs.map((g) => {
- /* marks ship at different viewBox sizes; normalise to 16px */
- const vb = Number(markViewBox(g.mark).split(" ")[2]);
- return (
- <path
- key={g.key}
+ })}
+ {!narrow && (
+ <text
className={s.fadeLate}
- d={BRAND_MARKS[g.mark]}
- transform={`translate(${g.lx - 8}, ${g.ly}) scale(${16 / vb})`}
- fill="var(--ink-muted)"
- />
- );
- });
- })()}
- </svg>
+ x={formatX(first.cost) - 14}
+ y={formatY(first.score) + 5}
+ textAnchor="end"
+ fontSize="14"
+ fontWeight={600}
+ fill={stroke}
+ stroke="var(--surface)"
+ strokeWidth={4}
+ paintOrder="stroke"
+ >
+ {formatLabel(format)}
+ </text>
+ )}
+ </g>
+ );
+ })}
+ {formatMarks.map((mark) => (
+ <path
+ key={mark.key}
+ className={s.fadeLate}
+ d={mark.d}
+ transform={`translate(${mark.x}, ${mark.y}) scale(${mark.scale})`}
+ fill="var(--ink-muted)"
+ />
+ ))}
+ </svg>
+ )}
- {hover && hovered ? (
+ {view === "board" && hoveredBoard ? (
<div
className={s.hoverCard}
style={{
- left: Math.min(Math.max(x(hovered.cost), 130), W - 130),
- top: y(hovered.score) - 14,
+ left: Math.min(
+ Math.max(boardX(modelBoardCostPerTask(hoveredBoard)), 138),
+ width - 138,
+ ),
+ top: boardY(hoveredBoard.score) - 12,
+ }}
+ >
+ <strong>{hoveredBoard.label}</strong>
+ <span>{hoveredBoard.provider}</span>
+ <span>
+ {hoveredBoard.score.toFixed(1)}% valid ·{" "}
+ {"unpriced" in hoveredBoard
+ ? "Self-hosted · cost not comparable"
+ : modelBoardCostPerTask(hoveredBoard) === 0
+ ? "Free"
+ : `$${modelBoardCostPerTask(hoveredBoard).toFixed(modelBoardCostPerTask(hoveredBoard) < 0.01 ? 4 : 3)} / task`}
+ </span>
+ </div>
+ ) : null}
+ {view === "formats" && formatHover && hoveredFormat ? (
+ <div
+ className={s.hoverCard}
+ style={{
+ left: Math.min(Math.max(formatX(hoveredFormat.cost), 130), width - 130),
+ top: formatY(hoveredFormat.score) - 14,
}}
>
<strong>
- {formatLabel(hover.f)} · {MODEL_NAME[hover.m]}
+ {formatLabel(formatHover.f)} · {MODEL_NAME[formatHover.m]}
</strong>
- {hovered.score.toFixed(1)}% structurally valid · ${hovered.cost.toFixed(2)} per pass
+ {hoveredFormat.score.toFixed(1)}% valid · ${hoveredFormat.cost.toFixed(2)} per pass
</div>
) : null}
</div>
- {narrow ? (
- <div className={`${s.legend} ${s.legendCenter} ${s.frontier}`}>
- {FORMATS.filter((f) => activeFormats.includes(f.id)).map((f) => (
- <span key={f.id} className={s.key}>
- <span className={s.dot} style={{ background: hue[f.id] }} aria-hidden />
- {f.label}
+ <div
+ id={view === "board" ? "benchmark-format-chart-panel" : "benchmark-model-chart-panel"}
+ role="tabpanel"
+ aria-labelledby={view === "board" ? "benchmark-formats-tab" : "benchmark-models-tab"}
+ hidden
+ >
+ {view === "board"
+ ? "Format comparison data is available in the chart data tables below."
+ : "Model comparison data is available in the chart data tables below."}
+ </div>
+
+ {view === "board" ? (
+ <div className={`${s.legend} ${s.legendCenter}`} aria-label="Model provider key">
+ <span className={s.key}>
+ <span className={s.paretoKeyLine} aria-hidden />
+ Pareto frontier
+ </span>
+ {visibleProviders.map((provider) => (
+ <span key={provider} className={s.key}>
+ <span
+ className={`${s.dot} ${s.providerHue}`}
+ style={{ background: PROVIDER_HUE[provider] }}
+ aria-hidden
+ />
+ {provider}
</span>
))}
</div>
+ ) : narrow ? (
+ /* The in-chart format labels do not fit at this width, so the formats
+ are keyed here; the models are keyed too, because their marks are on
+ the plot and nothing else names them. */
+ <>
+ <div className={`${s.legend} ${s.legendCenter} ${s.frontier}`} aria-label="Format key">
+ {FORMATS.filter((f) => activeFormats.includes(f.id)).map((f) => (
+ <span key={f.id} className={s.key}>
+ <span className={s.dot} style={{ background: FORMAT_HUE[f.id] }} aria-hidden />
+ {f.label}
+ </span>
+ ))}
+ </div>
+ <div className={`${s.legend} ${s.legendCenter} ${s.legendTight}`} aria-label="Model key">
+ {MODELS.filter((m) => pricedModels.includes(m.id)).map((m) => (
+ <span key={m.id} className={s.key}>
+ <Mark id={m.mark} />
+ {MODEL_NAME[m.id]}
+ </span>
+ ))}
+ </div>
+ </>
) : (
<div className={`${s.legend} ${s.legendCenter}`} aria-label="Model key">
{MODELS.filter((m) => pricedModels.includes(m.id)).map((m) => (
@@ -333,6 +1004,112 @@ export function ReliabilityByModel({
))}
</div>
)}
+
+ <ChartDataDisclosure label="View chart data">
+ <section className={s.dataSection} aria-labelledby="benchmark-model-data-heading">
+ <h3 id="benchmark-model-data-heading" className={s.dataSectionHeading}>
+ Model comparison data
+ </h3>
+ <DataTable>
+ <caption>
+ OpenUI model-board structural validity and measured cost for all models
+ </caption>
+ <thead>
+ <tr>
+ <th scope="col">Model</th>
+ <th scope="col">Provider</th>
+ <th scope="col">Family</th>
+ <th scope="col" data-numeric="true">
+ Valid
+ </th>
+ <th scope="col" data-numeric="true">
+ Cost per task
+ </th>
+ <th scope="col">Pricing</th>
+ <th scope="col">Frontier (all {OPENUI_MODEL_BOARD.length})</th>
+ <th scope="col">Shown in chart</th>
+ </tr>
+ </thead>
+ <tbody>
+ {OPENUI_MODEL_BOARD.map((point) => {
+ const family = modelBoardFamilyFor(point.id);
+ const unpriced = "unpriced" in point;
+ const cost = modelBoardCostPerTask(point);
+ return (
+ <tr key={`data-${point.id}`}>
+ <th scope="row">{point.label}</th>
+ <td>{point.provider}</td>
+ <td>{family?.label ?? "—"}</td>
+ <td data-numeric="true">{point.score.toFixed(1)}%</td>
+ <td data-numeric="true">
+ {unpriced
+ ? "Not comparable"
+ : cost === 0
+ ? "$0 / free"
+ : `$${cost.toFixed(4)}`}
+ </td>
+ <td>{unpriced ? "Self-hosted" : cost === 0 ? "Free" : "List price"}</td>
+ <td>
+ {!unpriced && BOARD_PARETO.some((row) => row.id === point.id) ? "Yes" : "No"}
+ </td>
+ <td>{selectedBoardIds.has(point.id) ? "Yes" : "No"}</td>
+ </tr>
+ );
+ })}
+ </tbody>
+ </DataTable>
+ <p className={s.dataNote}>
+ All {OPENUI_MODEL_BOARD.length} models remain in this table and the downloads, including
+ models hidden by the chart&rsquo;s default filter. Frontier membership here is computed
+ over the full board, so it does not shift with the chart&rsquo;s selection; the drawn
+ line is the frontier of the models currently shown. Self-hosted cost is unknown, not
+ zero. Filter state is preserved in this page&rsquo;s URL. Focused benchmark:{" "}
+ <a href="/benchmarks/language">language and model results</a>. Full dataset:{" "}
+ <a href="/benchmarks/data.json">JSON</a>, <a href="/benchmarks/data.csv">CSV</a>, or{" "}
+ <a href="/benchmarks/agent.md">agent Markdown</a>.
+ </p>
+ </section>
+ <section className={s.dataSection} aria-labelledby="benchmark-format-data-heading">
+ <h3 id="benchmark-format-data-heading" className={s.dataSectionHeading}>
+ Format comparison data
+ </h3>
+ <DataTable>
+ <caption>
+ Structural validity and cost per 46-screen pass for each model and format
+ </caption>
+ <thead>
+ <tr>
+ <th scope="col">Model</th>
+ <th scope="col">Format</th>
+ <th scope="col" data-numeric="true">
+ Valid
+ </th>
+ <th scope="col" data-numeric="true">
+ Cost per pass
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ {formatSeries.flatMap(({ format, points }) =>
+ points.map((point) => (
+ <tr key={`data-${format}-${point.model}`}>
+ <th scope="row">{MODEL_NAME[point.model]}</th>
+ <td>{formatLabel(format)}</td>
+ <td data-numeric="true">{point.score.toFixed(1)}%</td>
+ <td data-numeric="true">${point.cost.toFixed(2)}</td>
+ </tr>
+ )),
+ )}
+ </tbody>
+ </DataTable>
+ <p className={s.dataNote}>
+ Focused benchmark: <a href="/benchmarks/framework">framework comparison</a>. Download
+ this comparison as <a href="/benchmarks/framework/data.json">JSON</a>,{" "}
+ <a href="/benchmarks/framework/data.csv">CSV</a>, or{" "}
+ <a href="/benchmarks/framework/agent.md">agent Markdown</a>.
+ </p>
+ </section>
+ </ChartDataDisclosure>
</Chart>
);
}
diff --git a/docs/components/charts/render-split.tsx b/docs/components/charts/render-split.tsx
index b2acf7f67..4fa9a8d1b 100644
--- a/docs/components/charts/render-split.tsx
+++ b/docs/components/charts/render-split.tsx
@@ -10,7 +10,7 @@ import {
formatLabel,
runCounts,
} from "@/lib/benchmark-data";
-import { Chart, Chip, styles as s, slotClass } from "./primitives";
+import { Chart, ChartDataDisclosure, Chip, DataTable, styles as s, slotClass } from "./primitives";
/* Every run ends complete, partial, or blank. Each column keeps its natural
direction — a long "complete" bar is good, a long "blank" bar is not — and
@@ -130,6 +130,46 @@ export function RenderSplit({
);
})}
</div>
+ <ChartDataDisclosure label="View validity and render data">
+ <DataTable>
+ <caption>Structural validity and render success by format</caption>
+ <thead>
+ <tr>
+ <th scope="col">Format</th>
+ <th scope="col" data-numeric="true">
+ Valid runs
+ </th>
+ <th scope="col" data-numeric="true">
+ Partial runs
+ </th>
+ <th scope="col" data-numeric="true">
+ Blank runs
+ </th>
+ <th scope="col" data-numeric="true">
+ Validity
+ </th>
+ <th scope="col" data-numeric="true">
+ Render success
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ {shown.map((format) => {
+ const row = rates(format);
+ return (
+ <tr key={`data-${format}`}>
+ <th scope="row">{formatLabel(format)}</th>
+ <td data-numeric="true">{row.counts.complete}</td>
+ <td data-numeric="true">{row.counts.partial}</td>
+ <td data-numeric="true">{row.counts.blank}</td>
+ <td data-numeric="true">{row.inFull.toFixed(1)}%</td>
+ <td data-numeric="true">{row.atAll.toFixed(1)}%</td>
+ </tr>
+ );
+ })}
+ </tbody>
+ </DataTable>
+ </ChartDataDisclosure>
</Chart>
);
}
diff --git a/docs/components/charts/repair-funnel-flow.tsx b/docs/components/charts/repair-funnel-flow.tsx
index e69c8157a..7e02389fc 100644
--- a/docs/components/charts/repair-funnel-flow.tsx
+++ b/docs/components/charts/repair-funnel-flow.tsx
@@ -2,7 +2,7 @@
import { production, repairFunnel } from "@/lib/benchmark-data";
import { useEffect, useRef, useState } from "react";
-import { Chart, styles as s } from "./primitives";
+import { Chart, ChartDataDisclosure, DataTable, styles as s } from "./primitives";
/* Production repair as a step funnel: every stage shows how many screens are
still broken, and the caption under each step says what fixed the rest.
@@ -126,6 +126,32 @@ export function RepairFunnelFlow() {
</div>
))}
</div>
+ <ChartDataDisclosure label="View repair data">
+ <DataTable>
+ <caption>Production repair funnel as a share of all streaming generations</caption>
+ <thead>
+ <tr>
+ <th scope="col">Stage</th>
+ <th scope="col" data-numeric="true">
+ Share of generations
+ </th>
+ <th scope="col">What happens</th>
+ <th scope="col">Model call</th>
+ </tr>
+ </thead>
+ <tbody>
+ {stages.map((stage) => (
+ <tr key={`data-${stage.label}`}>
+ <th scope="row">{stage.label}</th>
+ <td data-numeric="true">{stage.value}%</td>
+ <td>{stage.sub}</td>
+ <td>{stage.llm ? "One model call" : "No model call"}</td>
+ </tr>
+ ))}
+ </tbody>
+ </DataTable>
+ <p className={s.dataNote}>Production traffic, not benchmark runs.</p>
+ </ChartDataDisclosure>
</Chart>
);
}
diff --git a/docs/components/charts/speed-tokens.tsx b/docs/components/charts/speed-tokens.tsx
index 28c8a7e00..e87ef6a6c 100644
--- a/docs/components/charts/speed-tokens.tsx
+++ b/docs/components/charts/speed-tokens.tsx
@@ -1,7 +1,7 @@
"use client";
import { FORMATS, FORMAT_ORDER, type FormatId, tokens } from "@/lib/benchmark-data";
-import { Chart, Row, styles as s, slotClass } from "./primitives";
+import { Chart, ChartDataDisclosure, DataTable, Row, styles as s, slotClass } from "./primitives";
/* Streaming speed, derived from the current benchmark: the mean output per
screen over all 1,104 scored runs, decoded at a fixed 50 tokens per second. */
@@ -70,6 +70,37 @@ export function SpeedTokens({ note }: { note?: React.ReactNode | null } = {}) {
);
})}
</div>
+ <ChartDataDisclosure label="View streaming data">
+ <DataTable>
+ <caption>
+ Estimated streaming time from mean output tokens at 50 tokens per second
+ </caption>
+ <thead>
+ <tr>
+ <th scope="col">Format</th>
+ <th scope="col" data-numeric="true">
+ Mean output tokens
+ </th>
+ <th scope="col" data-numeric="true">
+ Decode rate
+ </th>
+ <th scope="col" data-numeric="true">
+ Estimated seconds
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ {FORMAT_ORDER.map((format) => (
+ <tr key={`data-${format}`}>
+ <th scope="row">{FORMATS.find((item) => item.id === format)!.label}</th>
+ <td data-numeric="true">{tokens.outputPerScreen[format].toLocaleString()}</td>
+ <td data-numeric="true">{DECODE_TPS} tok/s</td>
+ <td data-numeric="true">{secs(format).toFixed(1)}s</td>
+ </tr>
+ ))}
+ </tbody>
+ </DataTable>
+ </ChartDataDisclosure>
</Chart>
);
}
diff --git a/docs/components/charts/viz.module.css b/docs/components/charts/viz.module.css
index 46945abca..87dd4db5c 100644
--- a/docs/components/charts/viz.module.css
+++ b/docs/components/charts/viz.module.css
@@ -14,9 +14,18 @@
--ink-note: var(--openui-text-neutral-tertiary, #8f8da3);
--rule: color-mix(in srgb, var(--ink) 6%, transparent);
--chip: color-mix(in srgb, var(--ink) 6%, transparent);
+ /* the page's own table tokens, so a disclosure table framed like the
+ marketing tables stays framed like them in both themes */
+ --table-rule: var(--openui-border-default, color-mix(in srgb, var(--ink) 12%, transparent));
+ --table-zebra: var(--openui-highlight-subtle, color-mix(in srgb, var(--ink) 3%, transparent));
/* the card surface token — distinct from the page background in both modes */
--surface: var(--openui-foreground, #fff);
+ /* Published so a full-bleed child can escape exactly this much and no
+ more. Any rule that changes the card's inline padding must change this
+ with it — the /benchmarks page sets both to 0. */
+ --card-pad: 28px;
+
margin: 2.5rem 0;
padding: 20px 28px;
border-radius: 18px;
@@ -130,6 +139,13 @@
flex: none;
}
+.paretoKeyLine {
+ width: 22px;
+ height: 0;
+ border-top: 1.5px solid var(--ink);
+ opacity: 0.62;
+}
+
/* rows -------------------------------------------------------------- */
.rows {
@@ -577,6 +593,8 @@
@media (max-width: 640px) {
.viz {
+ --card-pad: 18px;
+
padding: 16px 18px;
gap: 20px;
}
@@ -651,6 +669,15 @@
/* frontier ------------------------------------------------------------ */
+.chartControlRow {
+ position: relative;
+ z-index: 8;
+ display: flex;
+ align-items: flex-start;
+ justify-content: space-between;
+ gap: 16px;
+}
+
.axisToggle {
display: inline-flex;
align-self: flex-start;
@@ -676,6 +703,182 @@
box-shadow: 0 1px 2px rgba(0, 0, 0, 0.08);
}
+.modelSelect {
+ position: relative;
+ margin-left: auto;
+ color: var(--ink);
+ font-size: 0.875rem;
+}
+
+.modelSelect > summary {
+ display: flex;
+ align-items: center;
+ gap: 9px;
+ min-height: 34px;
+ padding: 6px 10px 6px 12px;
+ border: 1px solid var(--rule);
+ border-radius: 8px;
+ background: var(--surface);
+ cursor: pointer;
+ list-style: none;
+ user-select: none;
+}
+
+.modelSelect > summary::-webkit-details-marker {
+ display: none;
+}
+
+.modelSelect > summary:hover {
+ background: var(--chip);
+}
+
+.modelSelect > summary strong {
+ color: var(--ink-muted);
+ font-weight: 500;
+}
+
+.modelSelectChevron {
+ color: var(--ink-muted);
+ font-size: 0.75rem;
+ transition: transform 140ms ease;
+}
+
+.modelSelect[open] .modelSelectChevron {
+ transform: rotate(180deg);
+}
+
+.modelSelectMenu {
+ position: absolute;
+ top: calc(100% + 8px);
+ right: 0;
+ width: min(360px, calc(100vw - 32px));
+ padding: 10px;
+ border: 1px solid var(--rule);
+ border-radius: 10px;
+ background: var(--surface);
+ box-shadow: 0 16px 44px rgba(0, 0, 0, 0.14);
+}
+
+.modelSelectActions {
+ display: flex;
+ align-items: center;
+ gap: 6px;
+ padding: 2px 2px 10px;
+ border-bottom: 1px solid var(--rule);
+ color: var(--ink-muted);
+ font-size: 0.75rem;
+}
+
+.modelSelectActions span {
+ margin-right: auto;
+}
+
+.modelSelectActions button {
+ border: 0;
+ border-radius: 5px;
+ background: var(--chip);
+ color: var(--ink);
+ font: inherit;
+ padding: 4px 8px;
+ cursor: pointer;
+}
+
+.modelSelectActions button:hover {
+ background: color-mix(in srgb, var(--ink) 10%, var(--surface));
+}
+
+.modelSelectList {
+ max-height: 344px;
+ overflow-y: auto;
+ padding-top: 6px;
+ scrollbar-width: thin;
+}
+
+.modelSelectGroup {
+ padding: 4px 0 6px;
+}
+
+.modelSelectGroup + .modelSelectGroup {
+ margin-top: 5px;
+ padding-top: 10px;
+ border-top: 1px solid var(--rule);
+}
+
+.modelSelectGroupTitle {
+ display: flex;
+ align-items: center;
+ gap: 7px;
+ padding: 3px 6px 5px;
+ color: var(--ink-muted);
+ font-size: 0.75rem;
+ letter-spacing: 0.02em;
+}
+
+.modelSelectGroupTitle strong {
+ color: var(--ink-muted);
+ font-weight: 500;
+}
+
+.modelSelectGroupTitle .modelSelectDot {
+ width: 8px;
+ height: 8px;
+}
+
+.modelSelectGroupTitle > span:last-child {
+ margin-left: auto;
+ letter-spacing: 0;
+ opacity: 0.65;
+}
+
+.modelSelectList label {
+ display: grid;
+ grid-template-columns: 16px minmax(0, 1fr);
+ align-items: center;
+ gap: 8px;
+ min-height: 32px;
+ padding: 4px 6px;
+ border-radius: 6px;
+ cursor: pointer;
+}
+
+.modelSelectList label:hover {
+ background: var(--chip);
+}
+
+.modelSelectList input {
+ width: 14px;
+ height: 14px;
+ margin: 0;
+ accent-color: var(--ink);
+}
+
+.modelSelectDot {
+ width: 8px;
+ height: 8px;
+ border-radius: 50%;
+}
+
+@media (max-width: 620px) {
+ .chartControlRow {
+ align-items: center;
+ }
+
+ .axisBtn {
+ padding-inline: 9px;
+ font-size: 0.75rem;
+ }
+
+ .modelSelect > summary {
+ gap: 6px;
+ padding-inline: 9px;
+ font-size: 0.75rem;
+ }
+
+ .modelSelect > summary > span:first-child {
+ display: none;
+ }
+}
+
/* Holder for live-pixel SVG charts: until the ResizeObserver delivers the real
width (SSR, first paint, hidden tabs), the default-width SVG must clip
inside the card rather than widen the page. Vertical overflow stays open so
@@ -685,6 +888,211 @@
overflow-y: visible;
}
+.svgHolder.edgeAligned {
+ overflow-x: visible;
+}
+
+/* exact, server-rendered chart data -------------------------------------- */
+
+.dataDisclosure {
+ margin-top: 2px;
+ color: var(--ink);
+}
+
+.dataDisclosure > summary {
+ display: flex;
+ width: fit-content;
+ align-items: center;
+ gap: 6px;
+ padding: 7px 0 2px;
+ color: var(--ink-muted);
+ font-size: 0.75rem;
+ font-weight: 450;
+ opacity: 0.62;
+ cursor: pointer;
+ list-style: none;
+ transition:
+ color 140ms ease,
+ opacity 140ms ease;
+}
+
+/* a chevron, drawn from two borders so it inherits the summary's colour and
+ needs no asset: pointing down while collapsed, up once open */
+.dataDisclosure > summary::-webkit-details-marker {
+ display: none;
+}
+
+.dataDisclosure > summary::after {
+ content: "";
+ flex: none;
+ width: 7px;
+ height: 7px;
+ margin-top: -3px;
+ border-right: 1.5px solid currentColor;
+ border-bottom: 1.5px solid currentColor;
+ transform: rotate(45deg);
+ transition: transform 180ms ease;
+}
+
+.dataDisclosure[open] > summary::after {
+ margin-top: 3px;
+ transform: rotate(-135deg);
+}
+
+.dataDisclosure > summary:hover,
+.dataDisclosure > summary:focus-visible,
+.dataDisclosure[open] > summary {
+ color: var(--ink);
+ opacity: 0.9;
+}
+
+/* The body no longer scrolls: each table carries its own scroller, so a wide
+ table slides under headings that stay put. Spacing steps down from here —
+ disclosure 22, section 34, heading 14, note 12 — so the nesting is legible
+ without rules or indentation. */
+.dataDisclosureBody {
+ max-width: 100%;
+ padding: 22px 0 2px;
+}
+
+.dataSection + .dataSection {
+ margin-top: 34px;
+}
+
+.dataSectionHeading {
+ margin: 0 0 14px;
+ padding-left: 4px;
+ color: var(--ink);
+ font-size: 0.8125rem;
+ font-weight: 550;
+}
+
+/* One table, one scroller. No max-width here: a block element already fills
+ its container, and this rule sits after the mobile override in source
+ order, so a max-width would silently beat the full-bleed width there. */
+.dataScroller {
+ overflow-x: auto;
+ overscroll-behavior-x: contain;
+}
+
+/* Full bleed at the same width the marketing tables use, so every table on
+ the page reaches the screen edges together rather than at two different
+ breakpoints. Placed after the base rule: an earlier @media block loses to
+ a later plain rule at equal specificity. */
+@media (max-width: 767px) {
+ /* A wide table scrolls the full width of the phone instead of sliding
+ inside an inset window that reads as cutting it off on both sides. */
+ .dataScroller {
+ /* Break out to the screen edges regardless of how much page gutter,
+ card padding and section margin sit between here and the window.
+ 50% resolves against this element's own containing block, so
+ (50% - 50vw) is exactly the distance to the viewport edge whatever
+ that stack adds up to — no inset to hardcode and none to keep in sync.
+ The same amount is padded back in, so the first column still lines up
+ with the text above it while the last one can reach the screen. */
+ box-sizing: border-box;
+ width: 100vw;
+ margin-inline: calc(50% - 50vw);
+ padding-inline: calc(50vw - 50%);
+ scroll-padding-inline: calc(50vw - 50%);
+ }
+}
+
+/* Same frame as the marketing tables on the page — rounded outline, ruled
+ cells, alternating rows — one size down, so the exact numbers read as a
+ quieter member of the same family rather than raw output. The "best value
+ in the row" emphasis those tables carry does not apply here: these rows are
+ not competing series, and the model board marks its standouts with its own
+ Pareto frontier column. */
+.dataTable {
+ width: 100%;
+ min-width: 820px;
+ border: 1px solid var(--table-rule);
+ border-radius: 12px;
+ border-collapse: separate;
+ border-spacing: 0;
+ overflow: hidden;
+ font-size: 0.75rem;
+ font-variant-numeric: tabular-nums;
+}
+
+.dataTable th:nth-last-child(2),
+.dataTable td:nth-last-child(2) {
+ min-width: 104px;
+}
+
+.dataTable th:last-child,
+.dataTable td:last-child {
+ min-width: 112px;
+}
+
+.dataTable caption,
+.tableCaption {
+ position: absolute;
+ width: 1px;
+ height: 1px;
+ padding: 0;
+ overflow: hidden;
+ clip: rect(0 0 0 0);
+ clip-path: inset(50%);
+ white-space: nowrap;
+}
+
+.dataTable th,
+.dataTable td {
+ padding: 7px 12px;
+ border-right: 1px solid var(--table-rule);
+ border-bottom: 1px solid var(--table-rule);
+ text-align: left;
+ white-space: nowrap;
+}
+
+.dataTable th:last-child,
+.dataTable td:last-child {
+ border-right: 0;
+}
+
+.dataTable tbody tr:last-child th,
+.dataTable tbody tr:last-child td {
+ border-bottom: 0;
+}
+
+.dataTable tbody tr:nth-child(even) {
+ background: var(--table-zebra);
+}
+
+.dataTable th {
+ color: var(--ink-muted);
+ font-weight: 500;
+}
+
+/* the name anchors the row; every measured value stays grey */
+.dataTable tbody th {
+ color: var(--ink);
+}
+
+.dataTable tbody td {
+ color: var(--ink-muted);
+}
+
+.dataTable td[data-numeric="true"],
+.dataTable th[data-numeric="true"] {
+ text-align: right;
+}
+
+.dataNote {
+ margin: 12px 0 0;
+ padding-left: 4px;
+ color: var(--ink-muted);
+ font-size: 0.75rem;
+ line-height: 1.5;
+}
+
+/* a second legend row sits closer to the one above it than to the chart */
+.legendTight {
+ margin-top: -14px;
+}
+
/* vivid repair funnel */
.vivid .segBar {
height: 44px;
@@ -1894,3 +2302,25 @@
font-weight: 550;
color: var(--ink-muted);
}
+
+/* axis titles: one style for both hero charts, distinct from the in-plot
+ header so a rotated y title never reads as a heading */
+.axisTitle {
+ font-size: 12.5px;
+ fill: var(--ink-muted);
+ letter-spacing: 0.01em;
+}
+
+/* Provider hues are chosen against a white page, where they need to be dark
+ enough to read. On a dark surface that same darkness sinks into the
+ background, so every dot carrying one is lifted together. One rule, rather
+ than a second hand-tuned palette that would drift from the first. */
+@media (prefers-color-scheme: dark) {
+ :global(html:not([data-theme="light"]):not(.light)) .viz .providerHue {
+ filter: brightness(1.45) saturate(1.08);
+ }
+}
+:global(.dark) .viz .providerHue,
+:global([data-theme="dark"]) .viz .providerHue {
+ filter: brightness(1.45) saturate(1.08);
+}
diff --git a/docs/components/marketing-table.module.css b/docs/components/marketing-table.module.css
index a76ef7688..22b6e33e2 100644
--- a/docs/components/marketing-table.module.css
+++ b/docs/components/marketing-table.module.css
@@ -54,11 +54,18 @@
@media (max-width: 767px) {
.edgeToEdgeMobile {
+ /* Break out to the screen edges regardless of how much page gutter,
+ card padding and section margin sit between here and the window.
+ 50% resolves against this element's own containing block, so
+ (50% - 50vw) is exactly the distance to the viewport edge whatever
+ that stack adds up to — no inset to hardcode and none to keep in sync.
+ The same amount is padded back in, so the first column still lines up
+ with the text above it while the last one can reach the screen. */
box-sizing: border-box;
- width: calc(100% + 32px);
- margin-left: -16px;
- padding-inline: 16px;
- scroll-padding-inline: 16px;
+ width: 100vw;
+ margin-inline: calc(50% - 50vw);
+ padding-inline: calc(50vw - 50%);
+ scroll-padding-inline: calc(50vw - 50%);
}
.table {
diff --git a/docs/lib/benchmark-agent-data.ts b/docs/lib/benchmark-agent-data.ts
new file mode 100644
index 000000000..b256b6bf2
--- /dev/null
+++ b/docs/lib/benchmark-agent-data.ts
@@ -0,0 +1,500 @@
+import {
+ BENCHMARK_UPDATED,
+ BENCHMARK_VERSION,
+ blankScreens,
+ BRIEFS,
+ completionByDensity,
+ completionByModel,
+ completionOver,
+ costPerPass,
+ FORMAT_ORDER,
+ formatLabel,
+ FORMATS,
+ LINKS,
+ MODEL_BOARD_FAMILIES,
+ modelBoardCostPerTask,
+ modelBoardDefaultSelected,
+ modelBoardFamilyFor,
+ modelBoardFrontier,
+ MODELS,
+ OPENUI_MODEL_BOARD,
+ production,
+ repairFunnel,
+ runCounts,
+ RUNS_PER_FORMAT,
+ RUNS_TOTAL,
+ tokens,
+} from "@/lib/benchmark-data";
+import { BASE_URL } from "@/lib/source";
+
+export const BENCHMARK_CANONICAL_URL = `${BASE_URL}/benchmarks`;
+export const BENCHMARK_DATASET_NAME = "OpenUI Generative UI Benchmark";
+export const BENCHMARK_SCHEMA_URL = `${BENCHMARK_CANONICAL_URL}/data.schema.json`;
+export const LANGUAGE_BENCHMARK_URL = `${BENCHMARK_CANONICAL_URL}/language`;
+export const FRAMEWORK_BENCHMARK_URL = `${BENCHMARK_CANONICAL_URL}/framework`;
+export const MODEL_BOARD_UPDATED_ISO = "2026-08-25";
+export const BENCHMARK_UPDATED_ISO = "2026-08-18";
+
+const frontierIds = new Set(modelBoardFrontier().map((point) => point.id));
+
+export const modelBoardRows = OPENUI_MODEL_BOARD.map((point) => {
+ const family = modelBoardFamilyFor(point.id);
+ const familyOrder = family ? family.models.indexOf(point.id as never) + 1 : null;
+ const selfHosted = "unpriced" in point;
+ const free = !selfHosted && point.costPerPass === 0;
+
+ return {
+ model_id: point.id,
+ model_name: point.label,
+ provider: point.provider,
+ family_id: family?.id ?? point.id,
+ family_name: family?.label ?? point.label,
+ family_order: familyOrder,
+ connected_family: false,
+ default_selected: modelBoardDefaultSelected(point.id),
+ format: "openui",
+ validity_score_percent: point.score,
+ cost_per_task_usd: selfHosted ? null : modelBoardCostPerTask(point),
+ cost_per_46_screen_pass_usd: selfHosted ? null : point.costPerPass,
+ cost_type: selfHosted ? "self-hosted" : free ? "free" : "provider-list-price",
+ serving: selfHosted && "serving" in point ? point.serving : null,
+ pareto_frontier: !selfHosted && frontierIds.has(point.id),
+ briefs: BRIEFS,
+ generations_per_brief: 4,
+ dataset_updated: MODEL_BOARD_UPDATED_ISO,
+ };
+});
+
+export const formatComparisonRows = MODELS.flatMap((model) =>
+ FORMAT_ORDER.map((format) => ({
+ model_id: model.id,
+ model_name: "family" in model && model.family ? `${model.family} ${model.label}` : model.label,
+ provider: model.vendor,
+ format_id: format,
+ format_name: formatLabel(format),
+ valid_runs: runCounts[model.id][format].complete,
+ rendered_runs: runCounts[model.id][format].renderable,
+ total_runs: runCounts[model.id][format].runs,
+ validity_score_percent: completionByModel[model.id][format],
+ render_rate_percent:
+ (runCounts[model.id][format].renderable / runCounts[model.id][format].runs) * 100,
+ cost_per_46_screen_pass_usd: costPerPass[model.id]?.[format] ?? null,
+ })),
+);
+
+export const formatSummaryRows = FORMAT_ORDER.map((format) => ({
+ format_id: format,
+ format_name: formatLabel(format),
+ validity_score_percent: completionOver(format),
+ render_rate_percent: (1 - blankScreens[format] / RUNS_PER_FORMAT) * 100,
+ blank_screens: blankScreens[format],
+ runs: RUNS_PER_FORMAT,
+ system_prompt_tokens: tokens.systemPrompt[format],
+ mean_output_tokens_per_screen: tokens.outputPerScreen[format],
+ estimated_stream_seconds_at_50_tps: tokens.outputPerScreen[format] / 50,
+}));
+
+const bestValidity = Math.max(...modelBoardRows.map((row) => row.validity_score_percent));
+const bestValidityModels = modelBoardRows
+ .filter((row) => row.validity_score_percent === bestValidity)
+ .map((row) => row.model_name);
+const openuiSummary = formatSummaryRows.find((row) => row.format_id === "openui")!;
+const lowestPromptTokenFormat = [...formatSummaryRows].sort(
+ (a, b) => a.system_prompt_tokens - b.system_prompt_tokens,
+)[0];
+
+export const benchmarkAgentAnswers = [
+ {
+ id: "benchmark-scope",
+ question: "How large is the benchmark?",
+ answer: `${BRIEFS} interface briefs are used. The format comparison covers ${MODELS.length} models and ${FORMATS.length} formats for ${RUNS_TOTAL.toLocaleString("en-US")} scored runs; the OpenUI model board covers ${OPENUI_MODEL_BOARD.length} models.`,
+ evidence_paths: ["scope", "model_board", "format_comparison"],
+ },
+ {
+ id: "valid-definition",
+ question: "What counts as a valid generation?",
+ answer:
+ "A generation must parse, have a root, resolve every reference, contain no orphaned or invented components, use valid props, avoid truncation, and meet the component-count floor.",
+ evidence_paths: ["definitions.valid"],
+ },
+ {
+ id: "highest-model-validity",
+ question: "Which models have the highest OpenUI validity score?",
+ answer: `${bestValidityModels.join(" and ")} tie for the highest measured validity score at ${bestValidity.toFixed(1)}%. Cost determines which of these is Pareto-efficient; a high score alone does not imply frontier membership.`,
+ evidence_paths: ["model_board"],
+ },
+ {
+ id: "openui-format-result",
+ question: "How did OpenUI perform in the format comparison?",
+ answer: `OpenUI achieved ${openuiSummary.validity_score_percent.toFixed(1)}% structural validity and ${openuiSummary.render_rate_percent.toFixed(1)}% render success across ${openuiSummary.runs.toLocaleString("en-US")} runs, with ${openuiSummary.blank_screens} blank screen.`,
+ evidence_paths: ["format_summary"],
+ },
+ {
+ id: "prompt-token-result",
+ question: "Which format used the fewest system-prompt tokens?",
+ answer: `${lowestPromptTokenFormat.format_name} used the fewest system-prompt tokens at ${lowestPromptTokenFormat.system_prompt_tokens.toLocaleString("en-US")}.`,
+ evidence_paths: ["format_summary"],
+ },
+ {
+ id: "self-hosted-cost",
+ question: "Does a zero or missing cost mean a self-hosted model is free?",
+ answer:
+ "No. Self-hosted models have no comparable API list price, so their cost is null. A numeric zero is reserved for a source reporting zero marginal API price.",
+ evidence_paths: ["definitions.cost_type", "model_board"],
+ },
+ {
+ id: "pareto-meaning",
+ question: "What does Pareto frontier mean in this benchmark?",
+ answer:
+ "A priced model is on the frontier when no other comparably priced model is at least as valid while being strictly better on cost or validity. Frontier membership is a cost-quality trade-off, not a general model ranking.",
+ evidence_paths: ["definitions.pareto_frontier", "model_board"],
+ },
+ {
+ id: "chart-views",
+ question: "What do the two tabs at the top of the benchmark show?",
+ answer:
+ "Model comparison shows OpenUI structural validity and cost across ${MODEL_BOARD_SIZE} models as provider-coloured dots with one Pareto frontier. Models scoring below 70% structural validity are deselected by default but remain in the data; both axes rescale to whatever is selected, so selecting a lower-scoring model extends the vertical scale down to reach it. Format comparison shows structural validity and cost across OpenUI, A2UI, and json-render for six models. Exact values for both views are present in the page's server-rendered chart data tables and machine-readable distributions.",
+ evidence_paths: ["model_board", "model_families", "format_comparison", "distributions"],
+ },
+] as const;
+
+export const benchmarkAgentDataset = {
+ $schema: BENCHMARK_SCHEMA_URL,
+ schema_version: "1.0.0",
+ dataset: {
+ name: BENCHMARK_DATASET_NAME,
+ canonical_url: BENCHMARK_CANONICAL_URL,
+ version: BENCHMARK_VERSION,
+ benchmark_updated: BENCHMARK_UPDATED_ISO,
+ model_board_updated: MODEL_BOARD_UPDATED_ISO,
+ benchmark_updated_label: BENCHMARK_UPDATED,
+ currency: "USD",
+ pricing_basis: "Measured tokens at provider list prices unless cost_type says otherwise.",
+ maintainer: "OpenUI by Thesys",
+ first_party_disclosure: "OpenUI built and maintains this first-party benchmark.",
+ },
+ definitions: {
+ valid:
+ "Parses, has a root, resolves every reference, has no orphaned or invented components, uses valid props, is not truncated, and meets the component-count floor.",
+ render_success: "The format's own shipped renderer produced a non-blank screen.",
+ cost_per_task_usd: "Measured cost of one generated screen, in USD.",
+ cost_type: {
+ "provider-list-price":
+ "Comparable API price calculated from measured tokens and list prices.",
+ free: "The source reports zero marginal API price.",
+ "self-hosted": "No comparable API price; cost_per_task_usd is null, never zero.",
+ },
+ pareto_frontier:
+ "True when no other comparably priced model is at least as valid and strictly better on cost or validity.",
+ family_order:
+ "One-based catalog order within a named model family; null for singleton models. Families are not connected in the chart.",
+ connected_family:
+ "Whether members of a model family are connected by a chart line. False for every row in this chart version.",
+ default_selected:
+ "Whether the model is selected in the chart's opening view, which is every model at or above 70% structural validity.",
+ generation_condition:
+ "Four generations per brief, a 16,384-token output ceiling, temperature 0.7 where the provider accepts it, and minimal or no reasoning. Anthropic runs use the model default temperature because that API rejects the benchmark temperature setting for the tested model.",
+ scorer_regime:
+ "Every row is scored by one build of the shipped OpenUI parser, lang-core 0.2.16, whose parser validates enum and scalar prop values. The website and the benchmark repository are on the same regime; raw outputs and per-run verdicts are committed for independent rescoring.",
+ },
+ scope: {
+ briefs: BRIEFS,
+ formats: FORMATS.length,
+ comparison_models: MODELS.length,
+ model_board_models: OPENUI_MODEL_BOARD.length,
+ runs_per_format: RUNS_PER_FORMAT,
+ total_scored_runs: RUNS_TOTAL,
+ },
+ provenance: {
+ format_comparison: {
+ evidence_status: "published",
+ scorer_regime: "lang-core 0.2.16",
+ raw_results: LINKS.rawData,
+ raw_outputs: LINKS.rawOutputs,
+ harness: LINKS.harness,
+ briefs: LINKS.briefs,
+ },
+ model_board: {
+ evidence_status: "summary-only",
+ note: "The model-board measurements are published here as summary rows. Raw generations and per-run verdicts for every row are committed in the benchmark repository for independent rescoring.11 regime and still lack per-row evidence and pricing links. Do not treat current-main results as row-identical evidence for this board.",
+ raw_results: null,
+ },
+ },
+ model_families: MODEL_BOARD_FAMILIES,
+ model_board: modelBoardRows,
+ format_comparison: formatComparisonRows,
+ format_summary: formatSummaryRows,
+ complexity_bands: completionByDensity,
+ production_repair: {
+ ...production,
+ stages: repairFunnel.stages,
+ source_scope: "Recent OpenUI Cloud production traffic; not benchmark runs.",
+ },
+ methodology: `${BENCHMARK_CANONICAL_URL}/methodology`,
+ agent_answers: benchmarkAgentAnswers,
+ distributions: {
+ schema: BENCHMARK_SCHEMA_URL,
+ json: `${BENCHMARK_CANONICAL_URL}/data.json`,
+ csv: `${BENCHMARK_CANONICAL_URL}/data.csv`,
+ markdown: `${BENCHMARK_CANONICAL_URL}/agent.md`,
+ raw_results: LINKS.rawData,
+ harness: LINKS.harness,
+ briefs: LINKS.briefs,
+ language_benchmark: LANGUAGE_BENCHMARK_URL,
+ framework_benchmark: FRAMEWORK_BENCHMARK_URL,
+ },
+};
+
+const csvCell = (value: unknown) => {
+ if (value === null || value === undefined) return "";
+ const stringValue = String(value);
+ return /[",\n]/.test(stringValue) ? `"${stringValue.replaceAll('"', '""')}"` : stringValue;
+};
+
+export const modelBoardCsv = () => {
+ const headers = Object.keys(modelBoardRows[0]) as Array<keyof (typeof modelBoardRows)[number]>;
+ return [
+ headers.join(","),
+ ...modelBoardRows.map((row) => headers.map((header) => csvCell(row[header])).join(",")),
+ ].join("\n");
+};
+
+export const benchmarkAgentMarkdown = () => {
+ const summaryRows = benchmarkAgentDataset.format_summary
+ .map(
+ (row) =>
+ `| ${row.format_name} | ${row.validity_score_percent.toFixed(1)}% | ${row.render_rate_percent.toFixed(1)}% | ${row.blank_screens} | ${row.system_prompt_tokens.toLocaleString("en-US")} | ${row.mean_output_tokens_per_screen.toLocaleString("en-US")} |`,
+ )
+ .join("\n");
+ const boardRows = modelBoardRows
+ .map(
+ (row) =>
+ `| ${row.model_name} | ${row.provider} | ${row.family_name} | ${row.validity_score_percent.toFixed(1)}% | ${row.cost_per_task_usd === null ? "not comparable" : row.cost_per_task_usd === 0 ? "$0 / free" : `$${row.cost_per_task_usd.toFixed(4)}`} | ${row.cost_type} | ${row.pareto_frontier ? "yes" : "no"} | ${row.default_selected ? "yes" : "no"} |`,
+ )
+ .join("\n");
+ const answerRows = benchmarkAgentAnswers
+ .map((item) => `### ${item.question}\n\n${item.answer}`)
+ .join("\n\n");
+
+ return `# ${BENCHMARK_DATASET_NAME}
+
+> Agent-readable companion to the visual benchmark page. Version ${BENCHMARK_VERSION}; model board updated ${MODEL_BOARD_UPDATED_ISO}.
+
+Canonical page: ${BENCHMARK_CANONICAL_URL}
+
+## Scope and definitions
+
+- ${BRIEFS} briefs, four generations per brief, a 16,384-token output ceiling, temperature 0.7 where supported, and minimal or no reasoning.
+- The six-model format comparison contains ${RUNS_PER_FORMAT.toLocaleString("en-US")} runs per format.
+- Valid means: ${benchmarkAgentDataset.definitions.valid}
+- Render success means: ${benchmarkAgentDataset.definitions.render_success}
+- Self-hosted cost is unknown and represented as null, not zero.
+- This is a first-party benchmark maintained by OpenUI.
+- Every row is scored by one build of the shipped OpenUI parser, lang-core 0.2.16. Raw outputs and per-run verdicts are committed for independent rescoring.
+
+## Evidence status
+
+- Format comparison: published raw results, raw outputs, harness, and briefs are linked below.
+- OpenUI model board: summary measurements are published here, but row-level raw generations and pricing evidence are not yet linked. The format-comparison raw-results folder does not substantiate the model board.
+
+## Format summary
+
+| Format | Valid | Render rate | Blank screens | System prompt tokens | Mean output tokens |
+| --- | ---: | ---: | ---: | ---: | ---: |
+${summaryRows}
+
+## OpenUI model board
+
+Models retain explicit family metadata, but family members are not connected in the chart. The chart draws one Pareto frontier. Models below 70% structural validity are unselected by default so the vertical scale can focus on the range worth comparing; both axes rescale to fit whatever is selected.
+
+| Model | Provider | Family | Valid | Cost per task | Cost type | Pareto frontier | Shown by default |
+| --- | --- | --- | ---: | ---: | --- | --- | --- |
+${boardRows}
+
+## Direct answers
+
+${answerRows}
+
+## Machine-readable distributions
+
+- Focused language/model benchmark: ${LANGUAGE_BENCHMARK_URL}
+- Focused framework benchmark: ${FRAMEWORK_BENCHMARK_URL}
+- JSON Schema: ${BENCHMARK_SCHEMA_URL}
+- JSON: ${BENCHMARK_CANONICAL_URL}/data.json
+- CSV: ${BENCHMARK_CANONICAL_URL}/data.csv
+- Methodology: ${BENCHMARK_CANONICAL_URL}/methodology
+- Raw results: ${LINKS.rawData}
+- Harness: ${LINKS.harness}
+- Briefs: ${LINKS.briefs}
+`;
+};
+
+export const frameworkComparisonCsv = () => {
+ const headers = Object.keys(formatComparisonRows[0]) as Array<
+ keyof (typeof formatComparisonRows)[number]
+ >;
+ return [
+ headers.join(","),
+ ...formatComparisonRows.map((row) => headers.map((header) => csvCell(row[header])).join(",")),
+ ].join("\n");
+};
+
+export const languageBenchmarkDataset = {
+ dataset: {
+ name: "OpenUI language and model benchmark",
+ canonical_url: LANGUAGE_BENCHMARK_URL,
+ benchmark_type: "OpenUI language output across models",
+ version: BENCHMARK_VERSION,
+ updated: MODEL_BOARD_UPDATED_ISO,
+ },
+ question: "How reliably and economically do different models generate structurally valid OpenUI?",
+ metric: {
+ y: "Structural validity percentage",
+ x: "Measured cost per task in USD at provider list prices where comparable",
+ },
+ scope: {
+ models: modelBoardRows.length,
+ briefs: BRIEFS,
+ generations_per_brief: 4,
+ },
+ caveats: [
+ benchmarkAgentDataset.definitions.scorer_regime,
+ "Self-hosted cost is null because no comparable API list price is available; null does not mean free.",
+ "Five low-validity compact or local models are hidden in the default visual view but remain present in this dataset.",
+ "The website publishes summary rows for this board; row-level evidence links are not yet attached to every displayed value.",
+ ],
+ methodology: `${BENCHMARK_CANONICAL_URL}/methodology`,
+ data: modelBoardRows,
+ distributions: {
+ json: `${LANGUAGE_BENCHMARK_URL}/data.json`,
+ csv: `${LANGUAGE_BENCHMARK_URL}/data.csv`,
+ markdown: `${LANGUAGE_BENCHMARK_URL}/agent.md`,
+ combined_dataset: `${BENCHMARK_CANONICAL_URL}/data.json`,
+ },
+};
+
+export const frameworkBenchmarkDataset = {
+ dataset: {
+ name: "Generative UI framework benchmark",
+ canonical_url: FRAMEWORK_BENCHMARK_URL,
+ benchmark_type: "OpenUI versus A2UI versus json-render",
+ version: BENCHMARK_VERSION,
+ updated: BENCHMARK_UPDATED_ISO,
+ },
+ question:
+ "How do OpenUI, Google A2UI, and Vercel json-render compare under the same briefs and generation condition?",
+ scope: {
+ models: MODELS.length,
+ formats: FORMAT_ORDER.length,
+ briefs: BRIEFS,
+ generations_per_brief: 4,
+ runs_per_format: RUNS_PER_FORMAT,
+ total_scored_runs: RUNS_TOTAL,
+ },
+ definitions: {
+ structural_validity: benchmarkAgentDataset.definitions.valid,
+ render_success: benchmarkAgentDataset.definitions.render_success,
+ generation_condition: benchmarkAgentDataset.definitions.generation_condition,
+ scorer_regime: benchmarkAgentDataset.definitions.scorer_regime,
+ },
+ methodology: `${BENCHMARK_CANONICAL_URL}/methodology`,
+ evidence: benchmarkAgentDataset.provenance.format_comparison,
+ format_summary: formatSummaryRows,
+ data: formatComparisonRows,
+ distributions: {
+ json: `${FRAMEWORK_BENCHMARK_URL}/data.json`,
+ csv: `${FRAMEWORK_BENCHMARK_URL}/data.csv`,
+ markdown: `${FRAMEWORK_BENCHMARK_URL}/agent.md`,
+ combined_dataset: `${BENCHMARK_CANONICAL_URL}/data.json`,
+ },
+};
+
+export const languageBenchmarkMarkdown = () => {
+ const rows = modelBoardRows
+ .map(
+ (row) =>
+ `| ${row.model_name} | ${row.provider} | ${row.validity_score_percent.toFixed(1)}% | ${row.cost_per_task_usd === null ? "not comparable" : row.cost_per_task_usd === 0 ? "$0 / free" : `$${row.cost_per_task_usd.toFixed(4)}`} | ${row.cost_type} | ${row.pareto_frontier ? "yes" : "no"} | ${row.default_selected ? "yes" : "no"} |`,
+ )
+ .join("\n");
+
+ return `# OpenUI language and model benchmark
+
+Canonical page: ${LANGUAGE_BENCHMARK_URL}
+
+## What this benchmark answers
+
+How reliably and economically do different models generate structurally valid OpenUI?
+
+## Scope
+
+- ${modelBoardRows.length} models.
+- ${BRIEFS} interface briefs and four generations per brief.
+- Structural validity is the vertical metric; measured list-price cost per task is the horizontal metric where comparable.
+- Self-hosted cost is null, not zero.
+- Models below 70% structural validity are deselected by default in the visual chart but remain in this table.
+- Every row is scored by one build of the shipped OpenUI parser, lang-core 0.2.16.
+
+| Model | Provider | Structural validity | Cost per task | Cost type | Pareto frontier | Shown by default |
+| --- | --- | ---: | ---: | --- | --- | --- |
+${rows}
+
+## Links
+
+- Methodology: ${BENCHMARK_CANONICAL_URL}/methodology
+- JSON: ${LANGUAGE_BENCHMARK_URL}/data.json
+- CSV: ${LANGUAGE_BENCHMARK_URL}/data.csv
+- Combined benchmark: ${BENCHMARK_CANONICAL_URL}
+`;
+};
+
+export const frameworkBenchmarkMarkdown = () => {
+ const summary = formatSummaryRows
+ .map(
+ (row) =>
+ `| ${row.format_name} | ${row.validity_score_percent.toFixed(1)}% | ${row.render_rate_percent.toFixed(1)}% | ${row.blank_screens} | ${row.runs} |`,
+ )
+ .join("\n");
+ const rows = formatComparisonRows
+ .map(
+ (row) =>
+ `| ${row.model_name} | ${row.format_name} | ${row.valid_runs}/${row.total_runs} | ${row.validity_score_percent.toFixed(1)}% | ${row.render_rate_percent.toFixed(1)}% | ${row.cost_per_46_screen_pass_usd === null ? "not available" : `$${row.cost_per_46_screen_pass_usd.toFixed(2)}`} |`,
+ )
+ .join("\n");
+
+ return `# Generative UI framework benchmark
+
+Canonical page: ${FRAMEWORK_BENCHMARK_URL}
+
+## What this benchmark answers
+
+How do OpenUI, Google A2UI, and Vercel json-render compare under the same briefs and generation condition?
+
+## Scope and condition
+
+- ${BRIEFS} interface briefs, ${MODELS.length} models, ${FORMAT_ORDER.length} formats, and four generations per brief.
+- ${RUNS_PER_FORMAT.toLocaleString("en-US")} runs per format; ${RUNS_TOTAL.toLocaleString("en-US")} scored runs in total.
+- 16,384-token output ceiling, temperature 0.7 where supported, and minimal or no reasoning.
+- Each format uses its own SDK-generated prompt and shipped validation, plus the same shared completeness layer and component-count floor.
+- Every row is scored by one build of the shipped OpenUI parser, lang-core 0.2.16.
+
+## Format summary
+
+| Format | Structural validity | Render success | Blank screens | Runs |
+| --- | ---: | ---: | ---: | ---: |
+${summary}
+
+## Results by model and format
+
+| Model | Format | Valid runs | Structural validity | Render success | Cost per 46-screen pass |
+| --- | --- | ---: | ---: | ---: | ---: |
+${rows}
+
+## Links
+
+- Methodology: ${BENCHMARK_CANONICAL_URL}/methodology
+- JSON: ${FRAMEWORK_BENCHMARK_URL}/data.json
+- CSV: ${FRAMEWORK_BENCHMARK_URL}/data.csv
+- Raw results for the published scorer regime: ${LINKS.rawData}
+- Combined benchmark: ${BENCHMARK_CANONICAL_URL}
+`;
+};
diff --git a/docs/lib/benchmark-data-schema.ts b/docs/lib/benchmark-data-schema.ts
new file mode 100644
index 000000000..2b5efad1c
--- /dev/null
+++ b/docs/lib/benchmark-data-schema.ts
@@ -0,0 +1,250 @@
+import { BENCHMARK_SCHEMA_URL } from "@/lib/benchmark-agent-data";
+
+const nullableNumber = { type: ["number", "null"] } as const;
+const nullableString = { type: ["string", "null"] } as const;
+
+export const benchmarkDataSchema = {
+ $schema: "https://json-schema.org/draft/2020-12/schema",
+ $id: BENCHMARK_SCHEMA_URL,
+ title: "OpenUI Generative UI Benchmark dataset",
+ description:
+ "Schema for the versioned OpenUI model board, format comparison, summaries, definitions and agent-oriented answers.",
+ type: "object",
+ required: [
+ "$schema",
+ "schema_version",
+ "dataset",
+ "definitions",
+ "scope",
+ "provenance",
+ "model_families",
+ "model_board",
+ "format_comparison",
+ "format_summary",
+ "agent_answers",
+ "methodology",
+ "distributions",
+ ],
+ properties: {
+ $schema: { const: BENCHMARK_SCHEMA_URL },
+ schema_version: { type: "string", pattern: "^\\d+\\.\\d+\\.\\d+$" },
+ dataset: {
+ type: "object",
+ required: [
+ "name",
+ "canonical_url",
+ "version",
+ "benchmark_updated",
+ "model_board_updated",
+ "currency",
+ "pricing_basis",
+ "maintainer",
+ ],
+ properties: {
+ name: { type: "string" },
+ canonical_url: { type: "string", format: "uri" },
+ version: { type: "string" },
+ benchmark_updated: { type: "string", format: "date" },
+ model_board_updated: { type: "string", format: "date" },
+ benchmark_updated_label: { type: "string" },
+ currency: { const: "USD" },
+ pricing_basis: { type: "string" },
+ maintainer: { type: "string" },
+ first_party_disclosure: { type: "string" },
+ },
+ },
+ definitions: { type: "object" },
+ scope: {
+ type: "object",
+ required: [
+ "briefs",
+ "formats",
+ "comparison_models",
+ "model_board_models",
+ "runs_per_format",
+ "total_scored_runs",
+ ],
+ properties: {
+ briefs: { type: "integer", minimum: 1 },
+ formats: { type: "integer", minimum: 1 },
+ comparison_models: { type: "integer", minimum: 1 },
+ model_board_models: { type: "integer", minimum: 1 },
+ runs_per_format: { type: "integer", minimum: 1 },
+ total_scored_runs: { type: "integer", minimum: 1 },
+ },
+ },
+ provenance: {
+ type: "object",
+ required: ["format_comparison", "model_board"],
+ properties: {
+ format_comparison: {
+ type: "object",
+ required: ["evidence_status", "raw_results", "raw_outputs", "harness", "briefs"],
+ properties: {
+ evidence_status: { const: "published" },
+ raw_results: { type: "string", format: "uri" },
+ raw_outputs: { type: "string", format: "uri" },
+ harness: { type: "string", format: "uri" },
+ briefs: { type: "string", format: "uri" },
+ },
+ },
+ model_board: {
+ type: "object",
+ required: ["evidence_status", "note", "raw_results"],
+ properties: {
+ evidence_status: { const: "summary-only" },
+ note: { type: "string" },
+ raw_results: { type: "null" },
+ },
+ },
+ },
+ },
+ model_families: {
+ type: "array",
+ items: {
+ type: "object",
+ required: ["id", "label", "provider", "models"],
+ properties: {
+ id: { type: "string" },
+ label: { type: "string" },
+ provider: { type: "string" },
+ models: { type: "array", minItems: 2, items: { type: "string" } },
+ },
+ },
+ },
+ model_board: {
+ type: "array",
+ items: { $ref: "#/$defs/modelBoardRow" },
+ },
+ format_comparison: {
+ type: "array",
+ items: { $ref: "#/$defs/formatComparisonRow" },
+ },
+ format_summary: {
+ type: "array",
+ items: { $ref: "#/$defs/formatSummaryRow" },
+ },
+ complexity_bands: { type: "array", items: { type: "object" } },
+ production_repair: { type: "object" },
+ methodology: { type: "string", format: "uri" },
+ agent_answers: {
+ type: "array",
+ minItems: 1,
+ items: { $ref: "#/$defs/agentAnswer" },
+ },
+ distributions: {
+ type: "object",
+ required: ["schema", "json", "csv", "markdown", "raw_results", "harness", "briefs"],
+ additionalProperties: { type: "string", format: "uri" },
+ },
+ },
+ $defs: {
+ modelBoardRow: {
+ type: "object",
+ required: [
+ "model_id",
+ "model_name",
+ "provider",
+ "family_id",
+ "family_name",
+ "family_order",
+ "connected_family",
+ "default_selected",
+ "format",
+ "validity_score_percent",
+ "cost_per_task_usd",
+ "cost_per_46_screen_pass_usd",
+ "cost_type",
+ "serving",
+ "pareto_frontier",
+ "briefs",
+ "generations_per_brief",
+ "dataset_updated",
+ ],
+ properties: {
+ model_id: { type: "string" },
+ model_name: { type: "string" },
+ provider: { type: "string" },
+ family_id: { type: "string" },
+ family_name: { type: "string" },
+ family_order: { type: ["integer", "null"], minimum: 1 },
+ connected_family: { const: false },
+ default_selected: { type: "boolean" },
+ format: { const: "openui" },
+ validity_score_percent: { type: "number", minimum: 0, maximum: 100 },
+ cost_per_task_usd: nullableNumber,
+ cost_per_46_screen_pass_usd: nullableNumber,
+ cost_type: { enum: ["provider-list-price", "free", "self-hosted"] },
+ serving: nullableString,
+ pareto_frontier: { type: "boolean" },
+ briefs: { type: "integer", minimum: 1 },
+ generations_per_brief: { type: "integer", minimum: 1 },
+ dataset_updated: { type: "string", format: "date" },
+ },
+ },
+ formatComparisonRow: {
+ type: "object",
+ required: [
+ "model_id",
+ "model_name",
+ "provider",
+ "format_id",
+ "format_name",
+ "valid_runs",
+ "rendered_runs",
+ "total_runs",
+ "validity_score_percent",
+ "render_rate_percent",
+ "cost_per_46_screen_pass_usd",
+ ],
+ properties: {
+ model_id: { type: "string" },
+ model_name: { type: "string" },
+ provider: { type: "string" },
+ format_id: { enum: ["openui", "a2ui", "jsonRender"] },
+ format_name: { type: "string" },
+ valid_runs: { type: "integer", minimum: 0 },
+ rendered_runs: { type: "integer", minimum: 0 },
+ total_runs: { type: "integer", minimum: 1 },
+ validity_score_percent: { type: "number", minimum: 0, maximum: 100 },
+ render_rate_percent: { type: "number", minimum: 0, maximum: 100 },
+ cost_per_46_screen_pass_usd: nullableNumber,
+ },
+ },
+ formatSummaryRow: {
+ type: "object",
+ required: [
+ "format_id",
+ "format_name",
+ "validity_score_percent",
+ "render_rate_percent",
+ "blank_screens",
+ "runs",
+ "system_prompt_tokens",
+ "mean_output_tokens_per_screen",
+ "estimated_stream_seconds_at_50_tps",
+ ],
+ properties: {
+ format_id: { enum: ["openui", "a2ui", "jsonRender"] },
+ format_name: { type: "string" },
+ validity_score_percent: { type: "number", minimum: 0, maximum: 100 },
+ render_rate_percent: { type: "number", minimum: 0, maximum: 100 },
+ blank_screens: { type: "integer", minimum: 0 },
+ runs: { type: "integer", minimum: 1 },
+ system_prompt_tokens: { type: "integer", minimum: 0 },
+ mean_output_tokens_per_screen: { type: "number", minimum: 0 },
+ estimated_stream_seconds_at_50_tps: { type: "number", minimum: 0 },
+ },
+ },
+ agentAnswer: {
+ type: "object",
+ required: ["id", "question", "answer", "evidence_paths"],
+ properties: {
+ id: { type: "string" },
+ question: { type: "string" },
+ answer: { type: "string" },
+ evidence_paths: { type: "array", minItems: 1, items: { type: "string" } },
+ },
+ },
+ },
+} as const;
diff --git a/docs/lib/benchmark-data.ts b/docs/lib/benchmark-data.ts
index 7eae38b4a..52d4266c3 100644
--- a/docs/lib/benchmark-data.ts
+++ b/docs/lib/benchmark-data.ts
@@ -39,7 +39,7 @@ export const MODELS = [
{ id: "sol", mark: "openai", label: "Sol", family: "GPT-5.6", vendor: "OpenAI", gensPerBrief: 4 },
{ id: "opus", mark: "anthropic", label: "Claude Opus 4.8", vendor: "Anthropic", gensPerBrief: 4 },
{ id: "kimi", mark: "moonshot", label: "Kimi K3", vendor: "Moonshot", gensPerBrief: 4 },
- { id: "gemini", mark: "gemini", label: "Gemini 3.6 Flash", vendor: "Google", gensPerBrief: 4 },
+ { id: "gemini", mark: "gemini", label: "Gemini 3.7 Flash", vendor: "Google", gensPerBrief: 4 },
{ id: "qwen", mark: "qwen", label: "Qwen3.8 2.4T", vendor: "Alibaba", gensPerBrief: 4 },
{ id: "muse", mark: "meta", label: "Muse Spark 1.2", vendor: "Meta", gensPerBrief: 4 },
] as const;
@@ -82,14 +82,14 @@ export const runCounts: Record<ModelId, Record<FormatId, RunCount>> = {
jsonRender: { runs: 184, complete: 161, renderable: 184 },
},
kimi: {
- openui: { runs: 184, complete: 178, renderable: 184 },
+ openui: { runs: 184, complete: 177, renderable: 184 },
a2ui: { runs: 184, complete: 176, renderable: 179 },
jsonRender: { runs: 184, complete: 131, renderable: 184 },
},
gemini: {
- openui: { runs: 184, complete: 175, renderable: 184 },
- a2ui: { runs: 184, complete: 175, renderable: 180 },
- jsonRender: { runs: 184, complete: 142, renderable: 184 },
+ openui: { runs: 184, complete: 182, renderable: 184 },
+ a2ui: { runs: 184, complete: 173, renderable: 178 },
+ jsonRender: { runs: 184, complete: 171, renderable: 184 },
},
qwen: {
openui: { runs: 184, complete: 169, renderable: 183 },
@@ -97,7 +97,7 @@ export const runCounts: Record<ModelId, Record<FormatId, RunCount>> = {
jsonRender: { runs: 184, complete: 149, renderable: 180 },
},
muse: {
- openui: { runs: 184, complete: 178, renderable: 184 },
+ openui: { runs: 184, complete: 177, renderable: 184 },
a2ui: { runs: 184, complete: 178, renderable: 179 },
jsonRender: { runs: 184, complete: 150, renderable: 184 },
},
@@ -451,6 +451,341 @@ export const costPerPass: Partial<Record<ModelId, Record<FormatId, number>>> = {
export const costPer1kScreens = (modelId: ModelId, id: FormatId) =>
(costPerPass[modelId]![id] / BRIEFS) * 1000;
+/* ------------------------------------------------------------------ */
+/* OpenUI model board — the expanded, OpenUI-only provider comparison */
+/* ------------------------------------------------------------------ */
+
+/**
+ * API-model results from the 24–25 Aug 2026 OpenUI model-board run.
+ * Every row used the same 46 briefs, four generations per brief, frozen
+ * OpenUI prompt and validator. Cost is the measured price of one 46-screen
+ * pass; charts divide by BRIEFS so the horizontal axis stays cost per task.
+ *
+ * The two self-hosted starting-line models are intentionally absent because
+ * the source run reports serving hardware rather than a comparable API cost.
+ */
+export const OPENUI_MODEL_BOARD = [
+ {
+ id: "grok-4-6",
+ label: "Grok 4.6",
+ provider: "xAI",
+ score: 99.5,
+ costPerPass: 0.85,
+ },
+ {
+ id: "gpt-5-6-sol",
+ label: "GPT-5.6 Sol",
+ provider: "OpenAI",
+ score: 99.5,
+ costPerPass: 2.19,
+ },
+ {
+ id: "claude-opus-4-8",
+ label: "Claude Opus 4.8",
+ provider: "Anthropic",
+ score: 98.9,
+ costPerPass: 2.27,
+ },
+ {
+ id: "gemini-3-7-flash",
+ label: "Gemini 3.7 Flash",
+ provider: "Google",
+ score: 98.9,
+ costPerPass: 0.46,
+ },
+ {
+ id: "claude-sonnet-5",
+ label: "Claude Sonnet 5",
+ provider: "Anthropic",
+ score: 98.4,
+ costPerPass: 0.93,
+ },
+ {
+ id: "gpt-5-6-terra",
+ label: "GPT-5.6 Terra",
+ provider: "OpenAI",
+ score: 98.4,
+ costPerPass: 1.11,
+ },
+ {
+ id: "kimi-k3",
+ label: "Kimi K3",
+ provider: "Moonshot",
+ score: 96.2,
+ costPerPass: 1.53,
+ },
+ {
+ id: "claude-opus-5",
+ label: "Claude Opus 5",
+ provider: "Anthropic",
+ score: 96.2,
+ costPerPass: 3.56,
+ },
+ {
+ id: "muse-spark-1-2",
+ label: "Muse Spark 1.2",
+ provider: "Meta",
+ score: 96.2,
+ costPerPass: 0.72,
+ },
+ {
+ id: "claude-sonnet-4-6",
+ label: "Claude Sonnet 4.6",
+ provider: "Anthropic",
+ score: 92.9,
+ costPerPass: 2.07,
+ },
+ {
+ id: "qwen-3-8-2-4t",
+ label: "Qwen3.8 2.4T",
+ provider: "Alibaba",
+ score: 91.8,
+ costPerPass: 0.78,
+ },
+ {
+ id: "glm-5-3",
+ label: "GLM-5.3",
+ provider: "Zhipu",
+ score: 90.8,
+ costPerPass: 0.55,
+ },
+ {
+ id: "inkling-small",
+ label: "Inkling Small",
+ provider: "Thinking Machines",
+ score: 88.6,
+ costPerPass: 0.15,
+ },
+ {
+ id: "deepseek-v4-flash",
+ label: "DeepSeek V4 Flash",
+ provider: "DeepSeek",
+ score: 85.8,
+ costPerPass: 0.02,
+ },
+ {
+ id: "deepseek-v4-pro",
+ label: "DeepSeek V4 Pro",
+ provider: "DeepSeek",
+ score: 84.2,
+ costPerPass: 0.41,
+ },
+ {
+ id: "gpt-5-6-luna",
+ label: "GPT-5.6 Luna",
+ provider: "OpenAI",
+ score: 83.7,
+ costPerPass: 0.1,
+ },
+ {
+ id: "qwen-3-8-27b",
+ label: "Qwen3.8 27B",
+ provider: "Alibaba",
+ score: 78.8,
+ costPerPass: 0.25,
+ },
+ {
+ id: "gemini-3-6-flash",
+ label: "Gemini 3.6 Flash",
+ provider: "Google",
+ score: 78.8,
+ costPerPass: 0.42,
+ },
+ {
+ id: "gemini-3-5-flash-lite",
+ label: "Gemini 3.5 Flash-Lite",
+ provider: "Google",
+ score: 78.3,
+ costPerPass: 0.19,
+ },
+ {
+ id: "inkling",
+ label: "Inkling",
+ provider: "Thinking Machines",
+ score: 73.4,
+ costPerPass: 0.37,
+ },
+ {
+ id: "qwen-3-6-27b",
+ label: "Qwen3.6 27B",
+ provider: "Alibaba",
+ score: 68.5,
+ costPerPass: 0.29,
+ },
+ {
+ id: "qwen-3-6-35b-a3b",
+ label: "Qwen3.6 35B-A3B",
+ provider: "Alibaba",
+ score: 61.4,
+ costPerPass: 0.08,
+ },
+ {
+ id: "gemma-4-31b",
+ label: "Gemma 4 31B",
+ provider: "Google",
+ score: 46.7,
+ costPerPass: 0.04,
+ },
+ {
+ id: "phi-4",
+ label: "Phi-4",
+ provider: "Microsoft",
+ score: 44.0,
+ costPerPass: 0.02,
+ },
+ {
+ id: "gemma-4-26b-a4b",
+ label: "Gemma 4 26B-A4B",
+ provider: "Google",
+ score: 29.9,
+ costPerPass: 0.03,
+ },
+ {
+ id: "ministral-8b",
+ label: "Ministral 8B",
+ provider: "Mistral",
+ score: 27.2,
+ costPerPass: 0.04,
+ },
+ {
+ id: "granite-4-1-8b",
+ label: "Granite 4.1 8B",
+ provider: "IBM",
+ score: 14.7,
+ costPerPass: 0.01,
+ },
+ {
+ id: "lfm-2-5-2-6b",
+ label: "LFM 2.5 2.6B",
+ provider: "Liquid",
+ score: 3.3,
+ costPerPass: 0.0,
+ },
+ {
+ id: "diffusion-gemma-26b-a4b",
+ label: "DiffusionGemma 26B-A4B",
+ provider: "Google",
+ score: 13.0,
+ costPerPass: 0,
+ unpriced: true,
+ serving: "Self-hosted · A100 FP8",
+ },
+ {
+ id: "ling-3-tiny",
+ label: "Ling 3.0 Tiny",
+ provider: "InclusionAI",
+ score: 9.8,
+ costPerPass: 0,
+ unpriced: true,
+ serving: "Self-hosted · llama.cpp Q8",
+ },
+] as const;
+
+/**
+ * A model has to clear this structural-validity score to be selected in the
+ * chart's opening view. Below it, a model is not a candidate anyone is
+ * choosing between, and plotting it stretches the axis over empty space.
+ * A threshold rather than a hand-kept list, so the default follows the data
+ * whenever scores are regenerated. Every model stays in the table, the
+ * downloads and the filter, and selecting one expands the axis to fit it.
+ */
+export const MODEL_BOARD_DEFAULT_MIN_SCORE = 70;
+
+export const MODEL_BOARD_DEFAULT_HIDDEN_IDS = OPENUI_MODEL_BOARD.filter(
+ (point) => point.score < MODEL_BOARD_DEFAULT_MIN_SCORE,
+).map((point) => point.id);
+
+export const modelBoardDefaultSelected = (modelId: string) => {
+ const point = OPENUI_MODEL_BOARD.find((entry) => entry.id === modelId);
+ return point ? point.score >= MODEL_BOARD_DEFAULT_MIN_SCORE : true;
+};
+
+/** How many models the board carries. Derived so copy cannot drift from data. */
+export const MODEL_BOARD_SIZE = OPENUI_MODEL_BOARD.length;
+
+export type ModelBoardPoint = (typeof OPENUI_MODEL_BOARD)[number];
+export type ModelBoardProvider = ModelBoardPoint["provider"];
+export const modelBoardCostPerTask = (point: ModelBoardPoint) => point.costPerPass / BRIEFS;
+
+/**
+ * Explicit family membership for the model-board lines. Keeping this beside
+ * the measurements makes the relationship available to tables, data exports
+ * and agents instead of leaving it encoded only in SVG geometry.
+ */
+export const MODEL_BOARD_FAMILIES = [
+ {
+ id: "gpt-5-6",
+ label: "GPT-5.6",
+ provider: "OpenAI",
+ models: ["gpt-5-6-sol", "gpt-5-6-terra", "gpt-5-6-luna"],
+ },
+ {
+ id: "claude",
+ label: "Claude",
+ provider: "Anthropic",
+ models: ["claude-opus-5", "claude-opus-4-8", "claude-sonnet-5", "claude-sonnet-4-6"],
+ },
+ {
+ id: "gemini-flash",
+ label: "Gemini Flash",
+ provider: "Google",
+ models: ["gemini-3-7-flash", "gemini-3-6-flash", "gemini-3-5-flash-lite"],
+ },
+ {
+ id: "gemma-4",
+ label: "Gemma 4",
+ provider: "Google",
+ models: ["gemma-4-31b", "gemma-4-26b-a4b"],
+ },
+ {
+ id: "qwen-3-8",
+ label: "Qwen3.8",
+ provider: "Alibaba",
+ models: ["qwen-3-8-2-4t", "qwen-3-8-27b"],
+ },
+ {
+ id: "qwen-3-6",
+ label: "Qwen3.6",
+ provider: "Alibaba",
+ models: ["qwen-3-6-27b", "qwen-3-6-35b-a3b"],
+ },
+ {
+ id: "deepseek-v4",
+ label: "DeepSeek V4",
+ provider: "DeepSeek",
+ models: ["deepseek-v4-pro", "deepseek-v4-flash"],
+ },
+ {
+ id: "inkling",
+ label: "Inkling",
+ provider: "Thinking Machines",
+ models: ["inkling", "inkling-small"],
+ },
+] as const satisfies ReadonlyArray<{
+ id: string;
+ label: string;
+ provider: ModelBoardProvider;
+ models: readonly string[];
+}>;
+
+export const modelBoardFamilyFor = (modelId: string) =>
+ MODEL_BOARD_FAMILIES.find((family) => family.models.includes(modelId as never));
+
+export const modelBoardFrontier = () =>
+ OPENUI_MODEL_BOARD.filter(
+ (point) =>
+ !("unpriced" in point) &&
+ !OPENUI_MODEL_BOARD.some(
+ (other) =>
+ other !== point &&
+ !("unpriced" in other) &&
+ modelBoardCostPerTask(other) <= modelBoardCostPerTask(point) &&
+ other.score >= point.score &&
+ (modelBoardCostPerTask(other) < modelBoardCostPerTask(point) ||
+ other.score > point.score),
+ ),
+ ).sort((a, b) => modelBoardCostPerTask(b) - modelBoardCostPerTask(a));
+
/* ------------------------------------------------------------------ */
/* 6. Production: failure taxonomy + repair */
/* ------------------------------------------------------------------ */
@@ -522,7 +857,7 @@ export const userVisibleFailureRate = () => production.userVisibleShare;
export const CONFOUNDS = {
promptConditions:
- "One condition for all models and formats. OpenUI's prompt carries its component groups, two rules and two worked examples through lang-core's official generatePrompt options; json-render runs catalog.prompt() with three custom rules; A2UI runs its generator's prompt as-is. The competitors' official options got no worked examples, an asymmetry we plan to close in a follow-up.",
+ "One condition for all models and formats. Each format uses the system prompt generated by its own SDK, and all three carry the same two worked examples. The 70-component protocol catalogs are checked against one shared reference catalog.",
attachRule:
"Prompt content moves these numbers about as much as format choice does: one rule telling the model to attach every component it defines was worth 13 points to OpenUI on Kimi in earlier runs.",
scoring:
@@ -680,8 +1015,11 @@ export const frontierLine = (...args: Parameters<typeof frontierPoints>) =>
/* ------------------------------------------------------------------ */
export const BENCHMARK_VERSION = "v1";
-export const BENCHMARK_UPDATED = "18 Aug 2026";
+export const BENCHMARK_UPDATED = "25 Aug 2026";
+/* Provenance points at main rather than a pinned scorer tag: the published
+ numbers track the repository's current state, so there is one place to look
+ and no second regime to keep straight. */
export const REPO_ROOT = "https://github.com/thesysdev/generative-ui-bench";
const REPO_TREE = `${REPO_ROOT}/tree/main`;
export const LINKS = {
@@ -690,11 +1028,24 @@ export const LINKS = {
harness: REPO_ROOT,
briefs: `${REPO_TREE}/briefs/briefs.ts`,
catalog: `${REPO_TREE}/catalog/public-catalog.json`,
+ latestHarness: REPO_TREE,
+ latestReadme: `${REPO_TREE}/README.md`,
speedHarness: REPO_ROOT,
dispute: `${REPO_ROOT}/issues/new`,
};
export const CHANGELOG = [
+ {
+ date: "25 Aug 2026",
+ version: "v2",
+ entries: [
+ "Rescored 24–25 Aug 2026 against one build of the shipped OpenUI parser, lang-core 0.2.16, whose parser validates enum and scalar prop values. Every row on the site and in the repository is now on that single regime.",
+ "Google's seat in the cross-format slice moves from Gemini 3.6 Flash to Gemini 3.7 Flash. Gemini 3.6 Flash stays on the model board.",
+ "Rows 14 and 21–29 ran with an 8,192-token output ceiling for condition uniformity, because Phi-4's native context cannot fit the prompt plus a 16,384-token output request. Legs that truncated were rerun at the headline ceiling under a pre-registered replace-with-whatever-it-gives rule; no truncations remain except two on DiffusionGemma, which is served at a 16,384-token total window.",
+ "GPT-5.6 Terra reflects a rescore of its committed raws after the original results file was found to be from a different roll.",
+ "Ling 3.0 Flash was run and excluded: its serving returned empty outputs on 42 of 184 legs, which is not scoreable fairly.",
+ ],
+ },
{
date: "18 Aug 2026",
version: "v1",
diff --git a/docs/next.config.mjs b/docs/next.config.mjs
index d0a80d129..4c256fe37 100644
--- a/docs/next.config.mjs
+++ b/docs/next.config.mjs
@@ -102,6 +102,18 @@ const config = {
destination: "/blog/beyond-the-chatbar",
permanent: true,
},
+ // Preserve the founder-shared singular spelling while keeping one
+ // canonical benchmark namespace for search engines and agents.
+ {
+ source: "/benchmark/framework",
+ destination: "/benchmarks/framework",
+ permanent: true,
+ },
+ {
+ source: "/benchmark/language",
+ destination: "/benchmarks/language",
+ permanent: true,
+ },
];
},
async rewrites() {
diff --git a/docs/package.json b/docs/package.json
index 7ea3dbedc..62eae94c8 100644
--- a/docs/package.json
+++ b/docs/package.json
@@ -12,7 +12,10 @@
"postinstall": "fumadocs-mdx",
"lint": "eslint",
"format:fix": "prettier --write .",
- "format:check": "prettier --check ."
+ "format:check": "prettier --check .",
+ "check:benchmark-agents": "node scripts/check-benchmark-agent-readability.mjs && node scripts/check-benchmark-answer-contracts.mjs",
+ "check:benchmark-answer-contracts": "node scripts/check-benchmark-answer-contracts.mjs",
+ "audit:benchmark-publication": "node scripts/audit-benchmark-publication.mjs"
},
"dependencies": {
"@openuidev/cli": "workspace:*",
diff --git a/docs/scripts/audit-benchmark-publication.mjs b/docs/scripts/audit-benchmark-publication.mjs
new file mode 100644
index 000000000..8dc02cee3
--- /dev/null
+++ b/docs/scripts/audit-benchmark-publication.mjs
@@ -0,0 +1,93 @@
+const baseUrl = (process.env.BENCHMARK_BASE_URL ?? "http://localhost:3067").replace(/\/$/, "");
+
+const assert = (condition, message) => {
+ if (!condition) throw new Error(message);
+};
+
+const crawlerAgents = [
+ ["Googlebot", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"],
+ ["Bingbot", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"],
+ ["OAI-SearchBot", "OAI-SearchBot/1.0; +https://openai.com/searchbot"],
+];
+
+const challengeMarkers = [
+ "cf-chl-",
+ "Just a moment...",
+ "Enable JavaScript and cookies to continue",
+ "Attention Required! | Cloudflare",
+];
+
+const fetchPath = async (path, userAgent) => {
+ const response = await fetch(`${baseUrl}${path}`, {
+ headers: userAgent ? { "User-Agent": userAgent } : undefined,
+ redirect: "follow",
+ });
+ const text = await response.text();
+ assert(response.ok, `${path} returned ${response.status}`);
+ assert(
+ !challengeMarkers.some((marker) => text.includes(marker)),
+ `${path} returned an interstitial or bot challenge`,
+ );
+ return { response, text };
+};
+
+const { text: robots } = await fetchPath("/robots.txt");
+assert(robots.toLowerCase().includes("allow: /"), "robots.txt does not explicitly allow crawling");
+assert(!/disallow:\s*\/benchmarks(?:\s|$)/i.test(robots), "robots.txt disallows /benchmarks");
+
+for (const [name, userAgent] of crawlerAgents) {
+ const { text: html } = await fetchPath("/benchmarks", userAgent);
+ assert(html.includes("Generative UI"), `${name} cannot see the benchmark heading`);
+ assert(html.includes('"@type":"Dataset"'), `${name} cannot see Dataset JSON-LD`);
+ assert(html.includes('rel="canonical"'), `${name} cannot see the canonical link`);
+ assert(html.includes("View chart data"), `${name} cannot see server-rendered chart data`);
+ assert(
+ html.includes("Model comparison data") && html.includes("Format comparison data"),
+ `${name} cannot see both tab datasets`,
+ );
+
+ const { text: languageHtml } = await fetchPath("/benchmarks/language", userAgent);
+ assert(
+ languageHtml.includes("OpenUI language and model benchmark"),
+ `${name} cannot see the language benchmark`,
+ );
+ assert(languageHtml.includes('"@type":"Dataset"'), `${name} cannot see language Dataset JSON-LD`);
+
+ const { text: frameworkHtml } = await fetchPath("/benchmarks/framework", userAgent);
+ assert(
+ frameworkHtml.includes("Generative UI framework benchmark"),
+ `${name} cannot see the framework benchmark`,
+ );
+ assert(
+ frameworkHtml.includes('"@type":"Dataset"'),
+ `${name} cannot see framework Dataset JSON-LD`,
+ );
+}
+
+const endpointChecks = [
+ ["/benchmarks/data.json", "application/json"],
+ ["/benchmarks/data.schema.json", "application/schema+json"],
+ ["/benchmarks/data.csv", "text/csv"],
+ ["/benchmarks/agent.md", "text/markdown"],
+ ["/benchmarks/methodology", "text/html"],
+ ["/benchmarks/language", "text/html"],
+ ["/benchmarks/language/data.json", "application/json"],
+ ["/benchmarks/language/data.csv", "text/csv"],
+ ["/benchmarks/language/agent.md", "text/markdown"],
+ ["/benchmarks/framework", "text/html"],
+ ["/benchmarks/framework/data.json", "application/json"],
+ ["/benchmarks/framework/data.csv", "text/csv"],
+ ["/benchmarks/framework/agent.md", "text/markdown"],
+ ["/llms.txt", "text/plain"],
+ ["/sitemap.xml", "application/xml"],
+];
+
+for (const [path, contentType] of endpointChecks) {
+ const { response } = await fetchPath(path, crawlerAgents[2][1]);
+ assert(
+ response.headers.get("content-type")?.includes(contentType),
+ `${path} has unexpected content type: ${response.headers.get("content-type")}`,
+ );
+}
+
+console.log(`Benchmark publication audit passed for ${baseUrl}.`);
diff --git a/docs/scripts/check-benchmark-agent-readability.mjs b/docs/scripts/check-benchmark-agent-readability.mjs
new file mode 100644
index 000000000..62bc794fa
--- /dev/null
+++ b/docs/scripts/check-benchmark-agent-readability.mjs
@@ -0,0 +1,193 @@
+const baseUrl = process.env.BENCHMARK_BASE_URL ?? "http://localhost:3067";
+
+const assert = (condition, message) => {
+ if (!condition) throw new Error(message);
+};
+
+const fetchText = async (path) => {
+ const response = await fetch(`${baseUrl}${path}`);
+ assert(response.ok, `${path} returned ${response.status}`);
+ return { response, text: await response.text() };
+};
+
+const { text: html } = await fetchText("/benchmarks");
+assert(html.includes('"@type":"Dataset"'), "Benchmark page is missing Dataset JSON-LD");
+assert(html.includes('rel="canonical"'), "Benchmark page is missing a canonical link");
+assert(html.includes("/benchmarks/data.json"), "Benchmark page does not advertise JSON data");
+assert(html.includes("/benchmarks/data.csv"), "Benchmark page does not advertise CSV data");
+assert(
+ html.includes("/benchmarks/data.schema.json"),
+ "Benchmark page does not advertise its JSON Schema",
+);
+assert(html.includes("/benchmarks/agent.md"), "Benchmark page does not advertise agent Markdown");
+assert(html.includes('role="tabpanel"'), "Chart tabs are missing a tab panel");
+assert(
+ html.includes('aria-controls="benchmark-model-chart-panel"') &&
+ html.includes('aria-controls="benchmark-format-chart-panel"'),
+ "Each chart tab is not linked to its own panel",
+);
+assert(html.includes("View chart data"), "Hero chart is missing its structured data disclosure");
+assert(
+ html.includes("Model comparison data") && html.includes("Format comparison data"),
+ "Both tab datasets must be present in the server-rendered HTML",
+);
+assert(html.includes("View render data"), "Render chart is missing its structured data disclosure");
+assert(html.includes("View token data"), "Token chart is missing its structured data disclosure");
+assert(
+ html.includes("View complexity data"),
+ "Complexity chart is missing its structured data disclosure",
+);
+assert(html.includes("View repair data"), "Repair chart is missing its structured data disclosure");
+assert(
+ html.includes("/benchmarks/language") && html.includes("/benchmarks/framework"),
+ "Benchmark overview does not link to both focused benchmark pages",
+);
+
+const { text: languageHtml } = await fetchText("/benchmarks/language");
+assert(languageHtml.includes('rel="canonical"'), "Language benchmark is missing a canonical link");
+assert(languageHtml.includes('"@type":"Dataset"'), "Language benchmark is missing Dataset JSON-LD");
+assert(languageHtml.includes("All model results"), "Language benchmark is missing its exact table");
+
+const { text: frameworkHtml } = await fetchText("/benchmarks/framework");
+assert(
+ frameworkHtml.includes('rel="canonical"'),
+ "Framework benchmark is missing a canonical link",
+);
+assert(
+ frameworkHtml.includes('"@type":"Dataset"'),
+ "Framework benchmark is missing Dataset JSON-LD",
+);
+assert(
+ frameworkHtml.includes("Results by model and format"),
+ "Framework benchmark is missing its exact table",
+);
+
+const jsonResponse = await fetch(`${baseUrl}/benchmarks/data.json`);
+assert(jsonResponse.ok, `data.json returned ${jsonResponse.status}`);
+assert(
+ jsonResponse.headers.get("content-type")?.includes("application/json"),
+ "data.json has the wrong content type",
+);
+const data = await jsonResponse.json();
+assert(data.schema_version === "1.0.0", "Unexpected benchmark schema version");
+assert(data.$schema.endsWith("/benchmarks/data.schema.json"), "Dataset is missing $schema");
+/* The board grows. Assert the surfaces agree with each other rather than
+ with a number written here, so adding a model does not fail this check. */
+assert(data.model_board.length > 0, "Model board is empty");
+assert(
+ new Set(data.model_board.map((row) => row.model_id)).size === data.model_board.length,
+ "Model-board IDs are not unique",
+);
+assert(
+ data.model_board
+ .filter((row) => row.cost_type === "self-hosted")
+ .every((row) => row.cost_per_task_usd === null),
+ "Self-hosted cost must be null rather than zero",
+);
+assert(
+ data.model_board.every((row) => row.family_id && row.family_name),
+ "Every model needs explicit family metadata",
+);
+/* The default filter is a rule, not a list: anything below the published
+ structural-validity threshold starts deselected. Assert the rule, so
+ rescoring a model moves it between the two sets without failing here. */
+assert(
+ data.model_board.every((row) => row.default_selected === row.validity_score_percent >= 70),
+ "The chart's default filter no longer matches the published 70% threshold",
+);
+assert(
+ data.model_board.some((row) => !row.default_selected) &&
+ data.model_board.some((row) => row.default_selected),
+ "The default filter should hide some models and show others",
+);
+assert(
+ data.model_board.every((row) => row.connected_family === false),
+ "Model-family lines should not be encoded as connected",
+);
+assert(data.format_comparison.length === 18, "Expected six models × three formats");
+assert(data.agent_answers.length >= 8, "Expected direct agent answers");
+assert(
+ data.provenance.format_comparison.evidence_status === "published",
+ "Format-comparison provenance is missing",
+);
+assert(
+ data.provenance.model_board.evidence_status === "summary-only" &&
+ data.provenance.model_board.raw_results === null,
+ "Model-board evidence limitations are not explicit",
+);
+
+const { response: schemaResponse, text: schemaText } = await fetchText(
+ "/benchmarks/data.schema.json",
+);
+assert(
+ schemaResponse.headers.get("content-type")?.includes("application/schema+json"),
+ "JSON Schema has the wrong content type",
+);
+const schema = JSON.parse(schemaText);
+assert(schema.$id === data.$schema, "JSON Schema $id does not match dataset $schema");
+
+const { response: csvResponse, text: csv } = await fetchText("/benchmarks/data.csv");
+assert(
+ csvResponse.headers.get("content-type")?.includes("text/csv"),
+ "CSV has the wrong content type",
+);
+assert(
+ csv.trim().split("\n").length === data.model_board.length + 1,
+ `CSV should carry one header plus ${data.model_board.length} model rows`,
+);
+
+const { response: markdownResponse, text: markdown } = await fetchText("/benchmarks/agent.md");
+assert(
+ markdownResponse.headers.get("content-type")?.includes("text/markdown"),
+ "Agent document has the wrong content type",
+);
+assert(markdown.includes("## OpenUI model board"), "Agent document is missing the model board");
+assert(
+ markdown.includes("Self-hosted cost is unknown"),
+ "Agent document is missing the cost caveat",
+);
+assert(markdown.includes("## Direct answers"), "Agent document is missing direct answers");
+
+const languageDataResponse = await fetch(`${baseUrl}/benchmarks/language/data.json`);
+assert(languageDataResponse.ok, `language data.json returned ${languageDataResponse.status}`);
+const languageData = await languageDataResponse.json();
+assert(
+ languageData.data.length === data.model_board.length,
+ "Focused language dataset should carry one row per model-board model",
+);
+assert(
+ languageData.dataset.canonical_url.endsWith("/benchmarks/language"),
+ "Focused language dataset has the wrong canonical URL",
+);
+
+const frameworkDataResponse = await fetch(`${baseUrl}/benchmarks/framework/data.json`);
+assert(frameworkDataResponse.ok, `framework data.json returned ${frameworkDataResponse.status}`);
+const frameworkData = await frameworkDataResponse.json();
+assert(frameworkData.data.length === 18, "Focused framework dataset should contain 18 rows");
+assert(
+ frameworkData.dataset.canonical_url.endsWith("/benchmarks/framework"),
+ "Focused framework dataset has the wrong canonical URL",
+);
+
+const { response: languageMarkdownResponse, text: languageMarkdown } = await fetchText(
+ "/benchmarks/language/agent.md",
+);
+assert(
+ languageMarkdownResponse.headers.get("content-type")?.includes("text/markdown"),
+ "Focused language Markdown has the wrong content type",
+);
+assert(languageMarkdown.includes("## Scope"), "Focused language Markdown is missing scope");
+
+const { response: frameworkMarkdownResponse, text: frameworkMarkdown } = await fetchText(
+ "/benchmarks/framework/agent.md",
+);
+assert(
+ frameworkMarkdownResponse.headers.get("content-type")?.includes("text/markdown"),
+ "Focused framework Markdown has the wrong content type",
+);
+assert(
+ frameworkMarkdown.includes("## Format summary"),
+ "Focused framework Markdown is missing its summary",
+);
+
+console.log("Benchmark agent-readability checks passed.");
diff --git a/docs/scripts/check-benchmark-answer-contracts.mjs b/docs/scripts/check-benchmark-answer-contracts.mjs
new file mode 100644
index 000000000..ec3262462
--- /dev/null
+++ b/docs/scripts/check-benchmark-answer-contracts.mjs
@@ -0,0 +1,242 @@
+const baseUrl = process.env.BENCHMARK_BASE_URL ?? "http://localhost:3067";
+
+const assert = (condition, message) => {
+ if (!condition) throw new Error(message);
+};
+
+const fetchText = async (path) => {
+ const response = await fetch(`${baseUrl}${path}`);
+ assert(response.ok, `${path} returned ${response.status}`);
+ return { response, text: await response.text() };
+};
+
+const parseCsv = (source) => {
+ const rows = [];
+ let row = [];
+ let cell = "";
+ let quoted = false;
+
+ for (let index = 0; index < source.length; index += 1) {
+ const character = source[index];
+ if (quoted && character === '"' && source[index + 1] === '"') {
+ cell += '"';
+ index += 1;
+ } else if (character === '"') {
+ quoted = !quoted;
+ } else if (character === "," && !quoted) {
+ row.push(cell);
+ cell = "";
+ } else if (character === "\n" && !quoted) {
+ row.push(cell.replace(/\r$/, ""));
+ rows.push(row);
+ row = [];
+ cell = "";
+ } else {
+ cell += character;
+ }
+ }
+
+ if (cell || row.length) {
+ row.push(cell);
+ rows.push(row);
+ }
+
+ const [headers, ...values] = rows.filter((item) => item.some(Boolean));
+ return values.map((item) =>
+ Object.fromEntries(headers.map((header, index) => [header, item[index]])),
+ );
+};
+
+const { response: jsonResponse, text: jsonText } = await fetchText("/benchmarks/data.json");
+const data = JSON.parse(jsonText);
+const { response: schemaResponse, text: schemaText } = await fetchText(
+ "/benchmarks/data.schema.json",
+);
+const schema = JSON.parse(schemaText);
+const { text: csvText } = await fetchText("/benchmarks/data.csv");
+const { text: markdown } = await fetchText("/benchmarks/agent.md");
+const { text: html } = await fetchText("/benchmarks");
+const { text: languageJsonText } = await fetchText("/benchmarks/language/data.json");
+const { text: frameworkJsonText } = await fetchText("/benchmarks/framework/data.json");
+const languageData = JSON.parse(languageJsonText);
+const frameworkData = JSON.parse(frameworkJsonText);
+
+assert(
+ schemaResponse.headers.get("content-type")?.includes("application/schema+json"),
+ "Schema has the wrong content type",
+);
+assert(data.$schema === schema.$id, "Dataset $schema and schema $id do not match");
+assert(
+ jsonResponse.headers.get("link")?.includes('rel="describedby"'),
+ "Dataset response does not advertise its schema",
+);
+assert(
+ schema.required.every((key) => key in data),
+ "Dataset is missing a schema-required root key",
+);
+assert(html.includes("/benchmarks/data.schema.json"), "HTML does not advertise the JSON Schema");
+assert(
+ data.provenance.model_board.evidence_status === "summary-only",
+ "Model-board evidence status must remain explicit until raw evidence is published",
+);
+assert(
+ JSON.stringify(languageData.data) === JSON.stringify(data.model_board),
+ "Focused language dataset differs from the combined model board",
+);
+assert(
+ JSON.stringify(frameworkData.data) === JSON.stringify(data.format_comparison),
+ "Focused framework dataset differs from the combined format comparison",
+);
+assert(
+ JSON.stringify(frameworkData.format_summary) === JSON.stringify(data.format_summary),
+ "Focused framework summary differs from the combined format summary",
+);
+
+const csvRows = parseCsv(csvText);
+assert(csvRows.length === data.model_board.length, "CSV and JSON model counts differ");
+const csvById = new Map(csvRows.map((row) => [row.model_id, row]));
+for (const model of data.model_board) {
+ const csvRow = csvById.get(model.model_id);
+ assert(csvRow, `CSV is missing ${model.model_id}`);
+ assert(csvRow.model_name === model.model_name, `${model.model_id} name differs in CSV`);
+ assert(csvRow.provider === model.provider, `${model.model_id} provider differs in CSV`);
+ assert(
+ Number(csvRow.validity_score_percent) === model.validity_score_percent,
+ `${model.model_id} validity differs in CSV`,
+ );
+ assert(
+ (csvRow.cost_per_task_usd === "" ? null : Number(csvRow.cost_per_task_usd)) ===
+ model.cost_per_task_usd,
+ `${model.model_id} task cost differs in CSV`,
+ );
+ assert(markdown.includes(`| ${model.model_name} |`), `Markdown is missing ${model.model_name}`);
+}
+
+const answerById = new Map(data.agent_answers.map((item) => [item.id, item]));
+const answer = (id) => {
+ const item = answerById.get(id);
+ assert(item, `Missing agent answer: ${id}`);
+ assert(
+ markdown.includes(`### ${item.question}`),
+ `Markdown is missing question: ${item.question}`,
+ );
+ assert(markdown.includes(item.answer), `Markdown answer differs for: ${item.question}`);
+ return item.answer;
+};
+
+const openui = data.format_summary.find((row) => row.format_id === "openui");
+const bestScore = Math.max(...data.model_board.map((row) => row.validity_score_percent));
+const bestModels = data.model_board.filter((row) => row.validity_score_percent === bestScore);
+const cheapestPrompt = [...data.format_summary].sort(
+ (left, right) => left.system_prompt_tokens - right.system_prompt_tokens,
+)[0];
+
+const contracts = [
+ [
+ "scope includes brief count",
+ () => answer("benchmark-scope").includes(`${data.scope.briefs} interface briefs`),
+ ],
+ [
+ "scope includes format count",
+ () => answer("benchmark-scope").includes(`${data.scope.formats} formats`),
+ ],
+ [
+ "scope includes comparison-model count",
+ () => answer("benchmark-scope").includes(`${data.scope.comparison_models} models`),
+ ],
+ [
+ "scope includes model-board count",
+ () => answer("benchmark-scope").includes(`${data.scope.model_board_models} models`),
+ ],
+ [
+ "scope includes total run count",
+ () => answer("benchmark-scope").includes(data.scope.total_scored_runs.toLocaleString("en-US")),
+ ],
+ [
+ "validity answer states parsing requirement",
+ () => answer("valid-definition").includes("must parse"),
+ ],
+ [
+ "validity answer states reference requirement",
+ () => answer("valid-definition").includes("resolve every reference"),
+ ],
+ [
+ "best-score answer contains every tied model",
+ () => bestModels.every((row) => answer("highest-model-validity").includes(row.model_name)),
+ ],
+ [
+ "best-score answer contains measured score",
+ () => answer("highest-model-validity").includes(`${bestScore.toFixed(1)}%`),
+ ],
+ [
+ "OpenUI answer contains validity",
+ () => answer("openui-format-result").includes(`${openui.validity_score_percent.toFixed(1)}%`),
+ ],
+ [
+ "OpenUI answer contains render rate",
+ () => answer("openui-format-result").includes(`${openui.render_rate_percent.toFixed(1)}%`),
+ ],
+ [
+ "OpenUI answer contains blank-screen count",
+ () => answer("openui-format-result").includes(`${openui.blank_screens} blank screen`),
+ ],
+ [
+ "prompt answer identifies the minimum",
+ () => answer("prompt-token-result").includes(`${cheapestPrompt.format_name} used the fewest`),
+ ],
+ [
+ "self-hosted rows use null cost",
+ () =>
+ data.model_board
+ .filter((row) => row.cost_type === "self-hosted")
+ .every((row) => row.cost_per_task_usd === null),
+ ],
+ [
+ "numeric zero is restricted to free rows",
+ () =>
+ data.model_board
+ .filter((row) => row.cost_per_task_usd === 0)
+ .every((row) => row.cost_type === "free"),
+ ],
+ [
+ "cost answer distinguishes null from zero",
+ () =>
+ answer("self-hosted-cost").includes("cost is null") &&
+ answer("self-hosted-cost").includes("numeric zero"),
+ ],
+ [
+ "Pareto answer rejects general ranking",
+ () => answer("pareto-meaning").includes("not a general model ranking"),
+ ],
+ [
+ "family membership references valid models",
+ () =>
+ data.model_families
+ .flatMap((family) => family.models)
+ .every((id) => data.model_board.some((row) => row.model_id === id)),
+ ],
+ [
+ "tab answer names both views",
+ () =>
+ answer("chart-views").includes("Model comparison") &&
+ answer("chart-views").includes("Format comparison"),
+ ],
+ [
+ "tab answer names every compared format",
+ () => ["OpenUI", "A2UI", "json-render"].every((name) => answer("chart-views").includes(name)),
+ ],
+ [
+ "tab answer explains the dynamic validity scale",
+ () => answer("chart-views").includes("70%") && answer("chart-views").includes("rescale"),
+ ],
+ [
+ "the default filter matches the published 70% threshold",
+ () =>
+ data.model_board.every((row) => row.default_selected === row.validity_score_percent >= 70) &&
+ data.model_board.some((row) => !row.default_selected),
+ ],
+];
+
+for (const [name, check] of contracts) assert(check(), `Answer contract failed: ${name}`);
+
+console.log(`Benchmark answer contracts passed (${contracts.length}/${contracts.length}).`);