Skip to content

The Open Standard for Generative UI

9.5 KB
Raw Blame
import { Footer } from "@/app/(home)/sections/Footer/Footer";
import { BENCHMARK_CANONICAL_URL } from "@/lib/benchmark-agent-data";
import {
  BRIEFS,
  CATALOG_COMPONENTS,
  FORMATS,
  LINKS,
  MODELS,
  REPO_ROOT,
  RUNS_PER_FORMAT,
} from "@/lib/benchmark-data";
import { createPageMetadata } from "@/lib/page-metadata";
import { ArrowRight, ArrowUpRight } from "lucide-react";
import { BevelButton } from "../../components/Button/BevelButton";
import { PillLink } from "../../components/Button/Button";
import benchmarkStyles from "../benchmarks.module.css";
import page from "./metadata";

export const metadata = createPageMetadata(page, {
  openGraph: { type: "article", modifiedTime: "2026-08-26" },
});

const SECTIONS = [
  { id: "setup", label: "Benchmark setup" },
  { id: "scoring", label: "What counts as valid" },
  { id: "reproduce", label: "Reproduce the results" },
  { id: "notes", label: "Important notes" },
] as const;

const structuredData = {
  "@context": "https://schema.org",
  "@type": "TechArticle",
  "@id": `${BENCHMARK_CANONICAL_URL}/methodology#article`,
  headline: "Generative UI benchmark methodology",
  description: "How the OpenUI generative UI benchmark is generated, scored, and reproduced.",
  url: `${BENCHMARK_CANONICAL_URL}/methodology`,
  dateModified: "2026-08-26",
  author: { "@type": "Organization", name: "OpenUI by Thesys" },
  isPartOf: { "@id": `${BENCHMARK_CANONICAL_URL}#dataset` },
  codeRepository: REPO_ROOT,
};

export default function BenchmarkMethodologyPage() {
  return (
    <>
      <script
        type="application/ld+json"
        dangerouslySetInnerHTML={{
          __html: JSON.stringify(structuredData).replaceAll("<", "\\u003c"),
        }}
      />
      <div className="min-h-screen bg-[linear-gradient(to_bottom,var(--openui-foreground),var(--openui-background)_10rem,var(--openui-background)_calc(100%_-_10rem),var(--openui-foreground))] [[data-theme=dark]_&]:bg-[linear-gradient(to_bottom,#000,var(--swatch-neutral-950)_28rem,var(--swatch-neutral-950)_calc(100%_-_28rem),#000)]">
        <main className="mx-auto flex w-full max-w-[var(--home-container-wide)] gap-4 px-4 pt-16 pb-40 lg:gap-28 lg:pr-8 min-[1249px]:pl-0 min-[1024px]:max-[1248px]:pl-8">
          <aside className="hidden w-56 shrink-0 lg:block">
            <div className="sticky top-24">
              <p className="mb-3 text-sm font-medium text-fd-foreground">On this page</p>
              <ul className="space-y-2 text-sm text-[color:var(--openui-text-neutral-tertiary)]">
                {SECTIONS.map((section) => (
                  <li key={section.id}>
                    <a href={`#${section.id}`}>{section.label}</a>
                  </li>
                ))}
              </ul>
            </div>
          </aside>

          <div className="min-w-0 flex-1">
            <h1 className="mb-2 text-[length:var(--home-title-size)] font-[family-name:var(--home-font-display)] font-medium leading-[var(--home-title-leading)] tracking-[var(--home-title-tracking)] text-[color:var(--openui-text-neutral-primary)]">
              Methodology
            </h1>
            <p className="mb-4 text-[length:var(--home-lead-size)] font-[family-name:var(--home-font-text)] leading-[var(--home-lead-leading)] text-[color:var(--openui-text-neutral-secondary)]">
              One catalog, the same 46 interface briefs, and a shared scoring standard.
            </p>
            <nav
              aria-label="Methodology actions"
              className="flex flex-wrap items-center gap-2 border-b border-[color:var(--home-hairline)] pb-6"
            >
              <BevelButton
                href="/benchmarks"
                label="Benchmark"
                variant="secondary"
                className={benchmarkStyles.heroButton}
                badge={
                  <ArrowRight
                    className={benchmarkStyles.actionIcon}
                    strokeWidth={2.25}
                    aria-hidden="true"
                  />
                }
              />
              <PillLink
                href={REPO_ROOT}
                variant="ghost"
                external
                arrow={<ArrowUpRight size={16} strokeWidth={2.25} aria-hidden="true" />}
              >
                Source
              </PillLink>
            </nav>

            <article className="prose mt-8 min-w-0">
              <section id="setup" aria-labelledby="setup-heading">
                <h2 id="setup-heading">Benchmark setup</h2>
                <p>
                  We test how reliably models generate working interfaces, without telling them
                  which components or layouts to use.
                </p>
                <ul>
                  <li>
                    <strong>{BRIEFS} briefs</strong> across five complexity bands, from simple
                    screens to dense workspaces.
                  </li>
                  <li>
                    <strong>{CATALOG_COMPONENTS} equivalent components</strong> available to every
                    format.
                  </li>
                  <li>
                    <strong>Four generations per brief</strong>, with a 16,384-token output limit.
                  </li>
                  <li>
                    <strong>Temperature 0.7</strong> where supported, with minimal or no reasoning.
                  </li>
                </ul>
                <p>
                  For the framework comparison, the same briefs run across {MODELS.length} models
                  and {FORMATS.length} formats: OpenUI, Google A2UI, and Vercel json-render. That is{" "}
                  {RUNS_PER_FORMAT.toLocaleString("en-US")} generations per format.
                </p>
                <p>
                  Each format uses the prompt generated by its own SDK. All three receive the same
                  two worked examples, and their component catalogs are checked against one shared
                  reference catalog.
                </p>
              </section>

              <section id="scoring" aria-labelledby="scoring-heading">
                <h2 id="scoring-heading">What counts as valid</h2>
                <p>
                  Each format is parsed and validated by its own shipped SDK, followed by the same
                  completeness checks for all three formats.
                </p>
                <p>A generation passes structural validity when it:</p>
                <ul>
                  <li>parses and produces a root;</li>
                  <li>uses known components and valid required or enum props;</li>
                  <li>resolves every reference, with nothing orphaned;</li>
                  <li>does not end because it hit the output limit; and</li>
                  <li>
                    contains at least as many reachable components as the brief has requirements.
                  </li>
                </ul>
                <p>
                  <strong>Render success is separate.</strong> It records whether a non-blank screen
                  appeared at all. A screen can render and still fail structural validity.
                </p>
              </section>

              <section id="reproduce" aria-labelledby="reproduce-heading">
                <h2 id="reproduce-heading">Reproduce the results</h2>
                <p>
                  Raw model outputs and scored verdicts are committed to the benchmark repository.
                  The scorer runs offline, so the published outputs can be rescored without model
                  API keys. Recreating the committed result files with no diff is the integrity
                  check.
                </p>
                <p>
                  The repository README contains the exact Node and A2UI Python setup, plus commands
                  for rescoring one model or the full benchmark.
                </p>
                <p>
                  <a href={LINKS.latestReadme}>Read the reproduction instructions</a> or inspect the{" "}
                  <a href={LINKS.rawOutputs}>published raw outputs</a>.
                </p>
              </section>

              <section id="notes" aria-labelledby="notes-heading">
                <h2 id="notes-heading">Important notes</h2>
                <div className="not-prose my-6 border-l-2 border-[color:var(--openui-text-neutral-primary)] pl-5 text-sm leading-6 text-[color:var(--openui-text-neutral-secondary)]">
                  Every row is scored by one build of the shipped OpenUI parser,{` `}
                  <strong>lang-core 0.2.16</strong>, whose parser validates enum and scalar prop
                  values. Raw outputs and per-run verdicts are committed for independent rescoring.
                </div>
                <ul>
                  <li>
                    The component-count floor prevents tiny valid outputs from passing, but it does
                    not judge visual quality or prove every requirement was understood.
                  </li>
                  <li>
                    Cost uses measured tokens and list prices. Hidden reasoning tokens can make the
                    actual bill higher.
                  </li>
                  <li>
                    Self-hosted model costs are marked as not comparable rather than treated as
                    free.
                  </li>
                  <li>
                    OpenUI built this benchmark. The method and outputs are published so the results
                    can be inspected and challenged.
                  </li>
                </ul>
              </section>
            </article>
          </div>
        </main>
      </div>
      <Footer />
    </>
  );
}