The Open Standard for Generative UI
import { BENCHMARK_SCHEMA_URL } from "@/lib/benchmark-agent-data";
const nullableNumber = { type: ["number", "null"] } as const;
const nullableString = { type: ["string", "null"] } as const;
export const benchmarkDataSchema = {
$schema: "https://json-schema.org/draft/2020-12/schema",
$id: BENCHMARK_SCHEMA_URL,
title: "OpenUI Generative UI Benchmark dataset",
description:
"Schema for the versioned OpenUI model board, format comparison, summaries, definitions and agent-oriented answers.",
type: "object",
required: [
"$schema",
"schema_version",
"dataset",
"definitions",
"scope",
"provenance",
"model_families",
"model_board",
"format_comparison",
"format_summary",
"agent_answers",
"methodology",
"distributions",
],
properties: {
$schema: { const: BENCHMARK_SCHEMA_URL },
schema_version: { type: "string", pattern: "^\\d+\\.\\d+\\.\\d+$" },
dataset: {
type: "object",
required: [
"name",
"canonical_url",
"version",
"benchmark_updated",
"model_board_updated",
"currency",
"pricing_basis",
"maintainer",
],
properties: {
name: { type: "string" },
canonical_url: { type: "string", format: "uri" },
version: { type: "string" },
benchmark_updated: { type: "string", format: "date" },
model_board_updated: { type: "string", format: "date" },
benchmark_updated_label: { type: "string" },
currency: { const: "USD" },
pricing_basis: { type: "string" },
maintainer: { type: "string" },
first_party_disclosure: { type: "string" },
},
},
definitions: { type: "object" },
scope: {
type: "object",
required: [
"briefs",
"formats",
"comparison_models",
"model_board_models",
"runs_per_format",
"total_scored_runs",
],
properties: {
briefs: { type: "integer", minimum: 1 },
formats: { type: "integer", minimum: 1 },
comparison_models: { type: "integer", minimum: 1 },
model_board_models: { type: "integer", minimum: 1 },
runs_per_format: { type: "integer", minimum: 1 },
total_scored_runs: { type: "integer", minimum: 1 },
},
},
provenance: {
type: "object",
required: ["format_comparison", "model_board"],
properties: {
format_comparison: {
type: "object",
required: ["evidence_status", "raw_results", "raw_outputs", "harness", "briefs"],
properties: {
evidence_status: { const: "published" },
raw_results: { type: "string", format: "uri" },
raw_outputs: { type: "string", format: "uri" },
harness: { type: "string", format: "uri" },
briefs: { type: "string", format: "uri" },
},
},
model_board: {
type: "object",
required: ["evidence_status", "note", "raw_results"],
properties: {
evidence_status: { const: "summary-only" },
note: { type: "string" },
raw_results: { type: "null" },
},
},
},
},
model_families: {
type: "array",
items: {
type: "object",
required: ["id", "label", "provider", "models"],
properties: {
id: { type: "string" },
label: { type: "string" },
provider: { type: "string" },
models: { type: "array", minItems: 2, items: { type: "string" } },
},
},
},
model_board: {
type: "array",
items: { $ref: "#/$defs/modelBoardRow" },
},
format_comparison: {
type: "array",
items: { $ref: "#/$defs/formatComparisonRow" },
},
format_summary: {
type: "array",
items: { $ref: "#/$defs/formatSummaryRow" },
},
complexity_bands: { type: "array", items: { type: "object" } },
production_repair: { type: "object" },
methodology: { type: "string", format: "uri" },
agent_answers: {
type: "array",
minItems: 1,
items: { $ref: "#/$defs/agentAnswer" },
},
distributions: {
type: "object",
required: ["schema", "json", "csv", "markdown", "raw_results", "harness", "briefs"],
additionalProperties: { type: "string", format: "uri" },
},
},
$defs: {
modelBoardRow: {
type: "object",
required: [
"model_id",
"model_name",
"provider",
"family_id",
"family_name",
"family_order",
"connected_family",
"default_selected",
"format",
"validity_score_percent",
"cost_per_task_usd",
"cost_per_46_screen_pass_usd",
"cost_type",
"serving",
"pareto_frontier",
"briefs",
"generations_per_brief",
"dataset_updated",
],
properties: {
model_id: { type: "string" },
model_name: { type: "string" },
provider: { type: "string" },
family_id: { type: "string" },
family_name: { type: "string" },
family_order: { type: ["integer", "null"], minimum: 1 },
connected_family: { const: false },
default_selected: { type: "boolean" },
format: { const: "openui" },
validity_score_percent: { type: "number", minimum: 0, maximum: 100 },
cost_per_task_usd: nullableNumber,
cost_per_46_screen_pass_usd: nullableNumber,
cost_type: { enum: ["provider-list-price", "free", "self-hosted"] },
serving: nullableString,
pareto_frontier: { type: "boolean" },
briefs: { type: "integer", minimum: 1 },
generations_per_brief: { type: "integer", minimum: 1 },
dataset_updated: { type: "string", format: "date" },
},
},
formatComparisonRow: {
type: "object",
required: [
"model_id",
"model_name",
"provider",
"format_id",
"format_name",
"valid_runs",
"rendered_runs",
"total_runs",
"validity_score_percent",
"render_rate_percent",
"cost_per_46_screen_pass_usd",
],
properties: {
model_id: { type: "string" },
model_name: { type: "string" },
provider: { type: "string" },
format_id: { enum: ["openui", "a2ui", "jsonRender"] },
format_name: { type: "string" },
valid_runs: { type: "integer", minimum: 0 },
rendered_runs: { type: "integer", minimum: 0 },
total_runs: { type: "integer", minimum: 1 },
validity_score_percent: { type: "number", minimum: 0, maximum: 100 },
render_rate_percent: { type: "number", minimum: 0, maximum: 100 },
cost_per_46_screen_pass_usd: nullableNumber,
},
},
formatSummaryRow: {
type: "object",
required: [
"format_id",
"format_name",
"validity_score_percent",
"render_rate_percent",
"blank_screens",
"runs",
"system_prompt_tokens",
"mean_output_tokens_per_screen",
"estimated_stream_seconds_at_50_tps",
],
properties: {
format_id: { enum: ["openui", "a2ui", "jsonRender"] },
format_name: { type: "string" },
validity_score_percent: { type: "number", minimum: 0, maximum: 100 },
render_rate_percent: { type: "number", minimum: 0, maximum: 100 },
blank_screens: { type: "integer", minimum: 0 },
runs: { type: "integer", minimum: 1 },
system_prompt_tokens: { type: "integer", minimum: 0 },
mean_output_tokens_per_screen: { type: "number", minimum: 0 },
estimated_stream_seconds_at_50_tps: { type: "number", minimum: 0 },
},
},
agentAnswer: {
type: "object",
required: ["id", "question", "answer", "evidence_paths"],
properties: {
id: { type: "string" },
question: { type: "string" },
answer: { type: "string" },
evidence_paths: { type: "array", minItems: 1, items: { type: "string" } },
},
},
},
} as const;