Skip to content

The Open Standard for Generative UI

7.9 KB
Raw Blame
import { BENCHMARK_SCHEMA_URL } from "@/lib/benchmark-agent-data";

const nullableNumber = { type: ["number", "null"] } as const;
const nullableString = { type: ["string", "null"] } as const;

export const benchmarkDataSchema = {
  $schema: "https://json-schema.org/draft/2020-12/schema",
  $id: BENCHMARK_SCHEMA_URL,
  title: "OpenUI Generative UI Benchmark dataset",
  description:
    "Schema for the versioned OpenUI model board, format comparison, summaries, definitions and agent-oriented answers.",
  type: "object",
  required: [
    "$schema",
    "schema_version",
    "dataset",
    "definitions",
    "scope",
    "provenance",
    "model_families",
    "model_board",
    "format_comparison",
    "format_summary",
    "agent_answers",
    "methodology",
    "distributions",
  ],
  properties: {
    $schema: { const: BENCHMARK_SCHEMA_URL },
    schema_version: { type: "string", pattern: "^\\d+\\.\\d+\\.\\d+$" },
    dataset: {
      type: "object",
      required: [
        "name",
        "canonical_url",
        "version",
        "benchmark_updated",
        "model_board_updated",
        "currency",
        "pricing_basis",
        "maintainer",
      ],
      properties: {
        name: { type: "string" },
        canonical_url: { type: "string", format: "uri" },
        version: { type: "string" },
        benchmark_updated: { type: "string", format: "date" },
        model_board_updated: { type: "string", format: "date" },
        benchmark_updated_label: { type: "string" },
        currency: { const: "USD" },
        pricing_basis: { type: "string" },
        maintainer: { type: "string" },
        first_party_disclosure: { type: "string" },
      },
    },
    definitions: { type: "object" },
    scope: {
      type: "object",
      required: [
        "briefs",
        "formats",
        "comparison_models",
        "model_board_models",
        "runs_per_format",
        "total_scored_runs",
      ],
      properties: {
        briefs: { type: "integer", minimum: 1 },
        formats: { type: "integer", minimum: 1 },
        comparison_models: { type: "integer", minimum: 1 },
        model_board_models: { type: "integer", minimum: 1 },
        runs_per_format: { type: "integer", minimum: 1 },
        total_scored_runs: { type: "integer", minimum: 1 },
      },
    },
    provenance: {
      type: "object",
      required: ["format_comparison", "model_board"],
      properties: {
        format_comparison: {
          type: "object",
          required: ["evidence_status", "raw_results", "raw_outputs", "harness", "briefs"],
          properties: {
            evidence_status: { const: "published" },
            raw_results: { type: "string", format: "uri" },
            raw_outputs: { type: "string", format: "uri" },
            harness: { type: "string", format: "uri" },
            briefs: { type: "string", format: "uri" },
          },
        },
        model_board: {
          type: "object",
          required: ["evidence_status", "note", "raw_results"],
          properties: {
            evidence_status: { const: "summary-only" },
            note: { type: "string" },
            raw_results: { type: "null" },
          },
        },
      },
    },
    model_families: {
      type: "array",
      items: {
        type: "object",
        required: ["id", "label", "provider", "models"],
        properties: {
          id: { type: "string" },
          label: { type: "string" },
          provider: { type: "string" },
          models: { type: "array", minItems: 2, items: { type: "string" } },
        },
      },
    },
    model_board: {
      type: "array",
      items: { $ref: "#/$defs/modelBoardRow" },
    },
    format_comparison: {
      type: "array",
      items: { $ref: "#/$defs/formatComparisonRow" },
    },
    format_summary: {
      type: "array",
      items: { $ref: "#/$defs/formatSummaryRow" },
    },
    complexity_bands: { type: "array", items: { type: "object" } },
    production_repair: { type: "object" },
    methodology: { type: "string", format: "uri" },
    agent_answers: {
      type: "array",
      minItems: 1,
      items: { $ref: "#/$defs/agentAnswer" },
    },
    distributions: {
      type: "object",
      required: ["schema", "json", "csv", "markdown", "raw_results", "harness", "briefs"],
      additionalProperties: { type: "string", format: "uri" },
    },
  },
  $defs: {
    modelBoardRow: {
      type: "object",
      required: [
        "model_id",
        "model_name",
        "provider",
        "family_id",
        "family_name",
        "family_order",
        "connected_family",
        "default_selected",
        "format",
        "validity_score_percent",
        "cost_per_task_usd",
        "cost_per_46_screen_pass_usd",
        "cost_type",
        "serving",
        "pareto_frontier",
        "briefs",
        "generations_per_brief",
        "dataset_updated",
      ],
      properties: {
        model_id: { type: "string" },
        model_name: { type: "string" },
        provider: { type: "string" },
        family_id: { type: "string" },
        family_name: { type: "string" },
        family_order: { type: ["integer", "null"], minimum: 1 },
        connected_family: { const: false },
        default_selected: { type: "boolean" },
        format: { const: "openui" },
        validity_score_percent: { type: "number", minimum: 0, maximum: 100 },
        cost_per_task_usd: nullableNumber,
        cost_per_46_screen_pass_usd: nullableNumber,
        cost_type: { enum: ["provider-list-price", "free", "self-hosted"] },
        serving: nullableString,
        pareto_frontier: { type: "boolean" },
        briefs: { type: "integer", minimum: 1 },
        generations_per_brief: { type: "integer", minimum: 1 },
        dataset_updated: { type: "string", format: "date" },
      },
    },
    formatComparisonRow: {
      type: "object",
      required: [
        "model_id",
        "model_name",
        "provider",
        "format_id",
        "format_name",
        "valid_runs",
        "rendered_runs",
        "total_runs",
        "validity_score_percent",
        "render_rate_percent",
        "cost_per_46_screen_pass_usd",
      ],
      properties: {
        model_id: { type: "string" },
        model_name: { type: "string" },
        provider: { type: "string" },
        format_id: { enum: ["openui", "a2ui", "jsonRender"] },
        format_name: { type: "string" },
        valid_runs: { type: "integer", minimum: 0 },
        rendered_runs: { type: "integer", minimum: 0 },
        total_runs: { type: "integer", minimum: 1 },
        validity_score_percent: { type: "number", minimum: 0, maximum: 100 },
        render_rate_percent: { type: "number", minimum: 0, maximum: 100 },
        cost_per_46_screen_pass_usd: nullableNumber,
      },
    },
    formatSummaryRow: {
      type: "object",
      required: [
        "format_id",
        "format_name",
        "validity_score_percent",
        "render_rate_percent",
        "blank_screens",
        "runs",
        "system_prompt_tokens",
        "mean_output_tokens_per_screen",
        "estimated_stream_seconds_at_50_tps",
      ],
      properties: {
        format_id: { enum: ["openui", "a2ui", "jsonRender"] },
        format_name: { type: "string" },
        validity_score_percent: { type: "number", minimum: 0, maximum: 100 },
        render_rate_percent: { type: "number", minimum: 0, maximum: 100 },
        blank_screens: { type: "integer", minimum: 0 },
        runs: { type: "integer", minimum: 1 },
        system_prompt_tokens: { type: "integer", minimum: 0 },
        mean_output_tokens_per_screen: { type: "number", minimum: 0 },
        estimated_stream_seconds_at_50_tps: { type: "number", minimum: 0 },
      },
    },
    agentAnswer: {
      type: "object",
      required: ["id", "question", "answer", "evidence_paths"],
      properties: {
        id: { type: "string" },
        question: { type: "string" },
        answer: { type: "string" },
        evidence_paths: { type: "array", minItems: 1, items: { type: "string" } },
      },
    },
  },
} as const;