Skip to content

Open source CapCut alternative.

TypeScript

feat(site): typesafe judge route, adapter, pricing, setting

David Le046996a4e6dc
diff --git a/site/eslint.config.mjs b/site/eslint.config.mjs
index 417bb0df..ed79ceef 100644
--- a/site/eslint.config.mjs
+++ b/site/eslint.config.mjs
@@ -60,6 +60,10 @@ const eslintConfig = defineConfig([
],
message: "Name a model role (geminiModelRoleNames); the responses route resolves the id.",
},
+ {
+ name: "@/lib/inference/typesafe-models",
+ message: "The judge route pins the model; post to /api/inference/judge.",
+ },
],
},
],
diff --git a/site/package-lock.json b/site/package-lock.json
index 7858433a..6fffd6dc 100644
--- a/site/package-lock.json
+++ b/site/package-lock.json
@@ -45,6 +45,7 @@
"@tailwindcss/typography": "^0.5.19",
"@tanstack/react-query": "^5.101.0",
"@types/mdx": "^2.0.13",
+ "@typesafe-ai/sdk": "0.6.0",
"ai": "^7.0.14",
"better-auth": "1.6.11",
"better-call": "1.3.5",
@@ -10089,6 +10090,15 @@
"@types/node": "*"
}
},
+ "node_modules/@typesafe-ai/sdk": {
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/@typesafe-ai/sdk/-/sdk-0.6.0.tgz",
+ "integrity": "sha512-IddX+Q0XM+VagOUZFeP7wZjaO4SHMdvnh2zEBdrZZnXedWI3BNK1lKhMx3ayrkFWvVLbVcUHJy6AVZlY+e6Jaw==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=20"
+ }
+ },
"node_modules/@typescript-eslint/eslint-plugin": {
"version": "8.59.3",
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.59.3.tgz",
diff --git a/site/package.json b/site/package.json
index 94e98240..568e407a 100644
--- a/site/package.json
+++ b/site/package.json
@@ -77,6 +77,7 @@
"@tailwindcss/typography": "^0.5.19",
"@tanstack/react-query": "^5.101.0",
"@types/mdx": "^2.0.13",
+ "@typesafe-ai/sdk": "0.6.0",
"ai": "^7.0.14",
"better-auth": "1.6.11",
"better-call": "1.3.5",
diff --git a/site/src/app/api/inference/judge/route.ts b/site/src/app/api/inference/judge/route.ts
new file mode 100644
index 00000000..822ecbd6
--- /dev/null
+++ b/site/src/app/api/inference/judge/route.ts
@@ -0,0 +1,113 @@
+import { NextResponse } from "next/server";
+
+import {
+ creditErrorResponse,
+ creditUsageHeaders,
+ inferenceUsageRoutes,
+ recordFailedInferenceUsage,
+ recordInferenceUsage,
+ requireInferenceCredits,
+} from "@/lib/credits/inference";
+import { typesafeJudge } from "@/lib/inference/adapters/typesafe";
+import { judgeRequestSchema, type JudgeQuestion } from "@/lib/inference/judge";
+import { toJsonValue } from "@/lib/inference/json";
+import { InferenceProviderError } from "@/lib/inference/providers";
+import {
+ inferenceErrorCode,
+ inferenceProviderErrorResponse,
+ requireInferenceClientId,
+ validationErrorResponse,
+} from "@/lib/inference/responses";
+import {
+ INFERENCE_RATE_LIMIT,
+ shouldBypassDonkeyInferenceCredits,
+ withDonkeyAuth,
+} from "@/lib/donkey-api-auth";
+
+// A judgment: typed questions over one state, answered as probabilities by
+// TypeSafe's System One model. The page, the worker, and the eval post here
+// with the user's session; the route bills the tokens and pins the model.
+
+const requestKind = "judge";
+
+export const POST = withDonkeyAuth(async (request) => {
+ const client = requireInferenceClientId(request.donkey.clientId);
+ if (!client.ok) {
+ return client.response;
+ }
+
+ const parsed = judgeRequestSchema.safeParse(await request.json());
+ if (!parsed.success) {
+ return validationErrorResponse(parsed.error);
+ }
+
+ const judge = typesafeJudge();
+ const bypassCredits = shouldBypassDonkeyInferenceCredits(request.donkey);
+ if (!bypassCredits) {
+ const credits = await requireInferenceCredits({
+ model: judge.model,
+ provider: judge.id,
+ route: inferenceUsageRoutes.judge,
+ userId: request.donkey.userId,
+ enforceModelPrice: true,
+ });
+ if (!credits.ok) {
+ return credits.response;
+ }
+ }
+
+ try {
+ // Validated above: a score's criteria has at least two levels.
+ const result = await judge.askJudge(
+ { state: parsed.data.state, questions: parsed.data.questions as Record<string, JudgeQuestion> },
+ request.signal,
+ );
+
+ let usageHeaders: Record<string, string> = {};
+ if (bypassCredits) {
+ usageHeaders["X-Donkey-Dev-Auth-Bypass"] = "true";
+ } else {
+ const recordedUsage = await recordInferenceUsage({
+ clientId: client.clientId,
+ conversationId: request.donkey.conversationId,
+ model: result.model,
+ provider: judge.id,
+ requestKind,
+ route: inferenceUsageRoutes.judge,
+ status: "succeeded",
+ usage: toJsonValue(result.usage),
+ userId: request.donkey.userId,
+ });
+ usageHeaders = creditUsageHeaders(recordedUsage);
+ }
+
+ return NextResponse.json(result, {
+ headers: {
+ "X-Donkey-Inference-Provider": judge.id,
+ "X-Donkey-Inference-Model": result.model,
+ ...usageHeaders,
+ },
+ });
+ } catch (error) {
+ if (!bypassCredits) {
+ await recordFailedInferenceUsage({
+ clientId: client.clientId,
+ conversationId: request.donkey.conversationId,
+ errorCode: inferenceErrorCode(error),
+ model: judge.model,
+ provider: judge.id,
+ requestKind,
+ route: inferenceUsageRoutes.judge,
+ userId: request.donkey.userId,
+ });
+ }
+ const creditResponse = creditErrorResponse(error);
+ if (creditResponse) {
+ return creditResponse;
+ }
+ if (error instanceof InferenceProviderError) {
+ return inferenceProviderErrorResponse(error);
+ }
+ throw error;
+ }
+}, { allowRunner: true, rateLimit: INFERENCE_RATE_LIMIT });
diff --git a/site/src/lib/config/registry.ts b/site/src/lib/config/registry.ts
index 636d59ec..94c11eac 100644
--- a/site/src/lib/config/registry.ts
+++ b/site/src/lib/config/registry.ts
@@ -93,6 +93,50 @@ export const SETTINGS = defineSettings({
title: "Chat continuity",
description: "Project and conversation refresh cadence, scene ownership expiry, and saved engine transcript size.",
},
+ cutJudge: {
+ schema: z
+ .object({
+ // Preload the skill the judge picks for the turn; off leaves the model
+ // to read skills on its own.
+ skillSuggestion: z.boolean(),
+ // Mean of the "does this turn want a skill" questions below which no
+ // skill is attached.
+ skillGate: z.number().min(0).max(1),
+ // The winning skill's own fit below which it is dropped.
+ skillFits: z.number().min(0).max(1),
+ // Declare only the tool areas the judge routes to; off declares the
+ // whole catalog on every turn.
+ toolRouting: z.boolean(),
+ // An area's probability at or above which its tools are declared.
+ toolArea: z.number().min(0).max(1),
+ // How long the first model round waits for the judge before going out
+ // with the full catalog and no skill.
+ judgeWaitMs: z.number().int().min(0).max(2000),
+ // A stock candidate's fit below which it is left out of a search.
+ stockFit: z.number().min(0).max(1),
+ // A transcript word's disfluency probability at or above which
+ // find_filler reports it.
+ fillerCut: z.number().min(0).max(1),
+ // The confidence below which a described voice falls back to the default.
+ voicePick: z.number().min(0).max(1),
+ })
+ .strict(),
+ default: {
+ skillSuggestion: true,
+ skillGate: 0.4,
+ skillFits: 0.3,
+ toolRouting: true,
+ toolArea: 0.2,
+ judgeWaitMs: 400,
+ stockFit: 0.5,
+ fillerCut: 0.5,
+ voicePick: 0.4,
+ },
+ public: true,
+ title: "Chat judgments",
+ description:
+ "Thresholds for the typed judgments that route a chat turn (skill, tool areas), rank stock, find filler words, and resolve described voices.",
+ },
experimentResults: {
schema: z
.object({
diff --git a/site/src/lib/credits/inference.ts b/site/src/lib/credits/inference.ts
index c82ca7b5..60dfcfe6 100644
--- a/site/src/lib/credits/inference.ts
+++ b/site/src/lib/credits/inference.ts
@@ -17,6 +17,7 @@ export const inferenceUsageRoutes = {
assets: "/api/inference/assets/",
assetsRefresh: "/api/inference/assets/refresh/",
responses: "/api/inference/responses/",
+ judge: "/api/inference/judge/",
} as const;
type CreditsDatabase = PrismaClient | Prisma.TransactionClient;
diff --git a/site/src/lib/credits/provider-pricing.ts b/site/src/lib/credits/provider-pricing.ts
index 2d3dc2c5..c5df65bc 100644
--- a/site/src/lib/credits/provider-pricing.ts
+++ b/site/src/lib/credits/provider-pricing.ts
@@ -6,6 +6,7 @@ import {
elevenLabsModels,
type ElevenLabsRunModel,
} from "@/lib/inference/elevenlabs-models";
+import { typesafeModels, typesafeProviderId } from "@/lib/inference/typesafe-models";
import {
geminiModels,
geminiMusicModels,
@@ -72,7 +73,28 @@ export function providerCreditPricing(
if (normalizedProvider === "fal") {
return falMatteCreditPricing(normalizedModel);
}
+ // The judge id is hardcoded (typesafe-models.ts); a judgment bills its input
+ // tokens, and the answers are free.
+ if (normalizedProvider === typesafeProviderId) {
+ return typesafeCreditPricing(normalizedModel);
+ }
+
+ return undefined;
+}
+// TypeSafe System One (Jev): $0.042 per 1M input tokens, output free
+// (2026-09). A request for `jev-latest` reports the dated id it ran on, so
+// the family prefix matches both.
+const typesafeJudgePricing = textAudioTokenPricing({
+ input: "0.042",
+ cachedInput: "0.042",
+ output: "0",
+});
+
+function typesafeCreditPricing(model: string): ProviderCreditPricing | undefined {
+ if (model === typesafeModels.jev || modelMatches(model, "jev")) {
+ return typesafeJudgePricing;
+ }
return undefined;
}
diff --git a/site/src/lib/inference/adapters/typesafe.test.ts b/site/src/lib/inference/adapters/typesafe.test.ts
new file mode 100644
index 00000000..520ca2db
--- /dev/null
+++ b/site/src/lib/inference/adapters/typesafe.test.ts
@@ -0,0 +1,73 @@
+import { describe, expect, test } from "bun:test";
+
+import { createTypesafeJudge } from "./typesafe";
+import { choice, noul } from "@/lib/inference/judge";
+
+describe("typesafe judge", () => {
+ test("is unconfigured without a key and fails hard", async () => {
+ const judge = createTypesafeJudge({});
+ expect(judge.configured).toBe(false);
+ let code = "";
+ try {
+ await judge.askJudge({ state: "hi", questions: { q: noul("Is this a greeting?") } });
+ } catch (error) {
+ code = (error as { code: string }).code;
+ }
+ expect(code).toBe("missing_provider_credentials");
+ });
+
+ test("posts state and questions with the pinned model and returns typed answers", async () => {
+ let sent: { url: string; body: unknown; auth: string | null } | null = null;
+ const judge = createTypesafeJudge({ TYPESAFE_API_KEY: "secret" }, (async (
+ url: string | URL | Request,
+ init?: RequestInit,
+ ) => {
+ sent = {
+ url: String(url),
+ body: JSON.parse(String(init?.body)),
+ auth: new Headers(init?.headers).get("authorization"),
+ };
+ return new Response(
+ JSON.stringify({
+ model: "jev-1.12",
+ answers: {
+ intent: { type: "choice", choice: "chat", probabilities: { chat: 0.9, work: 0.1 }, confidence: 0.85 },
+ urgent: { type: "noul", noul: 0.05 },
+ },
+ usage: { input_tokens: 40, output_tokens: 2 },
+ }),
+ { status: 200, headers: { "content-type": "application/json" } },
+ );
+ }) as typeof fetch);
+
+ const result = await judge.askJudge({
+ state: "hello there",
+ questions: {
+ intent: choice("What kind of message?", { chat: "social filler", work: "an ask" }),
+ urgent: noul("Is it urgent?"),
+ },
+ });
+ expect(result.model).toBe("jev-1.12");
+ expect(result.answers.intent.choice).toBe("chat");
+ expect(result.answers.intent.probabilities.work).toBe(0.1);
+ expect(result.answers.urgent.noul).toBe(0.05);
+ expect(result.usage.input_tokens).toBe(40);
+ const s = sent as unknown as { url: string; body: { model: string; state: string }; auth: string | null };
+ expect(s.url).toBe("https://api.typesafe.ai/v1/systemone");
+ expect(s.body.model).toBe("jev-latest");
+ expect(s.body.state).toBe("hello there");
+ expect(s.auth).toBe("Bearer secret");
+ });
+
+ test("a provider failure surfaces as an inference provider error", async () => {
+ const judge = createTypesafeJudge({ TYPESAFE_API_KEY: "secret" }, (async () =>
+ new Response(JSON.stringify({ error: "bad" }), { status: 422 })) as typeof fetch);
+ let code = "";
+ try {
+ await judge.askJudge({ state: "x", questions: { q: noul("?") } });
+ } catch (error) {
+ code = (error as { code: string }).code;
+ }
+ expect(code).toBe("invalid_request");
+ });
+});
diff --git a/site/src/lib/inference/adapters/typesafe.ts b/site/src/lib/inference/adapters/typesafe.ts
new file mode 100644
index 00000000..0c9f52aa
--- /dev/null
+++ b/site/src/lib/inference/adapters/typesafe.ts
@@ -0,0 +1,106 @@
+import { APIConnectionError, APIError, TypeSafeClient } from "@typesafe-ai/sdk";
+
+import type { AdapterEnvironment } from "@/lib/inference/adapters/gemini-client";
+import type { Entry, JudgeQuestion, JudgeResult } from "@/lib/inference/judge";
+import { InferenceProviderError } from "@/lib/inference/providers";
+import { typesafeModels, typesafeProviderId } from "@/lib/inference/typesafe-models";
+
+// The one seam to TypeSafe's System One API. Every judgment — the chat turn's
+// route, queue triage, stock ranking, refusal classes — goes through
+// `askJudge` here, on the server, with the account's key. The model is pinned
+// from the registry; the page never names it.
+
+// A judgment gates a turn that is already running, so a stall is answered
+// fast: one attempt of 8 s, one retry.
+const requestTimeoutMs = 8_000;
+const maxRetries = 1;
+
+export interface TypesafeJudge {
+ readonly id: typeof typesafeProviderId;
+ readonly model: string;
+ readonly configured: boolean;
+ askJudge<Q extends Record<string, JudgeQuestion>>(
+ request: { state: Entry; questions: Q },
+ signal?: AbortSignal,
+ ): Promise<JudgeResult<Q>>;
+}
+
+export function createTypesafeJudge(
+ environment: AdapterEnvironment = process.env,
+ fetchImpl?: typeof fetch,
+): TypesafeJudge {
+ const apiKey = environment.TYPESAFE_API_KEY?.trim();
+ const model = typesafeModels.jev;
+ const client = apiKey
+ ? new TypeSafeClient({
+ apiKey,
+ defaultModel: model,
+ timeout: requestTimeoutMs,
+ retry: { maxRetries },
+ ...(fetchImpl ? { fetch: fetchImpl as never } : {}),
+ })
+ : null;
+
+ async function askJudge<Q extends Record<string, JudgeQuestion>>(
+ request: { state: Entry; questions: Q },
+ signal?: AbortSignal,
+ ): Promise<JudgeResult<Q>> {
+ if (!client) {
+ throw new InferenceProviderError("TypeSafe is not configured (TYPESAFE_API_KEY).", {
+ statusCode: 503,
+ code: "missing_provider_credentials",
+ });
+ }
+ try {
+ const result = await client.systemOne(
+ { state: request.state as never, questions: request.questions as never, model },
+ { signal },
+ );
+ return {
+ model: result.model,
+ answers: result.answers as unknown as JudgeResult<Q>["answers"],
+ usage: { input_tokens: result.usage.input_tokens, output_tokens: result.usage.output_tokens },
+ };
+ } catch (error) {
+ throw toProviderError(error);
+ }
+ }
+
+ return { id: typesafeProviderId, model, configured: client !== null, askJudge };
+}
+
+function toProviderError(error: unknown): InferenceProviderError {
+ if (error instanceof InferenceProviderError) return error;
+ if (error instanceof APIError) {
+ const status = error.status === 429 ? 429 : error.status >= 500 ? 502 : error.status === 401 ? 503 : 400;
+ return new InferenceProviderError(`TypeSafe request failed (${error.status}): ${error.message}`, {
+ statusCode: status,
+ code:
+ error.status === 401
+ ? "missing_provider_credentials"
+ : error.status === 429
+ ? "rate_limited"
+ : error.status === 422
+ ? "invalid_request"
+ : "provider_error",
+ details: typeof error.body === "object" ? (error.body as never) : null,
+ });
+ }
+ if (error instanceof APIConnectionError) {
+ return new InferenceProviderError(`TypeSafe unreachable: ${error.message}`, {
+ statusCode: 504,
+ code: "provider_timeout",
+ });
+ }
+ return new InferenceProviderError(error instanceof Error ? error.message : String(error), {
+ statusCode: 502,
+ code: "provider_error",
+ });
+}
+
+let shared: TypesafeJudge | null = null;
+
+/** The process-wide judge, built once from the environment. */
+export function typesafeJudge(): TypesafeJudge {
+ return (shared ??= createTypesafeJudge());
+}
diff --git a/site/src/lib/inference/judge.ts b/site/src/lib/inference/judge.ts
new file mode 100644
index 00000000..bc876e5b
--- /dev/null
+++ b/site/src/lib/inference/judge.ts
@@ -0,0 +1,136 @@
+import { z } from "zod";
+
+// The judgment contract shared by the judge route, the page, and the worker:
+// one state, named typed questions, one typed answer per question. The shapes
+// mirror TypeSafe's System One API (noul = P(yes), choice = a distribution over
+// named options, score = a distribution over ordered levels), so a caller reads
+// probabilities and code decides. Zod only, so it is importable anywhere.
+
+export type Entry = string | number | boolean | null | Entry[] | { [key: string]: Entry };
+
+const entrySchema: z.ZodType<Entry> = z.lazy(() =>
+ z.union([z.string(), z.number(), z.boolean(), z.null(), z.array(entrySchema), z.record(z.string(), entrySchema)]),
+);
+
+const noulQuestionSchema = z
+ .object({
+ type: z.literal("noul"),
+ instructions: entrySchema.optional(),
+ criteria: z.object({ true: entrySchema.optional(), false: entrySchema.optional() }).nullable().optional(),
+ })
+ .strict();
+
+const choiceQuestionSchema = z
+ .object({
+ type: z.literal("choice"),
+ instructions: entrySchema.optional(),
+ criteria: z.record(z.string().min(1).max(200), entrySchema).refine((c) => Object.keys(c).length >= 2, {
+ message: "A choice needs at least two options.",
+ }),
+ })
+ .strict();
+
+const scoreQuestionSchema = z
+ .object({
+ type: z.literal("score"),
+ instructions: entrySchema.optional(),
+ criteria: z.array(entrySchema).min(2).max(32),
+ })
+ .strict();
+
+export const judgeQuestionSchema = z.discriminatedUnion("type", [
+ noulQuestionSchema,
+ choiceQuestionSchema,
+ scoreQuestionSchema,
+]);
+
+/** The most questions one request carries. Per-item sweeps (stock candidates,
+ * transcript words) chunk under this. */
+export const JUDGE_MAX_QUESTIONS = 200;
+
+export const judgeRequestSchema = z
+ .object({
+ state: entrySchema,
+ questions: z
+ .record(z.string().min(1).max(120), judgeQuestionSchema)
+ .refine((q) => Object.keys(q).length >= 1, { message: "At least one question." })
+ .refine((q) => Object.keys(q).length <= JUDGE_MAX_QUESTIONS, {
+ message: `At most ${JUDGE_MAX_QUESTIONS} questions per request.`,
+ }),
+ })
+ .strict();
+
+export type JudgeRequest = z.infer<typeof judgeRequestSchema>;
+
+export interface NoulQuestion {
+ type: "noul";
+ instructions?: Entry;
+ criteria?: { true?: Entry; false?: Entry } | null;
+}
+export interface ChoiceQuestion<C extends Record<string, Entry> = Record<string, Entry>> {
+ type: "choice";
+ instructions?: Entry;
+ criteria: C;
+}
+export interface ScoreQuestion {
+ type: "score";
+ instructions?: Entry;
+ criteria: readonly [Entry, Entry, ...Entry[]];
+}
+export type JudgeQuestion = NoulQuestion | ChoiceQuestion | ScoreQuestion;
+
+export interface NoulAnswer {
+ type: "noul";
+ /** P(yes), 0..1. */
+ noul: number;
+}
+export interface ChoiceAnswer<C extends Record<string, Entry> = Record<string, Entry>> {
+ type: "choice";
+ choice: keyof C & string;
+ probabilities: { [K in keyof C]: number };
+ /** Concentration of the distribution, 0..1. */
+ confidence: number;
+}
+export interface ScoreAnswer {
+ type: "score";
+ /** Probability-weighted level, may fall between levels. */
+ score: number;
+ probabilities: Record<string, number>;
+ legend: Record<string, Entry>;
+ confidence: number;
+}
+export type JudgeAnswer = NoulAnswer | ChoiceAnswer | ScoreAnswer;
+
+export type AnswerFor<Q> = Q extends NoulQuestion
+ ? NoulAnswer
+ : Q extends ChoiceQuestion<infer C>
+ ? ChoiceAnswer<C>
+ : Q extends ScoreQuestion
+ ? ScoreAnswer
+ : never;
+
+export interface JudgeUsage {
+ input_tokens: number;
+ output_tokens: number;
+}
+
+export interface JudgeResult<Q extends Record<string, JudgeQuestion>> {
+ model: string;
+ answers: { [K in keyof Q]: AnswerFor<Q[K]> };
+ usage: JudgeUsage;
+}
+
+/** A yes/no question. */
+export function noul(instructions: Entry, criteria?: { true?: Entry; false?: Entry }): NoulQuestion {
+ return criteria ? { type: "noul", instructions, criteria } : { type: "noul", instructions };
+}
+
+/** One option out of a named set. */
+export function choice<const C extends Record<string, Entry>>(instructions: Entry, criteria: C): ChoiceQuestion<C> {
+ return { type: "choice", instructions, criteria };
+}
+
+/** A position along ordered levels. */
+export function score(instructions: Entry, criteria: readonly [Entry, Entry, ...Entry[]]): ScoreQuestion {
+ return { type: "score", instructions, criteria };
+}
diff --git a/site/src/lib/inference/typesafe-models.ts b/site/src/lib/inference/typesafe-models.ts
new file mode 100644
index 00000000..ab262716
--- /dev/null
+++ b/site/src/lib/inference/typesafe-models.ts
@@ -0,0 +1,10 @@
+// The TypeSafe System One model behind every judgment call. The judge route
+// pins it; the page and the worker never see it (eslint keeps the id out of
+// their bundles). Bump here when adopting a newer Jev.
+export const typesafeModels = {
+ jev: "jev-latest",
+} as const;
+
+export type TypesafeModel = (typeof typesafeModels)[keyof typeof typesafeModels];
+
+export const typesafeProviderId = "typesafe";