Open source CapCut alternative.
TypeScript
feat(site): typesafe judge route, adapter, pricing, setting
David Le046996a4e6dc
diff --git a/site/eslint.config.mjs b/site/eslint.config.mjsindex 417bb0df..ed79ceef 100644--- a/site/eslint.config.mjs+++ b/site/eslint.config.mjs@@ -60,6 +60,10 @@ const eslintConfig = defineConfig([],message: "Name a model role (geminiModelRoleNames); the responses route resolves the id.",},+ {+ name: "@/lib/inference/typesafe-models",+ message: "The judge route pins the model; post to /api/inference/judge.",+ },],},],diff --git a/site/package-lock.json b/site/package-lock.jsonindex 7858433a..6fffd6dc 100644--- a/site/package-lock.json+++ b/site/package-lock.json@@ -45,6 +45,7 @@"@tailwindcss/typography": "^0.5.19","@tanstack/react-query": "^5.101.0","@types/mdx": "^2.0.13",+ "@typesafe-ai/sdk": "0.6.0","ai": "^7.0.14","better-auth": "1.6.11","better-call": "1.3.5",@@ -10089,6 +10090,15 @@"@types/node": "*"}},+ "node_modules/@typesafe-ai/sdk": {+ "version": "0.6.0",+ "resolved": "https://registry.npmjs.org/@typesafe-ai/sdk/-/sdk-0.6.0.tgz",+ "integrity": "sha512-IddX+Q0XM+VagOUZFeP7wZjaO4SHMdvnh2zEBdrZZnXedWI3BNK1lKhMx3ayrkFWvVLbVcUHJy6AVZlY+e6Jaw==",+ "license": "MIT",+ "engines": {+ "node": ">=20"+ }+ },"node_modules/@typescript-eslint/eslint-plugin": {"version": "8.59.3","resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.59.3.tgz",diff --git a/site/package.json b/site/package.jsonindex 94e98240..568e407a 100644--- a/site/package.json+++ b/site/package.json@@ -77,6 +77,7 @@"@tailwindcss/typography": "^0.5.19","@tanstack/react-query": "^5.101.0","@types/mdx": "^2.0.13",+ "@typesafe-ai/sdk": "0.6.0","ai": "^7.0.14","better-auth": "1.6.11","better-call": "1.3.5",diff --git a/site/src/app/api/inference/judge/route.ts b/site/src/app/api/inference/judge/route.tsnew file mode 100644index 00000000..822ecbd6--- /dev/null+++ b/site/src/app/api/inference/judge/route.ts@@ -0,0 +1,113 @@+import { NextResponse } from "next/server";++import {+ creditErrorResponse,+ creditUsageHeaders,+ inferenceUsageRoutes,+ recordFailedInferenceUsage,+ recordInferenceUsage,+ requireInferenceCredits,+} from "@/lib/credits/inference";+import { typesafeJudge } from "@/lib/inference/adapters/typesafe";+import { judgeRequestSchema, type JudgeQuestion } from "@/lib/inference/judge";+import { toJsonValue } from "@/lib/inference/json";+import { InferenceProviderError } from "@/lib/inference/providers";+import {+ inferenceErrorCode,+ inferenceProviderErrorResponse,+ requireInferenceClientId,+ validationErrorResponse,+} from "@/lib/inference/responses";+import {+ INFERENCE_RATE_LIMIT,+ shouldBypassDonkeyInferenceCredits,+ withDonkeyAuth,+} from "@/lib/donkey-api-auth";++// A judgment: typed questions over one state, answered as probabilities by+// TypeSafe's System One model. The page, the worker, and the eval post here+// with the user's session; the route bills the tokens and pins the model.++const requestKind = "judge";++export const POST = withDonkeyAuth(async (request) => {+ const client = requireInferenceClientId(request.donkey.clientId);+ if (!client.ok) {+ return client.response;+ }++ const parsed = judgeRequestSchema.safeParse(await request.json());+ if (!parsed.success) {+ return validationErrorResponse(parsed.error);+ }++ const judge = typesafeJudge();+ const bypassCredits = shouldBypassDonkeyInferenceCredits(request.donkey);+ if (!bypassCredits) {+ const credits = await requireInferenceCredits({+ model: judge.model,+ provider: judge.id,+ route: inferenceUsageRoutes.judge,+ userId: request.donkey.userId,+ enforceModelPrice: true,+ });+ if (!credits.ok) {+ return credits.response;+ }+ }++ try {+ // Validated above: a score's criteria has at least two levels.+ const result = await judge.askJudge(+ { state: parsed.data.state, questions: parsed.data.questions as Record<string, JudgeQuestion> },+ request.signal,+ );++ let usageHeaders: Record<string, string> = {};+ if (bypassCredits) {+ usageHeaders["X-Donkey-Dev-Auth-Bypass"] = "true";+ } else {+ const recordedUsage = await recordInferenceUsage({+ clientId: client.clientId,+ conversationId: request.donkey.conversationId,+ model: result.model,+ provider: judge.id,+ requestKind,+ route: inferenceUsageRoutes.judge,+ status: "succeeded",+ usage: toJsonValue(result.usage),+ userId: request.donkey.userId,+ });+ usageHeaders = creditUsageHeaders(recordedUsage);+ }++ return NextResponse.json(result, {+ headers: {+ "X-Donkey-Inference-Provider": judge.id,+ "X-Donkey-Inference-Model": result.model,+ ...usageHeaders,+ },+ });+ } catch (error) {+ if (!bypassCredits) {+ await recordFailedInferenceUsage({+ clientId: client.clientId,+ conversationId: request.donkey.conversationId,+ errorCode: inferenceErrorCode(error),+ model: judge.model,+ provider: judge.id,+ requestKind,+ route: inferenceUsageRoutes.judge,+ userId: request.donkey.userId,+ });+ }+ const creditResponse = creditErrorResponse(error);+ if (creditResponse) {+ return creditResponse;+ }+ if (error instanceof InferenceProviderError) {+ return inferenceProviderErrorResponse(error);+ }+ throw error;+ }+}, { allowRunner: true, rateLimit: INFERENCE_RATE_LIMIT });diff --git a/site/src/lib/config/registry.ts b/site/src/lib/config/registry.tsindex 636d59ec..94c11eac 100644--- a/site/src/lib/config/registry.ts+++ b/site/src/lib/config/registry.ts@@ -93,6 +93,50 @@ export const SETTINGS = defineSettings({title: "Chat continuity",description: "Project and conversation refresh cadence, scene ownership expiry, and saved engine transcript size.",},+ cutJudge: {+ schema: z+ .object({+ // Preload the skill the judge picks for the turn; off leaves the model+ // to read skills on its own.+ skillSuggestion: z.boolean(),+ // Mean of the "does this turn want a skill" questions below which no+ // skill is attached.+ skillGate: z.number().min(0).max(1),+ // The winning skill's own fit below which it is dropped.+ skillFits: z.number().min(0).max(1),+ // Declare only the tool areas the judge routes to; off declares the+ // whole catalog on every turn.+ toolRouting: z.boolean(),+ // An area's probability at or above which its tools are declared.+ toolArea: z.number().min(0).max(1),+ // How long the first model round waits for the judge before going out+ // with the full catalog and no skill.+ judgeWaitMs: z.number().int().min(0).max(2000),+ // A stock candidate's fit below which it is left out of a search.+ stockFit: z.number().min(0).max(1),+ // A transcript word's disfluency probability at or above which+ // find_filler reports it.+ fillerCut: z.number().min(0).max(1),+ // The confidence below which a described voice falls back to the default.+ voicePick: z.number().min(0).max(1),+ })+ .strict(),+ default: {+ skillSuggestion: true,+ skillGate: 0.4,+ skillFits: 0.3,+ toolRouting: true,+ toolArea: 0.2,+ judgeWaitMs: 400,+ stockFit: 0.5,+ fillerCut: 0.5,+ voicePick: 0.4,+ },+ public: true,+ title: "Chat judgments",+ description:+ "Thresholds for the typed judgments that route a chat turn (skill, tool areas), rank stock, find filler words, and resolve described voices.",+ },experimentResults: {schema: z.object({diff --git a/site/src/lib/credits/inference.ts b/site/src/lib/credits/inference.tsindex c82ca7b5..60dfcfe6 100644--- a/site/src/lib/credits/inference.ts+++ b/site/src/lib/credits/inference.ts@@ -17,6 +17,7 @@ export const inferenceUsageRoutes = {assets: "/api/inference/assets/",assetsRefresh: "/api/inference/assets/refresh/",responses: "/api/inference/responses/",+ judge: "/api/inference/judge/",} as const;type CreditsDatabase = PrismaClient | Prisma.TransactionClient;diff --git a/site/src/lib/credits/provider-pricing.ts b/site/src/lib/credits/provider-pricing.tsindex 2d3dc2c5..c5df65bc 100644--- a/site/src/lib/credits/provider-pricing.ts+++ b/site/src/lib/credits/provider-pricing.ts@@ -6,6 +6,7 @@ import {elevenLabsModels,type ElevenLabsRunModel,} from "@/lib/inference/elevenlabs-models";+import { typesafeModels, typesafeProviderId } from "@/lib/inference/typesafe-models";import {geminiModels,geminiMusicModels,@@ -72,7 +73,28 @@ export function providerCreditPricing(if (normalizedProvider === "fal") {return falMatteCreditPricing(normalizedModel);}+ // The judge id is hardcoded (typesafe-models.ts); a judgment bills its input+ // tokens, and the answers are free.+ if (normalizedProvider === typesafeProviderId) {+ return typesafeCreditPricing(normalizedModel);+ }++ return undefined;+}+// TypeSafe System One (Jev): $0.042 per 1M input tokens, output free+// (2026-09). A request for `jev-latest` reports the dated id it ran on, so+// the family prefix matches both.+const typesafeJudgePricing = textAudioTokenPricing({+ input: "0.042",+ cachedInput: "0.042",+ output: "0",+});++function typesafeCreditPricing(model: string): ProviderCreditPricing | undefined {+ if (model === typesafeModels.jev || modelMatches(model, "jev")) {+ return typesafeJudgePricing;+ }return undefined;}diff --git a/site/src/lib/inference/adapters/typesafe.test.ts b/site/src/lib/inference/adapters/typesafe.test.tsnew file mode 100644index 00000000..520ca2db--- /dev/null+++ b/site/src/lib/inference/adapters/typesafe.test.ts@@ -0,0 +1,73 @@+import { describe, expect, test } from "bun:test";++import { createTypesafeJudge } from "./typesafe";+import { choice, noul } from "@/lib/inference/judge";++describe("typesafe judge", () => {+ test("is unconfigured without a key and fails hard", async () => {+ const judge = createTypesafeJudge({});+ expect(judge.configured).toBe(false);+ let code = "";+ try {+ await judge.askJudge({ state: "hi", questions: { q: noul("Is this a greeting?") } });+ } catch (error) {+ code = (error as { code: string }).code;+ }+ expect(code).toBe("missing_provider_credentials");+ });++ test("posts state and questions with the pinned model and returns typed answers", async () => {+ let sent: { url: string; body: unknown; auth: string | null } | null = null;+ const judge = createTypesafeJudge({ TYPESAFE_API_KEY: "secret" }, (async (+ url: string | URL | Request,+ init?: RequestInit,+ ) => {+ sent = {+ url: String(url),+ body: JSON.parse(String(init?.body)),+ auth: new Headers(init?.headers).get("authorization"),+ };+ return new Response(+ JSON.stringify({+ model: "jev-1.12",+ answers: {+ intent: { type: "choice", choice: "chat", probabilities: { chat: 0.9, work: 0.1 }, confidence: 0.85 },+ urgent: { type: "noul", noul: 0.05 },+ },+ usage: { input_tokens: 40, output_tokens: 2 },+ }),+ { status: 200, headers: { "content-type": "application/json" } },+ );+ }) as typeof fetch);++ const result = await judge.askJudge({+ state: "hello there",+ questions: {+ intent: choice("What kind of message?", { chat: "social filler", work: "an ask" }),+ urgent: noul("Is it urgent?"),+ },+ });+ expect(result.model).toBe("jev-1.12");+ expect(result.answers.intent.choice).toBe("chat");+ expect(result.answers.intent.probabilities.work).toBe(0.1);+ expect(result.answers.urgent.noul).toBe(0.05);+ expect(result.usage.input_tokens).toBe(40);+ const s = sent as unknown as { url: string; body: { model: string; state: string }; auth: string | null };+ expect(s.url).toBe("https://api.typesafe.ai/v1/systemone");+ expect(s.body.model).toBe("jev-latest");+ expect(s.body.state).toBe("hello there");+ expect(s.auth).toBe("Bearer secret");+ });++ test("a provider failure surfaces as an inference provider error", async () => {+ const judge = createTypesafeJudge({ TYPESAFE_API_KEY: "secret" }, (async () =>+ new Response(JSON.stringify({ error: "bad" }), { status: 422 })) as typeof fetch);+ let code = "";+ try {+ await judge.askJudge({ state: "x", questions: { q: noul("?") } });+ } catch (error) {+ code = (error as { code: string }).code;+ }+ expect(code).toBe("invalid_request");+ });+});diff --git a/site/src/lib/inference/adapters/typesafe.ts b/site/src/lib/inference/adapters/typesafe.tsnew file mode 100644index 00000000..0c9f52aa--- /dev/null+++ b/site/src/lib/inference/adapters/typesafe.ts@@ -0,0 +1,106 @@+import { APIConnectionError, APIError, TypeSafeClient } from "@typesafe-ai/sdk";++import type { AdapterEnvironment } from "@/lib/inference/adapters/gemini-client";+import type { Entry, JudgeQuestion, JudgeResult } from "@/lib/inference/judge";+import { InferenceProviderError } from "@/lib/inference/providers";+import { typesafeModels, typesafeProviderId } from "@/lib/inference/typesafe-models";++// The one seam to TypeSafe's System One API. Every judgment — the chat turn's+// route, queue triage, stock ranking, refusal classes — goes through+// `askJudge` here, on the server, with the account's key. The model is pinned+// from the registry; the page never names it.++// A judgment gates a turn that is already running, so a stall is answered+// fast: one attempt of 8 s, one retry.+const requestTimeoutMs = 8_000;+const maxRetries = 1;++export interface TypesafeJudge {+ readonly id: typeof typesafeProviderId;+ readonly model: string;+ readonly configured: boolean;+ askJudge<Q extends Record<string, JudgeQuestion>>(+ request: { state: Entry; questions: Q },+ signal?: AbortSignal,+ ): Promise<JudgeResult<Q>>;+}++export function createTypesafeJudge(+ environment: AdapterEnvironment = process.env,+ fetchImpl?: typeof fetch,+): TypesafeJudge {+ const apiKey = environment.TYPESAFE_API_KEY?.trim();+ const model = typesafeModels.jev;+ const client = apiKey+ ? new TypeSafeClient({+ apiKey,+ defaultModel: model,+ timeout: requestTimeoutMs,+ retry: { maxRetries },+ ...(fetchImpl ? { fetch: fetchImpl as never } : {}),+ })+ : null;++ async function askJudge<Q extends Record<string, JudgeQuestion>>(+ request: { state: Entry; questions: Q },+ signal?: AbortSignal,+ ): Promise<JudgeResult<Q>> {+ if (!client) {+ throw new InferenceProviderError("TypeSafe is not configured (TYPESAFE_API_KEY).", {+ statusCode: 503,+ code: "missing_provider_credentials",+ });+ }+ try {+ const result = await client.systemOne(+ { state: request.state as never, questions: request.questions as never, model },+ { signal },+ );+ return {+ model: result.model,+ answers: result.answers as unknown as JudgeResult<Q>["answers"],+ usage: { input_tokens: result.usage.input_tokens, output_tokens: result.usage.output_tokens },+ };+ } catch (error) {+ throw toProviderError(error);+ }+ }++ return { id: typesafeProviderId, model, configured: client !== null, askJudge };+}++function toProviderError(error: unknown): InferenceProviderError {+ if (error instanceof InferenceProviderError) return error;+ if (error instanceof APIError) {+ const status = error.status === 429 ? 429 : error.status >= 500 ? 502 : error.status === 401 ? 503 : 400;+ return new InferenceProviderError(`TypeSafe request failed (${error.status}): ${error.message}`, {+ statusCode: status,+ code:+ error.status === 401+ ? "missing_provider_credentials"+ : error.status === 429+ ? "rate_limited"+ : error.status === 422+ ? "invalid_request"+ : "provider_error",+ details: typeof error.body === "object" ? (error.body as never) : null,+ });+ }+ if (error instanceof APIConnectionError) {+ return new InferenceProviderError(`TypeSafe unreachable: ${error.message}`, {+ statusCode: 504,+ code: "provider_timeout",+ });+ }+ return new InferenceProviderError(error instanceof Error ? error.message : String(error), {+ statusCode: 502,+ code: "provider_error",+ });+}++let shared: TypesafeJudge | null = null;++/** The process-wide judge, built once from the environment. */+export function typesafeJudge(): TypesafeJudge {+ return (shared ??= createTypesafeJudge());+}diff --git a/site/src/lib/inference/judge.ts b/site/src/lib/inference/judge.tsnew file mode 100644index 00000000..bc876e5b--- /dev/null+++ b/site/src/lib/inference/judge.ts@@ -0,0 +1,136 @@+import { z } from "zod";++// The judgment contract shared by the judge route, the page, and the worker:+// one state, named typed questions, one typed answer per question. The shapes+// mirror TypeSafe's System One API (noul = P(yes), choice = a distribution over+// named options, score = a distribution over ordered levels), so a caller reads+// probabilities and code decides. Zod only, so it is importable anywhere.++export type Entry = string | number | boolean | null | Entry[] | { [key: string]: Entry };++const entrySchema: z.ZodType<Entry> = z.lazy(() =>+ z.union([z.string(), z.number(), z.boolean(), z.null(), z.array(entrySchema), z.record(z.string(), entrySchema)]),+);++const noulQuestionSchema = z+ .object({+ type: z.literal("noul"),+ instructions: entrySchema.optional(),+ criteria: z.object({ true: entrySchema.optional(), false: entrySchema.optional() }).nullable().optional(),+ })+ .strict();++const choiceQuestionSchema = z+ .object({+ type: z.literal("choice"),+ instructions: entrySchema.optional(),+ criteria: z.record(z.string().min(1).max(200), entrySchema).refine((c) => Object.keys(c).length >= 2, {+ message: "A choice needs at least two options.",+ }),+ })+ .strict();++const scoreQuestionSchema = z+ .object({+ type: z.literal("score"),+ instructions: entrySchema.optional(),+ criteria: z.array(entrySchema).min(2).max(32),+ })+ .strict();++export const judgeQuestionSchema = z.discriminatedUnion("type", [+ noulQuestionSchema,+ choiceQuestionSchema,+ scoreQuestionSchema,+]);++/** The most questions one request carries. Per-item sweeps (stock candidates,+ * transcript words) chunk under this. */+export const JUDGE_MAX_QUESTIONS = 200;++export const judgeRequestSchema = z+ .object({+ state: entrySchema,+ questions: z+ .record(z.string().min(1).max(120), judgeQuestionSchema)+ .refine((q) => Object.keys(q).length >= 1, { message: "At least one question." })+ .refine((q) => Object.keys(q).length <= JUDGE_MAX_QUESTIONS, {+ message: `At most ${JUDGE_MAX_QUESTIONS} questions per request.`,+ }),+ })+ .strict();++export type JudgeRequest = z.infer<typeof judgeRequestSchema>;++export interface NoulQuestion {+ type: "noul";+ instructions?: Entry;+ criteria?: { true?: Entry; false?: Entry } | null;+}+export interface ChoiceQuestion<C extends Record<string, Entry> = Record<string, Entry>> {+ type: "choice";+ instructions?: Entry;+ criteria: C;+}+export interface ScoreQuestion {+ type: "score";+ instructions?: Entry;+ criteria: readonly [Entry, Entry, ...Entry[]];+}+export type JudgeQuestion = NoulQuestion | ChoiceQuestion | ScoreQuestion;++export interface NoulAnswer {+ type: "noul";+ /** P(yes), 0..1. */+ noul: number;+}+export interface ChoiceAnswer<C extends Record<string, Entry> = Record<string, Entry>> {+ type: "choice";+ choice: keyof C & string;+ probabilities: { [K in keyof C]: number };+ /** Concentration of the distribution, 0..1. */+ confidence: number;+}+export interface ScoreAnswer {+ type: "score";+ /** Probability-weighted level, may fall between levels. */+ score: number;+ probabilities: Record<string, number>;+ legend: Record<string, Entry>;+ confidence: number;+}+export type JudgeAnswer = NoulAnswer | ChoiceAnswer | ScoreAnswer;++export type AnswerFor<Q> = Q extends NoulQuestion+ ? NoulAnswer+ : Q extends ChoiceQuestion<infer C>+ ? ChoiceAnswer<C>+ : Q extends ScoreQuestion+ ? ScoreAnswer+ : never;++export interface JudgeUsage {+ input_tokens: number;+ output_tokens: number;+}++export interface JudgeResult<Q extends Record<string, JudgeQuestion>> {+ model: string;+ answers: { [K in keyof Q]: AnswerFor<Q[K]> };+ usage: JudgeUsage;+}++/** A yes/no question. */+export function noul(instructions: Entry, criteria?: { true?: Entry; false?: Entry }): NoulQuestion {+ return criteria ? { type: "noul", instructions, criteria } : { type: "noul", instructions };+}++/** One option out of a named set. */+export function choice<const C extends Record<string, Entry>>(instructions: Entry, criteria: C): ChoiceQuestion<C> {+ return { type: "choice", instructions, criteria };+}++/** A position along ordered levels. */+export function score(instructions: Entry, criteria: readonly [Entry, Entry, ...Entry[]]): ScoreQuestion {+ return { type: "score", instructions, criteria };+}diff --git a/site/src/lib/inference/typesafe-models.ts b/site/src/lib/inference/typesafe-models.tsnew file mode 100644index 00000000..ab262716--- /dev/null+++ b/site/src/lib/inference/typesafe-models.ts@@ -0,0 +1,10 @@+// The TypeSafe System One model behind every judgment call. The judge route+// pins it; the page and the worker never see it (eslint keeps the id out of+// their bundles). Bump here when adopting a newer Jev.+export const typesafeModels = {+ jev: "jev-latest",+} as const;++export type TypesafeModel = (typeof typesafeModels)[keyof typeof typesafeModels];++export const typesafeProviderId = "typesafe";