Skip to content

Open source CapCut alternative.

TypeScript
12.2 KB
Raw Blame
/**
 * The eval runner: one case = one chat turn through the PRODUCTION loop —
 * streamCutChat, the pi-based turn runner the editor itself uses — with the
 * eval's own deps injected: the dev server as the inference proxy, sims and
 * stubs as the tool server, and fixture snapshots as the editor state.
 *
 * Every run is timed off the loop's own hooks: the gate, each LLM round and
 * its first visible delta, and each tool serve. Retries happen inside the
 * transport and are invisible here, so latency numbers are the turn as the
 * user experiences it.
 */

import type { UIMessage } from "ai";
import { AI_SKILLS } from "../../../src/cut/server/ai/catalog";
import { geminiModelRoleNames, resolveGeminiModel } from "../../../src/lib/inference/gemini-models";
import { streamCutChat, dropPiSession, type CutAgentDeps } from "../../../src/cut/lib/pi/cutAgent";
import { DEFAULT_LOCALE } from "../../../src/i18n/locales";
import type { TurnIntent } from "../../../src/cut/lib/turnJudge";
import { EDITOR_STATE, SAFE_TOOLS, serveSafeTool } from "./fixtures";
import { judgeReply, replyShapeNotes } from "./graders";
import type { EvalCase } from "./cases";

export interface RunConfig {
  /** Dev server base URL. */
  base: string;
  /** Model a simple-verdict turn's rounds run on. */
  simpleModel: string;
  /** Model a complex-verdict (or fail-open) turn's rounds run on. */
  complexModel: string;
  /** Whether the reply judge scores each turn (report-only). */
  judge: boolean;
  /** Threshold overrides for the turn judge; unset runs the registry defaults. */
  judgeSettings?: CutAgentDeps["judgeSettings"];
}

/** Production's config: the three-way gate routes between the two chat roles,
 * named the way the page names them so the route resolves the ids. */
export const defaultRunConfig = (base: string): RunConfig => ({
  base,
  simpleModel: geminiModelRoleNames.chatSimple,
  complexModel: geminiModelRoleNames.chat,
  judge: true,
});

export interface RoundTiming {
  /** Wall time of the round's request, request start → message settled. */
  roundMs: number;
  /** Round start → first visible (non-whitespace) text delta, if any text streamed. */
  firstDeltaMs: number | null;
  calls: { name: string; execMs: number }[];
}

export interface RunTimings {
  /** Wall time of the gate's classify call. */
  gateMs: number;
  /** True when attachments classified the turn "work" with no model call. */
  gateSkipped: boolean;
  /** Turn start → first visible text delta anywhere in the turn. */
  ttftMs: number | null;
  /** Σ round wall times. */
  modelMs: number;
  /** Σ tool-serve wall times (stubs and sims today, so ≈0 — kept separate so
   * real tool execution never pollutes model timing). */
  toolMs: number;
  /** gateMs + modelMs + toolMs — the turn's critical path.
   * The judge call is grading overhead, so judgeMs stays out of it. */
  totalMs: number;
  judgeMs?: number;
  /** Transport retries happen inside the loop's stream layer now; these stay
   * for the report schema and read 0. */
  retryMs: number;
  retries: number;
  rounds: RoundTiming[];
}

export interface TraceEntry {
  name: string;
  /** Zero-based round the call arrived in. */
  round: number;
  /** Turn start → this call being served. */
  atMs: number;
  execMs: number;
  /** The call's arguments, for a trace print. */
  args: Record<string, unknown>;
}

export interface CaseResult {
  pass: boolean;
  reply: string;
  trace: TraceEntry[];
  violations: string[];
  notes: string[];
  /** The judge's complaint (or transport note). Report-only: never flips pass. */
  judgeNote: string | null;
  intent: TurnIntent;
  /** The skill the judge attached, the areas it declared, and how many
   * tools that came to; request_tools calls widened the catalog (a miss). */
  skill: string | null;
  areas: string[];
  declaredTools: number;
  routingMisses: string[];
  /** The model the rounds ran on — differs per turn under a router config. */
  roundModel: string;
  /** The action the instant path carried the turn out with, if any. */
  instant: string | null;
  /** Each time the quality gate sent the turn back to work, in order. */
  qualityGate: string[];
  timings: RunTimings;
}

let runSeq = 0;

export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult> {
  const turnStart = performance.now();
  const messages = c.input();
  const lastUser = [...messages].reverse().find((m) => m.role === "user");
  const askText = lastUser?.parts.find((p) => p.type === "text")?.text ?? "";
  const sim = c.simulate?.();

  const trace: TraceEntry[] = [];
  const rounds: RoundTiming[] = [];
  const violations: string[] = [];
  const notes: string[] = [];
  let intent: TurnIntent = "complex";
  let gateMs = 0;
  let gateSkipped = false;
  let modelMs = 0;
  let toolMs = 0;
  let ttftMs: number | null = null;
  let reply = "";
  let streamError: string | null = null;
  let extensions = 0;
  let instant: string | null = null;
  const qualityGate: string[] = [];

  const devPost =
    (path: string) =>
    (payload: Record<string, unknown>, signal?: AbortSignal) =>
      fetch(`${cfg.base}${path}`, {
        method: "POST",
        headers: {
          "Content-Type": "application/json",
          "x-donkey-client-id": "donkey-cut-eval",
          "x-donkey-dev-auth-bypass": "1",
        },
        body: JSON.stringify(payload),
        signal,
      });
  let route: { skill: string | null; areas: string[]; declaredTools: number } | null = null;
  const routingMisses: string[] = [];
  const deps: CutAgentDeps = {
    post: devPost("/api/inference/responses"),
    judge: devPost("/api/inference/judge"),
    judgeSettings: cfg.judgeSettings,
    execTool: async (name, args) => {
      const atMs = performance.now() - turnStart;
      const e0 = performance.now();
      const record = (execMs: number) => {
        toolMs += execMs;
        const round = Math.max(0, rounds.length - 1);
        trace.push({ name, round, atMs, execMs, args });
        rounds[round]?.calls.push({ name, execMs });
      };
      try {
        const simulated = sim?.(name, args);
        if (name === "read_skill") {
          const doc = AI_SKILLS[String(args.name ?? "")];
          return doc ? { doc } : { error: "No such skill." };
        }
        if (simulated !== undefined) return simulated;
        if (c.stubs && name in c.stubs) {
          const stub = c.stubs[name];
          // A stub can script the tool FAILING, the way runAiTool throws.
          if (stub && typeof stub === "object" && "__error" in stub)
            throw new Error(String((stub as { __error: unknown }).__error));
          return stub;
        }
        if (SAFE_TOOLS.has(name)) return serveSafeTool(name, c.state ?? EDITOR_STATE, args);
        violations.push(name);
        return { error: "eval: this tool is disabled for this turn — do not retry it" };
      } finally {
        record(performance.now() - e0);
      }
    },
    models: { simple: cfg.simpleModel, complex: cfg.complexModel },
    buildContext: () => lastUser?.__state ?? c.state ?? EDITOR_STATE,
    resolveRefs: async () => lastUser?.__wireParts ?? [],
    debris: () => c.debris ?? [],
    limits: c.limits,
    hooks: {
      onGate: (i, ms, skipped, r) => {
        intent = i;
        gateMs = ms;
        gateSkipped = skipped;
        route = { skill: r.skill, areas: r.areas, declaredTools: r.declaredTools };
      },
      onInstant: (action) => {
        instant = action?.id ?? null;
      },
      onRequestTools: (areas) => routingMisses.push(...areas),
      onRound: (ms, firstDeltaMs) => {
        modelMs += ms;
        rounds.push({ roundMs: ms, firstDeltaMs, calls: [] });
      },
      onExtension: (n) => {
        extensions = n;
      },
      onQualityGate: (step) => qualityGate.push(step),
    },
  };

  // Each run is its own thread so pi sessions never leak between runs.
  const threadId = `eval-${c.name}-${++runSeq}`;
  const stream = streamCutChat({
    threadId,
    model: cfg.complexModel,
    messages: messages as unknown as UIMessage[],
    locale: DEFAULT_LOCALE,
    deps,
  });
  try {
    const reader = stream.getReader();
    for (;;) {
      const { done, value } = await reader.read();
      if (done) break;
      const chunk = value as unknown as { type: string; delta?: string; errorText?: string };
      if (chunk.type === "text-delta" && chunk.delta) {
        if (ttftMs === null && chunk.delta.trim()) ttftMs = performance.now() - turnStart;
        reply += chunk.delta;
      } else if (chunk.type === "text-start" && reply) {
        reply += "\n";
      } else if (chunk.type === "error" && chunk.errorText) {
        streamError = chunk.errorText;
      }
    }
  } finally {
    dropPiSession(threadId);
  }
  // Wall clock for the whole turn, captured at stream close. The gate runs
  // concurrently with the first model round, so summing the stage timings
  // would overstate what a user actually waits.
  const wallMs = performance.now() - turnStart;
  reply = reply.trim();
  if (streamError) throw new Error(streamError);

  // Rounds are timed off assistant messages; a round that requested tools has
  // its calls attached above. Cases assert the gate side of the verdict;
  // simple-vs-complex is routing, scored by latency, so either satisfies a
  // "work" expectation.
  const gateSide = intent === "chat" ? "chat" : "work";
  if (c.gate && gateSide !== c.gate) notes.push(`gate said ${intent}, expected ${c.gate}`);
  // An instant turn is the whole point of the path: the judgment settled it
  // and no model round ran. A case that pins one asserts both.
  if (c.instant !== undefined) {
    if (instant !== c.instant) notes.push(`instant was ${instant ?? "none"}, expected ${c.instant ?? "none"}`);
    if (c.instant !== null && rounds.length > 0) notes.push(`instant turn still spent ${rounds.length} model round(s)`);
  }
  if (!c.reply.test(reply)) notes.push(`reply did not match ${c.reply}`);
  for (const t of c.requiredTools ?? []) {
    if (!trace.some((e) => e.name === t)) notes.push(`required tool ${t} was never called`);
  }
  if (c.anyTools && !c.anyTools.some((t) => trace.some((e) => e.name === t)))
    notes.push(`none of [${c.anyTools.join(", ")}] were called`);
  if (c.maxToolCalls !== undefined && trace.length > c.maxToolCalls)
    notes.push(`slow: ${trace.length} tool calls (cap ${c.maxToolCalls})`);
  if (violations.length > 0) notes.push(`mutating tools called: ${violations.join(", ")}`);
  if (c.extensions?.min !== undefined && extensions < c.extensions.min)
    notes.push(`auto-continue extended ${extensions}× (needed ≥${c.extensions.min})`);
  if (c.extensions?.max !== undefined && extensions > c.extensions.max)
    notes.push(`auto-continue extended ${extensions}× (cap ${c.extensions.max})`);
  if (c.qualityGate?.min !== undefined && qualityGate.length < c.qualityGate.min)
    notes.push(
      `the quality gate let the turn close after ${qualityGate.length} return(s) (needed ≥${c.qualityGate.min})`
    );
  if (c.qualityGate?.max !== undefined && qualityGate.length > c.qualityGate.max)
    notes.push(`the quality gate sent the turn back ${qualityGate.length}× (cap ${c.qualityGate.max})`);
  const simVerify = (sim as unknown as { verify?: () => string[] } | undefined)?.verify;
  if (simVerify) notes.push(...simVerify());

  const mutated = trace.some(
    (e) =>
      !SAFE_TOOLS.has(e.name) &&
      e.name !== "read_skill" &&
      e.name !== "list_skills" &&
      !violations.includes(e.name)
  );
  notes.push(...replyShapeNotes(reply, { mutated, expectsQuestion: c.expectsQuestion }));

  let judgeNote: string | null = null;
  let judgeMs: number | undefined;
  if (cfg.judge && reply) {
    const verdict = await judgeReply({
      base: cfg.base,
      userAsk: askText,
      reply,
      didWork: mutated,
    });
    judgeNote = verdict.note;
    judgeMs = verdict.judgeMs;
  }

  const timings: RunTimings = {
    gateMs,
    gateSkipped,
    ttftMs,
    modelMs,
    toolMs,
    totalMs: wallMs,
    judgeMs,
    retryMs: 0,
    retries: 0,
    rounds,
  };
  return {
    pass: notes.length === 0,
    reply,
    trace,
    violations,
    notes,
    judgeNote,
    intent,
    skill: route?.skill ?? null,
    areas: route?.areas ?? [],
    declaredTools: route?.declaredTools ?? 0,
    routingMisses,
    instant,
    qualityGate,
    roundModel: resolveGeminiModel(intent === "simple" ? cfg.simpleModel : cfg.complexModel),
    timings,
  };
}