Open source CapCut alternative.
TypeScript
/**
* The eval's report: schema cut-chat-eval/v1. One report holds one or more
* model configs; each config holds per-case runs with full timings and
* per-bucket aggregates. Latency aggregates count passing runs only — a wrong
* answer's speed means nothing — with pass rate reported alongside so a
* fast-but-wrong config stays visible.
*/
import type { Bucket, EvalCase } from "./cases";
import type { CaseResult, RunTimings, TraceEntry } from "./harness";
export interface Agg {
p50: number;
p95: number;
mean: number;
n: number;
}
export interface RunReport {
pass: boolean;
intent: string;
skill?: string | null;
areas?: string[];
declaredTools?: number;
routingMisses?: string[];
/** The model the rounds ran on — differs per turn under a router config. */
roundModel?: string;
notes: string[];
/** The registry action the turn ran with no model round, if any. */
instant?: string | null;
/** Each time the quality gate sent the turn back to work, in order. */
qualityGate?: string[];
/** The voice/taste judge's complaint. Report-only: never part of pass. */
judgeNote?: string | null;
trace: TraceEntry[];
timings: RunTimings | null;
/** Set when the run threw (dev server down, non-retryable upstream error). */
error?: string;
}
export interface CaseLatency {
gateMs: Agg;
ttftMs: Agg | null;
totalMs: Agg;
modelMs: Agg;
toolMs: Agg;
rounds: { mean: number; max: number };
}
export interface CaseReport {
name: string;
bucket: Bucket;
passRate: number;
budgetBreaches: string[];
/** Passing runs only; null when no run passed. */
latency: CaseLatency | null;
runs: RunReport[];
}
export interface BucketSummary {
cases: number;
passRate: number;
gateMs: Agg | null;
ttftMs: Agg | null;
totalMs: Agg | null;
meanRounds: number | null;
}
export interface ConfigReport {
label: string;
chatModel: string;
buckets: Partial<Record<Bucket, BucketSummary>>;
cases: CaseReport[];
}
export interface LatencyReport {
schema: "cut-chat-eval/v1";
generatedAt: string;
base: string;
runsPerCase: number;
configs: ConfigReport[];
}
const r0 = (x: number) => Math.round(x);
const percentile = (sorted: number[], p: number): number => {
if (sorted.length === 0) return 0;
const idx = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);
return sorted[Math.max(0, idx)];
};
export function agg(values: number[]): Agg | null {
if (values.length === 0) return null;
const sorted = [...values].sort((a, b) => a - b);
return {
p50: r0(percentile(sorted, 50)),
p95: r0(percentile(sorted, 95)),
mean: r0(values.reduce((s, v) => s + v, 0) / values.length),
n: values.length,
};
}
export function toRunReport(result: CaseResult | null, error?: string): RunReport {
if (!result) return { pass: false, intent: "work", notes: [], trace: [], timings: null, error };
return {
pass: result.pass,
intent: result.intent,
skill: result.skill,
areas: result.areas,
declaredTools: result.declaredTools,
routingMisses: result.routingMisses,
roundModel: result.roundModel,
notes: result.notes,
instant: result.instant,
qualityGate: result.qualityGate,
judgeNote: result.judgeNote,
trace: result.trace,
timings: result.timings,
};
}
export function buildCaseReport(c: EvalCase, runs: RunReport[]): CaseReport {
const passing = runs.filter((r) => r.pass && r.timings).map((r) => r.timings as RunTimings);
const passRate = runs.length > 0 ? passing.length / runs.length : 0;
let latency: CaseLatency | null = null;
if (passing.length > 0) {
const ttfts = passing.map((t) => t.ttftMs).filter((v): v is number => v !== null);
latency = {
gateMs: agg(passing.map((t) => t.gateMs)) as Agg,
ttftMs: agg(ttfts),
totalMs: agg(passing.map((t) => t.totalMs)) as Agg,
modelMs: agg(passing.map((t) => t.modelMs)) as Agg,
toolMs: agg(passing.map((t) => t.toolMs)) as Agg,
rounds: {
mean: Math.round((passing.reduce((s, t) => s + t.rounds.length, 0) / passing.length) * 10) / 10,
max: Math.max(...passing.map((t) => t.rounds.length)),
},
};
}
const budgetBreaches: string[] = [];
if (c.budget && latency) {
const checks: [string, number | undefined, Agg | null][] = [
["totalMs", c.budget.totalMs, latency.totalMs],
["ttftMs", c.budget.ttftMs, latency.ttftMs],
["gateMs", c.budget.gateMs, latency.gateMs],
];
for (const [key, budget, a] of checks) {
if (budget !== undefined && a && a.p50 > budget)
budgetBreaches.push(`${key} p50 ${a.p50} > budget ${budget}`);
}
}
return { name: c.name, bucket: c.bucket, passRate, budgetBreaches, latency, runs };
}
export function buildBucketSummaries(
caseReports: CaseReport[]
): Partial<Record<Bucket, BucketSummary>> {
const buckets: Partial<Record<Bucket, BucketSummary>> = {};
for (const bucket of new Set(caseReports.map((c) => c.bucket))) {
const cs = caseReports.filter((c) => c.bucket === bucket);
const withLatency = cs.filter((c) => c.latency !== null);
const flat = <K extends "gateMs" | "totalMs" | "modelMs">(key: K) =>
agg(withLatency.map((c) => (c.latency as CaseLatency)[key].p50));
buckets[bucket] = {
cases: cs.length,
passRate:
Math.round((cs.reduce((s, c) => s + c.passRate, 0) / cs.length) * 100) / 100,
gateMs: flat("gateMs"),
ttftMs: agg(
withLatency
.map((c) => c.latency?.ttftMs?.p50)
.filter((v): v is number => typeof v === "number")
),
totalMs: flat("totalMs"),
meanRounds:
withLatency.length > 0
? Math.round(
(withLatency.reduce((s, c) => s + (c.latency as CaseLatency).rounds.mean, 0) /
withLatency.length) *
10
) / 10
: null,
};
}
return buckets;
}