TypeScript
/**
* 오케스트레이터 (PRD §18).
* expand+validate → subset → (조합×task×run) 실행(동시성, fail-soft) → 파서 →
* metrics/accuracy/flow → recorder → report+graph 렌더(언어 반영).
*/
import { readFileSync } from "node:fs";
import { join } from "node:path";
import {
difficultyMap,
instructionFor,
isLocalBench,
listLocalBenches,
taskIds,
} from "./benches/index";
import { expand, parseComboId } from "./combos";
import { resolveModelAlias } from "./config";
import { buildFlow, renderGraphHtml, writeFlowJson } from "./flow";
import { getJudge } from "./judge";
import { aggregateAll } from "./metrics";
import type { Combo, FlowGraph, RunOptions, RunResult } from "./models";
import { estimateCost, resolveCost } from "./pricing";
import { Recorder, makeRunId } from "./recorder";
import { renderReport } from "./report";
import { MockRunner } from "./runners/mock";
import { runnerFor } from "./runners/registry";
import { TerminalBenchRunner } from "./runners/terminalBench";
import { selectBalanced, selectSubset } from "./subset";
import { parserFor } from "./transcripts/index";
export interface ExecuteResult {
runId: string;
runDir: string;
reportPath: string;
graphPath: string;
metrics: ReturnType<typeof aggregateAll>;
graphs: FlowGraph[];
mock: boolean;
}
function loadCatalog(bench: string): string[] {
if (isLocalBench(bench)) return taskIds(bench);
try {
return new TerminalBenchRunner().loadTaskCatalog(bench);
} catch {
try {
return new MockRunner().loadTaskCatalog(bench);
} catch {
// 폴백까지 실패(미지의 벤치) — 원인/대안이 담긴 명확한 에러 하나로 승격.
throw new Error(
`unknown bench '${bench}' — available: ${[...listLocalBenches(), "terminal-bench-v2"].join(", ")}`,
);
}
}
}
async function pool<T>(items: T[], size: number, fn: (item: T) => Promise<void>): Promise<void> {
const n = Math.max(1, size);
let i = 0;
const workers = Array.from({ length: Math.min(n, items.length || 1) }, async () => {
while (i < items.length) {
const idx = i++;
await fn(items[idx]!);
}
});
await Promise.all(workers);
}
interface Job {
combo: Combo;
taskId: string;
runIdx: number;
}
export async function execute(opts: RunOptions): Promise<ExecuteResult> {
const { valid, excluded } = expand(opts.models, opts.agents, opts.harnesses);
// subset
const catalog = loadCatalog(opts.bench);
const tasks = opts.balanced
? selectBalanced(catalog, opts.tasks, opts.seed, difficultyMap(opts.bench))
: selectSubset(catalog, opts.tasks, opts.seed);
const runId = makeRunId();
const recorder = new Recorder(opts.outDir, runId);
const judge = getJudge(opts.judge);
// jobs
const jobs: Job[] = [];
for (const combo of valid)
for (const taskId of tasks)
for (let runIdx = 0; runIdx < Math.max(1, opts.runs); runIdx++)
jobs.push({ combo, taskId, runIdx });
const results: RunResult[] = [];
const graphs: FlowGraph[] = [];
let anyMock = opts.runner === "mock";
await pool(jobs, opts.concurrency, async (job) => {
const { combo, taskId, runIdx } = job;
const { agent, model } = parseComboId(combo.comboId);
const choice = runnerFor(agent, opts.runner, { bench: opts.bench }, opts.failRate ?? 0);
if (choice.mock) anyMock = true;
const workDir = recorder.workDir(combo.comboId, taskId, runIdx);
const inv = {
combo,
taskId,
runIdx,
workDir,
timeoutS: opts.timeoutS,
seed: opts.seed,
bench: opts.bench,
instruction: instructionFor(taskId),
};
let artifact: Awaited<ReturnType<typeof choice.runner.run>>;
try {
artifact = await choice.runner.run(inv);
} catch (e) {
// 계약상 throw 금지지만 방어적으로 fail-soft 결과 생성
artifact = {
invocation: inv,
success: false,
resultPath: join(workDir, "output.txt"),
transcriptPath: join(workDir, "transcript.jsonl"),
wallTimeMs: 0,
rawResolved: null,
totalCostUsd: null,
numTurns: null,
error: `runner threw: ${(e as Error).message}`,
};
}
const parsed = parserFor(agent).parse(artifact.transcriptPath, {
model: resolveModelAlias(model),
});
const estimated = estimateCost(
model,
parsed.inputTokens,
parsed.outputTokens,
parsed.cacheReadTokens,
parsed.cacheCreationTokens,
);
const { costUsd, costSource } = resolveCost(
model,
parsed.inputTokens,
parsed.outputTokens,
parsed.cacheReadTokens,
parsed.cacheCreationTokens,
artifact.totalCostUsd,
);
let qualityScore: number | null = null;
if (judge) {
let text = "";
try {
text = readFileSync(artifact.resultPath, "utf8");
} catch {
// ignore
}
qualityScore = await judge.score(text, inv.instruction);
}
const result: RunResult = {
comboId: combo.comboId,
taskId,
runIdx,
success: artifact.success,
resolved: artifact.rawResolved,
resultPath: artifact.resultPath,
inputTokens: parsed.inputTokens,
outputTokens: parsed.outputTokens,
cacheReadTokens: parsed.cacheReadTokens,
cacheCreationTokens: parsed.cacheCreationTokens,
totalTokens: parsed.totalTokens,
costUsd,
estimatedCostUsd: estimated,
costSource,
latencyMs: parsed.latencyMs || artifact.wallTimeMs,
wallTimeMs: artifact.wallTimeMs,
numTurns: artifact.numTurns ?? parsed.numTurns,
toolCalls: parsed.toolCalls,
subagentCount: parsed.subagentCount,
skillFires: parsed.skillFires,
mcpCalls: parsed.mcpCalls,
sessionId: parsed.sessionId,
transcriptPath: artifact.transcriptPath,
model: resolveModelAlias(model),
qualityScore,
error: artifact.error ?? null,
};
results.push(result);
recorder.appendResult(result);
// flow
const totalTok = parsed.totalTokens || 1;
const costForFlow = costUsd ?? estimated ?? 0;
const graph = buildFlow(parsed, combo.comboId, taskId, runIdx, {
harness: parseComboId(combo.comboId).harness,
costPerToken: costForFlow / totalTok,
success: artifact.success,
error: artifact.error,
});
writeFlowJson(graph, join(workDir, "flow.json"));
if (runIdx === 0) graphs.push(graph); // 리포트엔 run0 대표 그래프
});
// metrics & accuracy
const metrics = aggregateAll(results);
for (const m of metrics) recorder.writeAccuracy(m.comboId, m);
// meta
recorder.writeMeta({
runId,
bench: opts.bench,
tasks,
seed: opts.seed,
runs: opts.runs,
runner: opts.runner,
excluded,
createdAt: new Date().toISOString(),
language: opts.language,
mock: anyMock,
goal: opts.goal,
});
// report + graph
renderReport(
{
metrics,
graphs,
results,
excluded,
tasks,
bench: opts.bench,
language: opts.language,
judge: judge !== null,
score: opts.score,
mock: anyMock,
goal: opts.goal,
},
recorder.reportPath,
);
renderGraphHtml(graphs, recorder.graphPath, { language: opts.language });
await recorder.finalizeSqlite(results);
return {
runId,
runDir: recorder.runDir,
reportPath: recorder.reportPath,
graphPath: recorder.graphPath,
metrics,
graphs,
mock: anyMock,
};
}