Skip to content
TypeScript
7.3 KB
Raw Blame
/**
 * 오케스트레이터 (PRD §18).
 * expand+validate → subset → (조합×task×run) 실행(동시성, fail-soft) → 파서 →
 * metrics/accuracy/flow → recorder → report+graph 렌더(언어 반영).
 */
import { readFileSync } from "node:fs";
import { join } from "node:path";
import {
  difficultyMap,
  instructionFor,
  isLocalBench,
  listLocalBenches,
  taskIds,
} from "./benches/index";
import { expand, parseComboId } from "./combos";
import { resolveModelAlias } from "./config";
import { buildFlow, renderGraphHtml, writeFlowJson } from "./flow";
import { getJudge } from "./judge";
import { aggregateAll } from "./metrics";
import type { Combo, FlowGraph, RunOptions, RunResult } from "./models";
import { estimateCost, resolveCost } from "./pricing";
import { Recorder, makeRunId } from "./recorder";
import { renderReport } from "./report";
import { MockRunner } from "./runners/mock";
import { runnerFor } from "./runners/registry";
import { TerminalBenchRunner } from "./runners/terminalBench";
import { selectBalanced, selectSubset } from "./subset";
import { parserFor } from "./transcripts/index";

export interface ExecuteResult {
  runId: string;
  runDir: string;
  reportPath: string;
  graphPath: string;
  metrics: ReturnType<typeof aggregateAll>;
  graphs: FlowGraph[];
  mock: boolean;
}

function loadCatalog(bench: string): string[] {
  if (isLocalBench(bench)) return taskIds(bench);
  try {
    return new TerminalBenchRunner().loadTaskCatalog(bench);
  } catch {
    try {
      return new MockRunner().loadTaskCatalog(bench);
    } catch {
      // 폴백까지 실패(미지의 벤치) — 원인/대안이 담긴 명확한 에러 하나로 승격.
      throw new Error(
        `unknown bench '${bench}' — available: ${[...listLocalBenches(), "terminal-bench-v2"].join(", ")}`,
      );
    }
  }
}

async function pool<T>(items: T[], size: number, fn: (item: T) => Promise<void>): Promise<void> {
  const n = Math.max(1, size);
  let i = 0;
  const workers = Array.from({ length: Math.min(n, items.length || 1) }, async () => {
    while (i < items.length) {
      const idx = i++;
      await fn(items[idx]!);
    }
  });
  await Promise.all(workers);
}

interface Job {
  combo: Combo;
  taskId: string;
  runIdx: number;
}

export async function execute(opts: RunOptions): Promise<ExecuteResult> {
  const { valid, excluded } = expand(opts.models, opts.agents, opts.harnesses);

  // subset
  const catalog = loadCatalog(opts.bench);
  const tasks = opts.balanced
    ? selectBalanced(catalog, opts.tasks, opts.seed, difficultyMap(opts.bench))
    : selectSubset(catalog, opts.tasks, opts.seed);

  const runId = makeRunId();
  const recorder = new Recorder(opts.outDir, runId);
  const judge = getJudge(opts.judge);

  // jobs
  const jobs: Job[] = [];
  for (const combo of valid)
    for (const taskId of tasks)
      for (let runIdx = 0; runIdx < Math.max(1, opts.runs); runIdx++)
        jobs.push({ combo, taskId, runIdx });

  const results: RunResult[] = [];
  const graphs: FlowGraph[] = [];
  let anyMock = opts.runner === "mock";

  await pool(jobs, opts.concurrency, async (job) => {
    const { combo, taskId, runIdx } = job;
    const { agent, model } = parseComboId(combo.comboId);
    const choice = runnerFor(agent, opts.runner, { bench: opts.bench }, opts.failRate ?? 0);
    if (choice.mock) anyMock = true;

    const workDir = recorder.workDir(combo.comboId, taskId, runIdx);
    const inv = {
      combo,
      taskId,
      runIdx,
      workDir,
      timeoutS: opts.timeoutS,
      seed: opts.seed,
      bench: opts.bench,
      instruction: instructionFor(taskId),
    };

    let artifact: Awaited<ReturnType<typeof choice.runner.run>>;
    try {
      artifact = await choice.runner.run(inv);
    } catch (e) {
      // 계약상 throw 금지지만 방어적으로 fail-soft 결과 생성
      artifact = {
        invocation: inv,
        success: false,
        resultPath: join(workDir, "output.txt"),
        transcriptPath: join(workDir, "transcript.jsonl"),
        wallTimeMs: 0,
        rawResolved: null,
        totalCostUsd: null,
        numTurns: null,
        error: `runner threw: ${(e as Error).message}`,
      };
    }

    const parsed = parserFor(agent).parse(artifact.transcriptPath, {
      model: resolveModelAlias(model),
    });

    const estimated = estimateCost(
      model,
      parsed.inputTokens,
      parsed.outputTokens,
      parsed.cacheReadTokens,
      parsed.cacheCreationTokens,
    );
    const { costUsd, costSource } = resolveCost(
      model,
      parsed.inputTokens,
      parsed.outputTokens,
      parsed.cacheReadTokens,
      parsed.cacheCreationTokens,
      artifact.totalCostUsd,
    );

    let qualityScore: number | null = null;
    if (judge) {
      let text = "";
      try {
        text = readFileSync(artifact.resultPath, "utf8");
      } catch {
        // ignore
      }
      qualityScore = await judge.score(text, inv.instruction);
    }

    const result: RunResult = {
      comboId: combo.comboId,
      taskId,
      runIdx,
      success: artifact.success,
      resolved: artifact.rawResolved,
      resultPath: artifact.resultPath,
      inputTokens: parsed.inputTokens,
      outputTokens: parsed.outputTokens,
      cacheReadTokens: parsed.cacheReadTokens,
      cacheCreationTokens: parsed.cacheCreationTokens,
      totalTokens: parsed.totalTokens,
      costUsd,
      estimatedCostUsd: estimated,
      costSource,
      latencyMs: parsed.latencyMs || artifact.wallTimeMs,
      wallTimeMs: artifact.wallTimeMs,
      numTurns: artifact.numTurns ?? parsed.numTurns,
      toolCalls: parsed.toolCalls,
      subagentCount: parsed.subagentCount,
      skillFires: parsed.skillFires,
      mcpCalls: parsed.mcpCalls,
      sessionId: parsed.sessionId,
      transcriptPath: artifact.transcriptPath,
      model: resolveModelAlias(model),
      qualityScore,
      error: artifact.error ?? null,
    };
    results.push(result);
    recorder.appendResult(result);

    // flow
    const totalTok = parsed.totalTokens || 1;
    const costForFlow = costUsd ?? estimated ?? 0;
    const graph = buildFlow(parsed, combo.comboId, taskId, runIdx, {
      harness: parseComboId(combo.comboId).harness,
      costPerToken: costForFlow / totalTok,
      success: artifact.success,
      error: artifact.error,
    });
    writeFlowJson(graph, join(workDir, "flow.json"));
    if (runIdx === 0) graphs.push(graph); // 리포트엔 run0 대표 그래프
  });

  // metrics & accuracy
  const metrics = aggregateAll(results);
  for (const m of metrics) recorder.writeAccuracy(m.comboId, m);

  // meta
  recorder.writeMeta({
    runId,
    bench: opts.bench,
    tasks,
    seed: opts.seed,
    runs: opts.runs,
    runner: opts.runner,
    excluded,
    createdAt: new Date().toISOString(),
    language: opts.language,
    mock: anyMock,
    goal: opts.goal,
  });

  // report + graph
  renderReport(
    {
      metrics,
      graphs,
      results,
      excluded,
      tasks,
      bench: opts.bench,
      language: opts.language,
      judge: judge !== null,
      score: opts.score,
      mock: anyMock,
      goal: opts.goal,
    },
    recorder.reportPath,
  );
  renderGraphHtml(graphs, recorder.graphPath, { language: opts.language });

  await recorder.finalizeSqlite(results);

  return {
    runId,
    runDir: recorder.runDir,
    reportPath: recorder.reportPath,
    graphPath: recorder.graphPath,
    metrics,
    graphs,
    mock: anyMock,
  };
}