Skip to content
TypeScript
1.9 KB
Raw Blame
/**
 * Terminal-Bench 어댑터 (PRD §7). 외부 tb 호출만 위임(재구현 금지).
 * 미설치 시 registry 가 mock 폴백(리포트에 mock provenance 배너).
 */
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import type { RawRunArtifact, RunInvocation } from "../models";
import { type RunnerAdapter, RunnerUnavailable } from "./base";
import { commandExists, execCapture } from "./util";

export class TerminalBenchRunner implements RunnerAdapter {
  name = "tb";

  isAvailable(): boolean {
    return commandExists("tb");
  }

  loadTaskCatalog(bench: string): string[] {
    if (!this.isAvailable()) throw new RunnerUnavailable("tb not installed");
    if (bench !== "terminal-bench-v2") throw new RunnerUnavailable(`tb: unknown bench '${bench}'`);
    // 실제로는 tb 카탈로그를 질의해야 하나, 본 어댑터는 위임만 — 합성 id 사용.
    return Array.from({ length: 40 }, (_, i) => `tb-${String(i + 1).padStart(3, "0")}`);
  }

  async run(inv: RunInvocation): Promise<RawRunArtifact> {
    const work = inv.workDir;
    mkdirSync(work, { recursive: true });
    const resultPath = join(work, "output.txt");
    const transcriptPath = join(work, "transcript.jsonl");
    // 위임: tb run <task> (실제 인자 스펙은 tb 버전에 종속). fail-soft.
    const res = await execCapture("tb", ["run", "--task", inv.taskId], {
      cwd: work,
      timeoutS: inv.timeoutS,
    });
    writeFileSync(resultPath, res.stdout || res.stderr || "(no output)");
    writeFileSync(transcriptPath, res.stdout || "");
    return {
      invocation: inv,
      success: res.error === null && !res.timedOut && res.code === 0,
      resultPath,
      transcriptPath,
      wallTimeMs: res.wallTimeMs,
      rawResolved: null,
      totalCostUsd: null,
      numTurns: null,
      error: res.error ?? (res.timedOut ? "timeout" : null),
    };
  }
}