TypeScript
/**
* Terminal-Bench 어댑터 (PRD §7). 외부 tb 호출만 위임(재구현 금지).
* 미설치 시 registry 가 mock 폴백(리포트에 mock provenance 배너).
*/
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import type { RawRunArtifact, RunInvocation } from "../models";
import { type RunnerAdapter, RunnerUnavailable } from "./base";
import { commandExists, execCapture } from "./util";
export class TerminalBenchRunner implements RunnerAdapter {
name = "tb";
isAvailable(): boolean {
return commandExists("tb");
}
loadTaskCatalog(bench: string): string[] {
if (!this.isAvailable()) throw new RunnerUnavailable("tb not installed");
if (bench !== "terminal-bench-v2") throw new RunnerUnavailable(`tb: unknown bench '${bench}'`);
// 실제로는 tb 카탈로그를 질의해야 하나, 본 어댑터는 위임만 — 합성 id 사용.
return Array.from({ length: 40 }, (_, i) => `tb-${String(i + 1).padStart(3, "0")}`);
}
async run(inv: RunInvocation): Promise<RawRunArtifact> {
const work = inv.workDir;
mkdirSync(work, { recursive: true });
const resultPath = join(work, "output.txt");
const transcriptPath = join(work, "transcript.jsonl");
// 위임: tb run <task> (실제 인자 스펙은 tb 버전에 종속). fail-soft.
const res = await execCapture("tb", ["run", "--task", inv.taskId], {
cwd: work,
timeoutS: inv.timeoutS,
});
writeFileSync(resultPath, res.stdout || res.stderr || "(no output)");
writeFileSync(transcriptPath, res.stdout || "");
return {
invocation: inv,
success: res.error === null && !res.timedOut && res.code === 0,
resultPath,
transcriptPath,
wallTimeMs: res.wallTimeMs,
rawResolved: null,
totalCostUsd: null,
numTurns: null,
error: res.error ?? (res.timedOut ? "timeout" : null),
};
}
}