TypeScript
/**
* 오프라인 결정적 합성 러너 (PRD §7, §22.4).
* (comboId, taskId, runIdx, seed) 해시로 토큰/시간/서브에이전트 수 생성. Math.random 금지.
* 하네스 경향 반영. codex/gpt 는 서브에이전트 0(단일 루프).
* 로컬-스타일 벤치는 실제 task id + setup 훅 존중(verify 는 호출 안 하고 합성 rawResolved 사용).
*/
import { createHash } from "node:crypto";
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { isLocalBench, setupTask, taskIds } from "../benches/index";
import { parseComboId } from "../combos";
import { modelFamily } from "../config";
import type { RawRunArtifact, RunInvocation } from "../models";
import { type RunnerAdapter, RunnerUnavailable } from "./base";
class Det {
private buf: string;
private counter = 0;
constructor(...material: (string | number)[]) {
this.buf = material.join(":");
}
private hash(): bigint {
const h = createHash("sha256")
.update(`${this.buf}:${this.counter++}`)
.digest("hex")
.slice(0, 12);
return BigInt(`0x${h}`);
}
randint(lo: number, hi: number): number {
if (hi <= lo) return lo;
const span = BigInt(hi - lo + 1);
return lo + Number(this.hash() % span);
}
chance(p: number): boolean {
return this.randint(0, 9999) / 10000 < p;
}
}
const HARNESS_SUBAGENTS: Record<string, [number, number]> = {
none: [0, 0],
superpowers: [1, 3],
omc: [2, 5],
omx: [2, 5],
ecc: [0, 1],
lazycodex: [0, 1],
omo: [2, 5],
mattpocock: [1, 3],
ponytail: [0, 1],
};
// 미니멀리즘 하네스는 토큰/도구를 줄이는 경향(결정적 할인 계수)
const HARNESS_TOKEN_SCALE: Record<string, number> = {
ponytail: 0.5,
};
/**
* fan-out 가능 여부: 트리형 에이전트는 cc(모든 하네스), 그리고 opencode+omo(오케스트레이션).
* codex/openclaw/hermes 및 omo 외 opencode 는 단일 루프(0).
*/
function mockFanout(agent: string, harness: string, det: Det): number {
if (agent === "cc") {
const [lo, hi] = HARNESS_SUBAGENTS[harness] ?? [0, 0];
return det.randint(lo, hi);
}
if (agent === "opencode" && harness === "omo") {
const [lo, hi] = HARNESS_SUBAGENTS.omo ?? [0, 0];
return det.randint(lo, hi);
}
return 0;
}
const DIFFICULTY_SCALE: Record<string, number> = {
"subagent-": 1.8,
"local-": 1.0,
};
function scaleFor(taskId: string): number {
for (const [pre, s] of Object.entries(DIFFICULTY_SCALE)) if (taskId.startsWith(pre)) return s;
return 1.0;
}
export class MockRunner implements RunnerAdapter {
name = "mock";
constructor(private failRate = 0) {}
isAvailable(): boolean {
return true;
}
loadTaskCatalog(bench: string): string[] {
if (isLocalBench(bench)) return taskIds(bench);
// 외부 벤치도 합성 카탈로그(tb mock 폴백)
if (bench === "terminal-bench-v2")
return Array.from({ length: 40 }, (_, i) => `tb-${String(i + 1).padStart(3, "0")}`);
throw new RunnerUnavailable(`mock: unknown bench '${bench}'`);
}
async run(inv: RunInvocation): Promise<RawRunArtifact> {
const { agent, model, harness } = parseComboId(inv.combo.comboId);
const work = inv.workDir;
mkdirSync(work, { recursive: true });
const resultPath = join(work, "output.txt");
const transcriptPath = join(work, "transcript.jsonl");
// 로컬-스타일 벤치는 setup 픽스처 생성(러너 3개 공통)
if (isLocalBench(inv.bench)) setupTask(inv.taskId, work);
const det = new Det(inv.combo.comboId, inv.taskId, inv.runIdx, inv.seed);
const scale = scaleFor(inv.taskId);
// 실행 성공 여부(fail-soft 경로 검증용)
const failed = det.chance(this.failRate);
// 서브에이전트 수 — 하네스 오케스트레이션 × 에이전트 트리 능력으로 결정.
const fam = modelFamily(model);
const subagentCount = mockFanout(agent, harness, det);
// 토큰 (ponytail 등 미니멀리즘 하네스는 할인)
const hScale = scale * (HARNESS_TOKEN_SCALE[harness] ?? 1);
const inputTokens = Math.round(det.randint(8000, 30000) * hScale);
const outputTokens = Math.round(det.randint(1500, 6000) * hScale);
const cacheRead = Math.round(det.randint(20000, 120000) * hScale);
const cacheCreate = Math.round(det.randint(2000, 15000) * hScale);
const wallTimeMs = det.randint(3000, 90000);
const rawResolved = failed ? false : det.chance(0.7 - (scale - 1) * 0.15);
// transcript 합성 — cc 는 sidechain 트리, 그 외(codex/opencode/openclaw/hermes)는 평탄.
// 평탄 포맷에도 subagentCount 만큼 task 아이템을 실어 DefensiveParser 가 fan-out 을 집계.
const lines: string[] =
agent === "cc"
? this.ccTranscript(
det,
model,
harness,
subagentCount,
inputTokens,
outputTokens,
cacheRead,
cacheCreate,
)
: this.codexTranscript(
det,
model,
inputTokens,
outputTokens,
cacheRead,
cacheCreate,
subagentCount,
);
writeFileSync(transcriptPath, lines.join("\n"));
writeFileSync(
resultPath,
failed
? `mock failure for ${inv.taskId}`
: `mock result for ${inv.taskId} (${subagentCount} subagents)\nDone.`,
);
// claude family 는 실측 비용 시뮬레이션(actual), gpt 는 null(=> estimated/none)
const totalCostUsd = fam === "claude" ? Math.round((inputTokens / 1e6) * 15 * 100) / 100 : null;
return {
invocation: inv,
success: !failed,
resultPath,
transcriptPath,
wallTimeMs,
rawResolved,
totalCostUsd,
numTurns: det.randint(2, 12),
error: failed ? "mock: synthetic failure" : null,
};
}
private ccTranscript(
det: Det,
model: string,
harness: string,
subagents: number,
inT: number,
outT: number,
cr: number,
cc: number,
): string[] {
const lines: string[] = [];
let uuid = 0;
const nextId = () => `m${uuid++}`;
const tools = ["Read", "Bash", "Grep", "Edit", "Glob"];
let ts = 1_000;
const stamp = () => {
ts += det.randint(100, 2000);
return new Date(ts).toISOString();
};
// 메인 루프 첫 assistant
const rootId = nextId();
lines.push(
JSON.stringify({
type: "assistant",
uuid: rootId,
parentUuid: null,
isSidechain: false,
timestamp: stamp(),
message: {
role: "assistant",
model,
usage: {
input_tokens: Math.round(inT * 0.5),
output_tokens: Math.round(outT * 0.5),
cache_read_input_tokens: Math.round(cr * 0.5),
cache_creation_input_tokens: Math.round(cc * 0.5),
},
content: [{ type: "text", text: "Working on the task." }],
},
}),
);
// 메인 루프 tool_use 몇 개
let prev = rootId;
for (let i = 0; i < det.randint(2, 6); i++) {
const id = nextId();
const tool = tools[det.randint(0, tools.length - 1)]!;
lines.push(
JSON.stringify({
type: "assistant",
uuid: id,
parentUuid: prev,
isSidechain: false,
timestamp: stamp(),
message: {
role: "assistant",
model,
content: [{ type: "tool_use", id: `t${id}`, name: tool, input: { path: "x" } }],
usage: { input_tokens: 0, output_tokens: det.randint(50, 300) },
},
}),
);
prev = id;
}
// 스킬 사용 (하네스 있을 때)
if (harness !== "none") {
const id = nextId();
lines.push(
JSON.stringify({
type: "assistant",
uuid: id,
parentUuid: prev,
isSidechain: false,
timestamp: stamp(),
message: {
role: "assistant",
model,
content: [
{ type: "tool_use", id: `t${id}`, name: "Skill", input: { command: "planning" } },
],
usage: { input_tokens: 0, output_tokens: det.randint(20, 100) },
},
}),
);
prev = id;
}
// 서브에이전트 (Task tool_use + sidechain 대화)
for (let s = 0; s < subagents; s++) {
const taskId = nextId();
const taskUuid = taskId;
lines.push(
JSON.stringify({
type: "assistant",
uuid: taskUuid,
parentUuid: rootId,
isSidechain: false,
timestamp: stamp(),
message: {
role: "assistant",
model,
content: [
{ type: "tool_use", id: `task${s}`, name: "Task", input: { subagent_type: harness } },
],
usage: { input_tokens: 0, output_tokens: det.randint(50, 200) },
},
}),
);
// sidechain assistant 라인 (서브에이전트 내부)
const subTurns = det.randint(1, 3);
let subPrev = taskUuid;
for (let k = 0; k < subTurns; k++) {
const sid = nextId();
lines.push(
JSON.stringify({
type: "assistant",
uuid: sid,
parentUuid: subPrev,
isSidechain: true,
timestamp: stamp(),
message: {
role: "assistant",
model,
content: [
{ type: "text", text: `subagent ${s} step ${k}` },
{ type: "tool_use", id: `st${s}_${k}`, name: tools[det.randint(0, 4)]!, input: {} },
],
usage: {
input_tokens: Math.round((inT * 0.5) / Math.max(1, subagents * subTurns)),
output_tokens: Math.round((outT * 0.5) / Math.max(1, subagents * subTurns)),
cache_read_input_tokens: Math.round((cr * 0.5) / Math.max(1, subagents * subTurns)),
cache_creation_input_tokens: Math.round((cc * 0.5) / Math.max(1, subagents)),
},
},
}),
);
subPrev = sid;
}
}
// result 라인
lines.push(
JSON.stringify({
type: "result",
uuid: nextId(),
parentUuid: prev,
isSidechain: false,
timestamp: stamp(),
session_id: `sess-${det.randint(1000, 9999)}`,
message: { role: "assistant", content: [{ type: "text", text: "Done." }] },
}),
);
return lines;
}
private codexTranscript(
det: Det,
model: string,
inT: number,
outT: number,
cr: number,
cc: number,
subagents = 0,
): string[] {
const lines: string[] = [];
lines.push(
JSON.stringify({ type: "thread.started", thread_id: `th-${det.randint(1000, 9999)}` }),
);
// 평탄 포맷에서도 fan-out 을 표현: subagents 개의 task 아이템(DefensiveParser 가 집계).
for (let s = 0; s < subagents; s++) {
lines.push(
JSON.stringify({
type: "item.completed",
item: { type: "task", name: "Task", command: `subagent ${s}` },
}),
);
}
const turns = det.randint(2, 8);
for (let i = 0; i < turns; i++) {
lines.push(JSON.stringify({ type: "turn.started" }));
const nItems = det.randint(1, 4);
for (let k = 0; k < nItems; k++) {
lines.push(
JSON.stringify({
type: "item.completed",
item: {
type: det.chance(0.5) ? "command_execution" : "file_change",
command: "ls -la",
},
}),
);
}
}
lines.push(
JSON.stringify({
type: "turn.completed",
usage: {
input_tokens: inT,
output_tokens: outT,
cached_input_tokens: cr,
cache_creation_input_tokens: cc,
},
}),
);
return lines;
}
}