TypeScript
/**
* BR (동작) 검증 지표 — PRD §19.
* BR1 조합 유효성/제외 사유 · BR2 subset 결정성 · BR3 러너 never-throw ·
* BR4 하네스 per-run · BR5 벤치 setup+recompute-verify · BR6 멀티레벨 파싱 · BR7 fail-soft
*/
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { findTask, setupTask, verify } from "../src/benches/index";
import { expand } from "../src/combos";
import { harnessClaudeArgs, harnessCodexEnv, harnessStatus } from "../src/harness";
import { MockRunner } from "../src/runners/mock";
import { selectBalanced, selectSubset } from "../src/subset";
import { ClaudeCodeParser } from "../src/transcripts/claudeCode";
function tmp(): string {
return mkdtempSync(join(tmpdir(), "theseus-"));
}
describe("BR1 combo validity & exclude reasons", () => {
it("excludes incompatible agent×model and harness with reasons (no silent drop)", () => {
const { valid, excluded } = expand(["opus4.8", "gpt5.5"], ["cc", "codex"], ["none", "omc"]);
expect(valid.map((c) => c.comboId)).toContain("cc+opus4.8+none");
expect(valid.map((c) => c.comboId)).toContain("codex+gpt5.5+none");
// cc cannot use gpt model
const cross = excluded.find((c) => c.comboId === "cc+gpt5.5+none");
expect(cross?.excludeReason).toMatch(/incompatible/);
// omc is cc-only → codex+gpt5.5+omc excluded
const harnessBad = excluded.find((c) => c.comboId === "codex+gpt5.5+omc");
expect(harnessBad?.excludeReason).toMatch(/not compatible/);
// every excluded has a reason
expect(excluded.every((c) => !!c.excludeReason)).toBe(true);
});
});
describe("BR2 subset determinism", () => {
const catalog = Array.from({ length: 50 }, (_, i) => `task-${i}`);
it("same (catalog,k,seed) → identical subset; no Math.random", () => {
const a = selectSubset(catalog, 10, 42);
const b = selectSubset(catalog, 10, 42);
expect(a).toEqual(b);
expect(a).toHaveLength(10);
});
it("different seed → (generally) different ordering", () => {
const a = selectSubset(catalog, 10, 1);
const b = selectSubset(catalog, 10, 2);
expect(a).not.toEqual(b);
});
it("balanced subset is deterministic and spans buckets", () => {
const levels = ["easy", "medium", "hard"] as const;
const dm: Record<string, "easy" | "medium" | "hard"> = {};
catalog.forEach((t, i) => {
dm[t] = levels[i % 3]!;
});
const a = selectBalanced(catalog, 9, 7, dm);
const b = selectBalanced(catalog, 9, 7, dm);
expect(a).toEqual(b);
});
});
describe("BR3 runner never-throw", () => {
it("mock.run resolves with success flag even when fail injected", async () => {
const runner = new MockRunner(1.0); // always fail
const work = tmp();
const art = await runner.run({
combo: {
comboId: "cc+opus4.8+none",
agent: "cc",
model: "opus4.8",
harness: "none",
valid: true,
},
taskId: "local-hello",
runIdx: 0,
workDir: work,
timeoutS: 1,
seed: 0,
bench: "local",
});
expect(art.success).toBe(false);
expect(art.resultPath).toBeTruthy();
expect(art.transcriptPath).toBeTruthy();
});
});
describe("BR4 harness per-run application", () => {
it("none → no args/env; status reflects availability", () => {
expect(harnessClaudeArgs("none")).toEqual([]);
expect(harnessCodexEnv("none")).toEqual({});
expect(harnessStatus("none")).toBe("none");
// unavailable when not installed (offline)
expect(["applied", "unavailable"]).toContain(harnessStatus("omc"));
});
});
describe("BR5 bench setup + recompute-verify", () => {
it("setup creates fixtures; verify recomputes (no hardcoded golden)", () => {
const work = tmp();
setupTask("subagent-104", work);
// 정답을 직접(독립) 풀어서 기록 → verify 통과해야 함
const k = JSON.parse(readFileSync(join(work, "knapsack.json"), "utf8"));
// 독립 2D DP
const n = k.items.length;
const dp: number[][] = Array.from({ length: n + 1 }, () =>
new Array(k.capacity + 1).fill(0),
) as number[][];
const at = (i: number, c: number): number => dp[i]![c]!;
const set = (i: number, c: number, val: number) => {
dp[i]![c] = val;
};
for (let i = 1; i <= n; i++) {
const [w, v] = k.items[i - 1];
for (let c = 0; c <= k.capacity; c++)
set(i, c, c >= w ? Math.max(at(i - 1, c), at(i - 1, c - w) + v) : at(i - 1, c));
}
writeFileSync(join(work, "answer.txt"), String(at(n, k.capacity)));
expect(verify("subagent-104", work)).toBe(true);
// 틀린 답 → false
writeFileSync(join(work, "answer.txt"), String(at(n, k.capacity) + 1));
expect(verify("subagent-104", work)).toBe(false);
});
it("verify never throws on missing files", () => {
expect(verify("subagent-101", tmp())).toBe(false);
expect(verify("nonexistent", tmp())).toBe(false);
});
});
describe("BR6 multi-level subagent parsing (layout B)", () => {
it("links nested subagents via meta.toolUseId (depth 2)", () => {
const work = tmp();
const transcriptPath = join(work, "transcript.jsonl");
// main: spawns Task TU1
writeFileSync(
transcriptPath,
`${JSON.stringify({
type: "assistant",
uuid: "root1",
parentUuid: null,
isSidechain: false,
message: {
role: "assistant",
model: "claude-opus-4-8",
usage: { input_tokens: 1000, output_tokens: 200 },
content: [{ type: "tool_use", id: "TU1", name: "Task", input: {} }],
},
})}\n`,
);
const subDir = join(work, "transcript_subagents");
mkdirSync(subDir, { recursive: true });
// agent-1 spawns Task TU2
writeFileSync(
join(subDir, "agent-1.jsonl"),
`${JSON.stringify({
type: "assistant",
uuid: "a1",
message: {
role: "assistant",
usage: { input_tokens: 500, output_tokens: 100 },
content: [
{ type: "tool_use", id: "TU2", name: "Task", input: {} },
{ type: "tool_use", id: "r1", name: "Read", input: {} },
],
},
})}\n`,
);
writeFileSync(
join(subDir, "agent-1.meta.json"),
JSON.stringify({ toolUseId: "TU1", agentId: "agent-1" }),
);
// agent-2 child of agent-1
writeFileSync(
join(subDir, "agent-2.jsonl"),
`${JSON.stringify({
type: "assistant",
uuid: "a2",
message: {
role: "assistant",
usage: { input_tokens: 300, output_tokens: 50 },
content: [],
},
})}\n`,
);
writeFileSync(
join(subDir, "agent-2.meta.json"),
JSON.stringify({ toolUseId: "TU2", agentId: "agent-2" }),
);
const parsed = new ClaudeCodeParser().parse(transcriptPath);
expect(parsed.subagentCount).toBe(2); // 파일 2개 (인라인 중복 없음)
// 토큰 총합 = root + 두 서브에이전트
expect(parsed.inputTokens).toBe(1000 + 500 + 300);
expect(parsed.outputTokens).toBe(200 + 100 + 50);
// 부모 링크: agent-2 의 parentIdx 가 agent-1 이벤트를 가리켜 depth 2
const ev = parsed.events;
const a1 = ev.find((e) => e.name === "agent-1")!;
const a2 = ev.find((e) => e.name === "agent-2")!;
expect(ev[a2.parentIdx!]!.name).toBe("agent-1");
expect(ev[a1.parentIdx!]!.name).toBe("main");
});
});
describe("BR7 fail-soft (synthetic failure recorded, not dropped)", () => {
it("mock with failRate=1 produces success:false but still resolves", async () => {
const runner = new MockRunner(1.0);
const art = await runner.run({
combo: {
comboId: "cc+opus4.8+omc",
agent: "cc",
model: "opus4.8",
harness: "omc",
valid: true,
},
taskId: "subagent-101",
runIdx: 0,
workDir: tmp(),
timeoutS: 1,
seed: 3,
bench: "subagent",
});
expect(art.success).toBe(false);
expect(art.error).toBeTruthy();
});
});