TypeScript
/**
* 감사(audit)에서 확인된 결함들의 회귀 테스트.
* 파서 이중집계 · 미지 벤치 에러 · preflight 클램프/경고 · extractScore ·
* makeRunId 충돌 · costSource 일관성 · codex 룰 하네스 materialize.
*/
import { existsSync, mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { taskIds } from "../src/benches/index";
import { expand } from "../src/combos";
import { applyHarnessToWorkdir, ensureHarnessRulesStub } from "../src/harness";
import { extractScore } from "../src/judge";
import { aggregateCombo } from "../src/metrics";
import type { RunResult } from "../src/models";
import { execute } from "../src/orchestrator";
import { analyze } from "../src/preflight";
import { makeRunId } from "../src/recorder";
import { DefensiveParser } from "../src/transcripts/defensive";
function tmp(): string {
return mkdtempSync(join(tmpdir(), "theseus-audit-"));
}
describe("DefensiveParser — no double counting of wrapper+item events", () => {
it("counts one tool call per logical event; nested subagent marker still registers", () => {
const f = join(tmp(), "t.jsonl");
writeFileSync(
f,
[
JSON.stringify({ type: "item.completed", item: { type: "command_execution" } }),
JSON.stringify({ type: "item.completed", item: { type: "tool_use", name: "read_file" } }),
JSON.stringify({ type: "item.completed", item: { type: "subagent", name: "task" } }),
JSON.stringify({ type: "message", usage: { input_tokens: 100, output_tokens: 50 } }),
].join("\n"),
);
const p = new DefensiveParser("opencode").parse(f);
expect(p.toolCalls).toBe(3); // 이전 버그: 6 (래퍼+item 각각 집계)
expect(p.subagentCount).toBe(1);
expect(p.totalTokens).toBe(150);
});
});
describe("orchestrator — unknown bench raises one clear error", () => {
it("names the bench and lists the available ones", async () => {
await expect(
execute({
models: ["opus4.8"],
agents: ["cc"],
harnesses: ["none"],
bench: "definitely-not-a-bench",
tasks: 2,
runs: 1,
seed: 0,
runner: "mock",
concurrency: 1,
timeoutS: 5,
outDir: tmp(),
judge: "none",
score: false,
balanced: false,
language: "en",
}),
).rejects.toThrow(/unknown bench 'definitely-not-a-bench'.*subagent/);
});
});
describe("preflight — scale clamps to catalog, 0 tasks warns", () => {
const combos = () => expand(["opus4.8"], ["cc"], ["none"]);
it("clamps requested tasks to the local-bench catalog size", () => {
const { valid, excluded } = combos();
const pf = analyze(["opus4.8"], ["cc"], ["none"], valid, excluded, {
bench: "subagent",
runner: "mock",
tasks: 9999,
runs: 1,
});
expect(pf.scale).toBe(valid.length * taskIds("subagent").length);
});
it("warns when 0 tasks are selected", () => {
const { valid, excluded } = combos();
const pf = analyze(["opus4.8"], ["cc"], ["none"], valid, excluded, {
bench: "subagent",
runner: "mock",
tasks: 0,
runs: 3,
});
expect(pf.scale).toBe(0);
expect(pf.warnings.join(" ")).toMatch(/0 tasks/);
});
});
describe("judge.extractScore", () => {
it("extracts integers, decimals, and 10; clamps and rejects garbage", () => {
expect(extractScore("7")).toBe(7);
expect(extractScore("Score: 8.5 / 10")).toBe(8.5);
expect(extractScore("10")).toBe(10);
expect(extractScore("no digits here")).toBeNull();
expect(extractScore("")).toBeNull();
});
});
describe("recorder.makeRunId — same-millisecond collision", () => {
it("suffixes duplicate ids instead of overwriting", () => {
const d = new Date(1750000000000);
const a = makeRunId(d);
const b = makeRunId(d);
const c = makeRunId(d);
expect(a).not.toBe(b);
expect(b).not.toBe(c);
expect(b).toBe(`${a}-2`);
expect(c).toBe(`${a}-3`);
});
});
describe("metrics.aggregateCost — costSource derived from reported values", () => {
const run = (over: Partial<RunResult>): RunResult =>
({
comboId: "cc+opus4.8+none",
taskId: "t",
runIdx: 0,
success: true,
resolved: true,
subagentCount: 0,
totalTokens: 10,
inputTokens: 5,
outputTokens: 5,
cacheReadTokens: 0,
cacheCreationTokens: 0,
costUsd: null,
estimatedCostUsd: null,
costSource: "none",
wallTimeMs: 10,
toolCalls: 0,
skillFires: 0,
mcpCalls: 0,
numTurns: 1,
qualityScore: null,
error: null,
transcriptPath: "",
resultPath: "",
...over,
}) as RunResult;
it("a run *claiming* actual with a null cost no longer yields costSource=actual", () => {
const m = aggregateCombo("cc+opus4.8+none", [
run({ costSource: "actual", costUsd: null }),
run({ costSource: "none" }),
]);
expect(m.costUsd).toBeNull();
expect(m.costSource).toBe("none");
});
it("actual values present → actual; only estimates → estimated", () => {
expect(aggregateCombo("c", [run({ costUsd: 0.5, costSource: "actual" })]).costSource).toBe(
"actual",
);
expect(
aggregateCombo("c", [run({ estimatedCostUsd: 0.2, costSource: "estimated" })]).costSource,
).toBe("estimated");
});
});
describe("codex rules harnesses — materialized into the workdir (no silent baseline)", () => {
it("superpowers rules cache applies to a workdir via AGENTS.md", () => {
ensureHarnessRulesStub("superpowers", "# superpowers rules (test stub)\n");
const work = tmp();
expect(applyHarnessToWorkdir("superpowers", work)).toBe(true);
expect(existsSync(join(work, "AGENTS.md"))).toBe(true);
});
});