TypeScript
/**
* CO (정합) 검증 지표 — PRD §19.
* CO1 total=필드합 · CO2 서브카운트≤툴콜 · CO3 정렬 기본키 · CO4 실측>추정>none ·
* CO5 null≠0 · CO6 codex 실패 fail-soft · CO7 골든 정확성(독립 솔버 교차검증)
*/
import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { setupTask, verify } from "../src/benches/index";
import { type ComboMetrics, sortCombos } from "../src/metrics";
import { estimateCost, resolveCost } from "../src/pricing";
import { CodexCliRunner } from "../src/runners/codexCli";
import { MockRunner } from "../src/runners/mock";
import { parserFor } from "../src/transcripts/index";
function tmp(): string {
return mkdtempSync(join(tmpdir(), "theseus-"));
}
async function mockResult(
comboId: string,
agent: string,
model: string,
harness: string,
bench: string,
taskId: string,
) {
const runner = new MockRunner(0);
const work = tmp();
const art = await runner.run({
combo: { comboId, agent, model, harness, valid: true },
taskId,
runIdx: 0,
workDir: work,
timeoutS: 5,
seed: 1,
bench,
});
const parsed = parserFor(agent).parse(art.transcriptPath, { model });
return { art, parsed };
}
describe("CO1 totalTokens == sum of token fields", () => {
it("holds for cc and codex parsed transcripts", async () => {
for (const [agent, model, harness] of [
["cc", "opus4.8", "omc"],
["codex", "gpt5.5", "none"],
] as const) {
const { parsed } = await mockResult(
`${agent}+${model}+${harness}`,
agent,
model,
harness,
"subagent",
"subagent-102",
);
expect(parsed.totalTokens).toBe(
parsed.inputTokens +
parsed.outputTokens +
parsed.cacheReadTokens +
parsed.cacheCreationTokens,
);
}
});
});
describe("CO2 subagentCount <= toolCalls", () => {
it("subagents are a subset of tool calls", async () => {
const { parsed } = await mockResult(
"cc+opus4.8+omc",
"cc",
"opus4.8",
"omc",
"subagent",
"subagent-101",
);
expect(parsed.subagentCount).toBeLessThanOrEqual(parsed.toolCalls);
});
it("codex/gpt has zero subagents (single loop)", async () => {
const { parsed } = await mockResult(
"codex+gpt5.5+none",
"codex",
"gpt5.5",
"none",
"subagent",
"subagent-101",
);
expect(parsed.subagentCount).toBe(0);
});
});
describe("CO3 sort base key: subagentCount → totalTokens → wallTimeMs", () => {
it("orders ascending across the three keys", () => {
const base = (over: Partial<ComboMetrics>): ComboMetrics =>
({
comboId: "x",
runCount: 1,
accuracy: 0,
accuracyAtK: 0,
subagentCount: 0,
totalTokens: 0,
wallTimeMs: 0,
inputTokens: 0,
outputTokens: 0,
cacheReadTokens: 0,
cacheCreationTokens: 0,
costUsd: null,
estimatedCostUsd: null,
costSource: "none",
toolCalls: 0,
skillFires: 0,
mcpCalls: 0,
numTurns: 0,
qualityScore: null,
perTask: {},
...over,
}) as ComboMetrics;
const m = [
base({ comboId: "a", subagentCount: 2, totalTokens: 100, wallTimeMs: 100 }),
base({ comboId: "b", subagentCount: 0, totalTokens: 999, wallTimeMs: 50 }),
base({ comboId: "c", subagentCount: 0, totalTokens: 100, wallTimeMs: 80 }),
base({ comboId: "d", subagentCount: 0, totalTokens: 100, wallTimeMs: 10 }),
];
expect(sortCombos(m).map((x) => x.comboId)).toEqual(["d", "c", "b", "a"]);
});
});
describe("CO4 cost precedence: actual > estimated > none", () => {
it("uses actual when present", () => {
const r = resolveCost("opus4.8", 1000, 1000, 0, 0, 1.23);
expect(r.costSource).toBe("actual");
expect(r.costUsd).toBe(1.23);
});
it("falls back to estimate when no actual but pricing known", () => {
const r = resolveCost("gpt5.5", 1_000_000, 0, 0, 0, null);
expect(r.costSource).toBe("estimated");
expect(r.costUsd).toBeCloseTo(1.25, 5);
});
it("none when pricing unknown and no actual", () => {
const r = resolveCost("gpt5-codex", 1000, 1000, 0, 0, null); // resolved 'default' → no pricing
expect(r.costSource).toBe("none");
expect(r.costUsd).toBeNull();
});
});
describe("CO5 null != 0 (uncollected stays null)", () => {
it("estimateCost returns null (not 0) for unpriced models", () => {
expect(estimateCost("gpt5-codex", 1000, 1000, 0, 0)).toBeNull();
expect(estimateCost("fable5", 1000, 1000, 0, 0)).toBeNull();
});
});
describe("CO6 codex failure is fail-soft", () => {
it("missing codex binary → success:false, artifact still written", async () => {
const runner = new CodexCliRunner();
const work = tmp();
const art = await runner.run({
combo: {
comboId: "codex+gpt5.5+none",
agent: "codex",
model: "gpt5.5",
harness: "none",
valid: true,
},
taskId: "local-hello",
runIdx: 0,
workDir: work,
timeoutS: 2,
seed: 0,
bench: "local",
});
expect(art.success).toBe(false);
expect(art.error).toBeTruthy();
// transcript + output 항상 존재
expect(() => readFileSync(art.transcriptPath, "utf8")).not.toThrow();
expect(() => readFileSync(art.resultPath, "utf8")).not.toThrow();
});
});
describe("CO7 golden correctness via independent solvers", () => {
it("dijkstra (independent Bellman-Ford) matches verify", () => {
const work = tmp();
setupTask("subagent-101", work);
const g = JSON.parse(readFileSync(join(work, "graph.json"), "utf8"));
// 독립 Bellman-Ford
const dist = new Array(g.n).fill(Number.POSITIVE_INFINITY);
dist[0] = 0;
for (let i = 0; i < g.n - 1; i++)
for (const [u, v, w] of g.edges) if (dist[u] + w < dist[v]) dist[v] = dist[u] + w;
const ans = dist.map((d: number) => (d === Number.POSITIVE_INFINITY ? -1 : d));
writeFileSync(join(work, "distances.json"), JSON.stringify(ans));
expect(verify("subagent-101", work)).toBe(true);
});
it("ledger rollup (independent sum) matches verify", () => {
const work = tmp();
setupTask("subagent-102", work);
const bal: Record<string, number> = {};
for (let s = 0; s < 24; s++) {
const rows = JSON.parse(
readFileSync(join(work, "shards", `shard-${String(s).padStart(2, "0")}.json`), "utf8"),
);
for (const r of rows) bal[r.account] = (bal[r.account] ?? 0) + r.delta;
}
const out: Record<string, number> = {};
for (const [k, v] of Object.entries(bal)) if (v !== 0) out[k] = v;
writeFileSync(join(work, "balances.json"), JSON.stringify(out));
expect(verify("subagent-102", work)).toBe(true);
});
});