TypeScript
/**
* 조합별 서브에이전트 콜 검증 (PRD v2 §W5) — "각 조합 subagent 콜 잘 되는지 확인".
* 각 유효 조합을 위임 유도 과제(subagent-102, depth-2)로 1회 실행 → 흐름 파싱 →
* subagentCount/maxDepth 를 집계하고 기대 모델과 대조한다.
* 기대: 트리형 조합(cc+오케스트레이션, opencode+omo)은 fan-out 필요 / 단일 루프(codex·none·hermes)는 flat 정상.
* mock(오프라인·결정적, 기본) 또는 real(--yes 게이트). 절대 throw 금지.
*/
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expand, parseComboId } from "./combos";
import { HARNESSES } from "./config";
import { buildFlow } from "./flow";
import type { Combo } from "./models";
import { runnerFor } from "./runners/registry";
import { parserFor } from "./transcripts/index";
export type VerifyStatus = "captured" | "flat" | "gap";
export interface VerifyRow {
comboId: string;
agent: string;
harness: string;
subagents: number;
maxDepth: number;
expectFanout: boolean;
status: VerifyStatus; // captured=fan-out 관측, flat=단일루프(정상), gap=기대했으나 0(포착 누락 의심)
}
export interface VerifyOptions {
models: string[];
agents: string[];
harnesses: string[];
runner: string;
seed: number;
timeoutS: number;
task?: string;
attempts?: number; // 저(低)-fan-out 하네스(예: ecc 0~1)의 표본 노이즈 완화 — 최댓값 채택
}
/** 트리형 조합인가? (mock fan-out 규칙과 동일: cc+오케스트레이션, opencode+omo) */
export function expectsFanout(agent: string, harness: string): boolean {
if (agent === "cc") return HARNESSES[harness]?.orchestration === true;
if (agent === "opencode" && harness === "omo") return true;
return false;
}
export async function verifyCombos(opts: VerifyOptions): Promise<VerifyRow[]> {
const { valid } = expand(opts.models, opts.agents, opts.harnesses);
const task = opts.task ?? "subagent-102";
const attempts = Math.max(1, opts.attempts ?? 3);
const rows: VerifyRow[] = [];
for (const combo of valid as Combo[]) {
const { agent, model, harness } = parseComboId(combo.comboId);
const choice = runnerFor(agent, opts.runner, { bench: "subagent" }, 0);
let subagents = 0;
let maxDepth = 0;
// 여러 시도 중 최댓값 채택: fan-out 이 어느 한 번이라도 관측되면 "captured".
for (let attempt = 0; attempt < attempts; attempt++) {
const workDir = mkdtempSync(join(tmpdir(), "theseus-verify-"));
try {
const art = await choice.runner.run({
combo,
taskId: task,
runIdx: attempt,
workDir,
timeoutS: opts.timeoutS,
seed: opts.seed,
bench: "subagent",
});
const parsed = parserFor(agent).parse(art.transcriptPath, { model });
subagents = Math.max(subagents, parsed.subagentCount);
const flow = buildFlow(parsed, combo.comboId, task, attempt, {
harness,
success: art.success,
error: art.error,
});
maxDepth = Math.max(
maxDepth,
flow.nodes.reduce((m, n) => Math.max(m, n.depth), 0),
);
} catch {
// fail-soft: 누적 최댓값 유지
} finally {
// 시도당 임시 폴더 정리(attempts × combos 만큼 누적되는 것 방지)
rmSync(workDir, { recursive: true, force: true });
}
}
const expectFanout = expectsFanout(agent, harness);
const status: VerifyStatus = subagents > 0 ? "captured" : expectFanout ? "gap" : "flat";
rows.push({
comboId: combo.comboId,
agent,
harness,
subagents,
maxDepth,
expectFanout,
status,
});
}
return rows;
}