Skip to content
TypeScript

verify: sample multiple attempts per combo (max) to avoid false gaps

Claude8cdc04a1e9c6
diff --git a/src/verify.ts b/src/verify.ts
index 81f44db..b139ebe 100644
--- a/src/verify.ts
+++ b/src/verify.ts
@@ -35,6 +35,7 @@ export interface VerifyOptions {
seed: number;
timeoutS: number;
task?: string;
+ attempts?: number; // 저(低)-fan-out 하네스(예: ecc 0~1)의 표본 노이즈 완화 — 최댓값 채택
}
/** 트리형 조합인가? (mock fan-out 규칙과 동일: cc+오케스트레이션, opencode+omo) */
@@ -47,34 +48,41 @@ export function expectsFanout(agent: string, harness: string): boolean {
export async function verifyCombos(opts: VerifyOptions): Promise<VerifyRow[]> {
const { valid } = expand(opts.models, opts.agents, opts.harnesses);
const task = opts.task ?? "subagent-102";
+ const attempts = Math.max(1, opts.attempts ?? 3);
const rows: VerifyRow[] = [];
for (const combo of valid as Combo[]) {
const { agent, model, harness } = parseComboId(combo.comboId);
const choice = runnerFor(agent, opts.runner, { bench: "subagent" }, 0);
- const workDir = mkdtempSync(join(tmpdir(), "theseus-verify-"));
let subagents = 0;
let maxDepth = 0;
- try {
- const art = await choice.runner.run({
- combo,
- taskId: task,
- runIdx: 0,
- workDir,
- timeoutS: opts.timeoutS,
- seed: opts.seed,
- bench: "subagent",
- });
- const parsed = parserFor(agent).parse(art.transcriptPath, { model });
- subagents = parsed.subagentCount;
- const flow = buildFlow(parsed, combo.comboId, task, 0, {
- harness,
- success: art.success,
- error: art.error,
- });
- maxDepth = flow.nodes.reduce((m, n) => Math.max(m, n.depth), 0);
- } catch {
- // fail-soft: 0/0 로 남김
+ // 여러 시도 중 최댓값 채택: fan-out 이 어느 한 번이라도 관측되면 "captured".
+ for (let attempt = 0; attempt < attempts; attempt++) {
+ const workDir = mkdtempSync(join(tmpdir(), "theseus-verify-"));
+ try {
+ const art = await choice.runner.run({
+ combo,
+ taskId: task,
+ runIdx: attempt,
+ workDir,
+ timeoutS: opts.timeoutS,
+ seed: opts.seed,
+ bench: "subagent",
+ });
+ const parsed = parserFor(agent).parse(art.transcriptPath, { model });
+ subagents = Math.max(subagents, parsed.subagentCount);
+ const flow = buildFlow(parsed, combo.comboId, task, attempt, {
+ harness,
+ success: art.success,
+ error: art.error,
+ });
+ maxDepth = Math.max(
+ maxDepth,
+ flow.nodes.reduce((m, n) => Math.max(m, n.depth), 0),
+ );
+ } catch {
+ // fail-soft: 누적 최댓값 유지
+ }
}
const expectFanout = expectsFanout(agent, harness);
const status: VerifyStatus = subagents > 0 ? "captured" : expectFanout ? "gap" : "flat";