TypeScript
verify: sample multiple attempts per combo (max) to avoid false gaps
Claude8cdc04a1e9c6
diff --git a/src/verify.ts b/src/verify.tsindex 81f44db..b139ebe 100644--- a/src/verify.ts+++ b/src/verify.ts@@ -35,6 +35,7 @@ export interface VerifyOptions {seed: number;timeoutS: number;task?: string;+ attempts?: number; // 저(低)-fan-out 하네스(예: ecc 0~1)의 표본 노이즈 완화 — 최댓값 채택}/** 트리형 조합인가? (mock fan-out 규칙과 동일: cc+오케스트레이션, opencode+omo) */@@ -47,34 +48,41 @@ export function expectsFanout(agent: string, harness: string): boolean {export async function verifyCombos(opts: VerifyOptions): Promise<VerifyRow[]> {const { valid } = expand(opts.models, opts.agents, opts.harnesses);const task = opts.task ?? "subagent-102";+ const attempts = Math.max(1, opts.attempts ?? 3);const rows: VerifyRow[] = [];for (const combo of valid as Combo[]) {const { agent, model, harness } = parseComboId(combo.comboId);const choice = runnerFor(agent, opts.runner, { bench: "subagent" }, 0);- const workDir = mkdtempSync(join(tmpdir(), "theseus-verify-"));let subagents = 0;let maxDepth = 0;- try {- const art = await choice.runner.run({- combo,- taskId: task,- runIdx: 0,- workDir,- timeoutS: opts.timeoutS,- seed: opts.seed,- bench: "subagent",- });- const parsed = parserFor(agent).parse(art.transcriptPath, { model });- subagents = parsed.subagentCount;- const flow = buildFlow(parsed, combo.comboId, task, 0, {- harness,- success: art.success,- error: art.error,- });- maxDepth = flow.nodes.reduce((m, n) => Math.max(m, n.depth), 0);- } catch {- // fail-soft: 0/0 로 남김+ // 여러 시도 중 최댓값 채택: fan-out 이 어느 한 번이라도 관측되면 "captured".+ for (let attempt = 0; attempt < attempts; attempt++) {+ const workDir = mkdtempSync(join(tmpdir(), "theseus-verify-"));+ try {+ const art = await choice.runner.run({+ combo,+ taskId: task,+ runIdx: attempt,+ workDir,+ timeoutS: opts.timeoutS,+ seed: opts.seed,+ bench: "subagent",+ });+ const parsed = parserFor(agent).parse(art.transcriptPath, { model });+ subagents = Math.max(subagents, parsed.subagentCount);+ const flow = buildFlow(parsed, combo.comboId, task, attempt, {+ harness,+ success: art.success,+ error: art.error,+ });+ maxDepth = Math.max(+ maxDepth,+ flow.nodes.reduce((m, n) => Math.max(m, n.depth), 0),+ );+ } catch {+ // fail-soft: 누적 최댓값 유지+ }}const expectFanout = expectsFanout(agent, harness);const status: VerifyStatus = subagents > 0 ? "captured" : expectFanout ? "gap" : "flat";