Skip to content
TypeScript
3.7 KB
Raw Blame
/**
 * 조합별 서브에이전트 콜 검증 (PRD v2 §W5) — "각 조합 subagent 콜 잘 되는지 확인".
 * 각 유효 조합을 위임 유도 과제(subagent-102, depth-2)로 1회 실행 → 흐름 파싱 →
 * subagentCount/maxDepth 를 집계하고 기대 모델과 대조한다.
 * 기대: 트리형 조합(cc+오케스트레이션, opencode+omo)은 fan-out 필요 / 단일 루프(codex·none·hermes)는 flat 정상.
 * mock(오프라인·결정적, 기본) 또는 real(--yes 게이트). 절대 throw 금지.
 */
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expand, parseComboId } from "./combos";
import { HARNESSES } from "./config";
import { buildFlow } from "./flow";
import type { Combo } from "./models";
import { runnerFor } from "./runners/registry";
import { parserFor } from "./transcripts/index";

export type VerifyStatus = "captured" | "flat" | "gap";

export interface VerifyRow {
  comboId: string;
  agent: string;
  harness: string;
  subagents: number;
  maxDepth: number;
  expectFanout: boolean;
  status: VerifyStatus; // captured=fan-out 관측, flat=단일루프(정상), gap=기대했으나 0(포착 누락 의심)
}

export interface VerifyOptions {
  models: string[];
  agents: string[];
  harnesses: string[];
  runner: string;
  seed: number;
  timeoutS: number;
  task?: string;
  attempts?: number; // 저(低)-fan-out 하네스(예: ecc 0~1)의 표본 노이즈 완화 — 최댓값 채택
}

/** 트리형 조합인가? (mock fan-out 규칙과 동일: cc+오케스트레이션, opencode+omo) */
export function expectsFanout(agent: string, harness: string): boolean {
  if (agent === "cc") return HARNESSES[harness]?.orchestration === true;
  if (agent === "opencode" && harness === "omo") return true;
  return false;
}

export async function verifyCombos(opts: VerifyOptions): Promise<VerifyRow[]> {
  const { valid } = expand(opts.models, opts.agents, opts.harnesses);
  const task = opts.task ?? "subagent-102";
  const attempts = Math.max(1, opts.attempts ?? 3);
  const rows: VerifyRow[] = [];

  for (const combo of valid as Combo[]) {
    const { agent, model, harness } = parseComboId(combo.comboId);
    const choice = runnerFor(agent, opts.runner, { bench: "subagent" }, 0);
    let subagents = 0;
    let maxDepth = 0;
    // 여러 시도 중 최댓값 채택: fan-out 이 어느 한 번이라도 관측되면 "captured".
    for (let attempt = 0; attempt < attempts; attempt++) {
      const workDir = mkdtempSync(join(tmpdir(), "theseus-verify-"));
      try {
        const art = await choice.runner.run({
          combo,
          taskId: task,
          runIdx: attempt,
          workDir,
          timeoutS: opts.timeoutS,
          seed: opts.seed,
          bench: "subagent",
        });
        const parsed = parserFor(agent).parse(art.transcriptPath, { model });
        subagents = Math.max(subagents, parsed.subagentCount);
        const flow = buildFlow(parsed, combo.comboId, task, attempt, {
          harness,
          success: art.success,
          error: art.error,
        });
        maxDepth = Math.max(
          maxDepth,
          flow.nodes.reduce((m, n) => Math.max(m, n.depth), 0),
        );
      } catch {
        // fail-soft: 누적 최댓값 유지
      } finally {
        // 시도당 임시 폴더 정리(attempts × combos 만큼 누적되는 것 방지)
        rmSync(workDir, { recursive: true, force: true });
      }
    }
    const expectFanout = expectsFanout(agent, harness);
    const status: VerifyStatus = subagents > 0 ? "captured" : expectFanout ? "gap" : "flat";
    rows.push({
      comboId: combo.comboId,
      agent,
      harness,
      subagents,
      maxDepth,
      expectFanout,
      status,
    });
  }
  return rows;
}