Skip to content
TypeScript
8.0 KB
Raw Blame
/**
 * BR (동작) 검증 지표 — PRD §19.
 * BR1 조합 유효성/제외 사유 · BR2 subset 결정성 · BR3 러너 never-throw ·
 * BR4 하네스 per-run · BR5 벤치 setup+recompute-verify · BR6 멀티레벨 파싱 · BR7 fail-soft
 */
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { findTask, setupTask, verify } from "../src/benches/index";
import { expand } from "../src/combos";
import { harnessClaudeArgs, harnessCodexEnv, harnessStatus } from "../src/harness";
import { MockRunner } from "../src/runners/mock";
import { selectBalanced, selectSubset } from "../src/subset";
import { ClaudeCodeParser } from "../src/transcripts/claudeCode";

function tmp(): string {
  return mkdtempSync(join(tmpdir(), "theseus-"));
}

describe("BR1 combo validity & exclude reasons", () => {
  it("excludes incompatible agent×model and harness with reasons (no silent drop)", () => {
    const { valid, excluded } = expand(["opus4.8", "gpt5.5"], ["cc", "codex"], ["none", "omc"]);
    expect(valid.map((c) => c.comboId)).toContain("cc+opus4.8+none");
    expect(valid.map((c) => c.comboId)).toContain("codex+gpt5.5+none");
    // cc cannot use gpt model
    const cross = excluded.find((c) => c.comboId === "cc+gpt5.5+none");
    expect(cross?.excludeReason).toMatch(/incompatible/);
    // omc is cc-only → codex+gpt5.5+omc excluded
    const harnessBad = excluded.find((c) => c.comboId === "codex+gpt5.5+omc");
    expect(harnessBad?.excludeReason).toMatch(/not compatible/);
    // every excluded has a reason
    expect(excluded.every((c) => !!c.excludeReason)).toBe(true);
  });
});

describe("BR2 subset determinism", () => {
  const catalog = Array.from({ length: 50 }, (_, i) => `task-${i}`);
  it("same (catalog,k,seed) → identical subset; no Math.random", () => {
    const a = selectSubset(catalog, 10, 42);
    const b = selectSubset(catalog, 10, 42);
    expect(a).toEqual(b);
    expect(a).toHaveLength(10);
  });
  it("different seed → (generally) different ordering", () => {
    const a = selectSubset(catalog, 10, 1);
    const b = selectSubset(catalog, 10, 2);
    expect(a).not.toEqual(b);
  });
  it("balanced subset is deterministic and spans buckets", () => {
    const levels = ["easy", "medium", "hard"] as const;
    const dm: Record<string, "easy" | "medium" | "hard"> = {};
    catalog.forEach((t, i) => {
      dm[t] = levels[i % 3]!;
    });
    const a = selectBalanced(catalog, 9, 7, dm);
    const b = selectBalanced(catalog, 9, 7, dm);
    expect(a).toEqual(b);
  });
});

describe("BR3 runner never-throw", () => {
  it("mock.run resolves with success flag even when fail injected", async () => {
    const runner = new MockRunner(1.0); // always fail
    const work = tmp();
    const art = await runner.run({
      combo: {
        comboId: "cc+opus4.8+none",
        agent: "cc",
        model: "opus4.8",
        harness: "none",
        valid: true,
      },
      taskId: "local-hello",
      runIdx: 0,
      workDir: work,
      timeoutS: 1,
      seed: 0,
      bench: "local",
    });
    expect(art.success).toBe(false);
    expect(art.resultPath).toBeTruthy();
    expect(art.transcriptPath).toBeTruthy();
  });
});

describe("BR4 harness per-run application", () => {
  it("none → no args/env; status reflects availability", () => {
    expect(harnessClaudeArgs("none")).toEqual([]);
    expect(harnessCodexEnv("none")).toEqual({});
    expect(harnessStatus("none")).toBe("none");
    // unavailable when not installed (offline)
    expect(["applied", "unavailable"]).toContain(harnessStatus("omc"));
  });
});

describe("BR5 bench setup + recompute-verify", () => {
  it("setup creates fixtures; verify recomputes (no hardcoded golden)", () => {
    const work = tmp();
    setupTask("subagent-104", work);
    // 정답을 직접(독립) 풀어서 기록 → verify 통과해야 함
    const k = JSON.parse(readFileSync(join(work, "knapsack.json"), "utf8"));
    // 독립 2D DP
    const n = k.items.length;
    const dp: number[][] = Array.from({ length: n + 1 }, () =>
      new Array(k.capacity + 1).fill(0),
    ) as number[][];
    const at = (i: number, c: number): number => dp[i]![c]!;
    const set = (i: number, c: number, val: number) => {
      dp[i]![c] = val;
    };
    for (let i = 1; i <= n; i++) {
      const [w, v] = k.items[i - 1];
      for (let c = 0; c <= k.capacity; c++)
        set(i, c, c >= w ? Math.max(at(i - 1, c), at(i - 1, c - w) + v) : at(i - 1, c));
    }
    writeFileSync(join(work, "answer.txt"), String(at(n, k.capacity)));
    expect(verify("subagent-104", work)).toBe(true);
    // 틀린 답 → false
    writeFileSync(join(work, "answer.txt"), String(at(n, k.capacity) + 1));
    expect(verify("subagent-104", work)).toBe(false);
  });
  it("verify never throws on missing files", () => {
    expect(verify("subagent-101", tmp())).toBe(false);
    expect(verify("nonexistent", tmp())).toBe(false);
  });
});

describe("BR6 multi-level subagent parsing (layout B)", () => {
  it("links nested subagents via meta.toolUseId (depth 2)", () => {
    const work = tmp();
    const transcriptPath = join(work, "transcript.jsonl");
    // main: spawns Task TU1
    writeFileSync(
      transcriptPath,
      `${JSON.stringify({
        type: "assistant",
        uuid: "root1",
        parentUuid: null,
        isSidechain: false,
        message: {
          role: "assistant",
          model: "claude-opus-4-8",
          usage: { input_tokens: 1000, output_tokens: 200 },
          content: [{ type: "tool_use", id: "TU1", name: "Task", input: {} }],
        },
      })}\n`,
    );
    const subDir = join(work, "transcript_subagents");
    mkdirSync(subDir, { recursive: true });
    // agent-1 spawns Task TU2
    writeFileSync(
      join(subDir, "agent-1.jsonl"),
      `${JSON.stringify({
        type: "assistant",
        uuid: "a1",
        message: {
          role: "assistant",
          usage: { input_tokens: 500, output_tokens: 100 },
          content: [
            { type: "tool_use", id: "TU2", name: "Task", input: {} },
            { type: "tool_use", id: "r1", name: "Read", input: {} },
          ],
        },
      })}\n`,
    );
    writeFileSync(
      join(subDir, "agent-1.meta.json"),
      JSON.stringify({ toolUseId: "TU1", agentId: "agent-1" }),
    );
    // agent-2 child of agent-1
    writeFileSync(
      join(subDir, "agent-2.jsonl"),
      `${JSON.stringify({
        type: "assistant",
        uuid: "a2",
        message: {
          role: "assistant",
          usage: { input_tokens: 300, output_tokens: 50 },
          content: [],
        },
      })}\n`,
    );
    writeFileSync(
      join(subDir, "agent-2.meta.json"),
      JSON.stringify({ toolUseId: "TU2", agentId: "agent-2" }),
    );

    const parsed = new ClaudeCodeParser().parse(transcriptPath);
    expect(parsed.subagentCount).toBe(2); // 파일 2개 (인라인 중복 없음)
    // 토큰 총합 = root + 두 서브에이전트
    expect(parsed.inputTokens).toBe(1000 + 500 + 300);
    expect(parsed.outputTokens).toBe(200 + 100 + 50);
    // 부모 링크: agent-2 의 parentIdx 가 agent-1 이벤트를 가리켜 depth 2
    const ev = parsed.events;
    const a1 = ev.find((e) => e.name === "agent-1")!;
    const a2 = ev.find((e) => e.name === "agent-2")!;
    expect(ev[a2.parentIdx!]!.name).toBe("agent-1");
    expect(ev[a1.parentIdx!]!.name).toBe("main");
  });
});

describe("BR7 fail-soft (synthetic failure recorded, not dropped)", () => {
  it("mock with failRate=1 produces success:false but still resolves", async () => {
    const runner = new MockRunner(1.0);
    const art = await runner.run({
      combo: {
        comboId: "cc+opus4.8+omc",
        agent: "cc",
        model: "opus4.8",
        harness: "omc",
        valid: true,
      },
      taskId: "subagent-101",
      runIdx: 0,
      workDir: tmp(),
      timeoutS: 1,
      seed: 3,
      bench: "subagent",
    });
    expect(art.success).toBe(false);
    expect(art.error).toBeTruthy();
  });
});