Skip to content
TypeScript
6.8 KB
Raw Blame
/**
 * CO (정합) 검증 지표 — PRD §19.
 * CO1 total=필드합 · CO2 서브카운트≤툴콜 · CO3 정렬 기본키 · CO4 실측>추정>none ·
 * CO5 null≠0 · CO6 codex 실패 fail-soft · CO7 골든 정확성(독립 솔버 교차검증)
 */
import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { setupTask, verify } from "../src/benches/index";
import { type ComboMetrics, sortCombos } from "../src/metrics";
import { estimateCost, resolveCost } from "../src/pricing";
import { CodexCliRunner } from "../src/runners/codexCli";
import { MockRunner } from "../src/runners/mock";
import { parserFor } from "../src/transcripts/index";

function tmp(): string {
  return mkdtempSync(join(tmpdir(), "theseus-"));
}

async function mockResult(
  comboId: string,
  agent: string,
  model: string,
  harness: string,
  bench: string,
  taskId: string,
) {
  const runner = new MockRunner(0);
  const work = tmp();
  const art = await runner.run({
    combo: { comboId, agent, model, harness, valid: true },
    taskId,
    runIdx: 0,
    workDir: work,
    timeoutS: 5,
    seed: 1,
    bench,
  });
  const parsed = parserFor(agent).parse(art.transcriptPath, { model });
  return { art, parsed };
}

describe("CO1 totalTokens == sum of token fields", () => {
  it("holds for cc and codex parsed transcripts", async () => {
    for (const [agent, model, harness] of [
      ["cc", "opus4.8", "omc"],
      ["codex", "gpt5.5", "none"],
    ] as const) {
      const { parsed } = await mockResult(
        `${agent}+${model}+${harness}`,
        agent,
        model,
        harness,
        "subagent",
        "subagent-102",
      );
      expect(parsed.totalTokens).toBe(
        parsed.inputTokens +
          parsed.outputTokens +
          parsed.cacheReadTokens +
          parsed.cacheCreationTokens,
      );
    }
  });
});

describe("CO2 subagentCount <= toolCalls", () => {
  it("subagents are a subset of tool calls", async () => {
    const { parsed } = await mockResult(
      "cc+opus4.8+omc",
      "cc",
      "opus4.8",
      "omc",
      "subagent",
      "subagent-101",
    );
    expect(parsed.subagentCount).toBeLessThanOrEqual(parsed.toolCalls);
  });
  it("codex/gpt has zero subagents (single loop)", async () => {
    const { parsed } = await mockResult(
      "codex+gpt5.5+none",
      "codex",
      "gpt5.5",
      "none",
      "subagent",
      "subagent-101",
    );
    expect(parsed.subagentCount).toBe(0);
  });
});

describe("CO3 sort base key: subagentCount → totalTokens → wallTimeMs", () => {
  it("orders ascending across the three keys", () => {
    const base = (over: Partial<ComboMetrics>): ComboMetrics =>
      ({
        comboId: "x",
        runCount: 1,
        accuracy: 0,
        accuracyAtK: 0,
        subagentCount: 0,
        totalTokens: 0,
        wallTimeMs: 0,
        inputTokens: 0,
        outputTokens: 0,
        cacheReadTokens: 0,
        cacheCreationTokens: 0,
        costUsd: null,
        estimatedCostUsd: null,
        costSource: "none",
        toolCalls: 0,
        skillFires: 0,
        mcpCalls: 0,
        numTurns: 0,
        qualityScore: null,
        perTask: {},
        ...over,
      }) as ComboMetrics;
    const m = [
      base({ comboId: "a", subagentCount: 2, totalTokens: 100, wallTimeMs: 100 }),
      base({ comboId: "b", subagentCount: 0, totalTokens: 999, wallTimeMs: 50 }),
      base({ comboId: "c", subagentCount: 0, totalTokens: 100, wallTimeMs: 80 }),
      base({ comboId: "d", subagentCount: 0, totalTokens: 100, wallTimeMs: 10 }),
    ];
    expect(sortCombos(m).map((x) => x.comboId)).toEqual(["d", "c", "b", "a"]);
  });
});

describe("CO4 cost precedence: actual > estimated > none", () => {
  it("uses actual when present", () => {
    const r = resolveCost("opus4.8", 1000, 1000, 0, 0, 1.23);
    expect(r.costSource).toBe("actual");
    expect(r.costUsd).toBe(1.23);
  });
  it("falls back to estimate when no actual but pricing known", () => {
    const r = resolveCost("gpt5.5", 1_000_000, 0, 0, 0, null);
    expect(r.costSource).toBe("estimated");
    expect(r.costUsd).toBeCloseTo(1.25, 5);
  });
  it("none when pricing unknown and no actual", () => {
    const r = resolveCost("gpt5-codex", 1000, 1000, 0, 0, null); // resolved 'default' → no pricing
    expect(r.costSource).toBe("none");
    expect(r.costUsd).toBeNull();
  });
});

describe("CO5 null != 0 (uncollected stays null)", () => {
  it("estimateCost returns null (not 0) for unpriced models", () => {
    expect(estimateCost("gpt5-codex", 1000, 1000, 0, 0)).toBeNull();
    expect(estimateCost("fable5", 1000, 1000, 0, 0)).toBeNull();
  });
});

describe("CO6 codex failure is fail-soft", () => {
  it("missing codex binary → success:false, artifact still written", async () => {
    const runner = new CodexCliRunner();
    const work = tmp();
    const art = await runner.run({
      combo: {
        comboId: "codex+gpt5.5+none",
        agent: "codex",
        model: "gpt5.5",
        harness: "none",
        valid: true,
      },
      taskId: "local-hello",
      runIdx: 0,
      workDir: work,
      timeoutS: 2,
      seed: 0,
      bench: "local",
    });
    expect(art.success).toBe(false);
    expect(art.error).toBeTruthy();
    // transcript + output 항상 존재
    expect(() => readFileSync(art.transcriptPath, "utf8")).not.toThrow();
    expect(() => readFileSync(art.resultPath, "utf8")).not.toThrow();
  });
});

describe("CO7 golden correctness via independent solvers", () => {
  it("dijkstra (independent Bellman-Ford) matches verify", () => {
    const work = tmp();
    setupTask("subagent-101", work);
    const g = JSON.parse(readFileSync(join(work, "graph.json"), "utf8"));
    // 독립 Bellman-Ford
    const dist = new Array(g.n).fill(Number.POSITIVE_INFINITY);
    dist[0] = 0;
    for (let i = 0; i < g.n - 1; i++)
      for (const [u, v, w] of g.edges) if (dist[u] + w < dist[v]) dist[v] = dist[u] + w;
    const ans = dist.map((d: number) => (d === Number.POSITIVE_INFINITY ? -1 : d));
    writeFileSync(join(work, "distances.json"), JSON.stringify(ans));
    expect(verify("subagent-101", work)).toBe(true);
  });
  it("ledger rollup (independent sum) matches verify", () => {
    const work = tmp();
    setupTask("subagent-102", work);
    const bal: Record<string, number> = {};
    for (let s = 0; s < 24; s++) {
      const rows = JSON.parse(
        readFileSync(join(work, "shards", `shard-${String(s).padStart(2, "0")}.json`), "utf8"),
      );
      for (const r of rows) bal[r.account] = (bal[r.account] ?? 0) + r.delta;
    }
    const out: Record<string, number> = {};
    for (const [k, v] of Object.entries(bal)) if (v !== 0) out[k] = v;
    writeFileSync(join(work, "balances.json"), JSON.stringify(out));
    expect(verify("subagent-102", work)).toBe(true);
  });
});