Skip to content
TypeScript
5.8 KB
Raw Blame
/**
 * 감사(audit)에서 확인된 결함들의 회귀 테스트.
 * 파서 이중집계 · 미지 벤치 에러 · preflight 클램프/경고 · extractScore ·
 * makeRunId 충돌 · costSource 일관성 · codex 룰 하네스 materialize.
 */
import { existsSync, mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { taskIds } from "../src/benches/index";
import { expand } from "../src/combos";
import { applyHarnessToWorkdir, ensureHarnessRulesStub } from "../src/harness";
import { extractScore } from "../src/judge";
import { aggregateCombo } from "../src/metrics";
import type { RunResult } from "../src/models";
import { execute } from "../src/orchestrator";
import { analyze } from "../src/preflight";
import { makeRunId } from "../src/recorder";
import { DefensiveParser } from "../src/transcripts/defensive";

function tmp(): string {
  return mkdtempSync(join(tmpdir(), "theseus-audit-"));
}

describe("DefensiveParser — no double counting of wrapper+item events", () => {
  it("counts one tool call per logical event; nested subagent marker still registers", () => {
    const f = join(tmp(), "t.jsonl");
    writeFileSync(
      f,
      [
        JSON.stringify({ type: "item.completed", item: { type: "command_execution" } }),
        JSON.stringify({ type: "item.completed", item: { type: "tool_use", name: "read_file" } }),
        JSON.stringify({ type: "item.completed", item: { type: "subagent", name: "task" } }),
        JSON.stringify({ type: "message", usage: { input_tokens: 100, output_tokens: 50 } }),
      ].join("\n"),
    );
    const p = new DefensiveParser("opencode").parse(f);
    expect(p.toolCalls).toBe(3); // 이전 버그: 6 (래퍼+item 각각 집계)
    expect(p.subagentCount).toBe(1);
    expect(p.totalTokens).toBe(150);
  });
});

describe("orchestrator — unknown bench raises one clear error", () => {
  it("names the bench and lists the available ones", async () => {
    await expect(
      execute({
        models: ["opus4.8"],
        agents: ["cc"],
        harnesses: ["none"],
        bench: "definitely-not-a-bench",
        tasks: 2,
        runs: 1,
        seed: 0,
        runner: "mock",
        concurrency: 1,
        timeoutS: 5,
        outDir: tmp(),
        judge: "none",
        score: false,
        balanced: false,
        language: "en",
      }),
    ).rejects.toThrow(/unknown bench 'definitely-not-a-bench'.*subagent/);
  });
});

describe("preflight — scale clamps to catalog, 0 tasks warns", () => {
  const combos = () => expand(["opus4.8"], ["cc"], ["none"]);

  it("clamps requested tasks to the local-bench catalog size", () => {
    const { valid, excluded } = combos();
    const pf = analyze(["opus4.8"], ["cc"], ["none"], valid, excluded, {
      bench: "subagent",
      runner: "mock",
      tasks: 9999,
      runs: 1,
    });
    expect(pf.scale).toBe(valid.length * taskIds("subagent").length);
  });

  it("warns when 0 tasks are selected", () => {
    const { valid, excluded } = combos();
    const pf = analyze(["opus4.8"], ["cc"], ["none"], valid, excluded, {
      bench: "subagent",
      runner: "mock",
      tasks: 0,
      runs: 3,
    });
    expect(pf.scale).toBe(0);
    expect(pf.warnings.join(" ")).toMatch(/0 tasks/);
  });
});

describe("judge.extractScore", () => {
  it("extracts integers, decimals, and 10; clamps and rejects garbage", () => {
    expect(extractScore("7")).toBe(7);
    expect(extractScore("Score: 8.5 / 10")).toBe(8.5);
    expect(extractScore("10")).toBe(10);
    expect(extractScore("no digits here")).toBeNull();
    expect(extractScore("")).toBeNull();
  });
});

describe("recorder.makeRunId — same-millisecond collision", () => {
  it("suffixes duplicate ids instead of overwriting", () => {
    const d = new Date(1750000000000);
    const a = makeRunId(d);
    const b = makeRunId(d);
    const c = makeRunId(d);
    expect(a).not.toBe(b);
    expect(b).not.toBe(c);
    expect(b).toBe(`${a}-2`);
    expect(c).toBe(`${a}-3`);
  });
});

describe("metrics.aggregateCost — costSource derived from reported values", () => {
  const run = (over: Partial<RunResult>): RunResult =>
    ({
      comboId: "cc+opus4.8+none",
      taskId: "t",
      runIdx: 0,
      success: true,
      resolved: true,
      subagentCount: 0,
      totalTokens: 10,
      inputTokens: 5,
      outputTokens: 5,
      cacheReadTokens: 0,
      cacheCreationTokens: 0,
      costUsd: null,
      estimatedCostUsd: null,
      costSource: "none",
      wallTimeMs: 10,
      toolCalls: 0,
      skillFires: 0,
      mcpCalls: 0,
      numTurns: 1,
      qualityScore: null,
      error: null,
      transcriptPath: "",
      resultPath: "",
      ...over,
    }) as RunResult;

  it("a run *claiming* actual with a null cost no longer yields costSource=actual", () => {
    const m = aggregateCombo("cc+opus4.8+none", [
      run({ costSource: "actual", costUsd: null }),
      run({ costSource: "none" }),
    ]);
    expect(m.costUsd).toBeNull();
    expect(m.costSource).toBe("none");
  });

  it("actual values present → actual; only estimates → estimated", () => {
    expect(aggregateCombo("c", [run({ costUsd: 0.5, costSource: "actual" })]).costSource).toBe(
      "actual",
    );
    expect(
      aggregateCombo("c", [run({ estimatedCostUsd: 0.2, costSource: "estimated" })]).costSource,
    ).toBe("estimated");
  });
});

describe("codex rules harnesses — materialized into the workdir (no silent baseline)", () => {
  it("superpowers rules cache applies to a workdir via AGENTS.md", () => {
    ensureHarnessRulesStub("superpowers", "# superpowers rules (test stub)\n");
    const work = tmp();
    expect(applyHarnessToWorkdir("superpowers", work)).toBe(true);
    expect(existsSync(join(work, "AGENTS.md"))).toBe(true);
  });
});