Skip to content
TypeScript
3.0 KB
Raw Blame
/**
 * 통합 — 오케스트레이터 파이프라인 + 기록 레이아웃 + 결정성.
 */
import { existsSync, mkdtempSync, readFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { aggregateAll, sortCombos } from "../src/metrics";
import type { RunOptions } from "../src/models";
import { execute } from "../src/orchestrator";
import { loadMeta, loadResults } from "../src/recorder";

function opts(out: string, over: Partial<RunOptions> = {}): RunOptions {
  return {
    models: ["opus4.8"],
    agents: ["cc"],
    harnesses: ["none", "omc"],
    bench: "subagent",
    tasks: 3,
    runs: 2,
    seed: 11,
    runner: "mock",
    concurrency: 4,
    timeoutS: 5,
    outDir: out,
    judge: "heuristic",
    score: false,
    balanced: false,
    language: "kr",
    ...over,
  };
}

describe("orchestrator pipeline & recorder layout", () => {
  it("writes results.jsonl, meta.json, accuracy/, flow.json; results reload", async () => {
    const out = mkdtempSync(join(tmpdir(), "theseus-int-"));
    const res = await execute(opts(out));
    expect(existsSync(join(res.runDir, "results.jsonl"))).toBe(true);
    expect(existsSync(join(res.runDir, "meta.json"))).toBe(true);
    expect(existsSync(res.reportPath)).toBe(true);
    expect(existsSync(res.graphPath)).toBe(true);

    const meta = loadMeta(out, res.runId)!;
    expect(meta.bench).toBe("subagent");
    expect(meta.tasks).toHaveLength(3);
    expect(meta.mock).toBe(true);

    const results = loadResults(out, res.runId);
    // 2 combos × 3 tasks × 2 runs = 12
    expect(results).toHaveLength(12);
    // judge=heuristic → quality 채점됨(비-null)
    expect(results.every((r) => r.qualityScore !== null)).toBe(true);

    // flow.json 존재(한 work dir 샘플)
    const sample = results[0]!;
    const wd = join(res.runDir, "work");
    expect(existsSync(wd)).toBe(true);
    void sample;

    // accuracy 파일
    expect(existsSync(join(res.runDir, "accuracy"))).toBe(true);
  });

  it("determinism: same opts → identical aggregated metrics", async () => {
    const out1 = mkdtempSync(join(tmpdir(), "theseus-d1-"));
    const out2 = mkdtempSync(join(tmpdir(), "theseus-d2-"));
    const a = await execute(opts(out1));
    const b = await execute(opts(out2));
    const ma = sortCombos(a.metrics).map((m) => [
      m.comboId,
      m.subagentCount,
      m.totalTokens,
      m.accuracy,
    ]);
    const mb = sortCombos(b.metrics).map((m) => [
      m.comboId,
      m.subagentCount,
      m.totalTokens,
      m.accuracy,
    ]);
    expect(ma).toEqual(mb);
  });

  it("aggregateAll accuracy is bounded 0..100", async () => {
    const out = mkdtempSync(join(tmpdir(), "theseus-acc-"));
    const res = await execute(opts(out));
    for (const m of res.metrics) {
      expect(m.accuracy).toBeGreaterThanOrEqual(0);
      expect(m.accuracy).toBeLessThanOrEqual(100);
      expect(m.accuracyAtK).toBeGreaterThanOrEqual(0);
      expect(m.accuracyAtK).toBeLessThanOrEqual(100);
    }
  });
});