TypeScript
/**
* 통합 — 오케스트레이터 파이프라인 + 기록 레이아웃 + 결정성.
*/
import { existsSync, mkdtempSync, readFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { aggregateAll, sortCombos } from "../src/metrics";
import type { RunOptions } from "../src/models";
import { execute } from "../src/orchestrator";
import { loadMeta, loadResults } from "../src/recorder";
function opts(out: string, over: Partial<RunOptions> = {}): RunOptions {
return {
models: ["opus4.8"],
agents: ["cc"],
harnesses: ["none", "omc"],
bench: "subagent",
tasks: 3,
runs: 2,
seed: 11,
runner: "mock",
concurrency: 4,
timeoutS: 5,
outDir: out,
judge: "heuristic",
score: false,
balanced: false,
language: "kr",
...over,
};
}
describe("orchestrator pipeline & recorder layout", () => {
it("writes results.jsonl, meta.json, accuracy/, flow.json; results reload", async () => {
const out = mkdtempSync(join(tmpdir(), "theseus-int-"));
const res = await execute(opts(out));
expect(existsSync(join(res.runDir, "results.jsonl"))).toBe(true);
expect(existsSync(join(res.runDir, "meta.json"))).toBe(true);
expect(existsSync(res.reportPath)).toBe(true);
expect(existsSync(res.graphPath)).toBe(true);
const meta = loadMeta(out, res.runId)!;
expect(meta.bench).toBe("subagent");
expect(meta.tasks).toHaveLength(3);
expect(meta.mock).toBe(true);
const results = loadResults(out, res.runId);
// 2 combos × 3 tasks × 2 runs = 12
expect(results).toHaveLength(12);
// judge=heuristic → quality 채점됨(비-null)
expect(results.every((r) => r.qualityScore !== null)).toBe(true);
// flow.json 존재(한 work dir 샘플)
const sample = results[0]!;
const wd = join(res.runDir, "work");
expect(existsSync(wd)).toBe(true);
void sample;
// accuracy 파일
expect(existsSync(join(res.runDir, "accuracy"))).toBe(true);
});
it("determinism: same opts → identical aggregated metrics", async () => {
const out1 = mkdtempSync(join(tmpdir(), "theseus-d1-"));
const out2 = mkdtempSync(join(tmpdir(), "theseus-d2-"));
const a = await execute(opts(out1));
const b = await execute(opts(out2));
const ma = sortCombos(a.metrics).map((m) => [
m.comboId,
m.subagentCount,
m.totalTokens,
m.accuracy,
]);
const mb = sortCombos(b.metrics).map((m) => [
m.comboId,
m.subagentCount,
m.totalTokens,
m.accuracy,
]);
expect(ma).toEqual(mb);
});
it("aggregateAll accuracy is bounded 0..100", async () => {
const out = mkdtempSync(join(tmpdir(), "theseus-acc-"));
const res = await execute(opts(out));
for (const m of res.metrics) {
expect(m.accuracy).toBeGreaterThanOrEqual(0);
expect(m.accuracy).toBeLessThanOrEqual(100);
expect(m.accuracyAtK).toBeGreaterThanOrEqual(0);
expect(m.accuracyAtK).toBeLessThanOrEqual(100);
}
});
});