Skip to content
TypeScript
4.3 KB
Raw Blame
/**
 * L1 멀티모델 에이전트 (opencode / openclaw / hermes) — 추가 검증.
 * any-family 호환, 멀티모델 provider/model 매핑, 방어적 파서, registry 디스패치, fail-soft.
 */
import { mkdtempSync, readFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { expand } from "../src/combos";
import { agentFamilies, isMultiModelAgent, providerFor } from "../src/config";
import { MockRunner } from "../src/runners/mock";
import { MultiModelRunner } from "../src/runners/multiModel";
import { runnerFor } from "../src/runners/registry";
import { parserFor } from "../src/transcripts/index";

function tmp(): string {
  return mkdtempSync(join(tmpdir(), "theseus-mm-"));
}

describe("multi-model agents accept any family", () => {
  it("opencode/openclaw/hermes support both claude and gpt; cc/codex stay single-family", () => {
    for (const a of ["opencode", "openclaw", "hermes"]) {
      expect(agentFamilies(a).sort()).toEqual(["claude", "gpt", "open"]);
      expect(isMultiModelAgent(a)).toBe(true);
    }
    expect(agentFamilies("cc")).toEqual(["claude"]);
    expect(agentFamilies("codex")).toEqual(["gpt"]);
    expect(isMultiModelAgent("cc")).toBe(false);
  });

  it("expand: opencode valid with both families, none-only harness", () => {
    const { valid, excluded } = expand(
      ["opus4.8", "gpt5.5"],
      ["opencode", "openclaw", "hermes"],
      ["none", "omc"],
    );
    const ids = valid.map((c) => c.comboId);
    expect(ids).toContain("opencode+opus4.8+none");
    expect(ids).toContain("opencode+gpt5.5+none");
    expect(ids).toContain("hermes+gpt5.5+none");
    // omc is cc-only → excluded for these agents (with reason)
    const bad = excluded.find((c) => c.comboId === "opencode+opus4.8+omc");
    expect(bad?.excludeReason).toMatch(/not compatible/);
  });
});

describe("provider/model mapping", () => {
  it("maps families to providers", () => {
    expect(providerFor("claude")).toBe("anthropic");
    expect(providerFor("gpt")).toBe("openai");
  });
});

describe("registry dispatch", () => {
  it("forces the right MultiModelRunner per agent name", () => {
    for (const a of ["opencode", "openclaw", "hermes"]) {
      const choice = runnerFor(a, a, { bench: "subagent" });
      expect(choice.runner).toBeInstanceOf(MultiModelRunner);
      expect(choice.runner.name).toBe(a);
    }
  });
  it("auto dispatch falls back to mock when the agent binary is absent (local bench)", () => {
    const choice = runnerFor("opencode", "auto", { bench: "subagent" });
    // 오프라인: opencode 미설치 → mock 폴백
    expect(choice.mock).toBe(true);
  });
});

describe("defensive parser via mock (flat, tokens summed, no subagents)", () => {
  it("opencode+gpt5.5 yields tokens and zero subagents; total = sum", async () => {
    const runner = new MockRunner(0);
    const work = tmp();
    const art = await runner.run({
      combo: {
        comboId: "opencode+gpt5.5+none",
        agent: "opencode",
        model: "gpt5.5",
        harness: "none",
        valid: true,
      },
      taskId: "subagent-102",
      runIdx: 0,
      workDir: work,
      timeoutS: 5,
      seed: 2,
      bench: "subagent",
    });
    const parsed = parserFor("opencode").parse(art.transcriptPath, { model: "gpt-5.5" });
    expect(parsed.totalTokens).toBeGreaterThan(0);
    expect(parsed.totalTokens).toBe(
      parsed.inputTokens +
        parsed.outputTokens +
        parsed.cacheReadTokens +
        parsed.cacheCreationTokens,
    );
    expect(parsed.subagentCount).toBe(0);
  });
});

describe("MultiModelRunner fail-soft when binary missing", () => {
  it("returns success:false + artifacts (never throws)", async () => {
    const choice = runnerFor("hermes", "hermes", { bench: "local" });
    const art = await choice.runner.run({
      combo: {
        comboId: "hermes+opus4.8+none",
        agent: "hermes",
        model: "opus4.8",
        harness: "none",
        valid: true,
      },
      taskId: "local-hello",
      runIdx: 0,
      workDir: tmp(),
      timeoutS: 2,
      seed: 0,
      bench: "local",
    });
    expect(art.success).toBe(false);
    expect(() => readFileSync(art.transcriptPath, "utf8")).not.toThrow();
    expect(() => readFileSync(art.resultPath, "utf8")).not.toThrow();
  });
});