Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

47.9 KB
Raw Blame
/**
 * Black box tests for team-orchestrator.ts
 *
 * Tests are derived from:
 *   - requirements.md: FR3 (file convention), FR4 (anonymous IDs / shuffle),
 *     FR5 (per-model work dirs), FR6 (status tracking), FR8 (model list)
 *   - architecture.md: public API signatures, manifest.json schema,
 *     status.json schema, security (path validation), revision #5 (zero-padded IDs)
 *
 * Most runModels and judgeResponses behavior lives in integration tests; this file
 * keeps hermetic pane runs (the fake interactive child in a real headless magmux) to
 * pin the manifest identity, the pane argv and the per-slot environment.
 */

import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
import {
  existsSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
  readdirSync,
  realpathSync,
  rmSync,
  symlinkSync,
  writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { basename, join, resolve } from "node:path";
import { UPSTREAM_ERROR_LOG_ENV } from "./handlers/shared/upstream-error-capture.js";
import type { ModelStatus, TeamManifest, TeamStatus, VoteResult } from "./team-orchestrator.js";
import {
  MAGMUX,
  type PaneTestEnv,
  finishPaneTest,
  makePaneTestEnv,
  paneRunOptions,
  teamDirOf,
} from "./test-helpers/team-pane.js";

// ─── Dynamic imports (resolved at runtime so the module doesn't need to exist
//     until the tests actually run) ──────────────────────────────────────────

async function getOrchestrator() {
  return import("./team-orchestrator.js");
}

// ─── Helpers ─────────────────────────────────────────────────────────────────

/** Create a fresh isolated temp directory for each test. */
function makeTempDir(): string {
  return mkdtempSync(join(tmpdir(), "team-orch-test-"));
}

/** Parse JSON file from disk, or return null on failure. */
function readJson<T>(filePath: string): T {
  return JSON.parse(readFileSync(filePath, "utf-8")) as T;
}

// ─── Test state ───────────────────────────────────────────────────────────────

let tempDir: string;

/** The hermetic pane env of the current pane test (set by `usePaneHooks`). */
let t: PaneTestEnv;

/** Per-test pane env, and the no-orphan check after every pane test. */
function usePaneHooks(): void {
  beforeEach(() => {
    t = makePaneTestEnv();
  });
  afterEach(async () => {
    const report = await finishPaneTest(t);
    expect(report).toEqual({ processes: [], files: [] });
  });
}

beforeEach(() => {
  tempDir = makeTempDir();
});

afterEach(() => {
  if (tempDir && existsSync(tempDir)) {
    rmSync(tempDir, { recursive: true, force: true });
  }
});

// ─── Tests ────────────────────────────────────────────────────────────────────

describe("team-orchestrator", () => {
  describe("snippetHeadAndTail", () => {
    it("returns short text unchanged without an elision marker", async () => {
      const { snippetHeadAndTail } = await getOrchestrator();
      const text = "Short diagnostic output\nwith its original line breaks.";

      expect(snippetHeadAndTail(text)).toBe(text);
      expect(snippetHeadAndTail(text)).not.toContain("bytes omitted");
    });

    it("keeps the first 600 and last 1400 bytes and reports the omitted count", async () => {
      const { snippetHeadAndTail } = await getOrchestrator();
      const head = "H".repeat(600);
      const omitted = "M".repeat(37);
      const tail = "T".repeat(1400);

      expect(snippetHeadAndTail(`${head}${omitted}${tail}`)).toBe(
        `${head}\n\n… [37 bytes omitted] …\n\n${tail}`
      );
    });

    it("preserves a bolded FAIL verdict near-miss from the head of a long response", async () => {
      const { snippetHeadAndTail } = await getOrchestrator();
      const verdict = "**Verdict**: **FAIL**";
      const response = `${verdict}\n${"Detailed analysis without a matching verdict. ".repeat(80)}Unrelated tail prose.`;

      expect(response.match(/\*\*Verdict\*\*: (PASS|CONDITIONAL|FAIL)/)).toBeNull();
      expect(snippetHeadAndTail(response)).toContain(verdict);
      expect(snippetHeadAndTail(response)).toContain("Unrelated tail prose.");
    });
  });
  // ── FR3 / FR5: Directory structure ────────────────────────────────────────

  describe("setupSession — directory structure", () => {
    it("TEST-01: creates work/ and errors/ subdirectories", async () => {
      const { setupSession } = await getOrchestrator();

      setupSession(tempDir, ["model-a", "model-b"], "task content");

      expect(existsSync(join(tempDir, "work"))).toBe(true);
      expect(existsSync(join(tempDir, "errors"))).toBe(true);
    });

    it("TEST-02: creates one work subdirectory per model", async () => {
      const { setupSession } = await getOrchestrator();
      const models = ["model-a", "model-b", "model-c"];

      setupSession(tempDir, models, "task content");

      const workEntries = readdirSync(join(tempDir, "work"));
      expect(workEntries.length).toBe(models.length);
    });
  });

  // ── FR4: manifest.json ────────────────────────────────────────────────────

  describe("setupSession — manifest.json", () => {
    it("TEST-03: manifest.json has correct number of model entries", async () => {
      const { setupSession } = await getOrchestrator();
      const models = ["m1", "m2", "m3", "m4"];

      setupSession(tempDir, models, "task");

      const manifest = readJson<TeamManifest>(join(tempDir, "manifest.json"));
      expect(Object.keys(manifest.models).length).toBe(models.length);
    });

    it("TEST-04: anonymous IDs are zero-padded numeric strings (01, 02, ...)", async () => {
      // Architecture revision #5: use zero-padded numeric IDs to support >26 models
      const { setupSession } = await getOrchestrator();

      setupSession(tempDir, ["model-a", "model-b", "model-c"], "task");

      const manifest = readJson<TeamManifest>(join(tempDir, "manifest.json"));
      const ids = Object.keys(manifest.models);

      const zeroPaddedNumeric = /^\d{2,}$/;
      for (const id of ids) {
        expect(zeroPaddedNumeric.test(id)).toBe(true);
      }
    });

    it("TEST-05: manifest model entries contain all provided model names", async () => {
      const { setupSession } = await getOrchestrator();
      const models = ["model-alpha", "model-beta"];

      setupSession(tempDir, models, "task");

      const manifest = readJson<TeamManifest>(join(tempDir, "manifest.json"));
      const storedModelNames = Object.values(manifest.models).map((e) => e.model);

      // Order may differ due to shuffle; use set equality
      expect(storedModelNames.sort()).toEqual(models.sort());
    });

    it("TEST-06: manifest.json has a valid ISO 8601 created timestamp", async () => {
      const { setupSession } = await getOrchestrator();

      setupSession(tempDir, ["model-a"], "task");

      const manifest = readJson<TeamManifest>(join(tempDir, "manifest.json"));
      expect(typeof manifest.created).toBe("string");
      const parsed = new Date(manifest.created);
      // A valid ISO date parses without NaN
      expect(Number.isNaN(parsed.getTime())).toBe(false);
    });

    it("TEST-07: shuffle produces different order across multiple runs (statistical)", async () => {
      // With 6 models, probability of all 20 runs preserving original order is
      // (1/720)^20 ≈ 10^{-57} — effectively impossible if shuffle is implemented.
      const { setupSession } = await getOrchestrator();
      const models = ["m1", "m2", "m3", "m4", "m5", "m6"];

      // Collect the model-name arrays as ordered by the anonymous ID keys across runs
      const orderings: string[][] = [];

      for (let run = 0; run < 20; run++) {
        const runDir = mkdtempSync(join(tmpdir(), "team-shuffle-"));
        try {
          setupSession(runDir, models, "task");
          const manifest = readJson<TeamManifest>(join(runDir, "manifest.json"));
          // Sort by anonymous ID key to get a deterministic ordering per run
          const ordering = Object.keys(manifest.models)
            .sort()
            .map((k) => manifest.models[k].model);
          orderings.push(ordering);
        } finally {
          rmSync(runDir, { recursive: true, force: true });
        }
      }

      // At least one run should produce a different ordering from the first
      const first = orderings[0].join(",");
      const allIdentical = orderings.every((o) => o.join(",") === first);
      expect(allIdentical).toBe(false);
    });
  });

  // ── FR6: status.json ──────────────────────────────────────────────────────

  describe("setupSession — status.json", () => {
    it("TEST-08: all models start STARTING in status.json (the closed set has no PENDING)", async () => {
      const { setupSession } = await getOrchestrator();
      const models = ["model-a", "model-b", "model-c"];

      setupSession(tempDir, models, "task");

      const status = readJson<TeamStatus>(join(tempDir, "status.json"));
      const states = Object.values(status.models).map((m) => m.state);
      expect(states.every((s) => s === "STARTING")).toBe(true);
    });

    it("TEST-09: status.json model count matches input models array length", async () => {
      const { setupSession } = await getOrchestrator();
      const models = ["m1", "m2", "m3", "m4", "m5"];

      setupSession(tempDir, models, "task");

      const status = readJson<TeamStatus>(join(tempDir, "status.json"));
      expect(Object.keys(status.models).length).toBe(models.length);
    });
  });

  // ── FR3: input.md handling ────────────────────────────────────────────────

  describe("setupSession — input.md", () => {
    it("TEST-10: writes input.md with provided input text", async () => {
      const { setupSession } = await getOrchestrator();
      const inputText = "test task content for model evaluation";

      setupSession(tempDir, ["model-a"], inputText);

      const written = readFileSync(join(tempDir, "input.md"), "utf-8");
      expect(written).toBe(inputText);
    });

    it("TEST-11: succeeds when input.md already exists and no input text given", async () => {
      const { setupSession } = await getOrchestrator();
      const preExisting = "pre-existing task description";
      writeFileSync(join(tempDir, "input.md"), preExisting, "utf-8");

      // Must not throw
      expect(() => setupSession(tempDir, ["model-a"])).not.toThrow();

      // input.md content must be preserved
      const content = readFileSync(join(tempDir, "input.md"), "utf-8");
      expect(content).toBe(preExisting);
    });

    it("TEST-12: throws when no input.md exists and no input text is provided", async () => {
      const { setupSession } = await getOrchestrator();

      // No input.md in tempDir, no input argument
      expect(() => setupSession(tempDir, ["model-a"])).toThrow();
    });
  });

  // ── FR8: input validation — empty models ──────────────────────────────────

  // Moved from the deleted team-timeout-repro.test.ts (its bug #3).
  describe("setupSession — session directory overwrite protection", () => {
    it("rejects an existing session directory", async () => {
      const { setupSession } = await getOrchestrator();
      setupSession(tempDir, ["model-a"], "First run input");
      expect(() => setupSession(tempDir, ["model-b"], "Second run input")).toThrow(
        /Session already exists/
      );
    });

    it("preserves the session artifacts when a re-run is rejected", async () => {
      const { setupSession } = await getOrchestrator();
      setupSession(tempDir, ["model-a"], "First run input");
      const originalManifest = readFileSync(join(tempDir, "manifest.json"), "utf-8");
      const originalInput = readFileSync(join(tempDir, "input.md"), "utf-8");
      const originalStatus = readFileSync(join(tempDir, "status.json"), "utf-8");

      expect(() => setupSession(tempDir, ["model-b"], "DIFFERENT input")).toThrow();

      expect(readFileSync(join(tempDir, "manifest.json"), "utf-8")).toBe(originalManifest);
      expect(readFileSync(join(tempDir, "input.md"), "utf-8")).toBe(originalInput);
      expect(readFileSync(join(tempDir, "status.json"), "utf-8")).toBe(originalStatus);
    });
  });

  describe("setupSession — input validation", () => {
    it("TEST-13: throws for an empty models array", async () => {
      const { setupSession } = await getOrchestrator();

      expect(() => setupSession(tempDir, [], "task")).toThrow();
    });
  });

  // ── Native model slots ─────────────────────────────────────────────────────
  // Was "sentinel model rejection" (91ee9a8). That guard existed because
  // `internal`/`default` reached Claude Code as literal model names and it
  // "failed with cryptic model not found errors". The cause is fixed at the
  // `--model` boundary now (normalizeNativeModelSpec), and a native name is a
  // runnable slot — verified end to end: a team run with ["internal"] completed
  // exit 0 having spawned `claudish --model opus …`, and the same run with a
  // prompt that produced no vote block was reported EMPTY/shape_mismatch rather
  // than succeeded. Rejecting here would put the internal reviewer back outside
  // requirePattern, which is the guard that catches a voter that never voted.

  describe("setupSession — native model slots", () => {
    it("TEST-NS-01: accepts 'internal' as a runnable slot", async () => {
      const { setupSession } = await getOrchestrator();

      const manifest = setupSession(tempDir, ["internal"], "task");
      expect(Object.values(manifest.models).map((m) => m.model)).toEqual(["internal"]);
    });

    it("TEST-NS-02: accepts 'default' as a runnable slot", async () => {
      const { setupSession } = await getOrchestrator();

      const manifest = setupSession(tempDir, ["default"], "task");
      expect(Object.values(manifest.models).map((m) => m.model)).toEqual(["default"]);
    });

    it("TEST-NS-03: accepts Claude tier names (opus, sonnet, haiku)", async () => {
      const { setupSession } = await getOrchestrator();

      const manifest = setupSession(tempDir, ["opus", "sonnet", "haiku"], "task");
      expect(
        Object.values(manifest.models)
          .map((m) => m.model)
          .sort()
      ).toEqual(["haiku", "opus", "sonnet"]);
    });

    it("TEST-NS-04: accepts claude-* model IDs", async () => {
      const { setupSession } = await getOrchestrator();

      const manifest = setupSession(
        tempDir,
        ["claude-sonnet-4-6", "claude-3-opus-20240229"],
        "task"
      );
      expect(Object.values(manifest.models).map((m) => m.model)).toEqual([
        "claude-sonnet-4-6",
        "claude-3-opus-20240229",
      ]);
    });

    it("TEST-NS-05: preserves the caller's casing as the slot identity", async () => {
      const { setupSession } = await getOrchestrator();

      // The manifest is the run's identity and is echoed back in status//errors,
      // so it keeps the string the caller passed. Normalization happens in the
      // CHILD, at its own --model boundary.
      const manifest = setupSession(tempDir, ["Internal"], "task");
      expect(Object.values(manifest.models).map((m) => m.model)).toEqual(["Internal"]);
    });

    it("TEST-NS-06: accepts a native slot alongside external models in one manifest", async () => {
      const { setupSession } = await getOrchestrator();

      const manifest = setupSession(tempDir, ["gemini-2.0-flash", "internal", "gpt-4o"], "task");
      expect(Object.keys(manifest.models)).toHaveLength(3);
      expect(Object.values(manifest.models).map((m) => m.model)).toContain("internal");
    });

    it("TEST-NS-07: accepts valid external model names", async () => {
      const { setupSession } = await getOrchestrator();

      // These should NOT throw
      const manifest = setupSession(
        tempDir,
        ["gemini-2.0-flash", "gpt-4o", "or@deepseek/deepseek-r1"],
        "task"
      );
      expect(manifest).toBeDefined();
      expect(Object.keys(manifest.models)).toHaveLength(3);
    });
  });

  // ── Security: validateSessionPath ─────────────────────────────────────────

  describe("validateSessionPath", () => {
    it("TEST-14: throws when path resolves outside CWD", async () => {
      const { validateSessionPath } = await getOrchestrator();

      // /tmp is virtually always outside CWD (which is the project directory)
      const outsidePath = "/tmp/definitely-outside-cwd-test-path";

      // Only run if /tmp is actually outside CWD
      if (!resolve(outsidePath).startsWith(process.cwd())) {
        expect(() => validateSessionPath(outsidePath)).toThrow();
      } else {
        // CWD is /tmp or a subdir — skip this particular check
        console.warn("Skipping TEST-14: /tmp is inside CWD, cannot test outside-CWD rejection");
      }
    });

    it("TEST-15: accepts a path that resolves within CWD and returns resolved path", async () => {
      const { validateSessionPath } = await getOrchestrator();

      // Use a subdir of CWD that we know exists
      const insidePath = join(process.cwd(), "packages");

      const result = validateSessionPath(insidePath);

      // Should return the resolved absolute path without throwing
      expect(typeof result).toBe("string");
      expect(result.startsWith(process.cwd())).toBe(true);
    });
  });

  // ── FR6: getStatus ────────────────────────────────────────────────────────

  describe("getStatus", () => {
    it("TEST-16: returns parsed status.json with STARTING state after setupSession", async () => {
      const { setupSession, getStatus } = await getOrchestrator();

      setupSession(tempDir, ["model-a", "model-b"], "task");

      const status = getStatus(tempDir);

      expect(status).toBeDefined();
      expect(typeof status.models).toBe("object");

      const states = Object.values(status.models).map((m: ModelStatus) => m.state);
      expect(states.every((s) => s === "STARTING")).toBe(true);
    });

    it("TEST-17: getStatus throws when status.json does not exist", async () => {
      const { getStatus } = await getOrchestrator();

      // tempDir exists but has no status.json
      expect(() => getStatus(tempDir)).toThrow();
    });
  });

  describe.skipIf(!MAGMUX)("runModels on panes — spawn identity and argv", () => {
    usePaneHooks();

    it("keeps manifest identity and launches an interactive child with the pane argv", async () => {
      const { runModels, setupSession } = await getOrchestrator();
      const spawnPlanner = mock(async () => ({
        pinned: new Map([["vendor/model", "fake-env_probe"]]),
      }));
      const probe = join(t.tmp, "probe-{session}.json");
      const sessionPath = teamDirOf(t);
      setupSession(sessionPath, ["vendor/model"], "Analyze this input");

      const status = await runModels(sessionPath, {
        ...paneRunOptions(t, {}, { FAKE_PROBE_FILE: probe }),
        spawnPlanner,
      });
      const [anonId, slot] = Object.entries(status.models)[0] ?? [];
      expect(slot?.state).toBe("COMPLETED");
      expect(spawnPlanner).toHaveBeenCalledWith(["vendor/model"]);
      expect(slot?.model).toBe("vendor/model");
      expect(slot?.spawnModel).toBe("fake-env_probe");

      const { argv } = readJson<{ argv: string[] }>(
        probe.replace("{session}", slot?.sessionUuid as string)
      );
      // -i --model <spawnModel> -y --quiet --session-id <uuid> --add-dir <turn dir>
      expect(argv.slice(0, 6)).toEqual([
        "-i",
        "--model",
        "fake-env_probe",
        "-y",
        "--quiet",
        "--session-id",
      ]);
      expect(argv[6]).toBe(slot?.sessionUuid as string);
      expect(argv[7]).toBe("--add-dir");
      for (const banned of ["-p", "--print", "--stdin", "--output-format", "--verbose", "--json"])
        expect(argv).not.toContain(banned);

      // The manifest identity is re-used by the judge round; the pinned argv must never
      // replace it with the provider wire spec.
      const reread = readJson<TeamManifest>(join(sessionPath, "manifest.json"));
      expect(reread.models[anonId as string]?.model).toBe("vendor/model");
    }, 30_000);

    it("gives every model slot its own upstream-error log and token file", async () => {
      const { runModels, setupSession } = await getOrchestrator();
      const probe = join(t.tmp, "probe-{session}.json");
      const sessionPath = teamDirOf(t);
      setupSession(sessionPath, ["fake-env_probe-1", "fake-env_probe-2"], "Analyze this input");

      const status = await runModels(
        sessionPath,
        paneRunOptions(t, {}, { FAKE_PROBE_FILE: probe })
      );

      const paths = Object.entries(status.models).map(([anonId, m]) => {
        expect(m.state).toBe("COMPLETED");
        const { env } = readJson<{ env: Record<string, string> }>(
          probe.replace("{session}", m.sessionUuid as string)
        );
        const path = env[UPSTREAM_ERROR_LOG_ENV];
        expect(path).toBe(join(sessionPath, "errors", `${anonId}-upstream.jsonl`));
        expect(env.CLAUDISH_TOKEN_FILE).toBe(join(sessionPath, "stats", `${anonId}.json`));
        return path;
      });
      expect(new Set(paths).size).toBe(paths.length);
    }, 30_000);

    it("omits upstreamErrorLogPath from a recorded error when the child wrote no file", async () => {
      const { runModels, setupSession } = await getOrchestrator();
      const sessionPath = teamDirOf(t);
      setupSession(sessionPath, ["fake-exit_mid_turn"], "Analyze this input");
      await runModels(sessionPath, paneRunOptions(t));

      const recorded = readJson<TeamStatus>(join(sessionPath, "status.json"));
      const [anonId, modelStatus] = Object.entries(recorded.models)[0] ?? [];
      expect(modelStatus?.state).toBe("FAILED");
      expect(modelStatus?.error?.reason).toBe("child_exited");
      expect(modelStatus?.error).not.toHaveProperty("upstreamErrorLogPath");
      expect(existsSync(join(sessionPath, "errors", `${anonId}-upstream.jsonl`))).toBe(false);
      expect(existsSync(join(sessionPath, "errors", `${anonId}.log`))).toBe(true);
    }, 30_000);
  });

  describe.skipIf(!MAGMUX)("startModels on panes — team policy", () => {
    usePaneHooks();
    const TWO_LINES = "Review the change below.\nReply starting with VERDICT:.";

    it("returns once every slot left STARTING (D9)", async () => {
      const { startModels, setupSession, getStatus } = await getOrchestrator();
      const sessionPath = teamDirOf(t);
      setupSession(sessionPath, ["contract-fake-a", "contract-fake-b"], "@@HANG@@ hold");
      const handle = await startModels(sessionPath, paneRunOptions(t));
      const states = Object.values(getStatus(sessionPath).models).map((m) => m.state);
      expect(states).toEqual(["RUNNING", "RUNNING"]);
      const { cancelTeamRun } = await getOrchestrator();
      await cancelTeamRun(sessionPath);
      await handle.done;
    }, 30_000);

    it("fails a slot that stops on a question team cannot answer: FAILED blocked (D19)", async () => {
      const { runModels, setupSession } = await getOrchestrator();
      const sessionPath = teamDirOf(t);
      setupSession(sessionPath, ["fake-ask_user"], "Pick a fruit.");
      const status = await runModels(sessionPath, paneRunOptions(t));
      const [id, m] = Object.entries(status.models)[0] ?? [];
      expect(m?.state).toBe("FAILED");
      expect(m?.error?.reason).toBe("blocked");
      expect(m?.error?.detail.length).toBeGreaterThan(0);
      // The answer so far is kept for inspection.
      expect(existsSync(join(sessionPath, `response-${id}.md`))).toBe(true);
    }, 30_000);

    it("matches ^VERDICT: on the answer that starts after the task file was read (§2.13)", async () => {
      const { runModels, setupSession } = await getOrchestrator();
      const sessionPath = teamDirOf(t);
      setupSession(sessionPath, ["fake-narrate_then_read"], TWO_LINES);
      const status = await runModels(
        sessionPath,
        paneRunOptions(t, { requirePattern: "^VERDICT:" })
      );
      const [id, m] = Object.entries(status.models)[0] ?? [];
      expect(m?.state).toBe("COMPLETED");
      // The narration before the Read never reaches response-<id>.md.
      expect(readFileSync(join(sessionPath, `response-${id}.md`), "utf-8")).toBe("VERDICT: ok");
    }, 30_000);

    it("fails a slot that never read its task file: FAILED prompt_not_read", async () => {
      const { runModels, setupSession } = await getOrchestrator();
      const sessionPath = teamDirOf(t);
      setupSession(sessionPath, ["fake-no_read"], TWO_LINES);
      const status = await runModels(sessionPath, paneRunOptions(t));
      const m = Object.values(status.models)[0];
      expect(m?.state).toBe("FAILED");
      expect(m?.error?.reason).toBe("prompt_not_read");
      expect(m?.error?.detail).toContain("never read");
    }, 30_000);

    it("fails a slot whose child refuses --agent: FAILED agent_rejected (D11)", async () => {
      const { runModels, setupSession } = await getOrchestrator();
      const sessionPath = teamDirOf(t);
      setupSession(sessionPath, ["fake-answer"], "Reply with exactly PEAR");
      const status = await runModels(
        sessionPath,
        paneRunOptions(t, { claudeFlags: ["--agent", "zzz-missing"] })
      );
      const m = Object.values(status.models)[0];
      expect(m?.state).toBe("FAILED");
      expect(m?.error?.reason).toBe("agent_rejected");
    }, 30_000);

    it("names an open background shell in the slot's anomalies (R3-M5)", async () => {
      const { runModels, setupSession } = await getOrchestrator();
      const sessionPath = teamDirOf(t);
      setupSession(sessionPath, ["fake-bg_server"], "Start the dev server.");
      const status = await runModels(sessionPath, paneRunOptions(t));
      const m = Object.values(status.models)[0];
      expect(m?.state).toBe("COMPLETED");
      expect(m?.anomalies?.some((a) => a.startsWith("background_shell_open: "))).toBe(true);
    }, 30_000);

    it("refuses reserved claude_flags before anything is written or spawned", async () => {
      const { startModels, setupSession } = await getOrchestrator();
      const sessionPath = teamDirOf(t);
      setupSession(sessionPath, ["fake-answer"], "Reply with exactly PEAR");
      await expect(
        startModels(sessionPath, paneRunOptions(t, { claudeFlags: ["--max-budget-usd", "5"] }))
      ).rejects.toThrow(/^invalid_args: .*--max-budget-usd/);
      await expect(
        startModels(
          sessionPath,
          paneRunOptions(t, { claudeFlags: ["--allowedTools", "Read", "Bash"] })
        )
      ).rejects.toThrow(/^invalid_args: .*positional/);
      expect(existsSync(join(t.sockRoot, "panes"))).toBe(false);
    });

    it("refuses a non-plain prompt when the flags remove Read (§2.3 rule 4)", async () => {
      const { startModels, setupSession } = await getOrchestrator();
      const sessionPath = teamDirOf(t);
      setupSession(sessionPath, ["fake-answer"], TWO_LINES);
      await expect(
        startModels(sessionPath, paneRunOptions(t, { claudeFlags: ["--disallowedTools", "Read"] }))
      ).rejects.toThrow(/^invalid_args: .*Read tool is unavailable/);
    });
  });

  describe.skipIf(!MAGMUX)("runModels on panes — the whole turn is the answer", () => {
    usePaneHooks();

    it("keeps the voted answer AND the message a re-wake added after it", async () => {
      // Print mode kept only the final assistant message, so a late message replaced the
      // vote. A pane turn settles on Claude Code's own turn_duration, after the re-wake,
      // and its answer is every assistant text block of the turn.
      const { runModels, setupSession } = await getOrchestrator();
      const sessionPath = teamDirOf(t);
      setupSession(sessionPath, ["fake-rewake"], "Review and vote.");
      const status = await runModels(sessionPath, {
        ...paneRunOptions(
          t,
          { requirePattern: "^ANSWER fake-rewake " },
          { FAKE_GAP_MS_REWAKE: "1000" }
        ),
        paneTimings: { replStableMs: 200, secondaryQuietMs: 4_000 },
      });

      const [id, model] = Object.entries(status.models)[0] ?? [];
      const response = readFileSync(join(sessionPath, `response-${id}.md`), "utf-8");
      expect(model?.state).toBe("COMPLETED");
      expect(response).toStartWith("ANSWER fake-rewake ");
      expect(response).toEndWith("\n\nLATE second message");
      expect(model?.outputSize).toBe(Buffer.byteLength(response));
    }, 30_000);
  });

  describe("runModels — requirePattern validation", () => {
    it("rejects an invalid pattern before reading manifest.json", async () => {
      const { runModels } = await getOrchestrator();

      // tempDir deliberately exists without a manifest. The pattern error must
      // win before any filesystem read or child-process work can begin.
      await expect(runModels(tempDir, { requirePattern: "(" })).rejects.toThrow(
        /^Invalid requirePattern \/\(/
      );
    });
  });

  // ── Directory names match manifest IDs ───────────────────────────────────

  describe("setupSession — work directory names", () => {
    it("TEST-18: work directory names match manifest model IDs exactly", async () => {
      const { setupSession } = await getOrchestrator();
      const models = ["model-a", "model-b", "model-c"];

      setupSession(tempDir, models, "task");

      const manifest = readJson<TeamManifest>(join(tempDir, "manifest.json"));
      const manifestIds = Object.keys(manifest.models).sort();
      const workDirNames = readdirSync(join(tempDir, "work")).sort();

      expect(workDirNames).toEqual(manifestIds);
    });
  });

  // ── shuffleOrder in manifest ──────────────────────────────────────────────

  describe("setupSession — shuffleOrder in manifest", () => {
    it("TEST-19: manifest contains shuffleOrder field with correct length", async () => {
      const { setupSession } = await getOrchestrator();
      const models = ["model-a", "model-b", "model-c", "model-d"];

      setupSession(tempDir, models, "task");

      const manifest = readJson<TeamManifest>(join(tempDir, "manifest.json"));

      expect(Array.isArray(manifest.shuffleOrder)).toBe(true);
      expect(manifest.shuffleOrder!.length).toBe(models.length);
    });

    it("TEST-20: shuffleOrder contains all manifest IDs", async () => {
      const { setupSession } = await getOrchestrator();
      const models = ["model-a", "model-b", "model-c"];

      setupSession(tempDir, models, "task");

      const manifest = readJson<TeamManifest>(join(tempDir, "manifest.json"));
      const manifestIds = Object.keys(manifest.models).sort();

      expect([...manifest.shuffleOrder!].sort()).toEqual(manifestIds);
    });
  });

  // ── validateSessionPath: security ────────────────────────────────────────

  describe("validateSessionPath — additional security", () => {
    it("TEST-21: deterministic outside-CWD path throws", async () => {
      const { validateSessionPath } = await getOrchestrator();

      const outsidePath = resolve(process.cwd(), "..", "sibling-dir-that-does-not-exist");
      expect(() => validateSessionPath(outsidePath)).toThrow();
    });

    it("TEST-22: path traversal sequence ../../etc/hosts throws", async () => {
      const { validateSessionPath } = await getOrchestrator();

      expect(() => validateSessionPath("../../etc/hosts")).toThrow();
    });
  });

  // ── validateSessionPath: symlinked spellings of cwd ──────────────────────
  //
  // process.cwd() is a realpath; path.resolve() is lexical. On macOS os.tmpdir() is
  // /var/folders/… while its realpath is /private/var/…, so the same directory has two
  // spellings. Where the temp dir has no symlinked spelling (Linux), the test makes its own.

  describe("validateSessionPath — symlinks and alternate spellings of cwd", () => {
    const WITHIN = "Session path must be within current directory";
    let savedCwd: string;
    let base: string; // realpath of a fresh temp dir
    let proj: string; // the cwd under test, a realpath

    beforeEach(() => {
      savedCwd = process.cwd();
      base = realpathSync(mkdtempSync(join(tmpdir(), "vsp-")));
      proj = join(base, "proj");
      mkdirSync(proj);
      mkdirSync(join(base, "outside"));
      process.chdir(proj);
    });

    afterEach(() => {
      process.chdir(savedCwd);
      rmSync(base, { recursive: true, force: true });
    });

    /** Another spelling of `base`: the os.tmpdir() one when it differs, else a symlink we make. */
    function aliasOfBase(): string {
      const tmpSpelling = join(tmpdir(), basename(base));
      if (tmpSpelling !== base) return tmpSpelling;
      const link = join(base, "alias-of-base");
      symlinkSync(base, link);
      return link;
    }

    // REGRESSION: validateSessionPath rejects the os.tmpdir() (/var) spelling of a path inside a realpath cwd — Fixed in /dev:fix session dev-fix-20261020-182839-b87ae677
    it("accepts the alternate (os.tmpdir()) spelling of <cwd>/s and returns the caller's spelling", async () => {
      const { validateSessionPath } = await getOrchestrator();
      const aliased = join(aliasOfBase(), "proj", "s");
      expect(aliased.startsWith(`${proj}/`)).toBe(false); // really a different spelling

      expect(validateSessionPath(aliased)).toBe(resolve(aliased));
    });

    // REGRESSION: validateSessionPath compares a lexical path against a realpath cwd — Fixed in /dev:fix session dev-fix-20261020-182839-b87ae677
    it("accepts a nested non-existent path a/b/c and returns resolve() of it", async () => {
      const { validateSessionPath } = await getOrchestrator();

      expect(validateSessionPath("a/b/c")).toBe(resolve("a/b/c"));
      expect(validateSessionPath("a/b/c")).toBe(join(proj, "a", "b", "c"));
    });

    // REGRESSION: validateSessionPath lets a symlink inside cwd escape to a dir outside it — Fixed in /dev:fix session dev-fix-20261020-182839-b87ae677
    it("rejects link/s where link points to a directory outside cwd", async () => {
      const { validateSessionPath } = await getOrchestrator();
      symlinkSync(join(base, "outside"), join(proj, "link"));

      expect(() => validateSessionPath("link/s")).toThrow(WITHIN);
    });

    // REGRESSION: validateSessionPath accepts a path through a dangling symlink — Fixed in /dev:fix session dev-fix-20261020-182839-b87ae677
    it("rejects a path through a dangling symlink with the same 'must be within' message", async () => {
      const { validateSessionPath } = await getOrchestrator();
      symlinkSync(join(base, "nonexistent-outside", "dir"), join(proj, "dangling"));

      expect(() => validateSessionPath("dangling/s")).toThrow(WITHIN);
    });

    // REGRESSION: validateSessionPath must keep the path-separator boundary when comparing realpaths — Fixed in /dev:fix session dev-fix-20261020-182839-b87ae677
    it("rejects a sibling that shares the cwd's name as a prefix (projX)", async () => {
      const { validateSessionPath } = await getOrchestrator();
      mkdirSync(join(base, "projX"));

      expect(() => validateSessionPath(join(base, "projX", "s"))).toThrow(WITHIN);
    });

    // REGRESSION: validateSessionPath must keep rejecting traversal after the realpath change — Fixed in /dev:fix session dev-fix-20261020-182839-b87ae677
    it("rejects ../x", async () => {
      const { validateSessionPath } = await getOrchestrator();

      expect(() => validateSessionPath("../x")).toThrow(WITHIN);
    });
  });

  // ── judgeResponses: threshold ─────────────────────────────────────────────

  describe("judgeResponses — minimum responses", () => {
    it("TEST-23: throws when fewer than 2 response files are present", async () => {
      const { setupSession, judgeResponses } = await getOrchestrator();

      // Set up a session with two models but only write one response file
      setupSession(tempDir, ["model-a", "model-b"], "task");
      writeFileSync(join(tempDir, "response-01.md"), "Only one response", "utf-8");

      await expect(judgeResponses(tempDir)).rejects.toThrow("Need at least 2 responses");
    });
  });
});

// ─── Pure function unit tests ─────────────────────────────────────────────────

describe("fisherYatesShuffle", () => {
  async function getShuffle() {
    const { fisherYatesShuffle } = await getOrchestrator();
    return fisherYatesShuffle;
  }

  it("TEST-S1: empty array returns empty array without crash", async () => {
    const shuffle = await getShuffle();
    expect(shuffle([])).toEqual([]);
  });

  it("TEST-S2: single-element array returns same element", async () => {
    const shuffle = await getShuffle();
    expect(shuffle([42])).toEqual([42]);
  });

  it("TEST-S4: output is a permutation (sorted equals sorted input)", async () => {
    const shuffle = await getShuffle();
    const input = [1, 2, 3, 4, 5, 6, 7, 8];
    const result = shuffle([...input]);
    expect([...result].sort((a, b) => a - b)).toEqual([...input].sort((a, b) => a - b));
  });
});

describe("buildJudgePrompt", () => {
  async function getBuilder() {
    const { buildJudgePrompt } = await getOrchestrator();
    return buildJudgePrompt;
  }

  it("TEST-B1: contains the original input text", async () => {
    const build = await getBuilder();
    const prompt = build("my task description", { "01": "response body" });
    expect(prompt).toContain("my task description");
  });

  it("TEST-B2: contains all response IDs", async () => {
    const build = await getBuilder();
    const prompt = build("task", { "01": "resp-one", "02": "resp-two", "03": "resp-three" });
    expect(prompt).toContain("01");
    expect(prompt).toContain("02");
    expect(prompt).toContain("03");
  });

  it("TEST-B3: contains the vote block template", async () => {
    const build = await getBuilder();
    const prompt = build("task", { "01": "resp" });
    expect(prompt).toContain("```vote");
    expect(prompt).toContain("RESPONSE:");
    expect(prompt).toContain("VERDICT:");
    expect(prompt).toContain("CONFIDENCE:");
    expect(prompt).toContain("KEY_ISSUES:");
  });

  it("TEST-B4: contains correct number of response sections", async () => {
    const build = await getBuilder();
    const responses = { "01": "first", "02": "second", "03": "third" };
    const prompt = build("task", responses);
    // Each response has a "#### Response XX" heading
    const sectionMatches = prompt.match(/#### Response \d+/g);
    expect(sectionMatches?.length).toBe(3);
  });
});

describe("aggregateVerdict", () => {
  async function getAggregate() {
    const { aggregateVerdict } = await getOrchestrator();
    return aggregateVerdict;
  }

  it("TEST-A1: all APPROVE → score 1.0", async () => {
    const aggregate = await getAggregate();
    const votes: VoteResult[] = [
      {
        judgeId: "j1",
        responseId: "01",
        verdict: "APPROVE",
        confidence: 9,
        summary: "good",
        keyIssues: [],
      },
      {
        judgeId: "j2",
        responseId: "01",
        verdict: "APPROVE",
        confidence: 8,
        summary: "good",
        keyIssues: [],
      },
    ];
    const verdict = aggregate(votes, ["01"]);
    expect(verdict.responses["01"].score).toBe(1.0);
    expect(verdict.responses["01"].approvals).toBe(2);
    expect(verdict.responses["01"].rejections).toBe(0);
  });

  it("TEST-A2: all REJECT → score 0.0", async () => {
    const aggregate = await getAggregate();
    const votes: VoteResult[] = [
      {
        judgeId: "j1",
        responseId: "01",
        verdict: "REJECT",
        confidence: 3,
        summary: "bad",
        keyIssues: [],
      },
      {
        judgeId: "j2",
        responseId: "01",
        verdict: "REJECT",
        confidence: 2,
        summary: "bad",
        keyIssues: [],
      },
    ];
    const verdict = aggregate(votes, ["01"]);
    expect(verdict.responses["01"].score).toBe(0.0);
  });

  it("TEST-A3: mixed votes → correct percentages", async () => {
    const aggregate = await getAggregate();
    const votes: VoteResult[] = [
      {
        judgeId: "j1",
        responseId: "01",
        verdict: "APPROVE",
        confidence: 8,
        summary: "ok",
        keyIssues: [],
      },
      {
        judgeId: "j2",
        responseId: "01",
        verdict: "APPROVE",
        confidence: 7,
        summary: "ok",
        keyIssues: [],
      },
      {
        judgeId: "j3",
        responseId: "01",
        verdict: "REJECT",
        confidence: 4,
        summary: "no",
        keyIssues: [],
      },
    ];
    const verdict = aggregate(votes, ["01"]);
    // 2 approvals / (2 + 1 rejections) = 2/3
    expect(verdict.responses["01"].score).toBeCloseTo(2 / 3, 5);
    expect(verdict.responses["01"].approvals).toBe(2);
    expect(verdict.responses["01"].rejections).toBe(1);
  });

  it("TEST-A4: all ABSTAIN → score 0 (total=0 branch)", async () => {
    const aggregate = await getAggregate();
    const votes: VoteResult[] = [
      {
        judgeId: "j1",
        responseId: "01",
        verdict: "ABSTAIN",
        confidence: 5,
        summary: "unclear",
        keyIssues: [],
      },
    ];
    const verdict = aggregate(votes, ["01"]);
    expect(verdict.responses["01"].score).toBe(0);
    expect(verdict.responses["01"].abstentions).toBe(1);
  });

  it("TEST-A5: single response works correctly", async () => {
    const aggregate = await getAggregate();
    const votes: VoteResult[] = [
      {
        judgeId: "j1",
        responseId: "99",
        verdict: "APPROVE",
        confidence: 10,
        summary: "great",
        keyIssues: [],
      },
    ];
    const verdict = aggregate(votes, ["99"]);
    expect(verdict.ranking).toEqual(["99"]);
    expect(verdict.responses["99"].score).toBe(1.0);
  });

  it("TEST-A6: ranking is sorted by score descending", async () => {
    const aggregate = await getAggregate();
    const votes: VoteResult[] = [
      // "01" gets 1 approval, 1 rejection → 0.5
      {
        judgeId: "j1",
        responseId: "01",
        verdict: "APPROVE",
        confidence: 7,
        summary: "ok",
        keyIssues: [],
      },
      {
        judgeId: "j2",
        responseId: "01",
        verdict: "REJECT",
        confidence: 4,
        summary: "meh",
        keyIssues: [],
      },
      // "02" gets 2 approvals → 1.0
      {
        judgeId: "j1",
        responseId: "02",
        verdict: "APPROVE",
        confidence: 9,
        summary: "great",
        keyIssues: [],
      },
      {
        judgeId: "j2",
        responseId: "02",
        verdict: "APPROVE",
        confidence: 8,
        summary: "great",
        keyIssues: [],
      },
      // "03" gets 0 approvals, 2 rejections → 0.0
      {
        judgeId: "j1",
        responseId: "03",
        verdict: "REJECT",
        confidence: 2,
        summary: "bad",
        keyIssues: [],
      },
      {
        judgeId: "j2",
        responseId: "03",
        verdict: "REJECT",
        confidence: 1,
        summary: "bad",
        keyIssues: [],
      },
    ];
    const verdict = aggregate(votes, ["01", "02", "03"]);
    expect(verdict.ranking[0]).toBe("02"); // score 1.0
    expect(verdict.ranking[1]).toBe("01"); // score 0.5
    expect(verdict.ranking[2]).toBe("03"); // score 0.0
  });
});

describe("parseJudgeVotes", () => {
  let judgeDir: string;

  beforeEach(() => {
    judgeDir = mkdtempSync(join(tmpdir(), "judge-votes-test-"));
  });

  afterEach(() => {
    if (judgeDir && existsSync(judgeDir)) {
      rmSync(judgeDir, { recursive: true, force: true });
    }
  });

  async function getParser() {
    const { parseJudgeVotes } = await getOrchestrator();
    return parseJudgeVotes;
  }

  function writeResponse(filename: string, content: string) {
    writeFileSync(join(judgeDir, filename), content, "utf-8");
  }

  function makeVoteBlock(
    responseId: string,
    verdict: string,
    confidence = "8",
    summary = "Looks good",
    keyIssues = "None"
  ): string {
    return `\`\`\`vote\nRESPONSE: ${responseId}\nVERDICT: ${verdict}\nCONFIDENCE: ${confidence}\nSUMMARY: ${summary}\nKEY_ISSUES: ${keyIssues}\n\`\`\``;
  }

  it("TEST-P1: valid single vote block → 1 vote parsed correctly", async () => {
    const parse = await getParser();
    writeResponse("response-01.md", makeVoteBlock("r1", "APPROVE", "9", "Excellent work", "None"));

    const votes = parse(judgeDir, ["r1"]);

    expect(votes.length).toBe(1);
    expect(votes[0].judgeId).toBe("01");
    expect(votes[0].responseId).toBe("r1");
    expect(votes[0].verdict).toBe("APPROVE");
    expect(votes[0].confidence).toBe(9);
    expect(votes[0].summary).toBe("Excellent work");
    expect(votes[0].keyIssues).toEqual([]);
  });

  it("TEST-P2: multiple vote blocks in one file → all parsed", async () => {
    const parse = await getParser();
    const content = [
      makeVoteBlock("r1", "APPROVE"),
      makeVoteBlock("r2", "REJECT"),
      makeVoteBlock("r3", "ABSTAIN"),
    ].join("\n\n");
    writeResponse("response-01.md", content);

    const votes = parse(judgeDir, ["r1", "r2", "r3"]);
    expect(votes.length).toBe(3);
  });

  it("TEST-P3: unknown RESPONSE ID → filtered out (not in responseIds)", async () => {
    const parse = await getParser();
    writeResponse("response-01.md", makeVoteBlock("unknown-id", "APPROVE"));

    const votes = parse(judgeDir, ["r1", "r2"]);
    expect(votes.length).toBe(0);
  });

  it("TEST-P4: missing VERDICT field → vote skipped", async () => {
    const parse = await getParser();
    // Manually write a block without VERDICT
    const block = "```vote\nRESPONSE: r1\nCONFIDENCE: 7\nSUMMARY: Fine\nKEY_ISSUES: None\n```";
    writeResponse("response-01.md", block);

    const votes = parse(judgeDir, ["r1"]);
    expect(votes.length).toBe(0);
  });

  it("TEST-P5: non-numeric CONFIDENCE → defaults to 5", async () => {
    const parse = await getParser();
    // Write a block where CONFIDENCE is non-numeric
    const block =
      "```vote\nRESPONSE: r1\nVERDICT: APPROVE\nCONFIDENCE: high\nSUMMARY: Good\nKEY_ISSUES: None\n```";
    writeResponse("response-01.md", block);

    const votes = parse(judgeDir, ["r1"]);
    // CONFIDENCE regex requires \d+ so it won't match "high" → falls back to default "5"
    expect(votes.length).toBe(1);
    expect(votes[0].confidence).toBe(5);
  });

  it("TEST-P6: KEY_ISSUES 'None' → filtered to empty array", async () => {
    const parse = await getParser();
    writeResponse("response-01.md", makeVoteBlock("r1", "APPROVE", "7", "Summary", "None"));

    const votes = parse(judgeDir, ["r1"]);
    expect(votes[0].keyIssues).toEqual([]);
  });

  it("TEST-P7: KEY_ISSUES with multiple items → split correctly", async () => {
    const parse = await getParser();
    writeResponse(
      "response-01.md",
      makeVoteBlock("r1", "REJECT", "3", "Has issues", "bug in loop, off-by-one, missing test")
    );

    const votes = parse(judgeDir, ["r1"]);
    expect(votes[0].keyIssues).toEqual(["bug in loop", "off-by-one", "missing test"]);
  });

  it("TEST-P8: empty file → 0 votes", async () => {
    const parse = await getParser();
    writeResponse("response-01.md", "");

    const votes = parse(judgeDir, ["r1"]);
    expect(votes.length).toBe(0);
  });
});