Skip to content
TypeScript
11.6 KB
Raw Blame
/**
 * 오프라인 결정적 합성 러너 (PRD §7, §22.4).
 * (comboId, taskId, runIdx, seed) 해시로 토큰/시간/서브에이전트 수 생성. Math.random 금지.
 * 하네스 경향 반영. codex/gpt 는 서브에이전트 0(단일 루프).
 * 로컬-스타일 벤치는 실제 task id + setup 훅 존중(verify 는 호출 안 하고 합성 rawResolved 사용).
 */
import { createHash } from "node:crypto";
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { isLocalBench, setupTask, taskIds } from "../benches/index";
import { parseComboId } from "../combos";
import { modelFamily } from "../config";
import type { RawRunArtifact, RunInvocation } from "../models";
import { type RunnerAdapter, RunnerUnavailable } from "./base";

class Det {
  private buf: string;
  private counter = 0;
  constructor(...material: (string | number)[]) {
    this.buf = material.join(":");
  }
  private hash(): bigint {
    const h = createHash("sha256")
      .update(`${this.buf}:${this.counter++}`)
      .digest("hex")
      .slice(0, 12);
    return BigInt(`0x${h}`);
  }
  randint(lo: number, hi: number): number {
    if (hi <= lo) return lo;
    const span = BigInt(hi - lo + 1);
    return lo + Number(this.hash() % span);
  }
  chance(p: number): boolean {
    return this.randint(0, 9999) / 10000 < p;
  }
}

const HARNESS_SUBAGENTS: Record<string, [number, number]> = {
  none: [0, 0],
  superpowers: [1, 3],
  omc: [2, 5],
  omx: [2, 5],
  ecc: [0, 1],
  lazycodex: [0, 1],
  omo: [2, 5],
  mattpocock: [1, 3],
  ponytail: [0, 1],
};

// 미니멀리즘 하네스는 토큰/도구를 줄이는 경향(결정적 할인 계수)
const HARNESS_TOKEN_SCALE: Record<string, number> = {
  ponytail: 0.5,
};

/**
 * fan-out 가능 여부: 트리형 에이전트는 cc(모든 하네스), 그리고 opencode+omo(오케스트레이션).
 * codex/openclaw/hermes 및 omo 외 opencode 는 단일 루프(0).
 */
function mockFanout(agent: string, harness: string, det: Det): number {
  if (agent === "cc") {
    const [lo, hi] = HARNESS_SUBAGENTS[harness] ?? [0, 0];
    return det.randint(lo, hi);
  }
  if (agent === "opencode" && harness === "omo") {
    const [lo, hi] = HARNESS_SUBAGENTS.omo ?? [0, 0];
    return det.randint(lo, hi);
  }
  return 0;
}

const DIFFICULTY_SCALE: Record<string, number> = {
  "subagent-": 1.8,
  "local-": 1.0,
};
function scaleFor(taskId: string): number {
  for (const [pre, s] of Object.entries(DIFFICULTY_SCALE)) if (taskId.startsWith(pre)) return s;
  return 1.0;
}

export class MockRunner implements RunnerAdapter {
  name = "mock";
  constructor(private failRate = 0) {}

  isAvailable(): boolean {
    return true;
  }

  loadTaskCatalog(bench: string): string[] {
    if (isLocalBench(bench)) return taskIds(bench);
    // 외부 벤치도 합성 카탈로그(tb mock 폴백)
    if (bench === "terminal-bench-v2")
      return Array.from({ length: 40 }, (_, i) => `tb-${String(i + 1).padStart(3, "0")}`);
    throw new RunnerUnavailable(`mock: unknown bench '${bench}'`);
  }

  async run(inv: RunInvocation): Promise<RawRunArtifact> {
    const { agent, model, harness } = parseComboId(inv.combo.comboId);
    const work = inv.workDir;
    mkdirSync(work, { recursive: true });
    const resultPath = join(work, "output.txt");
    const transcriptPath = join(work, "transcript.jsonl");

    // 로컬-스타일 벤치는 setup 픽스처 생성(러너 3개 공통)
    if (isLocalBench(inv.bench)) setupTask(inv.taskId, work);

    const det = new Det(inv.combo.comboId, inv.taskId, inv.runIdx, inv.seed);
    const scale = scaleFor(inv.taskId);

    // 실행 성공 여부(fail-soft 경로 검증용)
    const failed = det.chance(this.failRate);

    // 서브에이전트 수 — 하네스 오케스트레이션 × 에이전트 트리 능력으로 결정.
    const fam = modelFamily(model);
    const subagentCount = mockFanout(agent, harness, det);

    // 토큰 (ponytail 등 미니멀리즘 하네스는 할인)
    const hScale = scale * (HARNESS_TOKEN_SCALE[harness] ?? 1);
    const inputTokens = Math.round(det.randint(8000, 30000) * hScale);
    const outputTokens = Math.round(det.randint(1500, 6000) * hScale);
    const cacheRead = Math.round(det.randint(20000, 120000) * hScale);
    const cacheCreate = Math.round(det.randint(2000, 15000) * hScale);
    const wallTimeMs = det.randint(3000, 90000);

    const rawResolved = failed ? false : det.chance(0.7 - (scale - 1) * 0.15);

    // transcript 합성 — cc 는 sidechain 트리, 그 외(codex/opencode/openclaw/hermes)는 평탄.
    // 평탄 포맷에도 subagentCount 만큼 task 아이템을 실어 DefensiveParser 가 fan-out 을 집계.
    const lines: string[] =
      agent === "cc"
        ? this.ccTranscript(
            det,
            model,
            harness,
            subagentCount,
            inputTokens,
            outputTokens,
            cacheRead,
            cacheCreate,
          )
        : this.codexTranscript(
            det,
            model,
            inputTokens,
            outputTokens,
            cacheRead,
            cacheCreate,
            subagentCount,
          );

    writeFileSync(transcriptPath, lines.join("\n"));
    writeFileSync(
      resultPath,
      failed
        ? `mock failure for ${inv.taskId}`
        : `mock result for ${inv.taskId} (${subagentCount} subagents)\nDone.`,
    );

    // claude family 는 실측 비용 시뮬레이션(actual), gpt 는 null(=> estimated/none)
    const totalCostUsd = fam === "claude" ? Math.round((inputTokens / 1e6) * 15 * 100) / 100 : null;

    return {
      invocation: inv,
      success: !failed,
      resultPath,
      transcriptPath,
      wallTimeMs,
      rawResolved,
      totalCostUsd,
      numTurns: det.randint(2, 12),
      error: failed ? "mock: synthetic failure" : null,
    };
  }

  private ccTranscript(
    det: Det,
    model: string,
    harness: string,
    subagents: number,
    inT: number,
    outT: number,
    cr: number,
    cc: number,
  ): string[] {
    const lines: string[] = [];
    let uuid = 0;
    const nextId = () => `m${uuid++}`;
    const tools = ["Read", "Bash", "Grep", "Edit", "Glob"];
    let ts = 1_000;
    const stamp = () => {
      ts += det.randint(100, 2000);
      return new Date(ts).toISOString();
    };

    // 메인 루프 첫 assistant
    const rootId = nextId();
    lines.push(
      JSON.stringify({
        type: "assistant",
        uuid: rootId,
        parentUuid: null,
        isSidechain: false,
        timestamp: stamp(),
        message: {
          role: "assistant",
          model,
          usage: {
            input_tokens: Math.round(inT * 0.5),
            output_tokens: Math.round(outT * 0.5),
            cache_read_input_tokens: Math.round(cr * 0.5),
            cache_creation_input_tokens: Math.round(cc * 0.5),
          },
          content: [{ type: "text", text: "Working on the task." }],
        },
      }),
    );

    // 메인 루프 tool_use 몇 개
    let prev = rootId;
    for (let i = 0; i < det.randint(2, 6); i++) {
      const id = nextId();
      const tool = tools[det.randint(0, tools.length - 1)]!;
      lines.push(
        JSON.stringify({
          type: "assistant",
          uuid: id,
          parentUuid: prev,
          isSidechain: false,
          timestamp: stamp(),
          message: {
            role: "assistant",
            model,
            content: [{ type: "tool_use", id: `t${id}`, name: tool, input: { path: "x" } }],
            usage: { input_tokens: 0, output_tokens: det.randint(50, 300) },
          },
        }),
      );
      prev = id;
    }

    // 스킬 사용 (하네스 있을 때)
    if (harness !== "none") {
      const id = nextId();
      lines.push(
        JSON.stringify({
          type: "assistant",
          uuid: id,
          parentUuid: prev,
          isSidechain: false,
          timestamp: stamp(),
          message: {
            role: "assistant",
            model,
            content: [
              { type: "tool_use", id: `t${id}`, name: "Skill", input: { command: "planning" } },
            ],
            usage: { input_tokens: 0, output_tokens: det.randint(20, 100) },
          },
        }),
      );
      prev = id;
    }

    // 서브에이전트 (Task tool_use + sidechain 대화)
    for (let s = 0; s < subagents; s++) {
      const taskId = nextId();
      const taskUuid = taskId;
      lines.push(
        JSON.stringify({
          type: "assistant",
          uuid: taskUuid,
          parentUuid: rootId,
          isSidechain: false,
          timestamp: stamp(),
          message: {
            role: "assistant",
            model,
            content: [
              { type: "tool_use", id: `task${s}`, name: "Task", input: { subagent_type: harness } },
            ],
            usage: { input_tokens: 0, output_tokens: det.randint(50, 200) },
          },
        }),
      );
      // sidechain assistant 라인 (서브에이전트 내부)
      const subTurns = det.randint(1, 3);
      let subPrev = taskUuid;
      for (let k = 0; k < subTurns; k++) {
        const sid = nextId();
        lines.push(
          JSON.stringify({
            type: "assistant",
            uuid: sid,
            parentUuid: subPrev,
            isSidechain: true,
            timestamp: stamp(),
            message: {
              role: "assistant",
              model,
              content: [
                { type: "text", text: `subagent ${s} step ${k}` },
                { type: "tool_use", id: `st${s}_${k}`, name: tools[det.randint(0, 4)]!, input: {} },
              ],
              usage: {
                input_tokens: Math.round((inT * 0.5) / Math.max(1, subagents * subTurns)),
                output_tokens: Math.round((outT * 0.5) / Math.max(1, subagents * subTurns)),
                cache_read_input_tokens: Math.round((cr * 0.5) / Math.max(1, subagents * subTurns)),
                cache_creation_input_tokens: Math.round((cc * 0.5) / Math.max(1, subagents)),
              },
            },
          }),
        );
        subPrev = sid;
      }
    }

    // result 라인
    lines.push(
      JSON.stringify({
        type: "result",
        uuid: nextId(),
        parentUuid: prev,
        isSidechain: false,
        timestamp: stamp(),
        session_id: `sess-${det.randint(1000, 9999)}`,
        message: { role: "assistant", content: [{ type: "text", text: "Done." }] },
      }),
    );
    return lines;
  }

  private codexTranscript(
    det: Det,
    model: string,
    inT: number,
    outT: number,
    cr: number,
    cc: number,
    subagents = 0,
  ): string[] {
    const lines: string[] = [];
    lines.push(
      JSON.stringify({ type: "thread.started", thread_id: `th-${det.randint(1000, 9999)}` }),
    );
    // 평탄 포맷에서도 fan-out 을 표현: subagents 개의 task 아이템(DefensiveParser 가 집계).
    for (let s = 0; s < subagents; s++) {
      lines.push(
        JSON.stringify({
          type: "item.completed",
          item: { type: "task", name: "Task", command: `subagent ${s}` },
        }),
      );
    }
    const turns = det.randint(2, 8);
    for (let i = 0; i < turns; i++) {
      lines.push(JSON.stringify({ type: "turn.started" }));
      const nItems = det.randint(1, 4);
      for (let k = 0; k < nItems; k++) {
        lines.push(
          JSON.stringify({
            type: "item.completed",
            item: {
              type: det.chance(0.5) ? "command_execution" : "file_change",
              command: "ls -la",
            },
          }),
        );
      }
    }
    lines.push(
      JSON.stringify({
        type: "turn.completed",
        usage: {
          input_tokens: inT,
          output_tokens: outT,
          cached_input_tokens: cr,
          cache_creation_input_tokens: cc,
        },
      }),
    );
    return lines;
  }
}