Skip to content
TypeScript
16.1 KB
Raw Blame
/**
 * 인터랙티브 세션 (REPL) — `theseus` 를 인자 없이 치면 열린다(claude/codex 처럼).
 *
 * 통일된 슬래시 인터페이스: 기능마다 `/<feature>` 하나.
 *   /goal <text>     달성하려는 목표(자유 입력도 목표에 누적)
 *   /models /agents /harness /bench   선택(L0×L1×L2) — 인자 없으면 목록, 있으면 설정
 *   /runs /tasks /runner /seed /judge /score /lang   파라미터
 *   /run             현재 선택으로 비교 실행 → 표 + report 경로
 *   /view            기록된 모든 트레이스를 localhost 갤러리로
 *   /config /help /clear /exit
 *
 * reduce() 는 순수 함수(동기 명령 = 상태 변경/출력). /run·/view·/results·/exit 은
 * 비동기 액션으로 분리해 루프에서 처리 → 테스트 가능.
 */
import * as readline from "node:readline";
import pc from "picocolors";
import { listLocalBenches } from "./benches/index";
import { logoBanner } from "./brand";
import { expand } from "./combos";
import { AGENTS, HARNESSES, MODELS, isKnownModel } from "./config";
import { normalizeLang, translator } from "./i18n";
import { bestByPerspective } from "./metrics";
import type { Lang, RunOptions, ViewMode } from "./models";
import { execute } from "./orchestrator";
import { analyze } from "./preflight";
import { listRuns } from "./recorder";
import { openUrl } from "./viz/open";
import { serveRuns } from "./viz/serve";
import { resultsTableString } from "./viz/table";

export interface SessionState {
  goal: string;
  models: string[];
  agents: string[];
  harnesses: string[];
  bench: string;
  tasks: number;
  runs: number;
  seed: number;
  runner: string;
  judge: string;
  score: boolean;
  concurrency: number;
  timeoutS: number;
  view: ViewMode;
  language: Lang;
  outDir: string;
  port: number;
}

export type Action =
  | { kind: "none" }
  | { kind: "run" }
  | { kind: "view"; open: boolean }
  | { kind: "results" }
  | { kind: "exit" };

export interface Reduced {
  state: SessionState;
  lines: string[];
  action: Action;
}

export function defaultState(language: Lang, outDir: string): SessionState {
  return {
    goal: "",
    models: ["opus4.8"],
    agents: ["cc"],
    harnesses: ["none"],
    bench: "subagent",
    tasks: 6,
    runs: 2,
    seed: 0,
    runner: "auto",
    judge: "none",
    score: false,
    concurrency: 4,
    timeoutS: 600,
    view: "inline",
    language,
    outDir,
    port: 7777,
  };
}

const splitList = (s: string): string[] =>
  s
    .split(/[\s,]+/)
    .map((x) => x.trim())
    .filter(Boolean);

const knownModel = (m: string): boolean => isKnownModel(m) || m.includes("/");

/** L0 모델 목록(family 별), 현재 선택 ● 표시. */
function listModels(state: SessionState): string[] {
  const byFam: Record<string, string[]> = { claude: [], gpt: [], open: [] };
  for (const [alias, def] of Object.entries(MODELS)) byFam[def.family]?.push(alias);
  const sel = new Set(state.models);
  const out = ["available L0 models — /models <a,b,…> to select:"];
  for (const fam of ["claude", "gpt", "open"]) {
    const items = byFam[fam]!.map((a) => (sel.has(a) ? pc.green(`●${a}`) : pc.dim(`○${a}`))).join(
      "  ",
    );
    out.push(`  ${pc.bold(fam)}: ${items}`);
  }
  out.push(pc.dim("  (open = OpenRouter, multi-model agents only; ':free' = $0)"));
  return out;
}

function listOf(title: string, keys: string[], sel: string[], hint: string): string[] {
  const set = new Set(sel);
  const items = keys.map((k) => (set.has(k) ? pc.green(`●${k}`) : pc.dim(`○${k}`))).join("  ");
  return [`${title} — ${hint}:`, `  ${items}`];
}

/** 인자 없으면 목록, 있으면 검증 후 설정. 미지(unknown)는 경고하고 건너뜀. */
function setSelection(
  raw: string,
  current: string[],
  valid: (x: string) => boolean,
  label: string,
): { value: string[]; lines: string[] } {
  const items = splitList(raw);
  const ok = items.filter(valid);
  const bad = items.filter((x) => !valid(x));
  const lines: string[] = [];
  if (bad.length) lines.push(pc.yellow(`  unknown ${label}: ${bad.join(", ")} (ignored)`));
  if (!ok.length) {
    lines.push(pc.red(`  no valid ${label} given — keeping ${current.join(", ") || "—"}`));
    return { value: current, lines };
  }
  lines.push(`  ${label}: ${pc.green(ok.join(", "))}`);
  return { value: ok, lines };
}

function configLines(state: SessionState): string[] {
  const { valid, excluded } = expand(state.models, state.agents, state.harnesses);
  const lines = [
    pc.bold("current plan"),
    `  goal     : ${state.goal ? pc.cyan(state.goal) : pc.dim("(none — type it, or /goal <text>)")}`,
    `  models   : ${state.models.join(", ")}`,
    `  agents   : ${state.agents.join(", ")}`,
    `  harness  : ${state.harnesses.join(", ")}`,
    `  bench    : ${state.bench}   tasks: ${state.tasks}   runs: ${state.runs}   seed: ${state.seed}`,
    `  runner   : ${state.runner}   judge: ${state.judge}   score: ${state.score}   lang: ${state.language}`,
    `  limits   : concurrency ${state.concurrency} · timeout ${state.timeoutS}s`,
    `  combos   : ${pc.green(`${valid.length} valid`)}${
      excluded.length ? pc.yellow(` · ${excluded.length} excluded`) : ""
    }`,
  ];
  for (const c of valid) lines.push(pc.dim(`    ✓ ${c.comboId}`));
  for (const c of excluded) lines.push(pc.dim(`    ✗ ${c.comboId} — ${c.excludeReason}`));
  return lines;
}

const HELP = [
  pc.bold("commands") + pc.dim("  (each feature = one slash; bare command shows current)"),
  `  ${pc.cyan("/goal")} <text>         what you want to achieve (plain text also appends)`,
  `  ${pc.cyan("/models")} <a,b,…>     L0 — select models`,
  `  ${pc.cyan("/agents")} <a,b,…>     L1 — cc,codex,opencode,openclaw,hermes`,
  `  ${pc.cyan("/harness")} <a,b,…>    L2 — none,superpowers,omc,ecc,omx,lazycodex,omo,ponytail,mattpocock`,
  `  ${pc.cyan("/bench")} <name>       ${listLocalBenches().join(",")} | terminal-bench-v2`,
  `  ${pc.cyan("/runs")} ${pc.cyan("/tasks")} ${pc.cyan("/seed")} ${pc.cyan("/concurrency")} <n>  ${pc.cyan("/timeout")} <s>   sweep size & limits`,
  `  ${pc.cyan("/runner")} <name>      auto|mock|cc|codex|opencode|openclaw|hermes|tb`,
  `  ${pc.cyan("/judge")} <name>  ${pc.cyan("/score")} on|off  ${pc.cyan("/lang")} en|kr|ja|zh`,
  `  ${pc.cyan("/run")}                 run the comparison → table + report.html`,
  `  ${pc.cyan("/view")} [open]         serve every recorded trace at localhost`,
  `  ${pc.cyan("/config")} ${pc.cyan("/clear")} ${pc.cyan("/help")} ${pc.cyan("/exit")}`,
];

/** 순수 reducer — 한 줄 입력 → 새 상태 + 출력 + (비동기) 액션. */
export function reduce(prev: SessionState, raw: string): Reduced {
  const line = raw.trim();
  const state = { ...prev };
  const lines: string[] = [];
  const none = (): Reduced => ({ state, lines, action: { kind: "none" } });

  if (!line) return none();

  // 자유 입력 = 목표 누적
  if (!line.startsWith("/")) {
    state.goal = state.goal ? `${state.goal} ${line}` : line;
    lines.push(pc.dim(`  goal ▸ ${state.goal}`));
    return none();
  }

  const [cmd, ...rest] = line.split(/\s+/);
  const arg = rest.join(" ").trim();

  switch (cmd) {
    case "/help":
    case "/?":
      lines.push(...HELP);
      return none();
    case "/config":
    case "/show":
      lines.push(...configLines(state));
      return none();
    case "/goal":
      if (!arg) lines.push(`  goal: ${state.goal ? pc.cyan(state.goal) : pc.dim("(none)")}`);
      else if (arg === "clear") {
        state.goal = "";
        lines.push(pc.dim("  goal cleared"));
      } else {
        state.goal = arg;
        lines.push(`  goal ▸ ${pc.cyan(state.goal)}`);
      }
      return none();
    case "/models": {
      if (!arg) {
        lines.push(...listModels(state));
        return none();
      }
      const r = setSelection(arg, state.models, knownModel, "models");
      state.models = r.value;
      lines.push(...r.lines);
      return none();
    }
    case "/agents": {
      if (!arg) {
        lines.push(...listOf("L1 agents", Object.keys(AGENTS), state.agents, "/agents <a,b,…>"));
        return none();
      }
      const r = setSelection(arg, state.agents, (x) => x in AGENTS, "agents");
      state.agents = r.value;
      lines.push(...r.lines);
      return none();
    }
    case "/harness": {
      if (!arg) {
        lines.push(
          ...listOf("L2 harnesses", Object.keys(HARNESSES), state.harnesses, "/harness <a,b,…>"),
        );
        return none();
      }
      const r = setSelection(arg, state.harnesses, (x) => x in HARNESSES, "harnesses");
      state.harnesses = r.value;
      lines.push(...r.lines);
      return none();
    }
    case "/bench":
      if (!arg) lines.push(`  bench: ${state.bench}  (${listLocalBenches().join(", ")})`);
      else {
        state.bench = arg;
        lines.push(`  bench: ${pc.green(arg)}`);
      }
      return none();
    case "/runs":
    case "/tasks":
    case "/seed":
    case "/concurrency": {
      const key = cmd.slice(1) as "runs" | "tasks" | "seed" | "concurrency";
      const n = Number.parseInt(arg, 10);
      if (Number.isFinite(n) && n >= 0) {
        state[key] = n;
        lines.push(`  ${key}: ${pc.green(String(n))}`);
      } else lines.push(`  ${key}: ${state[key]}`);
      return none();
    }
    case "/timeout": {
      const n = Number.parseInt(arg, 10);
      if (Number.isFinite(n) && n > 0) {
        state.timeoutS = n;
        lines.push(`  timeout: ${pc.green(`${n}s`)}`);
      } else lines.push(`  timeout: ${state.timeoutS}s`);
      return none();
    }
    case "/runner":
      if (arg) {
        state.runner = arg;
        lines.push(`  runner: ${pc.green(arg)}`);
      } else lines.push(`  runner: ${state.runner}`);
      return none();
    case "/judge":
      if (arg) {
        state.judge = arg;
        lines.push(`  judge: ${pc.green(arg)}`);
      } else lines.push(`  judge: ${state.judge}`);
      return none();
    case "/score":
      state.score = arg ? /^(on|true|yes|1)$/i.test(arg) : !state.score;
      lines.push(`  score: ${pc.green(String(state.score))}`);
      return none();
    case "/lang":
      state.language = normalizeLang(arg || state.language);
      lines.push(`  lang: ${pc.green(state.language)}`);
      return none();
    case "/clear": {
      const fresh = defaultState(state.language, state.outDir);
      lines.push(pc.dim("  selection reset to defaults"));
      return { state: fresh, lines, action: { kind: "none" } };
    }
    case "/run":
    case "/go":
      return { state, lines, action: { kind: "run" } };
    case "/view":
    case "/visualize":
      return { state, lines, action: { kind: "view", open: /open/i.test(arg) } };
    case "/results":
      return { state, lines, action: { kind: "results" } };
    case "/exit":
    case "/quit":
    case "/q":
      return { state, lines, action: { kind: "exit" } };
    default:
      lines.push(pc.yellow(`  unknown command ${cmd} — /help`));
      return none();
  }
}

// ── 비동기 액션 ──────────────────────────────────────────
function toRunOptions(s: SessionState): RunOptions {
  return {
    models: s.models,
    agents: s.agents,
    harnesses: s.harnesses.length ? s.harnesses : ["none"],
    bench: s.bench,
    tasks: s.tasks,
    runs: s.runs,
    seed: s.seed,
    runner: s.runner,
    concurrency: s.concurrency,
    timeoutS: s.timeoutS,
    outDir: s.outDir,
    judge: s.judge,
    score: s.score,
    balanced: false,
    language: s.language,
    view: "inline",
    port: s.port,
    goal: s.goal || undefined,
  };
}

async function doRun(state: SessionState): Promise<void> {
  const t = translator(state.language);
  if (!state.models.length || !state.agents.length) {
    console.log(pc.red("  set /models and /agents first"));
    return;
  }
  const { valid, excluded } = expand(state.models, state.agents, state.harnesses);
  if (!valid.length) {
    console.log(pc.red("  no valid combos — check /config"));
    return;
  }
  const pf = analyze(state.models, state.agents, state.harnesses, valid, excluded, {
    bench: state.bench,
    runner: state.runner,
    tasks: state.tasks,
    runs: state.runs,
    yes: false,
  });
  if (pf.gateBlocked) {
    console.log(pc.red(`  ${t("gate_blocked")} — try /runner mock`));
    return;
  }
  if (state.goal) console.log(pc.dim(`  goal ▸ ${state.goal}`));
  console.log(pc.dim(`  running ${pf.scale} …`));
  const res = await execute(toRunOptions(state));
  if (res.mock) console.log(pc.yellow(`  ${t("mock_banner")}`));
  console.log(resultsTableString(res.metrics, state.language));
  for (const [label, combo] of Object.entries(bestByPerspective(res.metrics, state.language)))
    console.log(`  ${pc.green("▸")} ${label}: ${combo}`);
  console.log(`  ${t("report_path")}: ${pc.bold(res.reportPath)}`);
  console.log(pc.dim("  /view to open the gallery · /run again after tweaks"));
}

interface ServeRef {
  handle: { url: string; close: () => void } | null;
}

async function doView(state: SessionState, open: boolean, srv: ServeRef): Promise<void> {
  const runs = listRuns(state.outDir);
  if (!runs.length) {
    console.log(pc.yellow(`  no runs yet in ${state.outDir}/ — /run first`));
    return;
  }
  if (!srv.handle) srv.handle = await serveRuns(state.outDir, state.port);
  console.log(
    `  ${pc.green("⛵ view")} — ${runs.length} run(s) at ${pc.bold(srv.handle.url)} ${pc.dim("(stays up; closes on /exit)")}`,
  );
  if (open) openUrl(srv.handle.url);
}

// ── REPL 루프 ────────────────────────────────────────────
export async function runSession(init: { language: Lang; outDir: string }): Promise<void> {
  let state = defaultState(init.language, init.outDir);
  const srv: ServeRef = { handle: null };

  console.log(logoBanner(state.language));
  console.log(
    pc.dim("  interactive session — describe your goal, then /run. /help for commands.\n"),
  );

  const rl = readline.createInterface({ input: process.stdin, output: process.stdout });

  // 라인 큐 — TTY 와 파이프(다중 라인) 모두에서 안정적으로 한 줄씩 소비.
  const queue: string[] = [];
  let waiter: ((v: string | null) => void) | null = null;
  let closed = false;
  rl.on("line", (l) => {
    if (waiter) {
      const w = waiter;
      waiter = null;
      w(l);
    } else queue.push(l);
  });
  rl.on("close", () => {
    closed = true;
    if (waiter) {
      const w = waiter;
      waiter = null;
      w(null);
    }
  });
  const next = (): Promise<string | null> =>
    queue.length
      ? Promise.resolve(queue.shift()!)
      : closed
        ? Promise.resolve(null)
        : new Promise((res) => {
            waiter = res;
          });

  rl.on("SIGINT", () => {
    srv.handle?.close();
    console.log(pc.dim("\n  bye ⛵"));
    process.exit(0);
  });

  const prompt = (): string =>
    `${pc.cyan("theseus")}${pc.dim(` [${state.models.length}m×${state.agents.length}a×${state.harnesses.length}h]`)} ▸ `;

  while (true) {
    rl.setPrompt(prompt());
    rl.prompt();
    const raw = await next();
    if (raw === null) break; // stream closed
    const r = reduce(state, raw);
    state = r.state;
    for (const l of r.lines) console.log(l);
    const a = r.action;
    if (a.kind === "exit") break;
    // REPL 은 어떤 명령이 실패해도 살아남는다 — 에러는 출력하고 프롬프트로 복귀.
    try {
      if (a.kind === "run") await doRun(state);
      else if (a.kind === "view") await doView(state, a.open, srv);
      else if (a.kind === "results") {
        const runs = listRuns(state.outDir);
        if (!runs.length) console.log(pc.yellow(`  no runs in ${state.outDir}/`));
        // 최근 run 은 report 로 — 여기서는 안내만(상세 표는 /run 직후 출력)
        else console.log(pc.dim(`  ${runs.length} run(s); use /view for the gallery`));
      }
    } catch (e) {
      console.log(pc.red(`  ${a.kind} failed: ${(e as Error).message}`));
      if (a.kind === "view" && /EADDRINUSE/.test(String(e))) {
        console.log(pc.dim(`  port ${state.port} is busy — the gallery may already be running`));
      }
    }
  }

  srv.handle?.close();
  rl.close();
  console.log(pc.dim("  bye ⛵"));
}