Skip to content
TypeScript
14.5 KB
Raw Blame
import Table from "cli-table3";
/**
 * CLI 엔트리 (PRD §17) — commander. compare(기본)/results/install/doctor/tui.
 * 멀티셀렉트는 콤마 split, 미선택=skip. 서브커맨드 없이 플래그만 주면 compare 로 동작.
 */
import { Command } from "commander";
import pc from "picocolors";
import { listLocalBenches } from "./benches/index";
import { logoBanner } from "./brand";
import { expand } from "./combos";
import { harnessStatus } from "./harness";
import { type Translator, normalizeLang, translator } from "./i18n";
import { runInstall } from "./install";
import { aggregateAll } from "./metrics";
import { bestByPerspective } from "./metrics";
import type { Lang, RunOptions } from "./models";
import { execute } from "./orchestrator";
import { analyze } from "./preflight";
import { listRuns, loadMeta, loadResults } from "./recorder";
import { getRunner } from "./runners/registry";
import { commandExists } from "./runners/util";
import { runSession } from "./session";
import { runTui } from "./tui";
import { verifyCombos } from "./verify";
import { openInBrowser, openUrl } from "./viz/open";
import { serveRuns } from "./viz/serve";
import { resultsTableString } from "./viz/table";
import { renderTerminal } from "./viz/terminal";

function splitList(s: string | undefined): string[] {
  if (!s) return [];
  return s
    .split(",")
    .map((x) => x.trim())
    .filter(Boolean);
}

function toRunOptions(o: any): RunOptions {
  return {
    models: splitList(o.model),
    agents: splitList(o.agent),
    harnesses: splitList(o.harness).length ? splitList(o.harness) : ["none"],
    bench: o.bench ?? "terminal-bench-v2",
    tasks: Number.parseInt(o.tasks ?? "20", 10),
    runs: Number.parseInt(o.runs ?? "1", 10),
    seed: Number.parseInt(o.seed ?? "0", 10),
    runner: o.runner ?? "auto",
    concurrency: Number.parseInt(o.concurrency ?? "4", 10),
    timeoutS: Number.parseInt(o.timeout ?? "600", 10),
    outDir: o.out ?? "runs",
    judge: o.judge ?? "none",
    score: Boolean(o.score),
    balanced: Boolean(o.balanced),
    language: normalizeLang(o.language),
    view: normalizeView(o.view),
    port: Number.parseInt(o.port ?? "7777", 10),
    goal: o.goal || undefined,
  };
}

function normalizeView(v: string | undefined): "inline" | "open" | "serve" | "term" {
  const s = (v ?? "inline").toLowerCase();
  if (s === "open" || s === "serve" || s === "term") return s;
  return "inline";
}

function addCompareOptions(cmd: Command): Command {
  return cmd
    .option("--model <list>", "L0 models (comma-separated)")
    .option("--agent <list>", "L1 agents: cc,codex,opencode,openclaw,hermes")
    .option(
      "--harness <list>",
      "L2 harnesses: none,superpowers,omc,ecc,omx,lazycodex,omo,ponytail,mattpocock",
      "none",
    )
    .option("--bench <name>", "benchmark", "terminal-bench-v2")
    .option("--tasks <n>", "task subset size", "20")
    .option("--runs <n>", "runs per task", "1")
    .option("--seed <n>", "subset seed (deterministic)", "0")
    .option("--runner <name>", "auto|mock|cc|codex|opencode|openclaw|hermes|tb", "auto")
    .option("--concurrency <n>", "parallel executions", "4")
    .option("--timeout <s>", "per-task timeout seconds", "600")
    .option("--out <dir>", "output dir", "runs")
    .option("--goal <text>", "what you want to achieve (recorded in meta + shown on the report)")
    .option("--judge <name>", "none|heuristic|cc", "none")
    .option("--score", "compute composite score (opt-in)", false)
    .option("--balanced", "difficulty-balanced subset", false)
    .option("--dry-run", "plan only, do not execute", false)
    .option("-y, --yes", "bypass real-runner gate", false)
    .option("--view <mode>", "visualization: inline|open|serve|term", "inline")
    .option("--port <n>", "port for --view serve", "7777")
    .option("--no-logo", "suppress the ASCII logo banner")
    .option("-L, --language <lang>", "en|kr|ja|zh", "kr");
}

function printPreflight(t: Translator, lang: Lang, opts: RunOptions, yes: boolean): boolean {
  const { valid, excluded } = expand(opts.models, opts.agents, opts.harnesses);
  const pf = analyze(opts.models, opts.agents, opts.harnesses, valid, excluded, {
    bench: opts.bench,
    runner: opts.runner,
    tasks: opts.tasks,
    runs: opts.runs,
    yes,
  });

  console.log(pc.bold(`\n${t("preflight_title")}`));
  console.log(`  ${t("valid_combos")}: ${valid.map((c) => c.comboId).join(", ") || "—"}`);
  if (excluded.length) {
    console.log(pc.yellow(`  ${t("excluded_combos")}:`));
    for (const c of excluded) console.log(`    - ${c.comboId}: ${c.excludeReason}`);
  }
  console.log(`  ${t("runner_label")}: ${opts.runner}`);
  console.log(`  ${t("scale")}: ${pf.scale}`);
  if (pf.warnings.length) {
    console.log(pc.yellow(`  ${t("warnings")}:`));
    for (const w of pf.warnings) console.log(`    ! ${w}`);
  }
  if (pf.gateBlocked) {
    console.log(pc.red(`\n${t("gate_blocked")}`));
    return false;
  }
  return true;
}

function printResultsTable(
  t: Translator,
  lang: Lang,
  results: ReturnType<typeof aggregateAll>,
): void {
  console.log(`\n${pc.bold(t("results_table"))}`);
  console.log(resultsTableString(results, lang));

  const best = bestByPerspective(results, lang);
  for (const [label, combo] of Object.entries(best))
    console.log(`  ${pc.green("▸")} ${label}: ${combo}`);
}

async function compareAction(o: any): Promise<void> {
  const opts = toRunOptions(o);
  const lang = opts.language;
  const t = translator(lang);
  if (o.logo !== false) console.log(logoBanner(lang));
  else console.log(pc.bold(pc.cyan(t("app_tagline"))));

  if (!opts.models.length || !opts.agents.length) {
    console.log(pc.red(`\n${t("no_models")}`));
    process.exitCode = 1;
    return;
  }

  const proceed = printPreflight(t, lang, opts, Boolean(o.yes));
  if (!proceed) {
    process.exitCode = 1;
    return;
  }
  if (o.dryRun) {
    console.log(pc.dim(`\n${t("dry_run_only")}`));
    return;
  }

  const res = await execute(opts);
  if (res.mock) console.log(pc.yellow(`\n${t("mock_banner")}`));

  // --view term: ANSI 차트 + 흐름 트리(브라우저 불필요)
  if (opts.view === "term") {
    console.log(renderTerminal(res.metrics, res.graphs, lang));
  } else {
    printResultsTable(t, lang, res.metrics);
  }

  console.log(`\n${t("run_complete")} — ${t("run_id")}: ${pc.bold(res.runId)}`);
  console.log(`  ${t("report_path")}: ${res.reportPath}`);
  console.log(`  ${t("graph_path")}: ${res.graphPath}`);

  // --view open: 브라우저 팝업
  if (opts.view === "open") {
    openInBrowser(res.reportPath);
    console.log(pc.dim(`  ${t("opening_browser")}`));
  }
  // --view serve: 로컬 서버(Ctrl-C 까지 유지)
  if (opts.view === "serve") {
    try {
      const handle = await serveRuns(opts.outDir, opts.port ?? 7777);
      console.log(
        `\n  ${pc.green("serving")} ${pc.bold(handle.url)}  ${pc.dim("(Ctrl-C to stop)")}`,
      );
      await new Promise<void>((resolve) => {
        process.on("SIGINT", () => {
          handle.close();
          resolve();
        });
      });
    } catch (e) {
      console.log(pc.red(`serve failed: ${(e as Error).message}`));
    }
  }
}

function buildProgram(): Command {
  const program = new Command();
  program
    .name("theseus")
    .description("compare model × agent × harness on your own tasks")
    .version("0.1.0");

  // compare = 기본 커맨드(isDefault). 서브커맨드 없이 플래그만 주면 compare 로 동작.
  // 루트에 옵션을 달지 않아 서브커맨드(tui/results)의 -L 등과 충돌하지 않음.
  addCompareOptions(
    program.command("compare", { isDefault: true }).description("run an N-combo comparison"),
  ).action(compareAction);

  program
    .command("session", { isDefault: false })
    .aliases(["chat", "repl"])
    .description("open an interactive session — slash commands to configure & run")
    .option("--out <dir>", "output dir", "runs")
    .option("-L, --language <lang>", "en|kr|ja|zh", "kr")
    .action((o) => runSession({ language: normalizeLang(o.language), outDir: o.out ?? "runs" }));

  program
    .command("results")
    .description("show recorded results")
    .option("--run <id>", "run id (default: latest)")
    .option("--out <dir>", "output dir", "runs")
    .option("-L, --language <lang>", "en|kr|ja|zh", "kr")
    .action((o) => {
      const lang = normalizeLang(o.language);
      const t = translator(lang);
      const runs = listRuns(o.out);
      if (!runs.length) {
        console.log(pc.yellow("no runs found"));
        return;
      }
      const runId = o.run ?? runs[0]!;
      const meta = loadMeta(o.out, runId);
      const results = loadResults(o.out, runId);
      console.log(pc.bold(`${t("run_id")}: ${runId}  (${meta?.bench ?? "?"})`));
      printResultsTable(t, lang, aggregateAll(results));
      console.log(pc.dim(`\nall runs: ${runs.join(", ")}`));
    });

  program
    .command("install [component]")
    .description("install agent/harness/bench (cc|codex|opencode|openclaw|hermes|tb|<harness>|all)")
    .option("--check", "show install status for all agents/harnesses", false)
    .action((component, o) => runInstall(o.check ? "--check" : (component ?? "")));

  program
    .command("doctor")
    .description("self-diagnose the execution environment")
    .action(() => {
      const table = new Table({ head: ["component", "status"] });
      table.push([
        "claude (cc)",
        commandExists("claude") ? pc.green("available") : pc.red("missing"),
      ]);
      table.push(["codex", commandExists("codex") ? pc.green("available") : pc.red("missing")]);
      for (const a of ["opencode", "openclaw", "hermes"])
        table.push([a, commandExists(a) ? pc.green("available") : pc.red("missing")]);
      table.push(["tb", commandExists("tb") ? pc.green("available") : pc.red("missing")]);
      table.push(["docker", commandExists("docker") ? pc.green("available") : pc.red("missing")]);
      table.push(["mock runner", pc.green("always available")]);
      for (const h of [
        "superpowers",
        "omc",
        "ecc",
        "omx",
        "lazycodex",
        "omo",
        "ponytail",
        "mattpocock",
      ])
        table.push([`harness:${h}`, harnessStatus(h)]);
      table.push(["local benches", listLocalBenches().join(", ")]);
      console.log(table.toString());
      // 러너 인스턴스화 확인(가용성)
      console.log(
        pc.dim(
          `runners: ${["mock", "cc", "codex", "opencode", "openclaw", "hermes", "tb"]
            .map((n) => `${n}=${getRunner(n).isAvailable()}`)
            .join(" ")}`,
        ),
      );
    });

  program
    .command("tui")
    .description("TUI wrapper (best-effort)")
    .option("--out <dir>", "output dir", "runs")
    .option("--rich", "force cli-table3 fallback", false)
    .option("--open", "open report.html in browser", false)
    .option("-L, --language <lang>", "en|kr|ja|zh", "kr")
    .action((o) =>
      runTui({
        out: o.out,
        rich: Boolean(o.rich),
        open: Boolean(o.open),
        language: normalizeLang(o.language),
      }),
    );

  program
    .command("view")
    .description("visualize every recorded trace — serve all runs at localhost")
    .option("--out <dir>", "output dir", "runs")
    .option("--port <n>", "port", "7777")
    .option("--open", "open the gallery in your browser", false)
    .action(async (o) => {
      const out = o.out ?? "runs";
      const runs = listRuns(out);
      if (!runs.length) {
        console.log(pc.yellow(`no runs found in ${out}/ — run a comparison first`));
        return;
      }
      try {
        const handle = await serveRuns(out, Number.parseInt(o.port ?? "7777", 10));
        console.log(
          `${pc.green("⛵ theseus view")} — ${runs.length} run(s) at ${pc.bold(handle.url)}  ${pc.dim("(Ctrl-C to stop)")}`,
        );
        if (o.open) openUrl(handle.url);
        await new Promise<void>((resolve) => {
          process.on("SIGINT", () => {
            handle.close();
            resolve();
          });
        });
      } catch (e) {
        console.log(pc.red(`view failed: ${(e as Error).message}`));
      }
    });

  program
    .command("verify")
    .description("check that each combo's subagent calls are actually captured")
    .option("--model <list>", "L0 models")
    .option("--agent <list>", "L1 agents")
    .option("--harness <list>", "L2 harnesses", "none")
    .option("--runner <name>", "auto|mock|cc|codex|opencode|openclaw|hermes", "mock")
    .option("--seed <n>", "seed", "0")
    .option("--timeout <s>", "per-task timeout seconds", "120")
    .option("-y, --yes", "allow real runner (not mock)", false)
    .option("-L, --language <lang>", "en|kr|ja|zh", "kr")
    .action(async (o) => {
      const lang = normalizeLang(o.language);
      const t = translator(lang);
      const models = splitList(o.model);
      const agents = splitList(o.agent);
      if (!models.length || !agents.length) {
        console.log(pc.red(t("no_models")));
        process.exitCode = 1;
        return;
      }
      const runner = o.runner ?? "mock";
      if (runner !== "mock" && !o.yes) {
        console.log(pc.red(t("gate_blocked")));
        process.exitCode = 1;
        return;
      }
      const rows = await verifyCombos({
        models,
        agents,
        harnesses: splitList(o.harness).length ? splitList(o.harness) : ["none"],
        runner,
        seed: Number.parseInt(o.seed ?? "0", 10),
        timeoutS: Number.parseInt(o.timeout ?? "120", 10),
      });
      const table = new Table({ head: ["combo", "subagents", "maxDepth", t("verify_captured")] });
      for (const r of rows) {
        const mark =
          r.status === "captured"
            ? pc.green("✓ captured")
            : r.status === "flat"
              ? pc.dim("— flat (expected)")
              : pc.red("✗ none (gap)");
        table.push([r.comboId, String(r.subagents), String(r.maxDepth), mark]);
      }
      console.log(table.toString());
      const gaps = rows.filter((r) => r.status === "gap").length;
      if (gaps) console.log(pc.red(`\n${gaps} combo(s) expected subagents but captured none.`));
      else console.log(pc.green("\nall combos captured subagent flow as expected."));
    });

  return program;
}

export async function main(argv = process.argv): Promise<void> {
  // 인자 없이 `theseus` → 인터랙티브 세션(claude/codex 처럼). 비대화형(파이프/CI)은 기존 도움말.
  if (argv.slice(2).length === 0 && process.stdin.isTTY) {
    await runSession({ language: "kr", outDir: "runs" });
    return;
  }
  await buildProgram().parseAsync(argv);
}

// 직접 실행 시
main().catch((e) => {
  console.error(pc.red(`error: ${(e as Error).message}`));
  process.exitCode = 1;
});