Skip to content
TypeScript
2.5 KB
Raw Blame
import { isLocalBench, taskIds } from "./benches/index";
import { isKnownModel, modelFamily } from "./config";
import { codexHomeStatus, harnessStatus } from "./harness";
/**
 * 사전점검 & 게이트 (PRD §15).
 * 유효/제외 조합 + 휴리스틱 경고 + 실행 규모 + real-run 게이트.
 */
import type { Combo } from "./models";

export interface PreflightInput {
  bench: string;
  runner: string;
  tasks: number;
  runs: number;
}

export interface PreflightResult {
  valid: Combo[];
  excluded: Combo[];
  warnings: string[];
  scale: number; // combos × tasks × runs
  isReal: boolean;
  gateBlocked: boolean; // real & scale>30 & !yes
}

const REAL_GATE = 30;

export function analyze(
  _models: string[],
  _agents: string[],
  _harnesses: string[],
  valid: Combo[],
  excluded: Combo[],
  opts: PreflightInput & { yes?: boolean },
): PreflightResult {
  const warnings: string[] = [];

  // 미등록 모델 alias
  for (const c of valid) {
    if (!isKnownModel(c.model))
      warnings.push(`unregistered model alias '${c.model}' (combo ${c.comboId})`);
  }

  // codex + 명시 gpt 모델(ChatGPT 계정 거부 가능)
  for (const c of valid) {
    if (c.agent === "codex" && modelFamily(c.model) === "gpt" && c.model !== "gpt5-codex") {
      warnings.push(
        `codex with explicit model '${c.model}': ChatGPT account may reject --model override; consider gpt5-codex`,
      );
    }
  }

  // 미설치 하네스 → base
  const harnessesSeen = new Set(valid.map((c) => c.harness));
  for (const h of harnessesSeen) {
    if (h === "none") continue;
    if (harnessStatus(h) === "unavailable")
      warnings.push(`harness '${h}' not installed → falls back to base (none)`);
    if (codexHomeStatus(h) === "pending")
      warnings.push(`codex harness '${h}' isolated CODEX_HOME not built (first run ~20s)`);
  }

  const isReal = opts.runner !== "mock";
  // 실제 실행 규모 = 요청 task 수가 아니라 카탈로그로 클램프된 subset 크기(게이트 과대평가 방지).
  let effTasks = Math.max(0, opts.tasks);
  if (isLocalBench(opts.bench)) {
    try {
      effTasks = Math.min(effTasks, taskIds(opts.bench).length);
    } catch {
      // 카탈로그 로드 실패 → 요청값 유지(경고는 실행 시점에)
    }
  }
  if (effTasks === 0) warnings.push("0 tasks selected — the run would be empty");
  const scale = valid.length * effTasks * Math.max(1, opts.runs);
  const gateBlocked = isReal && scale > REAL_GATE && !opts.yes;

  return { valid, excluded, warnings, scale, isReal, gateBlocked };
}