Skip to content
TypeScript
9.9 KB
Raw Blame
import { tr } from "./i18n";
/**
 * 메트릭 (PRD §13, §22.1).
 * 정렬 기본키 = subagentCount → totalTokens → wallTimeMs (한 곳: sortCombos).
 * 단일 종합점수 없음 — bestByPerspective(관점별 최적). compositeScore 는 opt-in 보조.
 */
import type { CostSource, Lang, RunResult } from "./models";

export interface ComboMetrics {
  comboId: string;
  runCount: number;
  accuracy: number; // 다수결 통과율 %
  accuracyAtK: number; // 평균 통과율 %
  subagentCount: number; // ★ 1순위 정렬키 (mean)
  totalTokens: number; // tie-break #2 (mean)
  wallTimeMs: number; // tie-break #3 (mean)
  inputTokens: number;
  outputTokens: number;
  cacheReadTokens: number;
  cacheCreationTokens: number;
  costUsd: number | null;
  estimatedCostUsd: number | null;
  costSource: CostSource;
  toolCalls: number;
  skillFires: number;
  mcpCalls: number;
  numTurns: number;
  qualityScore: number | null;
  // ── 사용자 관점 파생 지표 ──
  runsPassed?: number; // resolved===true 인 run 수 (k of k/n)
  consistency?: number; // 런 단위 통과율 % = 100*runsPassed/runCount (일관성/플레이키니스)
  effectiveCostUsd?: number | null; // 해결당 기대비용 = costUsd / (accuracyAtK/100); 0% 면 null
  perTask: Record<string, { passRate: number; resolved: boolean }>;
}

const round4 = (n: number) => Math.round(n * 1e4) / 1e4;
const mean = (xs: number[]) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0);

/** 정확도 (PRD §22.1) — task 별 run 들의 resolved 다수결. */
export function computeAccuracy(results: RunResult[]): {
  accuracy: number;
  accuracyAtK: number;
  perTask: Record<string, { passRate: number; resolved: boolean }>;
} {
  const byTask = new Map<string, RunResult[]>();
  for (const r of results) {
    const arr = byTask.get(r.taskId) ?? [];
    arr.push(r);
    byTask.set(r.taskId, arr);
  }
  const perTask: Record<string, { passRate: number; resolved: boolean }> = {};
  const passRates: number[] = [];
  let passedTasks = 0;
  for (const [taskId, runs] of byTask) {
    const passRate = mean(runs.map((r) => (r.resolved === true ? 1 : 0)));
    const resolved = passRate >= 0.5;
    perTask[taskId] = { passRate: round4(passRate), resolved };
    passRates.push(passRate);
    if (resolved) passedTasks += 1;
  }
  const nTasks = byTask.size;
  return {
    accuracy: nTasks ? round4((100 * passedTasks) / nTasks) : 0,
    accuracyAtK: nTasks ? round4(100 * mean(passRates)) : 0,
    perTask,
  };
}

function aggregateCost(results: RunResult[]): {
  costUsd: number | null;
  estimatedCostUsd: number | null;
  costSource: CostSource;
} {
  const actuals = results.map((r) => r.costUsd).filter((c): c is number => c !== null);
  const ests = results.map((r) => r.estimatedCostUsd).filter((c): c is number => c !== null);
  // 라벨은 실제 보고되는 값에서 도출 — costUsd:null 인데 costSource:"estimated" 같은 모순 방지.
  let costSource: CostSource;
  if (actuals.length) costSource = "actual";
  else if (ests.length) costSource = "estimated";
  else costSource = "none";
  return {
    costUsd: actuals.length ? round4(mean(actuals)) : null,
    estimatedCostUsd: ests.length ? round4(mean(ests)) : null,
    costSource,
  };
}

export function aggregateCombo(comboId: string, results: RunResult[]): ComboMetrics {
  const acc = computeAccuracy(results);
  const cost = aggregateCost(results);
  const quals = results.map((r) => r.qualityScore).filter((q): q is number => q !== null);
  const runsPassed = results.filter((r) => r.resolved === true).length;
  // 해결당 기대비용 = 1회 실행 비용 / 1회 성공확률(=accuracyAtK). 재시도까지 감안한 정직한 단가.
  const effectiveCostUsd =
    cost.costUsd !== null && acc.accuracyAtK > 0
      ? round4(cost.costUsd / (acc.accuracyAtK / 100))
      : null;
  return {
    comboId,
    runCount: results.length,
    accuracy: acc.accuracy,
    accuracyAtK: acc.accuracyAtK,
    perTask: acc.perTask,
    subagentCount: round4(mean(results.map((r) => r.subagentCount))),
    totalTokens: Math.round(mean(results.map((r) => r.totalTokens))),
    wallTimeMs: Math.round(mean(results.map((r) => r.wallTimeMs))),
    inputTokens: Math.round(mean(results.map((r) => r.inputTokens))),
    outputTokens: Math.round(mean(results.map((r) => r.outputTokens))),
    cacheReadTokens: Math.round(mean(results.map((r) => r.cacheReadTokens))),
    cacheCreationTokens: Math.round(mean(results.map((r) => r.cacheCreationTokens))),
    costUsd: cost.costUsd,
    estimatedCostUsd: cost.estimatedCostUsd,
    costSource: cost.costSource,
    toolCalls: round4(mean(results.map((r) => r.toolCalls))),
    skillFires: round4(mean(results.map((r) => r.skillFires))),
    mcpCalls: round4(mean(results.map((r) => r.mcpCalls))),
    numTurns: round4(mean(results.map((r) => r.numTurns))),
    qualityScore: quals.length ? round4(mean(quals)) : null,
    runsPassed,
    consistency: results.length ? round4((100 * runsPassed) / results.length) : 0,
    effectiveCostUsd,
  };
}

export function aggregateAll(results: RunResult[]): ComboMetrics[] {
  const byCombo = new Map<string, RunResult[]>();
  for (const r of results) {
    const arr = byCombo.get(r.comboId) ?? [];
    arr.push(r);
    byCombo.set(r.comboId, arr);
  }
  return [...byCombo.entries()].map(([id, rs]) => aggregateCombo(id, rs));
}

/** ★ 정렬 기본키 — subagentCount → totalTokens → wallTimeMs. 오름차순(적을수록 우선). */
export function sortCombos(metrics: ComboMetrics[]): ComboMetrics[] {
  return [...metrics].sort(
    (a, b) =>
      a.subagentCount - b.subagentCount ||
      a.totalTokens - b.totalTokens ||
      a.wallTimeMs - b.wallTimeMs ||
      (a.comboId < b.comboId ? -1 : a.comboId > b.comboId ? 1 : 0),
  );
}

// ── 관점별 최적 ─────────────────────────────────────────
type Dir = "max" | "min";
type Tier = "decision" | "diagnostic";
interface Perspective {
  key: string; // i18n persp_*
  metric: keyof ComboMetrics;
  dir: Dir;
  tier: Tier;
  requiresQuality?: boolean;
}
// decision = 사용자가 실제로 고르는 기준(헤드라인). diagnostic = "왜"를 설명(드릴다운).
const PERSPECTIVES: Perspective[] = [
  { key: "persp_accuracy", metric: "accuracy", dir: "max", tier: "decision" },
  { key: "persp_consistency", metric: "consistency", dir: "max", tier: "decision" },
  { key: "persp_effcost", metric: "effectiveCostUsd", dir: "min", tier: "decision" },
  { key: "persp_time", metric: "wallTimeMs", dir: "min", tier: "decision" },
  {
    key: "persp_quality",
    metric: "qualityScore",
    dir: "max",
    tier: "decision",
    requiresQuality: true,
  },
  { key: "persp_cost", metric: "costUsd", dir: "min", tier: "diagnostic" },
  { key: "persp_tokens", metric: "totalTokens", dir: "min", tier: "diagnostic" },
  { key: "persp_toolcalls", metric: "toolCalls", dir: "min", tier: "diagnostic" },
  { key: "persp_subagents", metric: "subagentCount", dir: "min", tier: "diagnostic" },
];

export interface PerspectiveBest {
  perspectiveKey: string;
  label: string;
  comboId: string;
  metric: keyof ComboMetrics;
  value: number;
}

export function bestByPerspectiveDetailed(
  metrics: ComboMetrics[],
  lang: Lang,
  opts?: { judge?: boolean; tier?: Tier },
): PerspectiveBest[] {
  const out: PerspectiveBest[] = [];
  for (const p of PERSPECTIVES) {
    if (opts?.tier && p.tier !== opts.tier) continue; // 헤드라인은 decision 만
    if (p.requiresQuality && !opts?.judge) continue; // judge off → 품질 관점 숨김
    // 값 None/미수집 조합은 제외
    const candidates = metrics.filter((m) => {
      const v = m[p.metric];
      return typeof v === "number" && Number.isFinite(v);
    });
    if (!candidates.length) continue;
    const best = candidates.reduce((a, b) => {
      const va = a[p.metric] as number;
      const vb = b[p.metric] as number;
      if (p.dir === "max") return vb > va ? b : a;
      return vb < va ? b : a;
    });
    out.push({
      perspectiveKey: p.key,
      label: tr(lang, p.key),
      comboId: best.comboId,
      metric: p.metric,
      value: best[p.metric] as number,
    });
  }
  return out;
}

export function bestByPerspective(
  metrics: ComboMetrics[],
  lang: Lang,
  opts?: { judge?: boolean },
): Record<string, string> {
  const out: Record<string, string> = {};
  for (const b of bestByPerspectiveDetailed(metrics, lang, opts)) out[b.label] = b.comboId;
  return out;
}

// ── compositeScore (opt-in, --score) ────────────────────
const SCORE_WEIGHTS: { metric: keyof ComboMetrics; dir: Dir; w: number }[] = [
  { metric: "accuracy", dir: "max", w: 0.4 },
  { metric: "costUsd", dir: "min", w: 0.15 },
  { metric: "totalTokens", dir: "min", w: 0.15 },
  { metric: "wallTimeMs", dir: "min", w: 0.1 },
  { metric: "toolCalls", dir: "min", w: 0.1 },
  { metric: "subagentCount", dir: "min", w: 0.1 },
];

export function compositeScore(metrics: ComboMetrics[]): Record<string, number> {
  const out: Record<string, number> = {};
  if (!metrics.length) return out;
  const ranges = new Map<keyof ComboMetrics, { min: number; max: number }>();
  for (const { metric } of SCORE_WEIGHTS) {
    const vals = metrics
      .map((m) => m[metric])
      .filter((v): v is number => typeof v === "number" && Number.isFinite(v));
    if (vals.length) ranges.set(metric, { min: Math.min(...vals), max: Math.max(...vals) });
  }
  for (const m of metrics) {
    let score = 0;
    let wsum = 0;
    for (const { metric, dir, w } of SCORE_WEIGHTS) {
      const r = ranges.get(metric);
      const v = m[metric];
      if (!r || typeof v !== "number" || !Number.isFinite(v)) continue;
      const span = r.max - r.min;
      let norm: number;
      if (span === 0) norm = 1;
      else norm = dir === "max" ? (v - r.min) / span : (r.max - v) / span;
      score += norm * w;
      wsum += w;
    }
    out[m.comboId] = wsum ? round4(score / wsum) : 0;
  }
  return out;
}