TypeScript
import { tr } from "./i18n";
/**
* 메트릭 (PRD §13, §22.1).
* 정렬 기본키 = subagentCount → totalTokens → wallTimeMs (한 곳: sortCombos).
* 단일 종합점수 없음 — bestByPerspective(관점별 최적). compositeScore 는 opt-in 보조.
*/
import type { CostSource, Lang, RunResult } from "./models";
export interface ComboMetrics {
comboId: string;
runCount: number;
accuracy: number; // 다수결 통과율 %
accuracyAtK: number; // 평균 통과율 %
subagentCount: number; // ★ 1순위 정렬키 (mean)
totalTokens: number; // tie-break #2 (mean)
wallTimeMs: number; // tie-break #3 (mean)
inputTokens: number;
outputTokens: number;
cacheReadTokens: number;
cacheCreationTokens: number;
costUsd: number | null;
estimatedCostUsd: number | null;
costSource: CostSource;
toolCalls: number;
skillFires: number;
mcpCalls: number;
numTurns: number;
qualityScore: number | null;
// ── 사용자 관점 파생 지표 ──
runsPassed?: number; // resolved===true 인 run 수 (k of k/n)
consistency?: number; // 런 단위 통과율 % = 100*runsPassed/runCount (일관성/플레이키니스)
effectiveCostUsd?: number | null; // 해결당 기대비용 = costUsd / (accuracyAtK/100); 0% 면 null
perTask: Record<string, { passRate: number; resolved: boolean }>;
}
const round4 = (n: number) => Math.round(n * 1e4) / 1e4;
const mean = (xs: number[]) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0);
/** 정확도 (PRD §22.1) — task 별 run 들의 resolved 다수결. */
export function computeAccuracy(results: RunResult[]): {
accuracy: number;
accuracyAtK: number;
perTask: Record<string, { passRate: number; resolved: boolean }>;
} {
const byTask = new Map<string, RunResult[]>();
for (const r of results) {
const arr = byTask.get(r.taskId) ?? [];
arr.push(r);
byTask.set(r.taskId, arr);
}
const perTask: Record<string, { passRate: number; resolved: boolean }> = {};
const passRates: number[] = [];
let passedTasks = 0;
for (const [taskId, runs] of byTask) {
const passRate = mean(runs.map((r) => (r.resolved === true ? 1 : 0)));
const resolved = passRate >= 0.5;
perTask[taskId] = { passRate: round4(passRate), resolved };
passRates.push(passRate);
if (resolved) passedTasks += 1;
}
const nTasks = byTask.size;
return {
accuracy: nTasks ? round4((100 * passedTasks) / nTasks) : 0,
accuracyAtK: nTasks ? round4(100 * mean(passRates)) : 0,
perTask,
};
}
function aggregateCost(results: RunResult[]): {
costUsd: number | null;
estimatedCostUsd: number | null;
costSource: CostSource;
} {
const actuals = results.map((r) => r.costUsd).filter((c): c is number => c !== null);
const ests = results.map((r) => r.estimatedCostUsd).filter((c): c is number => c !== null);
// 라벨은 실제 보고되는 값에서 도출 — costUsd:null 인데 costSource:"estimated" 같은 모순 방지.
let costSource: CostSource;
if (actuals.length) costSource = "actual";
else if (ests.length) costSource = "estimated";
else costSource = "none";
return {
costUsd: actuals.length ? round4(mean(actuals)) : null,
estimatedCostUsd: ests.length ? round4(mean(ests)) : null,
costSource,
};
}
export function aggregateCombo(comboId: string, results: RunResult[]): ComboMetrics {
const acc = computeAccuracy(results);
const cost = aggregateCost(results);
const quals = results.map((r) => r.qualityScore).filter((q): q is number => q !== null);
const runsPassed = results.filter((r) => r.resolved === true).length;
// 해결당 기대비용 = 1회 실행 비용 / 1회 성공확률(=accuracyAtK). 재시도까지 감안한 정직한 단가.
const effectiveCostUsd =
cost.costUsd !== null && acc.accuracyAtK > 0
? round4(cost.costUsd / (acc.accuracyAtK / 100))
: null;
return {
comboId,
runCount: results.length,
accuracy: acc.accuracy,
accuracyAtK: acc.accuracyAtK,
perTask: acc.perTask,
subagentCount: round4(mean(results.map((r) => r.subagentCount))),
totalTokens: Math.round(mean(results.map((r) => r.totalTokens))),
wallTimeMs: Math.round(mean(results.map((r) => r.wallTimeMs))),
inputTokens: Math.round(mean(results.map((r) => r.inputTokens))),
outputTokens: Math.round(mean(results.map((r) => r.outputTokens))),
cacheReadTokens: Math.round(mean(results.map((r) => r.cacheReadTokens))),
cacheCreationTokens: Math.round(mean(results.map((r) => r.cacheCreationTokens))),
costUsd: cost.costUsd,
estimatedCostUsd: cost.estimatedCostUsd,
costSource: cost.costSource,
toolCalls: round4(mean(results.map((r) => r.toolCalls))),
skillFires: round4(mean(results.map((r) => r.skillFires))),
mcpCalls: round4(mean(results.map((r) => r.mcpCalls))),
numTurns: round4(mean(results.map((r) => r.numTurns))),
qualityScore: quals.length ? round4(mean(quals)) : null,
runsPassed,
consistency: results.length ? round4((100 * runsPassed) / results.length) : 0,
effectiveCostUsd,
};
}
export function aggregateAll(results: RunResult[]): ComboMetrics[] {
const byCombo = new Map<string, RunResult[]>();
for (const r of results) {
const arr = byCombo.get(r.comboId) ?? [];
arr.push(r);
byCombo.set(r.comboId, arr);
}
return [...byCombo.entries()].map(([id, rs]) => aggregateCombo(id, rs));
}
/** ★ 정렬 기본키 — subagentCount → totalTokens → wallTimeMs. 오름차순(적을수록 우선). */
export function sortCombos(metrics: ComboMetrics[]): ComboMetrics[] {
return [...metrics].sort(
(a, b) =>
a.subagentCount - b.subagentCount ||
a.totalTokens - b.totalTokens ||
a.wallTimeMs - b.wallTimeMs ||
(a.comboId < b.comboId ? -1 : a.comboId > b.comboId ? 1 : 0),
);
}
// ── 관점별 최적 ─────────────────────────────────────────
type Dir = "max" | "min";
type Tier = "decision" | "diagnostic";
interface Perspective {
key: string; // i18n persp_*
metric: keyof ComboMetrics;
dir: Dir;
tier: Tier;
requiresQuality?: boolean;
}
// decision = 사용자가 실제로 고르는 기준(헤드라인). diagnostic = "왜"를 설명(드릴다운).
const PERSPECTIVES: Perspective[] = [
{ key: "persp_accuracy", metric: "accuracy", dir: "max", tier: "decision" },
{ key: "persp_consistency", metric: "consistency", dir: "max", tier: "decision" },
{ key: "persp_effcost", metric: "effectiveCostUsd", dir: "min", tier: "decision" },
{ key: "persp_time", metric: "wallTimeMs", dir: "min", tier: "decision" },
{
key: "persp_quality",
metric: "qualityScore",
dir: "max",
tier: "decision",
requiresQuality: true,
},
{ key: "persp_cost", metric: "costUsd", dir: "min", tier: "diagnostic" },
{ key: "persp_tokens", metric: "totalTokens", dir: "min", tier: "diagnostic" },
{ key: "persp_toolcalls", metric: "toolCalls", dir: "min", tier: "diagnostic" },
{ key: "persp_subagents", metric: "subagentCount", dir: "min", tier: "diagnostic" },
];
export interface PerspectiveBest {
perspectiveKey: string;
label: string;
comboId: string;
metric: keyof ComboMetrics;
value: number;
}
export function bestByPerspectiveDetailed(
metrics: ComboMetrics[],
lang: Lang,
opts?: { judge?: boolean; tier?: Tier },
): PerspectiveBest[] {
const out: PerspectiveBest[] = [];
for (const p of PERSPECTIVES) {
if (opts?.tier && p.tier !== opts.tier) continue; // 헤드라인은 decision 만
if (p.requiresQuality && !opts?.judge) continue; // judge off → 품질 관점 숨김
// 값 None/미수집 조합은 제외
const candidates = metrics.filter((m) => {
const v = m[p.metric];
return typeof v === "number" && Number.isFinite(v);
});
if (!candidates.length) continue;
const best = candidates.reduce((a, b) => {
const va = a[p.metric] as number;
const vb = b[p.metric] as number;
if (p.dir === "max") return vb > va ? b : a;
return vb < va ? b : a;
});
out.push({
perspectiveKey: p.key,
label: tr(lang, p.key),
comboId: best.comboId,
metric: p.metric,
value: best[p.metric] as number,
});
}
return out;
}
export function bestByPerspective(
metrics: ComboMetrics[],
lang: Lang,
opts?: { judge?: boolean },
): Record<string, string> {
const out: Record<string, string> = {};
for (const b of bestByPerspectiveDetailed(metrics, lang, opts)) out[b.label] = b.comboId;
return out;
}
// ── compositeScore (opt-in, --score) ────────────────────
const SCORE_WEIGHTS: { metric: keyof ComboMetrics; dir: Dir; w: number }[] = [
{ metric: "accuracy", dir: "max", w: 0.4 },
{ metric: "costUsd", dir: "min", w: 0.15 },
{ metric: "totalTokens", dir: "min", w: 0.15 },
{ metric: "wallTimeMs", dir: "min", w: 0.1 },
{ metric: "toolCalls", dir: "min", w: 0.1 },
{ metric: "subagentCount", dir: "min", w: 0.1 },
];
export function compositeScore(metrics: ComboMetrics[]): Record<string, number> {
const out: Record<string, number> = {};
if (!metrics.length) return out;
const ranges = new Map<keyof ComboMetrics, { min: number; max: number }>();
for (const { metric } of SCORE_WEIGHTS) {
const vals = metrics
.map((m) => m[metric])
.filter((v): v is number => typeof v === "number" && Number.isFinite(v));
if (vals.length) ranges.set(metric, { min: Math.min(...vals), max: Math.max(...vals) });
}
for (const m of metrics) {
let score = 0;
let wsum = 0;
for (const { metric, dir, w } of SCORE_WEIGHTS) {
const r = ranges.get(metric);
const v = m[metric];
if (!r || typeof v !== "number" || !Number.isFinite(v)) continue;
const span = r.max - r.min;
let norm: number;
if (span === 0) norm = 1;
else norm = dir === "max" ? (v - r.min) / span : (r.max - v) / span;
score += norm * w;
wsum += w;
}
out[m.comboId] = wsum ? round4(score / wsum) : 0;
}
return out;
}