Skip to content
TypeScript
2.2 KB
Raw Blame
/**
 * 품질 채점 (PRD §13, §22.2) — opt-in (--judge none|heuristic|cc).
 * 미채점(off)이면 quality 컬럼·품질 관점 숨김.
 */
import { commandExists, execCapture } from "./runners/util";

export interface Judge {
  name: string;
  score(text: string, instruction?: string): Promise<number | null>;
}

const NEGATIVE = [/error/i, /traceback/i, /exception/i, /\bfail/i, /실패/, /오류/];

export class HeuristicJudge implements Judge {
  name = "heuristic";
  async score(text: string): Promise<number | null> {
    if (!text || !text.trim()) return 0;
    let s = 5;
    if (NEGATIVE.some((re) => re.test(text))) s -= 3;
    const len = text.length;
    if (len >= 5 && len <= 600) s += 3;
    if (len > 2000) s -= 1;
    if (/[\n.!]$/.test(text.trimEnd()) || /[다요]$/.test(text.trimEnd())) s += 1;
    return Math.max(0, Math.min(10, s));
  }
}

export class ClaudeJudge implements Judge {
  name = "cc";
  async score(text: string, instruction?: string): Promise<number | null> {
    if (!commandExists("claude")) return null;
    const rubric = `Score the following task result from 0 to 10 (10 = perfect). Answer with ONLY a single integer 0-10, nothing else.\n\n${instruction ? `TASK: ${instruction}\n\n` : ""}RESULT:\n${text.slice(0, 4000)}`;
    try {
      const res = await execCapture(
        "claude",
        [
          "-p",
          rubric,
          "--output-format",
          "json",
          "--model",
          "claude-haiku-4-5",
          "--dangerously-skip-permissions",
        ],
        { timeoutS: 60 },
      );
      let out = res.stdout;
      try {
        out = JSON.parse(res.stdout).result ?? res.stdout;
      } catch {
        // raw
      }
      return extractScore(out);
    } catch {
      return null;
    }
  }
}

export function extractScore(text: string): number | null {
  const m = text.match(/\b(10(\.0)?|\d(\.\d)?)\b/);
  if (!m) return null;
  const v = Number.parseFloat(m[1]!);
  if (Number.isNaN(v)) return null;
  return Math.max(0, Math.min(10, v));
}

export function getJudge(name: string | null | undefined): Judge | null {
  if (!name || name === "none") return null;
  if (name === "heuristic") return new HeuristicJudge();
  if (name === "cc") return new ClaudeJudge();
  return null;
}