Skip to content
TypeScript
9.9 KB
Raw Blame
/**
 * 데이터 계약 (PRD §3). 순수 타입 + 외부 JSON 파싱용 zod 스키마.
 * 로직/IO 없음 — 모든 모듈이 여기서 import 한다.
 */
import { z } from "zod";

// ─────────────────────────────────────────────────────────────
// 공통
// ─────────────────────────────────────────────────────────────
// 'open' = OpenRouter-hosted models (Qwen, GLM, DeepSeek, Kimi, …; incl. free tier),
// usable by model-agnostic agents (opencode/openclaw/hermes).
export type Family = "claude" | "gpt" | "open";
export type Lang = "en" | "kr" | "ja" | "zh";
export type CostSource = "actual" | "estimated" | "none";
export type EventKind = "assistant" | "tool_use" | "subagent" | "result";
export type Status = "success" | "fail";

// ─────────────────────────────────────────────────────────────
// L0×L1×L2 조합
// ─────────────────────────────────────────────────────────────
export interface Combo {
  comboId: string; // `${agent}+${model}+${harness}`
  agent: string; // L1: 'cc' | 'codex'
  model: string; // L0 alias 또는 resolved id
  harness: string; // L2: none|superpowers|omc|ecc|omx|lazycodex
  valid: boolean;
  excludeReason?: string | null;
}

// ─────────────────────────────────────────────────────────────
// (combo, task, run_idx) 당 1개 결과
// ─────────────────────────────────────────────────────────────
export interface RunResult {
  comboId: string;
  taskId: string;
  runIdx: number;
  success: boolean; // 실행 완료(에러/타임아웃 아님). 품질 pass/fail 아님
  resolved?: boolean | null; // 숨은 검증 통과 — null=미채점
  resultPath: string;
  inputTokens: number;
  outputTokens: number;
  cacheReadTokens: number;
  cacheCreationTokens: number;
  totalTokens: number; // tie-break #2
  costUsd: number | null; // 단가 미상 모델은 null
  estimatedCostUsd: number | null;
  costSource: CostSource;
  latencyMs: number;
  wallTimeMs: number; // wall = tie-break #3
  numTurns: number;
  toolCalls: number;
  subagentCount: number; // ★ 1순위 정렬키
  skillFires: number;
  mcpCalls: number;
  sessionId: string;
  transcriptPath: string;
  model: string;
  qualityScore: number | null; // (opt-in) LLM-judge 0~10
  error?: string | null;
}

// ─────────────────────────────────────────────────────────────
// Transcript
// ─────────────────────────────────────────────────────────────
export interface TranscriptEvent {
  idx: number;
  tsMs: number;
  endMs: number;
  durationMs: number;
  kind: EventKind;
  name: string | null;
  parentIdx: number | null; // 중첩(서브에이전트 트리) 링크
  inputTokens: number;
  outputTokens: number;
  cacheReadTokens: number;
  cacheCreationTokens: number;
  summaryIn: string;
  summaryOut: string;
  status: Status;
  isSidechain: boolean;
  // 세분화
  toolCalls: number;
  skillFires: number;
  toolsUsed: string[]; // ['Read×3','Bash×5']
  skillsUsed: string[];
  steps: string[];
  logText: string;
}

export interface ParsedTranscript {
  agent: "cc" | "codex";
  events: TranscriptEvent[];
  sessionId: string;
  model: string;
  numTurns: number;
  toolCalls: number;
  subagentCount: number;
  skillFires: number;
  mcpCalls: number;
  inputTokens: number;
  outputTokens: number;
  cacheReadTokens: number;
  cacheCreationTokens: number;
  totalTokens: number;
  latencyMs: number;
  skillsUsed: string[];
  toolsUsed: string[];
  steps: string[];
}

// ─────────────────────────────────────────────────────────────
// Flow graph
// ─────────────────────────────────────────────────────────────
export interface FlowNode {
  id: string;
  label: string;
  startMs: number;
  endMs: number;
  durationMs: number;
  summaryIn: string;
  summaryOut: string;
  status: Status;
  tokens: number;
  toolCalls: number;
  costUsd: number;
  depth: number;
  highlight: boolean;
  harness: string;
  isMain: boolean;
  skillsUsed: string[];
  toolsUsed: string[];
  steps: string[];
  logText: string;
}
export interface FlowEdge {
  src: string;
  dst: string;
  order: number;
}
export interface FlowGraph {
  comboId: string;
  taskId: string;
  runIdx: number;
  nodes: FlowNode[];
  edges: FlowEdge[];
}

// ─────────────────────────────────────────────────────────────
// Runner I/O
// ─────────────────────────────────────────────────────────────
export interface RunInvocation {
  combo: Combo;
  taskId: string;
  runIdx: number;
  workDir: string;
  timeoutS: number;
  seed: number;
  bench: string;
  instruction?: string;
}

export interface RawRunArtifact {
  invocation: RunInvocation;
  success: boolean;
  resultPath: string;
  transcriptPath: string;
  wallTimeMs: number;
  rawResolved: boolean | null;
  totalCostUsd?: number | null;
  numTurns?: number | null;
  error?: string | null;
}

// ─────────────────────────────────────────────────────────────
// Orchestrator options
// ─────────────────────────────────────────────────────────────
export interface RunOptions {
  models: string[];
  agents: string[];
  harnesses: string[];
  bench: string;
  tasks: number;
  runs: number;
  seed: number;
  runner: string; // 'auto' | 'mock' | explicit
  concurrency: number;
  timeoutS: number;
  outDir: string;
  judge: string; // none|heuristic|cc
  score: boolean;
  balanced: boolean;
  language: Lang;
  view?: ViewMode; // 시각화 전달 모드 (PRD v2 §W2)
  port?: number; // serve 모드 포트
  failRate?: number; // mock 전용 (테스트)
  goal?: string; // 사용자가 밝힌 목표 — meta.json 에 기록되고 report 헤더에 표시
}

export type ViewMode = "inline" | "open" | "serve" | "term";

// ─────────────────────────────────────────────────────────────
// zod 스키마 — 외부(transcript/headless JSON) 방어적 파싱용
// ─────────────────────────────────────────────────────────────
const num = z.number().finite();

/** cc headless `claude -p --output-format json` 결과 */
export const CcHeadlessSchema = z
  .object({
    result: z.string().optional(),
    total_cost_usd: num.nullable().optional(),
    num_turns: num.nullable().optional(),
    session_id: z.string().optional(),
    is_error: z.boolean().optional(),
    usage: z
      .object({
        input_tokens: num.optional(),
        output_tokens: num.optional(),
        cache_read_input_tokens: num.optional(),
        cache_creation_input_tokens: num.optional(),
      })
      .partial()
      .optional(),
  })
  .passthrough();
export type CcHeadless = z.infer<typeof CcHeadlessSchema>;

/** cc transcript JSONL 한 줄 (방어적: 알 수 없는 필드 허용) */
export const CcLineSchema = z
  .object({
    type: z.string().optional(),
    uuid: z.string().optional(),
    parentUuid: z.string().nullable().optional(),
    isSidechain: z.boolean().optional(),
    timestamp: z.union([z.string(), z.number()]).optional(),
    message: z.unknown().optional(),
  })
  .passthrough();

/** cc 서브에이전트 meta.json */
export const CcSubagentMetaSchema = z
  .object({
    toolUseId: z.string().optional(),
    agentId: z.string().optional(),
  })
  .passthrough();

/** codex thread-event 한 줄 */
export const CodexEventSchema = z
  .object({
    type: z.string().optional(),
    // 일부 codex 빌드는 { msg: { type } } 형태
    msg: z.unknown().optional(),
  })
  .passthrough();

/** results.jsonl 한 줄 = RunResult */
export const RunResultSchema: z.ZodType<RunResult> = z
  .object({
    comboId: z.string(),
    taskId: z.string(),
    runIdx: num,
    success: z.boolean(),
    resolved: z.boolean().nullable().optional(),
    resultPath: z.string(),
    inputTokens: num,
    outputTokens: num,
    cacheReadTokens: num,
    cacheCreationTokens: num,
    totalTokens: num,
    costUsd: num.nullable(),
    estimatedCostUsd: num.nullable(),
    costSource: z.enum(["actual", "estimated", "none"]),
    latencyMs: num,
    wallTimeMs: num,
    numTurns: num,
    toolCalls: num,
    subagentCount: num,
    skillFires: num,
    mcpCalls: num,
    sessionId: z.string(),
    transcriptPath: z.string(),
    model: z.string(),
    qualityScore: num.nullable(),
    error: z.string().nullable().optional(),
  })
  .passthrough() as unknown as z.ZodType<RunResult>;