Skip to content
TypeScript
4.9 KB
Raw Blame
/**
 * 산출/기록 (PRD §14). run 디렉터리 레이아웃 + JSONL(1차 소스) + SQLite(질의 캐시, optional).
 */
import {
  appendFileSync,
  existsSync,
  mkdirSync,
  readFileSync,
  readdirSync,
  writeFileSync,
} from "node:fs";
import { join } from "node:path";
import type { ComboMetrics } from "./metrics";
import type { Combo, Lang, RunResult } from "./models";
import { RunResultSchema } from "./models";

export interface RunMeta {
  runId: string;
  bench: string;
  tasks: string[];
  seed: number;
  runs: number;
  runner: string;
  excluded: Combo[];
  createdAt: string;
  language: Lang;
  mock: boolean;
  goal?: string; // 사용자가 세션/CLI 에서 밝힌 목표 — run 의 의도를 기록
}

// 밀리초 해상도 충돌 방지 — 같은 ms 에 두 번 만들어지면 -2, -3 … 접미사(런 디렉터리 덮어쓰기 방지).
let lastBaseId = "";
let dupSeq = 1;

export function makeRunId(date = new Date()): string {
  const p = (n: number, w = 2) => String(n).padStart(w, "0");
  const base =
    `${date.getUTCFullYear()}${p(date.getUTCMonth() + 1)}${p(date.getUTCDate())}-` +
    `${p(date.getUTCHours())}${p(date.getUTCMinutes())}${p(date.getUTCSeconds())}-` +
    `${p(date.getUTCMilliseconds(), 3)}`;
  if (base === lastBaseId) {
    dupSeq += 1;
    return `${base}-${dupSeq}`;
  }
  lastBaseId = base;
  dupSeq = 1;
  return base;
}

function safe(s: string): string {
  return s.replace(/[^A-Za-z0-9._+-]/g, "_");
}

export class Recorder {
  readonly runDir: string;
  private resultsFile: string;

  constructor(
    private outDir: string,
    readonly runId: string,
  ) {
    this.runDir = join(outDir, runId);
    mkdirSync(this.runDir, { recursive: true });
    this.resultsFile = join(this.runDir, "results.jsonl");
  }

  workDir(comboId: string, taskId: string, runIdx: number): string {
    const d = join(this.runDir, "work", safe(comboId), safe(taskId), String(runIdx));
    mkdirSync(d, { recursive: true });
    return d;
  }

  writeMeta(meta: RunMeta): void {
    writeFileSync(join(this.runDir, "meta.json"), JSON.stringify(meta, null, 2));
  }

  appendResult(r: RunResult): void {
    appendFileSync(this.resultsFile, `${JSON.stringify(r)}\n`);
  }

  writeAccuracy(comboId: string, m: ComboMetrics): void {
    const dir = join(this.runDir, "accuracy");
    mkdirSync(dir, { recursive: true });
    writeFileSync(join(dir, `${safe(comboId)}.json`), JSON.stringify(m, null, 2));
  }

  get reportPath(): string {
    return join(this.runDir, "report.html");
  }
  get graphPath(): string {
    return join(this.runDir, "graph.html");
  }

  /** SQLite 질의 캐시(optional). 네이티브 미설치면 조용히 skip — JSONL 이 1차 소스. */
  async finalizeSqlite(results: RunResult[]): Promise<boolean> {
    try {
      const mod = await import("better-sqlite3");
      const Database = mod.default;
      const db = new Database(join(this.runDir, "theseus.sqlite"));
      db.exec(`CREATE TABLE IF NOT EXISTS results (
        comboId TEXT, taskId TEXT, runIdx INTEGER, success INTEGER, resolved INTEGER,
        totalTokens INTEGER, costUsd REAL, costSource TEXT, wallTimeMs INTEGER,
        subagentCount INTEGER, toolCalls INTEGER, skillFires INTEGER, mcpCalls INTEGER,
        model TEXT, qualityScore REAL
      )`);
      const stmt = db.prepare(`INSERT INTO results VALUES
        (@comboId,@taskId,@runIdx,@success,@resolved,@totalTokens,@costUsd,@costSource,
         @wallTimeMs,@subagentCount,@toolCalls,@skillFires,@mcpCalls,@model,@qualityScore)`);
      const tx = db.transaction((rows: RunResult[]) => {
        for (const r of rows)
          stmt.run({
            ...r,
            success: r.success ? 1 : 0,
            resolved: r.resolved === true ? 1 : r.resolved === false ? 0 : null,
          });
      });
      tx(results);
      db.close();
      return true;
    } catch {
      return false; // fail-soft
    }
  }
}

// ── results 커맨드용 로더 ─────────────────────────────────
export function listRuns(outDir: string): string[] {
  if (!existsSync(outDir)) return [];
  return readdirSync(outDir)
    .filter((d) => existsSync(join(outDir, d, "meta.json")))
    .sort()
    .reverse();
}

export function loadResults(outDir: string, runId: string): RunResult[] {
  const file = join(outDir, runId, "results.jsonl");
  if (!existsSync(file)) return [];
  const out: RunResult[] = [];
  for (const line of readFileSync(file, "utf8").split("\n")) {
    const s = line.trim();
    if (!s) continue;
    try {
      const parsed = RunResultSchema.safeParse(JSON.parse(s));
      if (parsed.success) out.push(parsed.data);
    } catch {
      // skip
    }
  }
  return out;
}

export function loadMeta(outDir: string, runId: string): RunMeta | null {
  const file = join(outDir, runId, "meta.json");
  if (!existsSync(file)) return null;
  try {
    return JSON.parse(readFileSync(file, "utf8")) as RunMeta;
  } catch {
    return null;
  }
}