Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
23.3 KB
Raw Blame
/**
 * v0.40.1.0 Track D / T6 — Nightly cross-modal quality probe phase.
 *
 * Once per 24h, runs the canonical quality pipeline:
 *   1. `gbrain eval longmemeval --by-type` against the committed nightly
 *      fixture (test/fixtures/longmemeval-nightly.jsonl) → JSONL output.
 *   2. `gbrain eval cross-modal --batch <jsonl> --max-usd $cap --yes`
 *      → batch summary with verdict.
 *   3. Audit JSONL row recording outcome / cost / pass-fail counts, the
 *      judge panel, the non-passing questions with per-judge scores and
 *      the run's metered chat spend (#5506).
 *
 * Both stages run under one BudgetTracker capped at `max_usd` (CEO B4).
 * A collapsed judge panel is inconclusive, and a non-pass run keeps its
 * receipts under the audit dir (#5506, D12).
 *
 * Default: DISABLED. Opt-in via `gbrain config set
 * autopilot.nightly_quality_probe.enabled true`. Doctor surfaces a
 * paste-ready enable hint when disabled.
 *
 * Embedding-key dependency: longmemeval needs `gateway.embedQuery()`.
 * Short-circuits with `outcome: no_embedding_key` + stderr warn when no
 * provider is configured (mirrors how the v0.31.12 model-routing infra
 * handles missing-provider cases).
 */

import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { tmpdir } from 'node:os';

import {
  logQualityProbeEvent,
  readRecentQualityProbeEvents,
  type QualityProbeAuditEvent,
  type QualityProbeFailure,
} from '../audit-quality-probe.ts';
import { withChatCallMeter, type ChatCallMeter } from '../ai/chat-usage.ts';
import { withBudgetTracker } from '../ai/gateway.ts';
import {
  BudgetExhausted,
  BudgetTracker,
  type BudgetActualUsage,
  type BudgetEstimate,
  type BudgetReservation,
  type PricingOverrides,
} from '../budget/budget-tracker.ts';
import { resolveAuditDir } from '../minions/handlers/shell-audit.ts';
import type { CrossModalBatchSummary } from './nightly-probe-adapters.ts';
import { NightlyProbeModelRoutesError, type NightlyProbeModelRoutes } from './nightly-probe-routes.ts';

/** Run-once gate window in ms. 24h matches the "nightly" cadence. */
const NIGHTLY_WINDOW_MS = 24 * 60 * 60 * 1000;

/**
 * Default run-level USD cap over every paid call of a probe run (LongMemEval
 * reader, extractor and query embeddings, then the judges). A default cap:
 * an unpriced model warns and runs; a configured `max_usd` is a user cap and
 * refuses an unpriced model with the shared no_pricing guidance.
 */
const DEFAULT_MAX_USD = 5.0;

/** Receipt directories (batch summary + LongMemEval output of a non-pass run) kept under the audit dir. */
const RECEIPTS_KEPT = 7;

/** Where a non-pass run's receipts go, under the audit dir (honors GBRAIN_AUDIT_DIR). */
const RECEIPTS_DIR = 'nightly-probe';

/** Committed fixture used as the probe's input dataset. */
const NIGHTLY_FIXTURE_REL_PATH = 'test/fixtures/longmemeval-nightly.jsonl';

export const NIGHTLY_PROBE_SEARCH_CONFIG_KEYS: ReadonlyArray<string> = Object.freeze([
  'search.mode',
  'search.reranker.enabled',
  'search.reranker.model',
  'search.reranker.top_n_in',
  'search.reranker.top_n_out',
  'search.reranker.timeout_ms',
]);

/** Result reported back to the cycle dispatcher / Minion handler. */
export interface NightlyProbeResult {
  outcome: 'pass' | 'fail' | 'inconclusive' | 'error' | 'budget_exceeded' | 'rate_limited' | 'no_embedding_key' | 'skipped' | 'disabled';
  exit_code: number;
  detail?: string;
}

export interface NightlyProbeDeps {
  /** Returns true when the feature config flag is on. */
  isEnabled: () => boolean | Promise<boolean>;
  /** Returns true when an embedding provider is configured + reachable. */
  hasEmbeddingProvider: () => boolean | Promise<boolean>;
  /** Resolves the run-level USD cap (config override OR DEFAULT_MAX_USD). */
  resolveMaxUsd: () => number | Promise<number>;
  /**
   * Where the cap came from and the brain's `pricing.overrides`: a `user` cap
   * (a configured max_usd) refuses an unpriced model, a `default` cap warns
   * and runs it. Absent: a default cap with no overrides.
   */
  resolveBudgetPolicy?: () => Promise<{ capSource: 'user' | 'default'; pricingOverrides?: PricingOverrides }>;
  /** Path of the LongMemEval fixture (the embedded asset, #5187); wins over resolveRepoRoot. */
  resolveFixturePath?: () => string | Promise<string>;
  /** Resolves a root holding test/fixtures/longmemeval-nightly.jsonl (used when resolveFixturePath is absent). */
  resolveRepoRoot?: () => string | Promise<string>;
  /** Resolves live search-mode/reranker overrides copied into the isolated benchmark brain. */
  resolveSearchConfigSnapshot?: () => Record<string, string> | Promise<Record<string, string>>;
  /**
   * Refreshes the gateway from the live brain and resolves the reader,
   * extractor and judge-slot routes (#5872). Runs only on a run that passed
   * the rate limit; a rejection becomes an `error` audit row.
   */
  resolveModelRoutes?: () => Promise<NightlyProbeModelRoutes>;
  /** Runs the longmemeval command; returns the path to the JSONL output. */
  runLongMemEval: (args: {
    fixturePath: string;
    outputPath: string;
    searchConfigSnapshot?: Record<string, string>;
    modelRoutes?: NightlyProbeModelRoutes;
  }) => Promise<void>;
  /** Runs the cross-modal batch; returns exit code (0/1/2). */
  runCrossModalBatch: (args: {
    batchPath: string;
    summaryPath: string;
    maxUsd: number;
    modelRoutes?: NightlyProbeModelRoutes;
  }) => Promise<{ exitCode: number; summary?: CrossModalBatchSummary }>;
  /** Now provider — overridable for tests of the 24h rate limit. */
  now: () => Date;
}

/**
 * Dual-plane flag resolution (same precedent as `mcp.publish_skills` in
 * serve-http.ts): the DB config row — what `gbrain config set` writes —
 * wins when present; the file plane (~/.gbrain/config.json) is the
 * fallback. Doctor's paste-ready enable hint says `gbrain config set
 * autopilot.nightly_quality_probe.enabled true`, so the gate MUST read
 * the DB plane — a file-only read turns that hint into a silent no-op.
 */
export function resolveProbeEnabled(
  dbVal: string | null | undefined,
  fileVal: unknown,
): boolean {
  if (dbVal != null) return dbVal === 'true';
  return fileVal === true;
}

/**
 * Same dual-plane rule for the run-level USD cap. Malformed or negative
 * values on either plane fall through to the next plane / the default.
 */
export function resolveProbeMaxUsd(
  dbVal: string | null | undefined,
  fileVal: unknown,
  fallback: number = DEFAULT_MAX_USD,
): number {
  return resolveProbeCap(dbVal, fileVal, fallback).maxUsd;
}

/** The run-level cap and its source: a valid value on either plane is a `user` cap, else the `default`. */
export function resolveProbeCap(
  dbVal: string | null | undefined,
  fileVal: unknown,
  fallback: number = DEFAULT_MAX_USD,
): { maxUsd: number; capSource: 'user' | 'default' } {
  for (const raw of [dbVal, fileVal]) {
    if (raw == null) continue;
    const n = Number(raw);
    if (Number.isFinite(n) && n >= 0) return { maxUsd: n, capSource: 'user' };
  }
  return { maxUsd: fallback, capSource: 'default' };
}

/**
 * The run's one BudgetTracker (CEO B4): every gateway chat and embed call of
 * both stages reserves against it, so the LongMemEval stage is capped too.
 * Keeps the first refusal so the phase can tell a budget stop from a crash
 * even when LongMemEval records the refusal as a per-question error.
 */
class ProbeBudgetTracker extends BudgetTracker {
  exhausted: BudgetExhausted | null = null;

  override reserve(estimate: BudgetEstimate): BudgetReservation | undefined {
    try {
      return super.reserve(estimate);
    } catch (err) {
      if (err instanceof BudgetExhausted) this.exhausted ??= err;
      throw err;
    }
  }

  override record(actual: BudgetActualUsage & { kind?: BudgetEstimate['kind'] }): void {
    try {
      super.record(actual);
    } catch (err) {
      if (err instanceof BudgetExhausted) this.exhausted ??= err;
      throw err;
    }
  }
}

/**
 * Pure function: decide whether the probe should run given the audit
 * history. Returns reason when skipping.
 */
export function shouldRunNightly(
  now: Date,
  recentEvents: ReadonlyArray<{ ts: string }>,
  windowMs: number = NIGHTLY_WINDOW_MS,
): { run: true } | { run: false; reason: 'rate_limited' } {
  const cutoff = now.getTime() - windowMs;
  for (const ev of recentEvents) {
    const ts = Date.parse(ev.ts);
    if (Number.isFinite(ts) && ts >= cutoff) {
      return { run: false, reason: 'rate_limited' };
    }
  }
  return { run: true };
}

function sha8File(p: string): string | undefined {
  try {
    const content = fs.readFileSync(p);
    return createHash('sha256').update(content).digest('hex').slice(0, 8);
  } catch {
    return undefined;
  }
}

/** A non-pass audit row lists at most this many questions; its digest counts them all. */
const AUDIT_MAX_FAILURES = 10;

function plural(n: number, noun: string): string {
  return `${n} ${noun}${n === 1 ? '' : 's'}`;
}

/**
 * The non-pass row's one-line `detail`: questions that did not pass out of
 * the total, the count per failing dimension and reason (per verdict for a
 * question with no failing dimension, and the malformed rows, which have
 * no failure entry), and the judge panel. Undefined when the batch names
 * no question that did not pass.
 */
function failureDigest(summary: CrossModalBatchSummary, failures: QualityProbeFailure[]): string | undefined {
  const tally = new Map<string, number>();
  for (const f of failures) {
    const keys = f.dimensions?.length ? f.dimensions.map(d => `${d.dimension} ${d.fail_reason}`) : [f.verdict];
    for (const key of keys) tally.set(key, (tally.get(key) ?? 0) + 1);
  }
  const malformed = summary.malformed_count ?? 0;
  if (malformed > 0) tally.set('malformed', malformed);
  const notPassed = failures.length + malformed;
  if (notPassed === 0) return undefined;
  const total = summary.total ?? summary.pass_count + notPassed;
  const reasons = [...tally].map(([key, n]) => `${key} x${n}`).join(', ');
  return `${notPassed}/${total} questions did not pass (${reasons})${panelDigest(summary)}`;
}

/** The digest's judge panel, naming each slot (A, B, C in slot order) that scored no question. */
function panelDigest(summary: CrossModalBatchSummary): string {
  const panel = summary.panel;
  if (!panel) return '';
  const silent = (panel.slot_scored_questions ?? []).flatMap((scored, i) => {
    if (scored > 0) return [];
    const model = summary.judge_models?.[i];
    return [`, slot ${String.fromCharCode(65 + i)}${model ? ` (${model})` : ''} scored no question`];
  });
  return `; judges: ${plural(panel.distinct_models, 'distinct model')} from ` +
    `${plural(panel.distinct_providers, 'provider')}${silent.join('')}`;
}

/** The judge panel of a completed batch: models in slot order, per-slot scored counts, distinct counts. */
function panelFields(summary: CrossModalBatchSummary): Partial<QualityProbeAuditEvent> {
  const panel = summary.panel;
  return {
    ...(summary.judge_models ? { judge_models: summary.judge_models } : {}),
    ...(panel?.slot_scored_questions ? { judge_scored_questions: panel.slot_scored_questions } : {}),
    ...(panel
      ? { distinct_judge_models: panel.distinct_models, distinct_judge_providers: panel.distinct_providers }
      : {}),
  };
}

/** Reader and extractor of a run whose routes resolved (#5872). */
function routeFields(routes: NightlyProbeModelRoutes | undefined): Partial<QualityProbeAuditEvent> {
  return routes ? { reader_model: routes.reader.model, extractor_model: routes.extractor.model } : {};
}

/** Metered chat spend of every call; `est_cost_usd` stays the batch's pre-flight estimate for its judges. */
function spendFields(meter: ChatCallMeter): Partial<QualityProbeAuditEvent> {
  return {
    chat_calls: meter.calls,
    chat_cost_usd: Math.round((meter.cost_usd ?? 0) * 1e6) / 1e6,
    unpriced_chat_calls: meter.unpriced_calls ?? 0,
  };
}

/**
 * Run the nightly probe. Pure DI surface — `deps` controls every external
 * effect so tests can stub long-running paths.
 */
export async function runNightlyQualityProbe(deps: NightlyProbeDeps): Promise<NightlyProbeResult> {
  const enabled = await deps.isEnabled();
  if (!enabled) {
    // Disabled-by-default; no audit row (doctor reads config separately).
    return { outcome: 'disabled', exit_code: 0, detail: 'feature flag off' };
  }

  // 24h rate limit — skip WITHOUT an audit row. The autopilot loop invokes
  // the probe every cycle (~5-10 min), so all but one invocation per day
  // lands here; logging each skip floods the audit file (~hundreds of
  // rows/day) and — because doctor treats any non-pass outcome as bad
  // signal — flips nightly_quality_probe_health to a permanent WARN the
  // moment the probe is enabled. A skip is a non-event: the real runs are
  // the signal, and their rows are what gates the next 24h window.
  const now = deps.now();
  const recent = readRecentQualityProbeEvents(2, now); // 2-day window is enough for 24h check
  const decision = shouldRunNightly(now, recent);
  if (!decision.run) {
    return { outcome: 'rate_limited', exit_code: 0, detail: 'already ran within 24h' };
  }

  // Embedding key check (longmemeval embeds queries).
  const hasEmbed = await deps.hasEmbeddingProvider();
  if (!hasEmbed) {
    process.stderr.write(
      `[nightly-quality-probe] no embedding provider configured; skipping. ` +
      `Configure VOYAGE_API_KEY / OPENAI_API_KEY and re-enable.\n`,
    );
    logQualityProbeEvent({
      outcome: 'no_embedding_key',
      exit_code: 0,
      pass_count: 0,
      fail_count: 0,
      inconclusive_count: 0,
      error_count: 0,
      est_cost_usd: 0,
      detail: 'no embedding provider configured',
    });
    return { outcome: 'no_embedding_key', exit_code: 0, detail: 'no embedding provider' };
  }

  const fixturePath = deps.resolveFixturePath
    ? await deps.resolveFixturePath()
    : path.join(deps.resolveRepoRoot ? await deps.resolveRepoRoot() : process.cwd(), NIGHTLY_FIXTURE_REL_PATH);
  if (!fs.existsSync(fixturePath)) {
    // A skip, not a runtime error (#5187): the probe could not run, so the
    // row says why instead of blaming a file in the user's brain repo.
    const detail =
      `the nightly fixture is not readable at ${fixturePath}, so the probe gave no quality signal. ` +
      `Reinstall gbrain (bun install -g github:garrytan/gbrain) and run gbrain doctor. See docs/eval-bench.md#nightly-cross-modal-quality-probe-opt-in-autopilot`;
    process.stderr.write(`[nightly-quality-probe] ${detail}\n`);
    logQualityProbeEvent({
      outcome: 'skipped',
      exit_code: 0,
      pass_count: 0,
      fail_count: 0,
      inconclusive_count: 0,
      error_count: 0,
      est_cost_usd: 0,
      reason: 'fixture_unavailable',
      detail,
    });
    return { outcome: 'skipped', exit_code: 0, detail };
  }

  const fixtureSha8 = sha8File(fixturePath);
  const maxUsd = (await deps.resolveMaxUsd()) ?? DEFAULT_MAX_USD;

  // Tempdir for the per-question hypothesis JSONL + batch summary.
  const workDir = fs.mkdtempSync(path.join(tmpdir(), 'nightly-probe-'));
  const lmeOutPath = path.join(workDir, 'lme-output.jsonl');
  const summaryPath = path.join(workDir, 'summary.json');

  // Set once a stage that makes model calls starts: the routes the run
  // uses (#5872), the meter pricing its chat calls (#5506) and the run
  // budget, so a row written after a part-way failure still carries them.
  let modelRoutes: NightlyProbeModelRoutes | undefined;
  let meter: ChatCallMeter | undefined;
  let tracker: ProbeBudgetTracker | undefined;
  let capFields: Partial<QualityProbeAuditEvent> = {};
  const stopped = (): Partial<QualityProbeAuditEvent> => {
    if (!tracker?.exhausted) return {};
    const reason = tracker.exhausted.reason;
    return {
      reason,
      detail:
        `stopped by the run budget before a verdict (${reason}): ${tracker.exhausted.message} ` +
        `No further paid call was made. Raise the cap with: gbrain config set autopilot.nightly_quality_probe.max_usd <usd>`,
    };
  };
  try {
    const searchConfigSnapshot = deps.resolveSearchConfigSnapshot
      ? await deps.resolveSearchConfigSnapshot()
      : undefined;
    if (deps.resolveModelRoutes) {
      try {
        modelRoutes = await deps.resolveModelRoutes();
      } catch (err) {
        throw err instanceof NightlyProbeModelRoutesError ? err : new NightlyProbeModelRoutesError(err);
      }
    }
    const policy = deps.resolveBudgetPolicy ? await deps.resolveBudgetPolicy() : { capSource: 'default' as const };
    tracker = new ProbeBudgetTracker({
      label: 'nightly_quality_probe',
      maxCostUsd: maxUsd,
      capSource: policy.capSource,
      ...(policy.pricingOverrides ? { pricingOverrides: policy.pricingOverrides } : {}),
    });
    capFields = { cap_usd: maxUsd, cap_source: policy.capSource };
    const runTracker = tracker;
    meter = { calls: 0, cost_usd: 0, unpriced_calls: 0, ...(policy.pricingOverrides ? { pricing_overrides: policy.pricingOverrides } : {}) };
    const runMeter = meter;
    await withBudgetTracker(runTracker, () => withChatCallMeter(runMeter, () =>
      deps.runLongMemEval({ fixturePath, outputPath: lmeOutPath, searchConfigSnapshot, modelRoutes })));
    if (runTracker.exhausted) {
      const fields = stopped();
      process.stderr.write(`[nightly-quality-probe] ${fields.detail}\n`);
      logQualityProbeEvent({
        outcome: 'budget_exceeded',
        exit_code: 1,
        pass_count: 0,
        fail_count: 0,
        inconclusive_count: 0,
        error_count: 0,
        est_cost_usd: 0,
        fixture_sha8: fixtureSha8,
        ...fields,
        ...routeFields(modelRoutes),
        ...spendFields(runMeter),
        ...capFields,
        ...keepReceipts(workDir, deps.now()),
      });
      return { outcome: 'budget_exceeded', exit_code: 1, detail: fields.detail };
    }
    const { exitCode, summary } = await withBudgetTracker(runTracker, () => withChatCallMeter(runMeter, () =>
      deps.runCrossModalBatch({
        batchPath: lmeOutPath,
        summaryPath,
        maxUsd,
        modelRoutes,
      })));

    const verdict: NightlyProbeResult['outcome'] = (() => {
      if (runTracker.exhausted) return 'budget_exceeded';
      if (summary) {
        if (summary.verdict === 'pass') return 'pass';
        if (summary.verdict === 'fail') return 'fail';
        if (summary.verdict === 'inconclusive') return 'inconclusive';
        if (summary.verdict === 'error') return 'error';
      }
      // If exit code is 1 with no summary, the batch refused (budget).
      if (exitCode === 1) return 'budget_exceeded';
      return 'error';
    })();
    const collapse = (verdict === 'pass' || verdict === 'fail') && summary ? collapsedPanel(summary) : undefined;
    const outcome: NightlyProbeResult['outcome'] = collapse ? 'inconclusive' : verdict;

    const failures = outcome !== 'pass' ? summary?.failures : undefined;
    const digest = summary && outcome !== 'pass' ? failureDigest(summary, failures ?? []) : undefined;
    const detail = collapse
      ? `${collapse} (batch verdict: ${verdict})${digest ? `; ${digest}` : ''}`
      : digest;
    logQualityProbeEvent({
      outcome,
      exit_code: exitCode,
      pass_count: summary?.pass_count ?? 0,
      fail_count: summary?.fail_count ?? 0,
      inconclusive_count: summary?.inconclusive_count ?? 0,
      error_count: summary?.error_count ?? 0,
      est_cost_usd: summary?.est_cost_usd ?? 0,
      fixture_sha8: fixtureSha8,
      ...(detail ? { detail } : {}),
      ...(collapse ? { reason: 'panel_collapsed' } : {}),
      ...stopped(),
      ...routeFields(modelRoutes),
      ...(summary ? panelFields(summary) : {}),
      ...(failures?.length ? { failures: failures.slice(0, AUDIT_MAX_FAILURES) } : {}),
      ...spendFields(runMeter),
      ...capFields,
      ...(outcome !== 'pass' ? keepReceipts(workDir, deps.now()) : {}),
    });

    return { outcome, exit_code: exitCode, ...(detail ? { detail } : {}) };
  } catch (err) {
    const budget = stopped();
    const outcome = budget.reason ? 'budget_exceeded' : 'error';
    const detail = budget.detail ?? (err instanceof Error ? err.message : String(err));
    process.stderr.write(`[nightly-quality-probe] ${outcome === 'error' ? 'runtime error: ' : ''}${detail}\n`);
    logQualityProbeEvent({
      outcome,
      exit_code: 1,
      pass_count: 0,
      fail_count: 0,
      inconclusive_count: 0,
      error_count: 0,
      est_cost_usd: 0,
      fixture_sha8: fixtureSha8,
      detail,
      ...(budget.reason ? { reason: budget.reason } : {}),
      ...routeFields(modelRoutes),
      ...(meter ? spendFields(meter) : {}),
      ...capFields,
      ...keepReceipts(workDir, deps.now()),
    });
    return { outcome, exit_code: 1, detail };
  } finally {
    try {
      fs.rmSync(workDir, { recursive: true, force: true });
    } catch { /* best-effort */ }
  }
}

/**
 * A panel that is not three independent judges (#5506, D12): among the
 * slots that scored, one model holding two or more slots (its votes count
 * more than once), or fewer than two distinct models. Returns the detail's
 * problem + next step, or undefined for a sound panel or a summary without
 * panel fields (older batches keep their verdict).
 */
function collapsedPanel(summary: CrossModalBatchSummary): string | undefined {
  const models = summary.judge_models;
  const panel = summary.panel;
  if (!models || !panel) return undefined;
  const scored = panel.slot_scored_questions;
  const judged = models.flatMap((model, i) =>
    scored && !((scored[i] ?? 0) > 0) ? [] : [{ model, id: String.fromCharCode(65 + i) }]);
  const remedy =
    'Set three different judge models: gbrain config set models.eval.cross_modal.slot_a <model> ' +
    '(and slot_b, slot_c; one provider is enough). See docs/eval-bench.md#nightly-cross-modal-quality-probe-opt-in-autopilot';
  if (panel.distinct_models < 2) {
    return `judge panel collapsed: ${plural(panel.distinct_models, 'distinct model')} judged, so there is no cross-check. ${remedy}`;
  }
  const slotsByModel = new Map<string, string[]>();
  for (const { model, id } of judged) slotsByModel.set(model, [...(slotsByModel.get(model) ?? []), id]);
  const shared = [...slotsByModel].filter(([, ids]) => ids.length > 1);
  if (shared.length === 0) return undefined;
  const holders = shared.map(([model, ids]) => `${model} holds slots ${ids.join(', ')}`).join('; ');
  return `judge panel collapsed: ${holders}, so its votes count more than once. ${remedy}`;
}

/**
 * Keep a non-pass run's batch summary and LongMemEval output (#5506) under
 * `<audit dir>/nightly-probe/<ts>/`, pruning to the newest RECEIPTS_KEPT.
 * The fixture is synthetic, so receipts hold no user data. Best-effort:
 * returns `receipt_dir` only when at least one file was kept.
 */
function keepReceipts(workDir: string, now: Date): Partial<QualityProbeAuditEvent> {
  try {
    const files = ['summary.json', 'lme-output.jsonl'].filter(f => fs.existsSync(path.join(workDir, f)));
    if (files.length === 0) return {};
    const root = path.join(resolveAuditDir(), RECEIPTS_DIR);
    const dir = path.join(root, now.toISOString().replace(/[:.]/g, '-'));
    fs.mkdirSync(dir, { recursive: true });
    for (const f of files) fs.copyFileSync(path.join(workDir, f), path.join(dir, f));
    const kept = fs.readdirSync(root).sort();
    for (const old of kept.slice(0, Math.max(0, kept.length - RECEIPTS_KEPT))) {
      fs.rmSync(path.join(root, old), { recursive: true, force: true });
    }
    return { receipt_dir: dir };
  } catch (err) {
    process.stderr.write(`[nightly-quality-probe] could not keep the run's receipts: ${err instanceof Error ? err.message : String(err)}\n`);
    return {};
  }
}