Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
14.4 KB
Raw Blame
/**
 * Bridge between `NightlyProbeDeps` (object-shape) and the existing CLI
 * functions (argv-shape) for `runEvalLongMemEval` + `runEvalCrossModal`.
 *
 * Per eng-D2: the existing CLI functions take argv arrays, not the object
 * shape the nightly-probe phase expects. The adapter converts; the CLI
 * functions stay unchanged.
 *
 * Per codex round-2 #1: `runEvalCrossModal --batch` only writes the summary
 * to `--output` (or its own default path). The adapter MUST pass
 * `--output summaryPath` so the file lands where the caller expects.
 *
 * Per codex round-2 #12: in-process invocation avoids the gbrain-version-
 * drift bug class. The adapter calls the CLI functions directly (not via
 * subprocess), so the workspace gbrain runs — not whatever's installed.
 */

import { readFileSync, existsSync } from 'node:fs';

import type { RunOpts } from '../../commands/eval-longmemeval.ts';
import type { RunCrossModalOpts } from '../../commands/eval-cross-modal.ts';
import { dimensionScoreKey } from '../cross-modal-eval/runner.ts';
import { redactSecrets } from '../../eval/longmemeval/run-config.ts';
import type { QualityProbeFailure } from '../audit-quality-probe.ts';
import type { NightlyProbeModelRoutes, NightlyProbeSlotId } from './nightly-probe-routes.ts';

/** Arguments accepted by the longmemeval adapter. */
export interface LongMemEvalProbeArgs {
  fixturePath: string;
  outputPath: string;
  searchConfigSnapshot?: Record<string, string>;
  /** Brain-resolved routes (#5872); absent leaves the command's own defaults. */
  modelRoutes?: NightlyProbeModelRoutes;
}

/** Arguments accepted by the cross-modal adapter. */
export interface CrossModalProbeArgs {
  batchPath: string;
  summaryPath: string;
  maxUsd: number;
  /** Brain-resolved routes (#5872); absent leaves the command's own defaults. */
  modelRoutes?: NightlyProbeModelRoutes;
}

/** Cross-modal batch summary shape (matches `runEvalCrossModal --batch --json`'s envelope). */
export interface CrossModalBatchSummary {
  pass_count: number;
  fail_count: number;
  inconclusive_count: number;
  error_count: number;
  est_cost_usd: number;
  verdict: string;
  /** Questions in the batch denominator (scored, upstream-error and malformed rows). */
  total?: number;
  /** Rows in `total` that had no question or hypothesis, so no `per_question` entry. */
  malformed_count?: number;
  /** Judge models in slot order (#5506). */
  judge_models?: string[];
  /** Counts over the slots that scored; `slot_scored_questions` is per slot in slot order. */
  panel?: { distinct_models: number; distinct_providers: number; slot_scored_questions?: number[] };
  /** Every non-passing question in summary order; the audit row keeps the first 10. */
  failures?: QualityProbeFailure[];
}

/** Error text copied into the audit row is redacted, then cut to this many characters. */
const AUDIT_ERROR_MAX_CHARS = 200;

/**
 * Adapter errors append raw model output after this marker
 * (claude-cli-language-model.ts); the audit row keeps the text before it.
 */
const RAW_OUTPUT_MARKER = '--- raw ---';

/** Name recorded for a failing dimension that is not one of the probe's own. */
const UNRECOGNIZED_DIMENSION = 'unrecognized';

/**
 * Adapter for `runEvalLongMemEval`. Builds the argv shape the CLI expects
 * and calls it in-process.
 *
 * The CLI's first positional arg is `<dataset.jsonl>` (fixturePath).
 * `--output PATH` writes per-question rows.
 *
 * Embedded failures throw so the nightly phase can audit the failure
 * without terminating autopilot. Standalone CLI invocations still exit.
 */
export async function runLongMemEvalForProbe(args: LongMemEvalProbeArgs): Promise<void> {
  const { runEvalLongMemEval } = await import('../../commands/eval-longmemeval.ts');
  const { argv, runOpts } = buildLongMemEvalProbeCall(args);
  await runEvalLongMemEval(argv, runOpts);
}

/**
 * argv + RunOpts for the LongMemEval call. The brain-resolved reader rides
 * `--model` and the extractor `RunOpts.extractorModel`. `--no-embed-cache`
 * (C-N5): the cache swaps the gateway's process-global embed transport for
 * the run, which inside the daemon would also catch every other embed call.
 * `--reader-mode direct` (#6627): the judge grades DIRECTNESS, and the notes
 * reader's whole notes-plus-reasoning text becomes the hypothesis, so the
 * probe uses the original terse reader. The shared reader prompts stay as-is.
 */
export function buildLongMemEvalProbeCall(args: LongMemEvalProbeArgs): { argv: string[]; runOpts: RunOpts } {
  const argv = [args.fixturePath, '--output', args.outputPath, '--no-embed-cache', '--reader-mode', 'direct'];
  const runOpts: RunOpts = { searchConfigSnapshot: args.searchConfigSnapshot, exitOnError: false };
  if (args.modelRoutes) {
    argv.push('--model', args.modelRoutes.reader.model);
    runOpts.extractorModel = args.modelRoutes.extractor.model;
  }
  return { argv, runOpts };
}

/**
 * QA-shaped judge dimensions for the nightly probe. The batch judge's
 * DEFAULT_DIMENSIONS rubric (DEPTH / SOURCING / SPECIFICITY / …) is built
 * for rich agent responses; LongMemEval hypotheses are deliberately terse
 * factual answers ("in widget-co") that can never score ≥7 on DEPTH or
 * SOURCING — so with the default rubric the probe FAILs every night even
 * when retrieval + answering are perfectly healthy. The probe owns its
 * invocation of the eval tool and passes dimensions matching the
 * fixture's QA shape instead.
 *
 * NOTE: the `--dimensions` CLI flag splits on commas, so these dimension
 * descriptions must stay comma-free.
 */
export const PROBE_QA_DIMENSIONS: string[] = [
  // No faithfulness/grounding dimension on purpose: the judge never sees
  // the haystack, so any accurate detail beyond the terse gold label reads
  // as "invented" and correct answers fail (verified empirically — a
  // correct "before + dates" answer scored 4/10 on such a dimension).
  'CORRECTNESS — Does the hypothesis state the same fact as the expected answer? A terse direct answer is ideal.',
  'DIRECTNESS — Does it answer THIS question without hedging or padding or answering something else?',
];

/**
 * The probe's dimension names as the aggregate keys them (the judge's
 * score key, trimmed and lowercased). Any other failing dimension name is
 * judge-chosen text and reaches the audit row as `UNRECOGNIZED_DIMENSION`.
 */
const PROBE_DIMENSION_NAMES: ReadonlySet<string> = new Set(
  PROBE_QA_DIMENSIONS.map(d => dimensionScoreKey(d).toLowerCase()),
);

const SLOT_FLAGS: ReadonlyArray<readonly [NightlyProbeSlotId, string]> = [
  ['A', '--slot-a-model'],
  ['B', '--slot-b-model'],
  ['C', '--slot-c-model'],
];

/**
 * argv + options for the cross-modal batch. Each slot route rides its
 * `--slot-<x>-model` flag, which wins over the #4636 substitution, and with
 * routes the batch keeps the gateway the caller refreshed from the brain
 * (`useConfiguredGateway`) instead of rebuilding it from the file plane.
 * With no routes the call is the pre-#5872 one. `--concurrent 1` (#7534):
 * every judge call reserves its worst case against the run cap until it
 * settles, so questions run one at a time (each question's judges still run
 * together) and a cap that pays for the probe is not refused for holding
 * several questions' worst cases at once.
 */
export function buildCrossModalProbeCall(args: CrossModalProbeArgs): { argv: string[]; opts: RunCrossModalOpts } {
  const argv = [
    '--batch',
    args.batchPath,
    '--output',
    args.summaryPath,
    '--max-usd',
    String(args.maxUsd),
    '--concurrent',
    '1',
    '--dimensions',
    PROBE_QA_DIMENSIONS.join(','),
    '--yes',
    '--json',
  ];
  if (!args.modelRoutes) return { argv, opts: {} };
  for (const [id, flag] of SLOT_FLAGS) {
    const model = args.modelRoutes.slots[id];
    if (model) argv.push(flag, model);
  }
  return { argv, opts: { useConfiguredGateway: true } };
}

/**
 * Adapter for `runEvalCrossModal --batch`. Threads `--output` so the
 * summary lands at the caller-controlled path (codex round-2 #1 fix),
 * then reads + parses the summary from that path.
 *
 * Returns `{ exitCode, summary }` shape so the caller can both surface the
 * verdict and decide what to do with non-zero exit codes (cost overrun,
 * gate failure, etc).
 *
 * Throws if `summaryPath` is missing after the run (caller misconfigured
 * the batch input) or unparseable (cross-modal wrote garbage). Both
 * cases are paste-ready in the error message.
 */
export async function runCrossModalBatchForProbe(
  args: CrossModalProbeArgs,
): Promise<{ exitCode: number; summary: CrossModalBatchSummary }> {
  const { runEvalCrossModal } = await import('../../commands/eval-cross-modal.ts');
  const { argv, opts } = buildCrossModalProbeCall(args);
  const exitCode = await runEvalCrossModal(argv, opts);

  if (!existsSync(args.summaryPath)) {
    throw new Error(
      `nightly-probe-adapter: cross-modal --batch finished (exit ${exitCode}) but ` +
      `summary file is missing at ${args.summaryPath}. ` +
      `Hint: confirm the batch input JSONL is valid and writable.`,
    );
  }

  let raw: string;
  try {
    raw = readFileSync(args.summaryPath, 'utf-8');
  } catch (err) {
    throw new Error(
      `nightly-probe-adapter: could not read cross-modal summary at ${args.summaryPath}: ` +
      `${(err as Error).message}`,
    );
  }

  let parsed: unknown;
  try {
    parsed = JSON.parse(raw);
  } catch (err) {
    throw new Error(
      `nightly-probe-adapter: cross-modal summary at ${args.summaryPath} is malformed JSON: ` +
      `${(err as Error).message}. First 200 chars: ${raw.slice(0, 200)}`,
    );
  }

  if (typeof parsed !== 'object' || parsed === null) {
    throw new Error(
      `nightly-probe-adapter: cross-modal summary at ${args.summaryPath} is not a JSON object`,
    );
  }

  // Cross-modal --batch --json wraps the summary as a top-level object;
  // pick the fields we care about and pass through. Tolerate the shape
  // being slightly larger (e.g. per-question receipts inline).
  const obj = parsed as Record<string, unknown>;
  const summary: CrossModalBatchSummary = {
    pass_count: Number(obj.pass_count ?? 0),
    fail_count: Number(obj.fail_count ?? 0),
    inconclusive_count: Number(obj.inconclusive_count ?? 0),
    error_count: Number(obj.error_count ?? 0),
    est_cost_usd: Number(obj.est_cost_usd ?? 0),
    verdict: typeof obj.verdict === 'string' ? obj.verdict : 'unknown',
  };
  const failures = parseBatchFailures(obj.per_question);
  if (failures.length > 0) summary.failures = failures;
  if (isCount(obj.total)) summary.total = obj.total;
  if (isCount(obj.malformed_count)) summary.malformed_count = obj.malformed_count;
  if (Array.isArray(obj.slots)) {
    const models = obj.slots.map(s => (isRecord(s) && typeof s.model === 'string' ? s.model : null));
    if (models.every((m): m is string => m !== null)) summary.judge_models = models;
  }
  const panel = obj.panel;
  if (isRecord(panel) && isCount(panel.distinct_models) && isCount(panel.distinct_providers)) {
    const scored = panel.slot_scored_questions;
    summary.panel = {
      distinct_models: panel.distinct_models,
      distinct_providers: panel.distinct_providers,
      ...(Array.isArray(scored) && scored.every(isCount) ? { slot_scored_questions: scored } : {}),
    };
  }

  return { exitCode, summary };
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return typeof value === 'object' && value !== null && !Array.isArray(value);
}

function isCount(value: unknown): value is number {
  return typeof value === 'number' && Number.isInteger(value) && value >= 0;
}

/** Error text for the audit row: the part before any raw model output, redacted and cut. */
function auditErrorText(text: string): string {
  const rawAt = text.indexOf(RAW_OUTPUT_MARKER);
  const head = rawAt === -1 ? text : text.slice(0, rawAt).trimEnd();
  return redactSecrets(head).slice(0, AUDIT_ERROR_MAX_CHARS);
}

/**
 * One failure entry per non-passing `per_question` entry, in summary order.
 * A malformed entry yields what can be read, never a throw: an entry
 * without `final_aggregate` (an error, an older receipt) has no
 * dimensions. Per-slot scores come from the batch's slot-indexed
 * `slot_scores`, never from the aggregate's `scores`, which drop a slot
 * that gave no score. Judge feedback and raw text are never copied.
 */
function parseBatchFailures(perQuestion: unknown): QualityProbeFailure[] {
  if (!Array.isArray(perQuestion)) return [];
  const failures: QualityProbeFailure[] = [];
  for (const raw of perQuestion) {
    const entry = isRecord(raw) ? raw : {};
    const verdict = typeof entry.verdict === 'string' ? entry.verdict : 'unknown';
    if (verdict === 'pass') continue;
    const failure: QualityProbeFailure = {
      question_id: typeof entry.question_id === 'string' ? entry.question_id : 'unknown',
      verdict,
    };
    if (typeof entry.error === 'string') failure.error = auditErrorText(entry.error);
    const aggregate = isRecord(entry.final_aggregate) ? entry.final_aggregate : {};
    const dimensions = parseFailingDimensions(aggregate.dimensions, entry.slot_scores);
    if (dimensions.length > 0) failure.dimensions = dimensions;
    else if (Array.isArray(aggregate.errors)) {
      const slotErrors = aggregate.errors.filter(isRecord).map(e => ({
        model: typeof e.modelId === 'string' ? e.modelId : 'unknown',
        error: auditErrorText(typeof e.error === 'string' ? e.error : ''),
      }));
      if (slotErrors.length > 0) failure.slot_errors = slotErrors;
    }
    failures.push(failure);
  }
  return failures;
}

/**
 * The aggregate's dimensions that carry a fail reason, with their
 * slot-indexed scores. A name outside the probe's dimensions is recorded
 * as `UNRECOGNIZED_DIMENSION`, never copied.
 */
function parseFailingDimensions(
  dimensions: unknown,
  slotScores: unknown,
): NonNullable<QualityProbeFailure['dimensions']> {
  if (!isRecord(dimensions)) return [];
  const scoresByDimension = isRecord(slotScores) ? slotScores : {};
  const failing: NonNullable<QualityProbeFailure['dimensions']> = [];
  for (const [dimension, roll] of Object.entries(dimensions)) {
    if (!isRecord(roll) || typeof roll.failReason !== 'string') continue;
    const scores = scoresByDimension[dimension];
    failing.push({
      dimension: PROBE_DIMENSION_NAMES.has(dimension) ? dimension : UNRECOGNIZED_DIMENSION,
      ...(typeof roll.mean === 'number' && Number.isFinite(roll.mean) ? { mean: roll.mean } : {}),
      ...(Array.isArray(scores)
        ? { scores: scores.map(s => (typeof s === 'number' && Number.isFinite(s) ? s : null)) }
        : {}),
      fail_reason: roll.failReason,
    });
  }
  return failing;
}