Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
5.9 KB
Raw Blame
/**
 * decide-lane.ts — System One arms for `gbrain eval longmemeval` (shared
 * plumbing in src/eval/decide-eval-flags.ts; this module holds the
 * LongMemEval-specific parts so the harness facade only delegates):
 *
 *   - the `--decide*` flags and the eval-only `--eval-pool-depth N` (recall
 *     experiment: lifts the per-arm cap to N for this run, and sets
 *     `search.reranker.top_n_in=N` unless a `--search-pin` already set it);
 *   - eval-half restriction: with `--decide-dataset`, only question ids whose
 *     family (the longmemeval builder uses family = question_id) is in the
 *     dataset's eval half run;
 *   - eval validity: a `--decide` arm refuses trajectory routing, which reads
 *     the dataset's question_type labels (run both arms with --no-trajectory);
 *   - the per-row `decide` receipt, and `--decide answerable=on` harness-reader
 *     abstention (the S4 verdict `abstain` replaces the reader call).
 *
 * All-off: every helper is a no-op when no slot is on or shadow.
 */
import type { BrainEngine } from '../../core/engine.ts';
import type { HybridSearchMeta } from '../../core/types.ts';
import type { ReaderAnswer } from './reader.ts';
import { EVAL_POOL_DEPTH_MAX, setEvalPoolDepth } from '../../core/search/eval-pool-depth.ts';
import {
  DECIDE_EVAL_FLAGS, applyDecideEvalFlag, decideRowReceipt, decideSpendTotals, prepareDecideEval, spendDelta,
  type DecideEvalOptions, type DecideEvalRun, type DecideSpend,
} from '../decide-eval-flags.ts';

export { configureDecideBrain, newDecideEvalOptions, summarizeDecideReceipts, type DecideEvalOptions, type DecideEvalRun } from '../decide-eval-flags.ts';

interface LmeDecideArgs {
  decide: DecideEvalOptions;
  evalPoolDepth?: number;
  searchPins?: Record<string, string>;
  noTrajectory: boolean;
}

/** LME_FLAGS entries (the harness spreads these into its one flag table). */
export const LME_DECIDE_FLAGS: Array<{ name: string; arg?: string; help: string[]; apply: (o: LmeDecideArgs, value: string) => void }> = [
  ...DECIDE_EVAL_FLAGS.map((f) => ({ ...f, apply: (o: LmeDecideArgs, v: string) => { applyDecideEvalFlag(o.decide, f.name, v); } })),
  { name: '--eval-pool-depth', arg: 'N', help: [
      `Eval-only recall experiment: every retrieval arm fetches N candidates (N <= ${EVAL_POOL_DEPTH_MAX};`,
      'production caps each arm at 100) and search.reranker.top_n_in=N unless a',
      '--search-pin sets it. Folds into retrieval_config_hash. Pair with --capture-pool',
      'for pool_recall at fused depths 30/50/100/300.'],
    apply: (o, v) => {
      const n = Number(v);
      if (!Number.isInteger(n) || n < 1 || n > EVAL_POOL_DEPTH_MAX) throw new Error(`--eval-pool-depth must be an integer from 1 to ${EVAL_POOL_DEPTH_MAX} (got: ${v})`);
      o.evalPoolDepth = n;
      o.searchPins = { 'search.reranker.top_n_in': String(n), ...(o.searchPins ?? {}) };
    } },
];

/**
 * Before pins resolve: set the eval pool depth, prepare the decide run, add
 * its search pins, and restrict the questions to the dataset's eval half.
 * Throws (flag errors, eval validity, split_mismatch) with the message to print.
 */
export function prepareLmeDecide<Q extends { question_id: string }>(opts: LmeDecideArgs, questions: Q[]): { run: DecideEvalRun | null; questions: Q[] } {
  setEvalPoolDepth(opts.evalPoolDepth ?? null);
  const run = prepareDecideEval(opts.decide, { command: 'gbrain eval longmemeval', throwaway: true });
  if (!run) return { run, questions };
  if (!opts.noTrajectory) {
    throw new Error('--decide arms need --no-trajectory: trajectory routing reads the dataset question_type labels, and a matched pair must route with text-only classifiers (run the baseline arm with --no-trajectory too)');
  }
  opts.searchPins = { ...(opts.searchPins ?? {}), ...run.searchPins };
  if (!run.dataset) return { run, questions };
  const kept = questions.filter((q) => run.dataset!.evalFamilies.has(q.question_id));
  process.stderr.write(`[longmemeval] decide split holdout: ${kept.length}/${questions.length} question(s) in the eval half of ${run.dataset.name}\n`);
  return { run, questions: kept };
}

/** decide_spend totals before a question's search (null when no arm is active). */
export async function lmeSpendBefore(engine: BrainEngine, run: DecideEvalRun | null | undefined): Promise<DecideSpend | null> {
  return run ? decideSpendTotals(engine) : null;
}

/** hybridSearch opts the arm needs (S4 answerability on the query path). */
export function lmeDecideSearchOpts(run: DecideEvalRun | null | undefined): { decide?: { answerability: true } } {
  return run?.slots.answerable ? { decide: { answerability: true } } : {};
}

/** The row's `decide` field (absent on all-off rows). */
export async function lmeDecideRow(engine: BrainEngine, run: DecideEvalRun | null | undefined, meta: HybridSearchMeta | undefined, before: DecideSpend | null): Promise<Record<string, unknown>> {
  if (!run || !before) return {};
  const receipt = decideRowReceipt(run, meta?.decide, spendDelta(before, await decideSpendTotals(engine)));
  return { decide: { ...receipt, ...(meta?.answerability ? { answerability: meta.answerability } : {}), ...(lmeAbstention(run, meta) ? { reader_abstained: true } : {}) } };
}

export const LME_ABSTAIN_HYPOTHESIS = 'I cannot answer this: the conversation history does not contain the information needed.';

/**
 * `--decide answerable=on`: the harness reader abstains when S4 acted with
 * verdict abstain (scored by the judge on the `_abs` questions). The returned
 * answer stands in for the reader call; the row's decide block says so.
 */
export function lmeAbstention(run: DecideEvalRun | null | undefined, meta: HybridSearchMeta | undefined): ReaderAnswer | null {
  if (run?.slots.answerable !== 'on' || meta?.decide?.answerable?.effective !== 'on' || meta.answerability?.verdict !== 'abstain') return null;
  return { text: LME_ABSTAIN_HYPOTHESIS, finish_reason: 'end_turn', usage: null, response_model: null, context_chars: 0, context_sessions: 0, sessions_truncated: 0 };
}