Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
2.0 KB
Raw Blame
/**
 * #6627 (D30): the nightly probe's judge grades DIRECTNESS ("without hedging
 * or padding"), and the eval reader's default notes protocol makes the whole
 * notes-plus-reasoning text the hypothesis, so healthy answers failed
 * directness. The probe asks for the direct reader; the shared reader prompts
 * (pinned by READER_PROMPT_SHA in benchmark receipts) stay unchanged.
 */
import { describe, expect, test } from 'bun:test';
import { buildLongMemEvalProbeCall, PROBE_QA_DIMENSIONS } from '../src/core/cycle/nightly-probe-adapters.ts';
import { READER_MAX_TOKENS, READER_SYSTEM_TEXT, resolveReaderConfig } from '../src/eval/longmemeval/reader.ts';

const flagValue = (argv: string[], flag: string) => {
  const at = argv.indexOf(flag);
  return at >= 0 ? argv[at + 1] : undefined;
};

describe('#6627 nightly probe reader mode', () => {
  test('the probe passes --reader-mode direct, with and without brain-resolved model routes', () => {
    const base = { fixturePath: '/f.jsonl', outputPath: '/o.jsonl' };
    const routes = { reader: { model: 'anthropic:claude-x' }, extractor: { model: 'anthropic:claude-y' } } as never;
    for (const args of [base, { ...base, modelRoutes: routes }]) {
      const { argv } = buildLongMemEvalProbeCall(args);
      expect(flagValue(argv, '--reader-mode')).toBe('direct');
      expect(argv.filter(a => a === '--reader-mode')).toHaveLength(1);
      expect(argv[0]).toBe('/f.jsonl');
    }
  });

  test('direct mode is the original terse reader prompt and budget, so the directness rubric can pass it', () => {
    const { argv } = buildLongMemEvalProbeCall({ fixturePath: '/f.jsonl', outputPath: '/o.jsonl' });
    const config = resolveReaderConfig({ mode: flagValue(argv, '--reader-mode') });
    expect(config.system).toBe(READER_SYSTEM_TEXT);
    expect(config.system).not.toContain('First extract all the relevant information');
    expect(config.maxTokens).toBe(READER_MAX_TOKENS);
    expect(PROBE_QA_DIMENSIONS.some(d => d.startsWith('DIRECTNESS'))).toBe(true);
  });
});