Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.8 KB
Raw Blame
/**
 * GBRA-156: a remote `search` or `query` without `limit` returns every lexically
 * matched page and at most `search.remote_page_cap` distinct pages in total.
 *
 * 1. Protects: with an embedding model on, the vector arm filled the mode's
 *    row limit with every page it could reach, so one remote question read
 *    most of a small brain (a median 12 pages a call against keyword's 2 on
 *    the harness ladder). Vector-only pages now only fill the result up to
 *    the cap; keyword, title and identity matches are never cut.
 * 2. Fails when: a remote call without limit returns more vector-only pages
 *    than the cap, a lexically matched page is dropped, an explicit limit, a
 *    trusted local call or a cap of 0 is cut, or the cut is not reported in
 *    `_meta.retrieval.page_cap`.
 * 3. Existing coverage: autocut and adaptive return trim results only with a
 *    reranker or an opt-in config; nothing bounded a remote default.
 * 4. No production seam: production serve-http app and real tokens; the
 *    embedding provider is a deterministic in-process transport.
 */
import { afterAll, beforeAll, describe, expect, spyOn, test } from 'bun:test';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { configureGateway, resetGateway, __setEmbedTransportForTests } from '../src/core/ai/gateway.ts';
import { operationsByName } from '../src/core/operations.ts';
import { DEFAULT_REMOTE_PAGE_CAP, REMOTE_PAGE_CAP_KEY, capVectorOnlyPages, pageCapLine, remotePageCap } from '../src/core/search/remote-page-cap.ts';
import { captureRetrievalMeta, formatResult, resetRetrievalMetaForTests } from '../src/cli/main.ts';
import type { SearchResult } from '../src/core/types.ts';
import { installFixtureChunks } from './helpers/page-projection.ts';
import { callTool, legacyToken, startServeHttp, type LiveServeHttp, type ToolResultWire } from './helpers/live-mcp-servers.ts';
import { testBackends } from './helpers/test-backends.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';

const VEC = Array.from({ length: 1536 }, (_, i) => (i === 0 ? 1 : 0.01));
const LEXICAL = ['alpha', 'bravo', 'charlie', 'delta', 'echo'];
const VECTOR_ONLY = ['foxtrot', 'golf', 'hotel', 'india', 'juliet', 'kilo', 'lima'];

function slugsOf(out: ToolResultWire): string[] {
  return [...new Set((JSON.parse(out.content[0]!.text) as Array<{ slug: string }>).map(r => r.slug))];
}

function row(slug: string, extra: Partial<SearchResult> = {}): SearchResult {
  return { slug, page_id: 1, title: slug, type: 'note', chunk_text: '', chunk_source: 'compiled_truth', chunk_id: 1, chunk_index: 0, score: 1, stale: false, source_id: 'default', ...extra } as SearchResult;
}

describe('capVectorOnlyPages', () => {
  test('keeps every lexical page and fills with vector-only pages up to the cap, in rank order', () => {
    const rows = [row('v1'), row('k1', { keyword_hit: true }), row('v2'), row('v1', { chunk_id: 2 }), row('v3'), row('t1', { alias_hit: true }), row('v4')];
    const out = capVectorOnlyPages(rows, 3);
    expect(out.rows.map(r => r.slug)).toEqual(['v1', 'k1', 'v1', 't1']);
    expect(out.meta).toEqual({ cap: 3, kept_pages: 3, dropped_pages: 3 });
  });

  test('a qualified alias hit is lexical evidence and is kept past the cap', () => {
    const rows = [row('v1'), row('v2'), row('q1', { qualified_alias_hit: true })];
    expect(capVectorOnlyPages(rows, 2).rows.map(r => r.slug)).toEqual(['v1', 'q1']);
  });

  test('more lexical pages than the cap are all kept and no vector-only page is added', () => {
    const rows = [row('k1', { keyword_hit: true }), row('v1'), row('k2', { exact_lookup: 'slug' }), row('k3', { relational_pinned: true }), row('k4', { keyword_hit: true, keyword_relaxed: true })];
    expect(capVectorOnlyPages(rows, 2).rows.map(r => r.slug)).toEqual(['k1', 'k2', 'k3', 'k4']);
  });

  test('pages are distinct per source, and a null cap or a set under the cap is returned untouched', () => {
    const rows = [row('same', { source_id: 'source-a' }), row('same', { source_id: 'source-b' }), row('other')];
    expect(capVectorOnlyPages(rows, 2).rows.map(r => r.source_id)).toEqual(['source-a', 'source-b']);
    expect(capVectorOnlyPages(rows, null)).toEqual({ rows });
    expect(capVectorOnlyPages(rows, 5)).toEqual({ rows });
  });
});

// Generated by gstack /ship Step 9 pass 2 (GBRA-156).
// Value: protects=host-run subagent tools (remote trust boundary, viaSubagent) keep today's rows, and the cap line claims only what the cap left out, naming the CLI flag on a thin-client CLI;
// fails_when=remotePageCap caps delegated host jobs silently, the line counts pages before later stages add or drop some, or a thin-client CLI drops the line;
// why_new=no test covers a subagent context or the CLI renderer; seam=none; no_claim=the fan-out and arm stages themselves
describe('remotePageCap callers and the cap line', () => {
  const engine = { getConfig: async () => null } as never;

  test('a delegated host subagent call is not capped; a remote call is', async () => {
    expect(await remotePageCap({ remote: true, viaSubagent: true, engine } as never, {})).toBeNull();
    expect(await remotePageCap({ remote: true, engine } as never, {})).toBe(DEFAULT_REMOTE_PAGE_CAP);
  });

  test('a subagent job delegated by a remote OAuth client stays capped', async () => {
    const auth = { token: '', clientId: 'client-example', principal: { kind: 'oauth_client', id: 'client-example' } };
    expect(await remotePageCap({ remote: true, viaSubagent: true, auth, engine } as never, {})).toBe(DEFAULT_REMOTE_PAGE_CAP);
  });

  test('the line names only the pages the cap left out, and the flag the reader passes', () => {
    const retrieval = { page_cap: { cap: 3, kept_pages: 3, dropped_pages: 9 } };
    expect(pageCapLine(retrieval)).toBe('[gbrain search] 9 more pages matched only by meaning and were left out (default remote cap 3). Pass limit to read them.');
    expect(pageCapLine(retrieval, '--limit')).toBe('[gbrain search] 9 more pages matched only by meaning and were left out (default remote cap 3). Pass --limit to read them.');
    expect(pageCapLine({})).toBeNull();
  });

  test('a thin-client CLI search prints the cap line on stderr with --limit', () => {
    const err: string[] = [];
    const write = spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => { err.push(String(chunk)); return true; }) as never);
    try {
      captureRetrievalMeta('retrieval', { page_cap: { cap: 3, kept_pages: 3, dropped_pages: 9 } });
      formatResult('search', [row('a', { chunk_text: 'x' })], {});
      formatResult('search', [row('a', { chunk_text: 'x' })], { json: true });
    } finally {
      write.mockRestore();
      resetRetrievalMetaForTests();
    }
    expect(err.filter(line => line.includes('9 more pages matched only by meaning') && line.includes('Pass --limit'))).toHaveLength(2);
  });
});

for (const kind of testBackends()) {
  describe(`remote page cap (${kind})`, () => {
    let engine: BrainEngine;
    let close: () => Promise<void>;
    let serve: LiveServeHttp;
    let token: string;

    beforeAll(async () => {
      if (kind === 'postgres') ({ engine, close } = await isolatedPersistencePostgres(process.env.DATABASE_URL!));
      else { engine = new PGLiteEngine(); await engine.connect({ engine: 'pglite' } as never); await engine.initSchema(); close = () => engine.disconnect(); }
      for (const name of [...LEXICAL, ...VECTOR_ONLY]) {
        const text = LEXICAL.includes(name) ? `The quillmarsh ledger, page ${name}.` : `Notes on page ${name}.`;
        await engine.putPage(`notes/${name}`, { type: 'note', title: `Page ${name}`, compiled_truth: text, timeline: '' });
        await installFixtureChunks(engine, `notes/${name}`, [{ chunk_index: 0, chunk_text: text, chunk_source: 'compiled_truth', embedding: new Float32Array(VEC), modality: 'text' }]);
      }
      configureGateway({ embedding_model: 'openai:text-embedding-3-large', embedding_dimensions: 1536, env: { OPENAI_API_KEY: 'sk-test' } });
      __setEmbedTransportForTests(async (args: any) => ({ embeddings: args.values.map(() => VEC) }) as any);
      serve = await startServeHttp(engine, { surface: 'full' });
      token = await legacyToken(engine, ['read']);
    }, 180_000);

    afterAll(async () => {
      __setEmbedTransportForTests(null);
      resetGateway();
      await serve?.close();
      await close?.();
    }, 60_000);

    for (const tool of ['search', 'query'] as const) {
      const base = tool === 'query' ? { expand: false } : {};

      test(`${tool}: a query no page matches lexically returns the top vector pages up to the default cap, and says so in _meta and to the model`, async () => {
        const out = await callTool(serve.base, token, tool, { query: 'saltpan orbit', ...base });
        expect(slugsOf(out)).toHaveLength(DEFAULT_REMOTE_PAGE_CAP);
        const dropped = LEXICAL.length + VECTOR_ONLY.length - DEFAULT_REMOTE_PAGE_CAP;
        expect((out._meta as any)?.retrieval?.page_cap).toEqual({ cap: DEFAULT_REMOTE_PAGE_CAP, kept_pages: DEFAULT_REMOTE_PAGE_CAP, dropped_pages: dropped });
        expect(out.content.slice(1).map(c => c.text)).toContain(`[gbrain search] ${dropped} more pages matched only by meaning and were left out (default remote cap ${DEFAULT_REMOTE_PAGE_CAP}). Pass limit to read them.`);
      }, 60_000);

      test(`${tool}: a call the cap leaves alone carries no cap line`, async () => {
        const out = await callTool(serve.base, token, tool, { query: 'saltpan orbit', limit: 20, ...base });
        expect((out._meta as any)?.retrieval?.page_cap).toBeUndefined();
        expect(out.content.slice(1).some(c => c.text.includes('default remote cap'))).toBe(false);
      }, 60_000);

      test(`${tool}: every lexically matched page survives the cap, and vector-only pages are left out`, async () => {
        const slugs = slugsOf(await callTool(serve.base, token, tool, { query: 'quillmarsh ledger', ...base }));
        expect(slugs.sort()).toEqual(LEXICAL.map(n => `notes/${n}`).sort());
      }, 60_000);

      test(`${tool}: with no embedding model every keyword match comes back, past the cap`, async () => {
        resetGateway();
        try {
          const out = await callTool(serve.base, token, tool, { query: 'quillmarsh ledger', ...base });
          expect(slugsOf(out).sort()).toEqual(LEXICAL.map(n => `notes/${n}`).sort());
          expect((out._meta as any)?.retrieval?.page_cap).toBeUndefined();
        } finally {
          configureGateway({ embedding_model: 'openai:text-embedding-3-large', embedding_dimensions: 1536, env: { OPENAI_API_KEY: 'sk-test' } });
          __setEmbedTransportForTests(async (args: any) => ({ embeddings: args.values.map(() => VEC) }) as any);
        }
      }, 60_000);

      test(`${tool}: an explicit limit keeps today's rows`, async () => {
        const slugs = slugsOf(await callTool(serve.base, token, tool, { query: 'saltpan orbit', limit: 20, ...base }));
        expect(slugs).toHaveLength(LEXICAL.length + VECTOR_ONLY.length);
      }, 60_000);

      test(`${tool}: search.remote_page_cap sets the cap and 0 turns it off`, async () => {
        try {
          await engine.setConfig(REMOTE_PAGE_CAP_KEY, '5');
          expect(slugsOf(await callTool(serve.base, token, tool, { query: 'saltpan orbit', ...base }))).toHaveLength(5);
          await engine.setConfig(REMOTE_PAGE_CAP_KEY, '0');
          expect(slugsOf(await callTool(serve.base, token, tool, { query: 'saltpan orbit', ...base })).length).toBeGreaterThan(5);
        } finally {
          await engine.unsetConfig(REMOTE_PAGE_CAP_KEY);
        }
      }, 60_000);

      // Generated by gstack /ship Step 7 coverage audit (GBRA-156).
      // Value: protects=a malformed search.remote_page_cap (negative, fractional, text, blank) falls back to the default cap;
      // fails_when=the integer guard is dropped and a bad value returns no vector page or the wrong count;
      // why_new=only '5' and '0' were covered; seam=none;
      // no_claim=config written through engine.setConfig, not the CLI config validator (PGLite unit only)
      test(`${tool}: a malformed search.remote_page_cap falls back to the default cap`, async () => {
        try {
          for (const raw of ['-2', '2.5', 'many', ' ']) {
            await engine.setConfig(REMOTE_PAGE_CAP_KEY, raw);
            const out = await callTool(serve.base, token, tool, { query: 'saltpan orbit', ...base });
            expect({ raw, pages: slugsOf(out).length, cap: (out._meta as any)?.retrieval?.page_cap?.cap })
              .toEqual({ raw, pages: DEFAULT_REMOTE_PAGE_CAP, cap: DEFAULT_REMOTE_PAGE_CAP });
          }
        } finally {
          await engine.unsetConfig(REMOTE_PAGE_CAP_KEY);
        }
      }, 60_000);

      // Generated by gstack /ship Step 7 coverage audit (GBRA-156).
      // Value: protects=a paged remote call (offset > 0, no limit) keeps today's rows and carries no page_cap;
      // fails_when=the offset exemption is removed and page 2 of a remote search is cut to the cap;
      // why_new=existing tests cover limit, local and cap 0, never offset; seam=none;
      // no_claim=page boundaries across offsets (deterministic in-process embedding; PGLite)
      test(`${tool}: a paged call (offset > 0) is not capped`, async () => {
        const out = await callTool(serve.base, token, tool, { query: 'saltpan orbit', offset: 1, ...base });
        expect(out.isError).not.toBe(true);
        expect(slugsOf(out).length).toBeGreaterThan(DEFAULT_REMOTE_PAGE_CAP);
        expect((out._meta as any)?.retrieval?.page_cap).toBeUndefined();
      }, 60_000);

      test(`${tool}: a trusted local call is not capped`, async () => {
        const ctx = { engine, config: null, logger: console, dryRun: false, remote: false } as never;
        const rows = await operationsByName[tool]!.handler(ctx, { query: 'saltpan orbit', ...base }) as Array<{ slug: string }>;
        expect(new Set(rows.map(r => r.slug)).size).toBeGreaterThan(DEFAULT_REMOTE_PAGE_CAP);
      }, 60_000);
    }
  });
}