Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
11.4 KB
Raw Blame
/**
 * Large pages (Wave A): get_chunks pages by chunk_index.
 *
 * 1. Protects: get_chunks never returns every chunk of a large page in one
 *    call. It returns [from_index, from_index + limit) (default 0 and 50),
 *    keeps the bare-array result, and carries total_chunks / next_from_index
 *    on `_meta.pagination`; remote callers also get the model-visible
 *    `listing_truncated` notice naming the next call, are capped at 500, and
 *    the trusted local CLI gets a stderr line instead.
 * 2. Fails when: the handler reads every chunk again, a page boundary skips or
 *    repeats a chunk, total_chunks counts the wrong rows, a remote limit is
 *    not capped, or an invalid from_index / limit is answered.
 * 3. No new seam: the real handler over PGLite, the engine's getChunkPage
 *    (PGLite, and Postgres when DATABASE_URL is set), and one MCP dispatch
 *    round trip.
 */
import { afterAll, beforeAll, describe, expect, spyOn, test } from 'bun:test';
import { mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { operationsByName, type OperationContext } from '../src/core/operations.ts';
import type { Notice } from '../src/core/agent-output.ts';
import { dispatchToolCall, __resetBackupNoticeForTests } from '../src/mcp/dispatch.ts';
import { __resetFactsDrainNoticesForTests } from '../src/core/facts/drain.ts';
import { __resetBehaviorNoticeForTests } from '../src/core/behavior-change-notice.ts';
import { withEnv } from './helpers/with-env.ts';
import { SAFE_FENCE_CHUNKER_VERSION } from '../src/core/search/safe-chunks.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';

const BIG = 'notes/big-example';
const SMALL = 'notes/small-example';
const BIG_CHUNKS = 620;
let engine: PGLiteEngine;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
  await seedChunks(engine);
}, 120_000);

async function seedChunks(e: BrainEngine): Promise<void> {
  await e.putPage(BIG, { type: 'note', title: 'Big', compiled_truth: 'big body' });
  await e.upsertChunks(BIG, Array.from({ length: BIG_CHUNKS }, (_, i) => ({ chunk_index: i, chunk_text: `chunk ${i} of alice-example notes`, chunk_source: 'compiled_truth' as const })));
  await e.putPage(SMALL, { type: 'note', title: 'Small', compiled_truth: 'small body' });
  await e.upsertChunks(SMALL, [0, 1, 2].map(i => ({ chunk_index: i, chunk_text: `small ${i}`, chunk_source: 'compiled_truth' as const })));
  // What a sealed markdown import leaves: the chunks match the current revision under the safe fence chunker.
  await e.executeRaw(`UPDATE pages SET text_projection_revision = knowledge_revision, chunker_version = ${SAFE_FENCE_CHUNKER_VERSION} WHERE slug = ANY($1::text[])`, [[BIG, SMALL]]);
}
afterAll(async () => { await engine.disconnect(); });

interface Captured { notices: Notice[]; meta: Record<string, unknown> }

function ctx(remote: boolean, captured?: Captured): OperationContext {
  return {
    engine, config: { engine: 'pglite' } as never, sourceId: 'default', remote, dryRun: false,
    logger: { info() {}, warn() {}, error() {} },
    ...(captured ? { emitNotice: (n: Notice) => { captured.notices.push(n); }, emitResponseMeta: (k: string, v: unknown) => { captured.meta[k] = v; } } : {}),
  } as OperationContext;
}

async function chunks(remote: boolean, params: Record<string, unknown>, captured?: Captured): Promise<Array<{ chunk_index: number }>> {
  return await operationsByName.get_chunks.handler(ctx(remote, captured), params) as Array<{ chunk_index: number }>;
}

const indices = (rows: Array<{ chunk_index: number }>) => rows.map(r => r.chunk_index);
const range = (from: number, to: number) => Array.from({ length: to - from }, (_, i) => from + i);
const fresh = (): Captured => ({ notices: [], meta: {} });

describe('get_chunks paging', () => {
  test('default: the first 50 chunks, total_chunks and next_from_index on _meta, a notice for remote callers', async () => {
    const c = fresh();
    expect(indices(await chunks(true, { slug: BIG }, c))).toEqual(range(0, 50));
    expect(c.meta.pagination).toEqual({ from_index: 0, limit: 50, returned: 50, total_chunks: BIG_CHUNKS, next_from_index: 50 });
    expect(c.notices).toHaveLength(1);
    expect(c.notices[0].code).toBe('listing_truncated');
    expect(c.notices[0].fix?.mcp).toEqual({ tool: 'get_chunks', arguments: { slug: BIG, from_index: 50, limit: 50 } });
  });

  test('a default call on a page with more than 50 chunks marks the cut in the array itself (clients that ignore _meta)', async () => {
    for (const remote of [true, false]) {
      const rows = await chunks(remote, { slug: BIG }) as Array<{ chunk_index: number; truncated?: boolean; next_from_index?: number }>;
      expect(rows).toHaveLength(50);
      expect(rows[49]).toMatchObject({ chunk_index: 49, truncated: true, next_from_index: 50 });
      expect(rows.slice(0, 49).some(row => 'truncated' in row || 'next_from_index' in row)).toBe(false);
    }
    const whole = await chunks(true, { slug: SMALL }) as Array<Record<string, unknown>>;
    expect(whole.some(row => 'truncated' in row || 'next_from_index' in row)).toBe(false);
    const last = await chunks(true, { slug: BIG, from_index: 600, limit: 50 }) as Array<Record<string, unknown>>;
    expect(last.some(row => 'truncated' in row)).toBe(false);
  });

  test('paging boundaries: consecutive pages cover every chunk exactly once', async () => {
    const seen: number[] = [];
    let from: number | undefined = 0;
    let calls = 0;
    while (from !== undefined) {
      const c = fresh();
      seen.push(...indices(await chunks(true, { slug: BIG, from_index: from, limit: 200 }, c)));
      from = (c.meta.pagination as { next_from_index?: number }).next_from_index;
      calls++;
    }
    expect(calls).toBe(4);
    expect(seen).toEqual(range(0, BIG_CHUNKS));
  });

  test('the last page has no next_from_index and no notice', async () => {
    const c = fresh();
    expect(indices(await chunks(true, { slug: BIG, from_index: 600, limit: 50 }, c))).toEqual(range(600, BIG_CHUNKS));
    expect(c.meta.pagination).toEqual({ from_index: 600, limit: 50, returned: 20, total_chunks: BIG_CHUNKS });
    expect(c.notices).toEqual([]);
  });

  test('a page with fewer chunks than the limit returns them all, no notice', async () => {
    const c = fresh();
    expect(indices(await chunks(true, { slug: SMALL }, c))).toEqual([0, 1, 2]);
    expect(c.meta.pagination).toEqual({ from_index: 0, limit: 50, returned: 3, total_chunks: 3 });
    expect(c.notices).toEqual([]);
  });

  test('from_index past the end returns [] with total_chunks', async () => {
    const c = fresh();
    expect(await chunks(true, { slug: BIG, from_index: 5000 }, c)).toEqual([]);
    expect(c.meta.pagination).toEqual({ from_index: 5000, limit: 50, returned: 0, total_chunks: BIG_CHUNKS });
    expect(c.notices).toEqual([]);
  });

  test('a missing page reads as [] with total_chunks 0', async () => {
    const c = fresh();
    expect(await chunks(true, { slug: 'notes/missing-example' }, c)).toEqual([]);
    expect(c.meta.pagination).toEqual({ from_index: 0, limit: 50, returned: 0, total_chunks: 0 });
  });

  test('remote limit is capped at 500; trusted local honors any explicit limit', async () => {
    const c = fresh();
    expect(indices(await chunks(true, { slug: BIG, limit: 10_000 }, c))).toEqual(range(0, 500));
    expect(c.meta.pagination).toMatchObject({ limit: 500, clamped_from: 10_000, next_from_index: 500 });
    expect(c.notices[0].why).toContain('limit 10000 was capped at 500');
    expect(indices(await chunks(false, { slug: BIG, limit: 10_000 }))).toEqual(range(0, BIG_CHUNKS));
  });

  test('trusted local: default 50 plus one stderr line naming --limit/--from-index, no notice; an explicit limit is silent', async () => {
    const err = spyOn(console, 'error').mockImplementation(() => {});
    try {
      const c = fresh();
      expect(indices(await chunks(false, { slug: BIG }, c))).toEqual(range(0, 50));
      expect(c.notices).toEqual([]);
      expect(err).toHaveBeenCalledTimes(1);
      const line = String(err.mock.calls[0][0]);
      expect(line).toContain('--limit');
      expect(line).toContain('--from-index 50');
      await chunks(false, { slug: BIG, limit: 10 });
      expect(err).toHaveBeenCalledTimes(1);
    } finally { err.mockRestore(); }
  });

  test('invalid from_index or limit is invalid_params', async () => {
    for (const from_index of [-1, 1.5, 'ten', Number.NaN]) {
      await expect(chunks(true, { slug: BIG, from_index })).rejects.toThrow('from_index must be a non-negative integer');
    }
    for (const limit of [0, -5, 2.5, 'all']) {
      await expect(chunks(true, { slug: BIG, limit })).rejects.toThrow('limit must be a positive integer');
    }
  });

  test('MCP dispatch: content[0] stays the bare array; the notice names the next call', async () => {
    __resetBackupNoticeForTests();
    __resetFactsDrainNoticesForTests();
    await engine.executeRaw('UPDATE sources SET created_at = now()');
    __resetBehaviorNoticeForTests();
    const home = mkdtempSync(join(tmpdir(), 'gbrain-get-chunks-notice-'));
    await withEnv({ GBRAIN_BACKUP_CHECK: 'off', GBRAIN_NO_ONBOARD_NUDGE: '1', GBRAIN_HOME: home }, async () => {
      const res = await dispatchToolCall(engine as never, 'get_chunks', { slug: BIG, limit: 100 }, { remote: true, transport: 'stdio', sourceId: 'default' });
      expect(res.isError).toBeUndefined();
      const body = JSON.parse(res.content[0].text);
      expect(Array.isArray(body)).toBe(true);
      expect(body).toHaveLength(100);
      expect(body[99]).toMatchObject({ chunk_index: 99, truncated: true, next_from_index: 100 });
      expect(res._meta?.pagination).toEqual({ from_index: 0, limit: 100, returned: 100, total_chunks: BIG_CHUNKS, next_from_index: 100 });
      const notice = res.content.slice(1).map(b => b.text).find(t => t.startsWith('[gbrain notice listing_truncated'));
      expect(notice).toContain(`fix: get_chunks {"slug":"${BIG}","from_index":100,"limit":100}`);
    });
  }, 30_000);
});

describe('engine getChunkPage', () => {
  test('returns the chunk_index range, the admitted count and the highest index (PGLite, and Postgres when DATABASE_URL is set)', async () => {
    const engines: Array<{ engine: BrainEngine; close?: () => Promise<void> }> = [{ engine }];
    if (process.env.DATABASE_URL) {
      const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL);
      await seedChunks(pg.engine);
      engines.push({ engine: pg.engine, close: pg.close });
    }
    try {
      for (const { engine: e } of engines) {
        const page = await e.getChunkPage(BIG, { sourceId: 'default', fromIndex: 10, limit: 5 });
        expect(page.chunks.map(c => c.chunk_index)).toEqual([10, 11, 12, 13, 14]);
        expect(page.total_chunks).toBe(BIG_CHUNKS);
        expect(page.max_chunk_index).toBe(BIG_CHUNKS - 1);
        expect(page.chunks[0].chunk_text).toBe('chunk 10 of alice-example notes');
        expect(await e.getChunkPage(BIG, { sourceId: 'default', fromIndex: 5000, limit: 5 })).toEqual({ chunks: [], total_chunks: BIG_CHUNKS, max_chunk_index: BIG_CHUNKS - 1 });
        expect(await e.getChunkPage('notes/missing-example', { sourceId: 'default', fromIndex: 0, limit: 5 })).toEqual({ chunks: [], total_chunks: 0, max_chunk_index: null });
      }
    } finally { for (const { close } of engines) if (close) await close(); }
  }, 120_000);
});