Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
12.6 KB
Raw Blame
/**
 * v0.32.x search-lite \u2014 token budget enforcement.
 *
 * Pure module under test (no DB, no LLM). Confirms:
 *   - char/4 heuristic estimates correctly
 *   - greedy walk preserves caller ordering
 *   - meta accounting (used / kept / dropped) matches the actual cut
 *   - undefined / <=0 budget is a no-op
 *   - first-result-too-big keeps ONE truncated copy (WP2/T3 minKeep:1);
 *     GBRAIN_SEARCH_SALVAGE=off restores the strict [] (ENG-7)
 *   - packToBudget's strict [] edge is FROZEN for the memory-verb
 *     consumers (recall/entity/context_pack) \u2014 pinned directly (FOV-2)
 *
 * Lives in test/token-budget.test.ts to mirror existing search/* test naming.
 */

import { describe, test, expect } from 'bun:test';
import {
  enforceTokenBudget,
  estimateTokens,
  packToBudget,
  resultTokens,
  searchSalvageEnabled,
} from '../src/core/search/token-budget.ts';
import type { SearchResult } from '../src/core/types.ts';
import { withEnv } from './helpers/with-env.ts';

function makeResult(overrides: Partial<SearchResult> = {}): SearchResult {
  return {
    slug: 'test-page',
    page_id: 1,
    title: 'Test Title',
    type: 'concept',
    chunk_text: 'test chunk text',
    chunk_source: 'compiled_truth',
    chunk_id: 1,
    chunk_index: 0,
    score: 1.0,
    stale: false,
    ...overrides,
  };
}

describe('estimateTokens', () => {
  test('empty / nullish strings cost 0 tokens', () => {
    expect(estimateTokens('')).toBe(0);
    expect(estimateTokens(undefined)).toBe(0);
    expect(estimateTokens(null)).toBe(0);
  });

  test('rounds up so a single char still costs 1 token', () => {
    expect(estimateTokens('a')).toBe(1);
    expect(estimateTokens('abc')).toBe(1);
    expect(estimateTokens('abcd')).toBe(1);
    expect(estimateTokens('abcde')).toBe(2);
  });

  test('scales linearly at ~4 chars / token', () => {
    // 100 chars \u2192 25 tokens
    expect(estimateTokens('x'.repeat(100))).toBe(25);
    // 401 chars \u2192 ceil(401/4) = 101
    expect(estimateTokens('x'.repeat(401))).toBe(101);
  });
});

describe('resultTokens', () => {
  test('sums title + chunk_text', () => {
    const r = makeResult({ title: 'abcd', chunk_text: 'efgh' });  // 1 + 1 = 2
    expect(resultTokens(r)).toBe(2);
  });
});

describe('enforceTokenBudget', () => {
  test('undefined budget is a no-op', () => {
    const results = [makeResult({ slug: 'a' }), makeResult({ slug: 'b' })];
    const { results: kept, meta } = enforceTokenBudget(results, undefined);
    expect(kept).toHaveLength(2);
    expect(meta.dropped).toBe(0);
    expect(meta.kept).toBe(2);
    expect(meta.budget).toBe(0);  // safe-budget normalization
  });

  test('zero / negative budget is a no-op', () => {
    const results = [makeResult({ slug: 'a' })];
    expect(enforceTokenBudget(results, 0).results).toHaveLength(1);
    expect(enforceTokenBudget(results, -5).results).toHaveLength(1);
  });

  test('empty input returns empty', () => {
    const { results, meta } = enforceTokenBudget([], 100);
    expect(results).toHaveLength(0);
    expect(meta.kept).toBe(0);
    expect(meta.dropped).toBe(0);
  });

  test('greedy top-down: stops as soon as cumulative cost would exceed budget', () => {
    // Each result: title 'a' (1 tok) + chunk_text 'xxxx' (1 tok) = 2 tokens each
    const results = [
      makeResult({ slug: 'a', title: 'a', chunk_text: 'xxxx' }),
      makeResult({ slug: 'b', title: 'a', chunk_text: 'xxxx' }),
      makeResult({ slug: 'c', title: 'a', chunk_text: 'xxxx' }),
      makeResult({ slug: 'd', title: 'a', chunk_text: 'xxxx' }),
    ];
    // Budget 5 \u2192 fits 2 results (cost 2+2=4); a 3rd would push to 6.
    const { results: kept, meta } = enforceTokenBudget(results, 5);
    expect(kept).toHaveLength(2);
    expect(kept.map(r => r.slug)).toEqual(['a', 'b']);
    expect(meta.used).toBe(4);
    expect(meta.dropped).toBe(2);
    expect(meta.kept).toBe(2);
    expect(meta.budget).toBe(5);
  });

  test('preserves caller ordering (never re-ranks)', () => {
    const results = [
      makeResult({ slug: 'low-score', score: 0.1, title: 'a', chunk_text: 'xxxx' }),
      makeResult({ slug: 'high-score', score: 0.9, title: 'a', chunk_text: 'xxxx' }),
    ];
    const { results: kept } = enforceTokenBudget(results, 2);
    // 2 tokens fits exactly one result; budget pass should keep the FIRST,
    // even though it has a worse score \u2014 ordering is caller's contract.
    expect(kept).toHaveLength(1);
    expect(kept[0].slug).toBe('low-score');
  });

  // WP2/T3 (ENG-2/FOV-2): the strict [] expectation flipped to the minKeep
  // shape \u2014 the search wrapper never returns empty just because the top hit
  // is big; it keeps one truncated COPY and reports the cut honestly.
  test('first result exceeds budget alone \u2192 keeps ONE truncated copy (minKeep:1)', () => {
    const big = makeResult({ slug: 'big', title: 'a', chunk_text: 'x'.repeat(1000) });
    const small = makeResult({ slug: 'small', title: 'a', chunk_text: 'xxxx' });
    const { results: kept, meta } = enforceTokenBudget([big, small], 5);
    expect(kept).toHaveLength(1);
    expect(kept[0].slug).toBe('big');
    // title 'a' costs 1 token \u2192 4 tokens left \u2192 16 chars of chunk_text.
    expect(kept[0].chunk_text).toBe('x'.repeat(16));
    expect(resultTokens(kept[0])).toBeLessThanOrEqual(5);
    expect(meta.kept).toBe(1);
    expect(meta.dropped).toBe(1); // N-1
    expect(meta.truncated).toBe(true);
    expect(meta.used).toBeLessThanOrEqual(5);
  });

  test('minKeep truncates a COPY \u2014 the shared SearchResult is never mutated', () => {
    const big = makeResult({ slug: 'big', title: 'a', chunk_text: 'x'.repeat(1000) });
    const { results: kept } = enforceTokenBudget([big], 5);
    expect(kept[0]).not.toBe(big);
    expect(big.chunk_text).toBe('x'.repeat(1000)); // flows on to cache write + eval capture
    expect(kept[0].chunk_text.length).toBeLessThan(big.chunk_text.length);
  });

  test('budget below title-only cost \u2192 slices the title so used <= budget', () => {
    // Title alone costs 25 tokens (100 chars); budget 5 can't even fit it.
    // A hard cap that can be exceeded is not a cap: the title is sliced to
    // budget*4 chars (costs exactly `budget` under ceil(len/4)).
    const big = makeResult({ slug: 'big', title: 'T'.repeat(100), chunk_text: 'x'.repeat(1000) });
    const { results: kept, meta } = enforceTokenBudget([big], 5);
    expect(kept).toHaveLength(1);
    expect(kept[0].chunk_text).toBe('');
    expect(kept[0].title).toBe('T'.repeat(20));
    expect(meta.truncated).toBe(true);
    expect(meta.used).toBe(5);
    expect(meta.used).toBeLessThanOrEqual(meta.budget);
  });

  test('truncated flag absent on ordinary cuts and no-op passes', () => {
    const results = [
      makeResult({ slug: 'a', title: 'a', chunk_text: 'xxxx' }),
      makeResult({ slug: 'b', title: 'a', chunk_text: 'xxxx' }),
    ];
    expect(enforceTokenBudget(results, 2).meta.truncated).toBeUndefined();
    expect(enforceTokenBudget(results, undefined).meta.truncated).toBeUndefined();
  });
});

describe('GBRAIN_SEARCH_SALVAGE=off \u2014 strict budget wrapper restored (ENG-7)', () => {
  test('searchSalvageEnabled reads the env kill switch', async () => {
    await withEnv({ GBRAIN_SEARCH_SALVAGE: undefined }, () => {
      expect(searchSalvageEnabled()).toBe(true);
    });
    await withEnv({ GBRAIN_SEARCH_SALVAGE: 'off' }, () => {
      expect(searchSalvageEnabled()).toBe(false);
    });
  });

  test('first-result-exceeds returns [] again under the kill switch', async () => {
    await withEnv({ GBRAIN_SEARCH_SALVAGE: 'off' }, () => {
      const big = makeResult({ slug: 'big', title: 'a', chunk_text: 'x'.repeat(1000) });
      const small = makeResult({ slug: 'small', title: 'a', chunk_text: 'xxxx' });
      const { results: kept, meta } = enforceTokenBudget([big, small], 5);
      expect(kept).toHaveLength(0);
      expect(meta.dropped).toBe(2);
      expect(meta.kept).toBe(0);
      expect(meta.truncated).toBeUndefined();
    });
  });
});

// FOV-2 \u2014 direct packToBudget edge pins. The generic packer feeds the FROZEN
// memory-verb paths (recall operations.ts, entity, and context_pack); its
// strict first-item-exceeds [] edge must NOT inherit the search wrapper's
// minKeep salvage. No direct test existed at HEAD.
describe('packToBudget \u2014 frozen strict edge (memory-verb consumers)', () => {
  test('first item alone exceeds budget \u2192 [] (strict, regardless of salvage env)', async () => {
    // salvage ON \u2014 must not leak into the frozen packer edge.
    await withEnv({ GBRAIN_SEARCH_SALVAGE: undefined }, () => {
      const big = makeResult({ slug: 'big', title: 'a', chunk_text: 'x'.repeat(1000) });
      const { items, meta } = packToBudget([big], resultTokens, 5);
      expect(items).toHaveLength(0);
      expect(meta.kept).toBe(0);
      expect(meta.dropped).toBe(1);
      expect(meta.truncated).toBeUndefined();
    });
  });

  test('budget undefined / <= 0 passes items through unchanged', () => {
    const items = [makeResult({ slug: 'a' }), makeResult({ slug: 'b' })];
    expect(packToBudget(items, resultTokens, undefined).items).toHaveLength(2);
    expect(packToBudget(items, resultTokens, 0).items).toHaveLength(2);
    expect(packToBudget(items, resultTokens, -1).items).toHaveLength(2);
  });

  test('context_pack-shaped fixture: page-list packing keeps a contiguous prefix, strict on oversize head', () => {
    // Mirrors the context_pack cost model (operations.ts): pages costed by
    // `${title} ${slug}` via the generic <T> API.
    type PackPage = { title: string; slug: string };
    const cost = (pg: PackPage) => estimateTokens(`${pg.title} ${pg.slug}`);
    const pages: PackPage[] = [
      { title: 'Quarterly planning notes', slug: 'notes/quarterly-planning' }, // ~13 tokens
      { title: 'Acme seed memo', slug: 'deals/acme-seed' },                    // ~8 tokens
      { title: 'Widget series A', slug: 'deals/widget-series-a' },             // ~10 tokens
    ];
    // Budget fits the first two, not the third \u2192 contiguous prefix of 2.
    const budget = cost(pages[0]) + cost(pages[1]);
    const packed = packToBudget(pages, cost, budget);
    expect(packed.items.map((p) => p.slug)).toEqual([
      'notes/quarterly-planning',
      'deals/acme-seed',
    ]);
    expect(packed.meta.dropped).toBe(1);

    // Oversize head: the first page alone exceeds the budget \u2192 STRICT [];
    // context_pack's budget contract never fabricates a truncated page row.
    const tiny = packToBudget(pages, cost, 2);
    expect(tiny.items).toHaveLength(0);
    expect(tiny.meta.kept).toBe(0);
    expect(tiny.meta.dropped).toBe(3);
  });
});

describe('read-path audit #17 — packer and estimator', () => {
  test('one oversized result mid-list is skipped, not a cut of everything after it', () => {
    const results = [
      makeResult({ slug: 'a', title: 'a', chunk_text: 'xxxx' }),
      makeResult({ slug: 'huge', title: 'a', chunk_text: 'x'.repeat(4000) }),
      makeResult({ slug: 'c', title: 'a', chunk_text: 'xxxx' }),
      makeResult({ slug: 'd', title: 'a', chunk_text: 'xxxx' }),
    ];
    const { results: kept, meta } = enforceTokenBudget(results, 10);
    expect(kept.map(r => r.slug)).toEqual(['a', 'c', 'd']);
    expect(meta).toMatchObject({ used: 6, kept: 3, dropped: 1 });
  });

  test('CJK text costs about one token per character, not one per four', () => {
    expect(estimateTokens('知识管理系统')).toBe(6);
    expect(estimateTokens('こんにちは世界')).toBe(7);
    expect(estimateTokens('한국어 텍스트')).toBe(7);
    expect(estimateTokens('abcd知识')).toBe(3);
  });

  test('a CJK result is budgeted at its real cost', () => {
    const cjk = makeResult({ slug: 'cjk', title: '', chunk_text: '知'.repeat(40) });
    const tail = makeResult({ slug: 'tail', title: '', chunk_text: 'xxxx' });
    const { results: kept } = enforceTokenBudget([tail, cjk], 20);
    expect(kept.map(r => r.slug)).toEqual(['tail']);
  });

  test('the salvage truncation never splits a surrogate pair and stays within budget', () => {
    const emoji = makeResult({ slug: 'e', title: '', chunk_text: 'abc' + '😀'.repeat(50) });
    const { results: kept, meta } = enforceTokenBudget([emoji], 3);
    expect(meta.truncated).toBe(true);
    const text = kept[0].chunk_text;
    expect(text.isWellFormed()).toBe(true);
    expect(resultTokens(kept[0])).toBeLessThanOrEqual(3);
    expect(text.length).toBeGreaterThan(0);
  });

  test('the salvage truncation of CJK text stays within budget', () => {
    const cjk = makeResult({ slug: 'cjk', title: '知识', chunk_text: '知'.repeat(200) });
    const { results: kept, meta } = enforceTokenBudget([cjk], 10);
    expect(meta.truncated).toBe(true);
    expect(resultTokens(kept[0])).toBeLessThanOrEqual(10);
    expect(kept[0].chunk_text).toBe('知'.repeat(8));
  });
});