Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
9.1 KB
Raw Blame
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { installPageProjection, readProjectionSnapshot } from '../src/core/page-state/projections.ts';
/**
 * SUP-3874 — heal already-stored chunks that exceed the embedding input cap.
 */

import { describe, test, expect, beforeAll, beforeEach, afterAll } from 'bun:test';
import {
  healOversizedChunks,
  healedChunksToStaleRows,
  healOversizedPageChunks,
} from '../src/core/embed-oversize-heal.ts';
import type { Chunk } from '../src/core/types.ts';
import { estimateEmbedTokens } from '../src/core/chunkers/token-estimate.ts';
import { EMBED_INPUT_SAFETY } from '../src/core/embedding-input-limit.ts';
import { configureGateway, resetGateway } from '../src/core/ai/gateway.ts';

beforeEach(() => {
  resetGateway();
});

afterAll(() => {
  resetGateway();
  configureGateway({
    embedding_model: 'openai:text-embedding-3-large',
    embedding_dimensions: 1536,
    env: { ...process.env },
  });
});

const MXBAI_CAP = Math.floor(512 * EMBED_INPUT_SAFETY); // 307

function fatParagraph(n: number): string {
  const para =
    'The SuperAICoach SEO implementation plan covers technical setup, content calendars, schema markup, and local Philadelphia keyword clusters that must remain searchable after re-embedding. ';
  return Array.from({ length: n }, () => para).join('\n\n');
}

function chunkRow(overrides: Partial<Chunk>): Chunk {
  return {
    id: 1,
    page_id: 42,
    chunk_index: 0,
    chunk_text: 'body',
    chunk_source: 'compiled_truth',
    embedding: null,
    model: 'openai:text-embedding-3-large',
    token_count: 2,
    embedded_at: null,
    ...overrides,
  };
}

describe('healedChunksToStaleRows', () => {
  test('passes chunk_source through unchanged — fenced_code is NEVER coerced (D20-T4)', () => {
    const rows = healedChunksToStaleRows(
      [
        chunkRow({ chunk_index: 0, chunk_source: 'compiled_truth' }),
        chunkRow({ id: 2, chunk_index: 1, chunk_source: 'fenced_code' }),
        chunkRow({ id: 3, chunk_index: 2, chunk_source: 'timeline' }),
      ],
      'notes/some-page',
      'src-a',
    );
    expect(rows.map((r) => r.chunk_source)).toEqual(['compiled_truth', 'fenced_code', 'timeline']);
    for (const r of rows) {
      expect(r.slug).toBe('notes/some-page');
      expect(r.source_id).toBe('src-a');
    }
  });

  test('keeps only rows still needing embeddings (unembedded or NULL vector)', () => {
    const rows = healedChunksToStaleRows(
      [
        chunkRow({ chunk_index: 0, embedded_at: null }),
        chunkRow({ id: 2, chunk_index: 1, embedded_at: new Date(), embedding_is_null: false }),
        chunkRow({ id: 3, chunk_index: 2, embedded_at: new Date(), embedding_is_null: true }),
      ],
      'p',
      'default',
    );
    expect(rows.map((r) => r.chunk_index)).toEqual([0, 2]);
  });

  test('maps model/token_count to null when absent and returns [] for fully-embedded pages', () => {
    const rows = healedChunksToStaleRows(
      [chunkRow({ model: undefined as unknown as string, token_count: null })],
      'p',
      'default',
    );
    expect(rows).toHaveLength(1);
    expect(rows[0].model).toBeNull();
    expect(rows[0].token_count).toBeNull();
    expect(
      healedChunksToStaleRows(
        [chunkRow({ embedded_at: new Date(), embedding_is_null: false })],
        'p',
        'default',
      ),
    ).toEqual([]);
  });
});

describe('healOversizedChunks', () => {
  test('no-op when every chunk already fits', () => {
    const chunks = [
      { chunk_index: 0, chunk_text: 'short one', chunk_source: 'compiled_truth' as const, token_count: 2 },
      { chunk_index: 1, chunk_text: 'short two', chunk_source: 'compiled_truth' as const, token_count: 2 },
    ];
    const result = healOversizedChunks(chunks, MXBAI_CAP);
    expect(result.changed).toBe(false);
    expect(result.splitCount).toBe(0);
    expect(result.chunks).toHaveLength(2);
    expect(result.chunks.map((c) => c.chunk_text)).toEqual(['short one', 'short two']);
  });

  test('splits only the oversized row and keeps siblings', () => {
    const oversized = fatParagraph(40);
    expect(estimateEmbedTokens(oversized)).toBeGreaterThan(MXBAI_CAP);

    const chunks = [
      { chunk_index: 0, chunk_text: 'lead-in', chunk_source: 'compiled_truth' as const, token_count: 2 },
      { chunk_index: 1, chunk_text: oversized, chunk_source: 'compiled_truth' as const, token_count: 5000 },
      { chunk_index: 2, chunk_text: 'closing', chunk_source: 'timeline' as const, token_count: 2 },
    ];
    const result = healOversizedChunks(chunks, MXBAI_CAP);
    expect(result.changed).toBe(true);
    expect(result.splitCount).toBe(1);
    expect(result.chunks.length).toBeGreaterThan(3);
    expect(result.chunks[0].chunk_text).toBe('lead-in');
    expect(result.chunks[result.chunks.length - 1].chunk_text).toBe('closing');
    expect(result.chunks[result.chunks.length - 1].chunk_source).toBe('timeline');
    for (const c of result.chunks) {
      expect(estimateEmbedTokens(c.chunk_text)).toBeLessThanOrEqual(MXBAI_CAP);
      expect(c.chunk_index).toBe(result.chunks.indexOf(c));
    }
    // Split, not truncated: joined body of middle pieces still covers the original.
    const middle = result.chunks.slice(1, -1).map((c) => c.chunk_text).join('');
    expect(middle.length).toBeGreaterThanOrEqual(Math.floor(oversized.length * 0.9));
  });

  test('reindexes contiguously after a split', () => {
    const oversized = fatParagraph(40);
    const result = healOversizedChunks(
      [{ chunk_index: 7, chunk_text: oversized, chunk_source: 'compiled_truth' as const, token_count: null }],
      MXBAI_CAP,
    );
    expect(result.changed).toBe(true);
    expect(result.chunks.map((c) => c.chunk_index)).toEqual(
      result.chunks.map((_, i) => i),
    );
  });

  test('preserves modality and code metadata on every split piece', () => {
    const oversized = fatParagraph(40);
    const result = healOversizedChunks([{
      chunk_index: 0,
      chunk_text: oversized,
      chunk_source: 'fenced_code' as const,
      token_count: 5000,
      modality: 'image' as const,
      language: 'typescript',
      symbol_name: 'buildIndex',
      symbol_type: 'function',
      start_line: 10,
      end_line: 90,
      parent_symbol_path: ['SearchEngine'],
      doc_comment: 'Build the searchable index.',
      symbol_name_qualified: 'SearchEngine.buildIndex',
    }], MXBAI_CAP);

    expect(result.changed).toBe(true);
    expect(result.chunks.length).toBeGreaterThan(1);
    for (const chunk of result.chunks) {
      expect(chunk.modality).toBe('image');
      expect(chunk.language).toBe('typescript');
      expect(chunk.symbol_name).toBe('buildIndex');
      expect(chunk.symbol_type).toBe('function');
      expect(chunk.start_line).toBe(10);
      expect(chunk.end_line).toBe(90);
      expect(chunk.parent_symbol_path).toEqual(['SearchEngine']);
      expect(chunk.doc_comment).toBe('Build the searchable index.');
      expect(chunk.symbol_name_qualified).toBe('SearchEngine.buildIndex');
    }
  });
});

describe('healOversizedPageChunks with guarded database projections', () => {
  let engine: PGLiteEngine;
  beforeAll(async () => {
    engine = new PGLiteEngine();
    await engine.connect({});
    await engine.initSchema();
  }, 120_000);
  afterAll(async () => { await engine.disconnect(); });

  async function seed(slug: string, body: string) {
    await engine.putPage(slug, { type: 'note', title: 'Example', compiled_truth: body });
    const prepared = (await readProjectionSnapshot(engine, slug, 'default', { allowUnsealed: true }))!;
    await installPageProjection(engine, prepared,
      [{ chunk_index: 0, chunk_text: body, chunk_source: 'compiled_truth' }], { seal: true });
    return prepared.snapshot;
  }

  test('small chunks retain their stored identity and canonical revision', async () => {
    const snapshot = await seed('oversize-small', 'A short example.');
    const before = await engine.getChunks('oversize-small');
    const result = await healOversizedPageChunks(engine, 'oversize-small', { maxTokens: MXBAI_CAP });
    expect(result.changed).toBe(false);
    expect(result.chunks.map(c => c.id)).toEqual(before.map(c => c.id));
    expect((await engine.readPageSnapshot('oversize-small'))!.revision).toBe(snapshot.revision);
  });

  test('oversized chunks split atomically and remain readable under the same canonical revision', async () => {
    const snapshot = await seed('oversize-large', fatParagraph(40));
    const splits: number[] = [];
    const result = await healOversizedPageChunks(engine, 'oversize-large', { maxTokens: MXBAI_CAP, onSplit: n => splits.push(n) });
    expect(result.changed).toBe(true);
    expect(splits).toEqual([1]);
    expect(result.chunks.length).toBeGreaterThan(1);
    for (const c of result.chunks) expect(estimateEmbedTokens(c.chunk_text)).toBeLessThanOrEqual(MXBAI_CAP);
    expect(await engine.getChunks('oversize-large', { requireSafeChunks: true })).toHaveLength(result.chunks.length);
    expect((await engine.readPageSnapshot('oversize-large'))!.revision).toBe(snapshot.revision);
  });
  // Stale-revision and same-revision/different-chunk races are exercised on
  // both real engines in page-projection-concurrency.test.ts.
});