Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #2704 — a single markdown page whose compiled_truth exceeds Postgres's
* hard 1,048,575-byte tsvector cap made update_page_search_vector() throw
* "string is too long for tsvector" INSIDE the pages UPSERT transaction,
* blocking the whole source's sync checkpoint (Sync BLOCKED) even though
* every other file in the run imported fine.
*
* v124 (migrate.ts) drops compiled_truth from the trigger — it was
* already redundant with content_chunks.search_vector (chunk-grain,
* populated separately and well under the tsvector cap), which is what
* searchKeyword() actually queries.
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { installFixtureChunks } from './helpers/page-projection.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
// #2704: the 1,048,575-byte tsvector cap is on to_tsvector's SERIALIZED
// OUTPUT (lexemes + position lists), not the raw input byte length —
// repeating the same few words produces a tiny deduplicated vector
// regardless of input size (verified: a 2.2MB string of 5 repeated words
// does NOT overflow). Genuinely diverse, mostly-unique tokens are what
// blows the output past the cap, matching a real large export (a Google
// Docs dump, a long mailing-list thread) where the words don't repeat
// like lorem-ipsum filler does.
const OVERSIZED_BODY = Array.from({ length: 200_000 }, (_, i) => `token${i.toString(36)}`).join(' '); // ~2MB, ~2.7MB serialized tsvector
describe('#2704: oversized page body no longer overflows pages.search_vector', () => {
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
}, 60_000);
afterAll(async () => {
await engine.disconnect();
}, 60_000);
beforeEach(async () => {
await resetPgliteState(engine);
});
test('putPage with a >1MB compiled_truth succeeds (previously threw "string is too long for tsvector")', async () => {
expect(OVERSIZED_BODY.length).toBeGreaterThan(1_048_575);
const page = await engine.putPage('oversized-page', {
type: 'note',
title: 'Oversized Page',
compiled_truth: OVERSIZED_BODY,
});
expect(page).not.toBeNull();
expect(page.slug).toBe('oversized-page');
}, 30_000);
test('an oversized page is still keyword-searchable via chunk-grain search after import', async () => {
// Mirrors import-file.ts: chunking is what actually feeds
// content_chunks.search_vector, independent of the pages-level
// trigger this fix touches. A distinctive token near the start proves
// the chunk (not just the page row) is queryable.
const distinctiveBody = `zzdistinctivetoken2704 ${OVERSIZED_BODY}`;
await engine.putPage('oversized-searchable', {
type: 'note',
title: 'Oversized Searchable',
compiled_truth: distinctiveBody,
});
const { chunkText } = await import('../src/core/chunkers/recursive.ts');
let chunkIndex = 0;
const chunks = chunkText(distinctiveBody).map((c) => ({
chunk_index: chunkIndex++,
chunk_text: c.text,
chunk_source: 'compiled_truth' as const,
}));
await installFixtureChunks(engine, 'oversized-searchable', chunks);
const results = await engine.searchKeyword('zzdistinctivetoken2704');
expect(results.some((r) => r.slug === 'oversized-searchable')).toBe(true);
}, 30_000);
test('normal-sized page search_vector still carries title/timeline signal (not fully inert)', async () => {
await engine.putPage('small-page', {
type: 'note',
title: 'zzTitleToken2704',
compiled_truth: 'short body',
});
const rows = await engine.executeRaw<{ has_vector: boolean }>(
`SELECT search_vector IS NOT NULL AS has_vector FROM pages WHERE slug = 'small-page'`,
);
expect(rows[0]?.has_vector).toBe(true);
}, 30_000);
});