Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(sync): sum stale chunk text through disjoint index-friendly arms (#8811)

garrytan1ddc1e963c87
diff --git a/src/core/engine-sql/chunks.ts b/src/core/engine-sql/chunks.ts
index 26bf7a1d48..f57c0ade8e 100644
--- a/src/core/engine-sql/chunks.ts
+++ b/src/core/engine-sql/chunks.ts
@@ -501,12 +501,22 @@ export async function getChunkPage(
* embedding_signature drift (NULL grandfathered). `includeNullSignature`
* (#3391) lifts the grandfather clause so pre-stamp pages count as stale
* too (provider-migration paths). Shared by countStaleChunks +
- * sumStaleChunkChars so they can't drift.
+ * sumStaleChunkChars so they can't drift. #8811: `arm` narrows it to one of
+ * its two disjoint halves, an unembedded chunk or an embedded chunk of a page
+ * whose signature drifted.
*/
-function staleChunkWhere(column: string, opts?: StaleChunkOpts): SqlFragment {
+function staleChunkWhere(column: string, opts?: StaleChunkOpts, arm?: 'unembedded' | 'drifted'): SqlFragment {
const staleColRef = trustedSql(`cc.${quoteIdentifier(column)}`);
const conds: SqlFragment[] = [sqlFragment`p.deleted_at IS NULL`, sqlFragment`${trustedSql(chunkGenerationFilter('p', 'cc'))}`];
- if (opts?.signature !== undefined) {
+ if (arm === 'unembedded') {
+ conds.push(sqlFragment`${staleColRef} IS NULL`);
+ } else if (arm === 'drifted') {
+ conds.push(
+ opts?.includeNullSignature
+ ? sqlFragment`${staleColRef} IS NOT NULL AND (p.embedding_signature IS NULL OR p.embedding_signature <> ${opts.signature})`
+ : sqlFragment`${staleColRef} IS NOT NULL AND p.embedding_signature <> ${opts?.signature}`,
+ );
+ } else if (opts?.signature !== undefined) {
conds.push(
opts.includeNullSignature
? sqlFragment`(${staleColRef} IS NULL OR p.embedding_signature IS NULL OR p.embedding_signature <> ${opts.signature})`
@@ -538,10 +548,16 @@ export async function countStaleChunks(exec: ScopedRead, column: string, opts?:
export async function sumStaleChunkChars(exec: LegacyUnscopedRead, column: string, opts?: StaleChunkOpts): Promise<number> {
// Sibling of countStaleChunks: same stale predicate, summing chunk_text
// length for the sync cost preview. ::bigint guards int4 overflow.
- const { text, params } = renderFragment(sqlFragment`SELECT COALESCE(SUM(LENGTH(cc.chunk_text)), 0)::bigint AS chars
+ // #8811: with a signature the predicate ORs a chunk column with a page
+ // column, which no index serves, so every content_chunks row was read on
+ // each sync. Its two disjoint halves are summed apart instead: unembedded
+ // chunks through the partial stale index, drifted pages' chunks by page.
+ const stale = (arm?: 'unembedded' | 'drifted') => sqlFragment`SELECT LENGTH(cc.chunk_text) AS chars
FROM content_chunks cc
JOIN pages p ON p.id = cc.page_id
- WHERE ${staleChunkWhere(column, opts)}`);
+ WHERE ${staleChunkWhere(column, opts, arm)}`;
+ const { text, params } = renderFragment(sqlFragment`SELECT COALESCE(SUM(chars), 0)::bigint AS chars FROM (${
+ opts?.signature === undefined ? stale() : sqlFragment`${stale('unembedded')} UNION ALL ${stale('drifted')}`}) stale`);
const { rows } = await exec.unsafe<{ chars?: number | string }>(text, params);
return Number(rows[0]?.chars ?? 0);
}
diff --git a/test/e2e/sum-stale-chunk-chars-8811-postgres.test.ts b/test/e2e/sum-stale-chunk-chars-8811-postgres.test.ts
new file mode 100644
index 0000000000..5a519d8df2
--- /dev/null
+++ b/test/e2e/sum-stale-chunk-chars-8811-postgres.test.ts
@@ -0,0 +1,3 @@
+import { registerPostgresTests } from '../helpers/test-backends.ts';
+
+await registerPostgresTests(() => import('../sum-stale-chunk-chars-8811.test.ts'));
diff --git a/test/sum-stale-chunk-chars-8811.test.ts b/test/sum-stale-chunk-chars-8811.test.ts
new file mode 100644
index 0000000000..616d2d23de
--- /dev/null
+++ b/test/sum-stale-chunk-chars-8811.test.ts
@@ -0,0 +1,83 @@
+/**
+ * #8811: `sumStaleChunkChars` (engine-sql/chunks.ts), the sync cost gate's and the migration plan's stale-text total.
+ * With a signature it sums the stale predicate's two disjoint halves (an unembedded chunk, or an embedded chunk of a
+ * page whose signature drifted) apart, so each can use its own index. This pins the totals for every option the
+ * callers pass over a mix of matching, drifted, unsigned and embed_skip pages, on PGLite and on Postgres when
+ * DATABASE_URL is set. Each chunk's text length is a distinct power of two, so a total names exactly the chunks it
+ * counted: a chunk summed twice or missed changes it.
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
+import { testBackends } from './helpers/test-backends.ts';
+
+const CURRENT = 'example:model-current:dims';
+const OLD = 'example:model-old:dims';
+
+const PAGES: Array<{ slug: string; source: string; signature: string | null; skip?: boolean; embedded: number[]; unembedded: number[] }> = [
+ { slug: 'notes/current', source: 'default', signature: CURRENT, embedded: [1, 2], unembedded: [4] },
+ { slug: 'notes/drifted', source: 'default', signature: OLD, embedded: [8, 16], unembedded: [32] },
+ { slug: 'notes/unsigned', source: 'default', signature: null, embedded: [64], unembedded: [128] },
+ { slug: 'notes/skipped', source: 'default', signature: OLD, skip: true, embedded: [256], unembedded: [512] },
+ { slug: 'notes/elsewhere', source: 'source-b', signature: OLD, embedded: [1024], unembedded: [2048] },
+];
+
+for (const kind of testBackends()) {
+ describe(`sumStaleChunkChars totals (${kind}, #8811)`, () => {
+ let engine: BrainEngine;
+ let close: () => Promise<void>;
+
+ beforeAll(async () => {
+ if (kind === 'pglite') {
+ const pglite = new PGLiteEngine();
+ await pglite.connect({});
+ await pglite.initSchema();
+ engine = pglite;
+ close = () => pglite.disconnect();
+ } else {
+ const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!);
+ engine = pg.engine;
+ close = pg.close;
+ await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
+ }
+ await engine.executeRaw(`INSERT INTO sources (id, name, config) VALUES ('source-b', 'source-b', '{}') ON CONFLICT (id) DO NOTHING`);
+ const [{ dim }] = await engine.executeRaw<{ dim: number }>(
+ `SELECT atttypmod AS dim FROM pg_attribute WHERE attrelid = 'content_chunks'::regclass AND attname = 'embedding' AND attnum > 0`);
+ for (const page of PAGES) {
+ await engine.putPage(page.slug, { type: 'note', title: page.slug, compiled_truth: `# ${page.slug}` }, { sourceId: page.source });
+ const lengths = [...page.embedded, ...page.unembedded];
+ await engine.upsertChunks(page.slug, lengths.map((n, i) => ({ chunk_index: i, chunk_text: 'x'.repeat(n), chunk_source: 'compiled_truth' as const, token_count: 1 })), { sourceId: page.source });
+ await engine.executeRaw(
+ `UPDATE content_chunks SET embedding = ('[' || array_to_string(array_fill(0.01::real, ARRAY[$1::int]), ',') || ']')::vector
+ WHERE page_id = (SELECT id FROM pages WHERE slug = $2 AND source_id = $3) AND length(chunk_text) = ANY($4::int[])`,
+ [Number(dim), page.slug, page.source, page.embedded]);
+ await engine.executeRaw(`UPDATE pages SET embedding_signature = $1, frontmatter = CASE WHEN $2 THEN '{"embed_skip": true}'::jsonb ELSE frontmatter END
+ WHERE slug = $3 AND source_id = $4`, [page.signature, page.skip === true, page.slug, page.source]);
+ }
+ }, 120_000);
+ afterAll(async () => { await close?.(); });
+
+ test('without a signature only unembedded chunks count', async () => {
+ expect(await engine.sumStaleChunkChars()).toBe(4 + 32 + 128 + 2048);
+ expect(await engine.sumStaleChunkChars({ sourceId: 'default' })).toBe(4 + 32 + 128);
+ });
+
+ test('a signature adds embedded chunks of drifted pages once, never unsigned or embed_skip pages', async () => {
+ expect(await engine.sumStaleChunkChars({ signature: CURRENT })).toBe(4 + 32 + 128 + 2048 + 8 + 16 + 1024);
+ expect(await engine.sumStaleChunkChars({ signature: CURRENT, sourceId: 'source-b' })).toBe(2048 + 1024);
+ });
+
+ test('includeNullSignature also counts embedded chunks of unsigned pages', async () => {
+ expect(await engine.sumStaleChunkChars({ signature: CURRENT, includeNullSignature: true })).toBe(4 + 32 + 128 + 2048 + 8 + 16 + 1024 + 64);
+ expect(await engine.sumStaleChunkChars({ signature: CURRENT, includeNullSignature: true, sourceId: 'default' })).toBe(4 + 32 + 128 + 8 + 16 + 64);
+ });
+
+ test('countStaleChunks counts the same chunks the totals name', async () => {
+ const rows = await engine.executeRaw<{ n: number }>(`SELECT length(chunk_text) AS n FROM content_chunks`);
+ expect(rows.length).toBe(PAGES.reduce((sum, page) => sum + page.embedded.length + page.unembedded.length, 0));
+ expect(await engine.countStaleChunks({ signature: CURRENT })).toBe(7);
+ expect(await engine.countStaleChunks({ signature: CURRENT, includeNullSignature: true })).toBe(8);
+ });
+ });
+}