Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
Eval capture keeps fact results, and a re-created source keeps its rows
garrytan18044e51462b
diff --git a/src/core/feedback/store.ts b/src/core/feedback/store.tsindex d7efb947ff..aeab0c2a9c 100644--- a/src/core/feedback/store.ts+++ b/src/core/feedback/store.ts@@ -334,7 +334,8 @@ export async function scrubPurgedSlugs(* #9068: insert one captured eval candidate with only the slugs a page of its sources (any source when it names none)* still holds, in retrieval order. Each probe takes the feedback writers' key-share lock on the page row, so a capture* deferred past its query's response either commits before a purge takes the row, and the purge's scrub strips it, or- * waits for the purge and drops the slug. Both engines run this statement.+ * waits for the purge and drops the slug. A facts-arm result (`facts/<id>`, facts-arm.ts) names a fact, not a page:+ * it stays while that fact row does. Both engines run this statement.*/export async function insertEvalCandidate(engine: Pick<BrainEngine, 'executeRaw'>, input: EvalCandidateInput): Promise<number> {const rows = await engine.executeRaw<{ id: number | string }>(@@ -346,6 +347,8 @@ export async function insertEvalCandidate(engine: Pick<BrainEngine, 'executeRaw'ARRAY(SELECT u.s FROM unnest($3::text[]) WITH ORDINALITY AS u(s, n)WHERE EXISTS (SELECT 1 FROM pages pg WHERE pg.slug = u.sAND (cardinality($5::text[]) = 0 OR pg.source_id = ANY($5::text[])) FOR KEY SHARE)+ OR CASE WHEN u.s ~ '^facts/[0-9]{1,18}$' THEN EXISTS (SELECT 1 FROM facts f WHERE f.id = substr(u.s, 7)::bigint+ AND (cardinality($5::text[]) = 0 OR f.source_id = ANY($5::text[]))) ELSE false ENDORDER BY u.n),$4::int[], $5::text[], $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)RETURNING id`,diff --git a/src/core/source-delete.ts b/src/core/source-delete.tsindex 20e398411b..e7d18647ba 100644--- a/src/core/source-delete.ts+++ b/src/core/source-delete.ts@@ -19,7 +19,9 @@ export const unverifiedArchiveReason = (id: string) =>* claimed. A binding of another incarnation is never touched here; `gbrain* repair orphan-bindings` removes those. The same statement runs* removedSourceScrubCtes, and a second statement repeats it once the row is- * gone. Returns whether a row was deleted.+ * gone, unless a source with this id exists again by then: a replacement+ * created between the two statements keeps its rows. Returns whether a row+ * was deleted.*/export async function deleteSourceRow(engine: Pick<BrainEngine, 'executeRaw'>,@@ -38,7 +40,7 @@ export async function deleteSourceRow([id],);if (!rows.length) return false;- await engine.executeRaw(`WITH ${removedSourceScrubCtes('true')} SELECT 1`, [id]);+ await engine.executeRaw(`WITH ${removedSourceScrubCtes('NOT EXISTS (SELECT 1 FROM sources WHERE id = $1)')} SELECT 1`, [id]);return true;}@@ -48,8 +50,8 @@ export async function deleteSourceRow(* context_volunteer_events rows, and each eval candidate's captured slugs that no other source the candidate covers* still has a page for (retrieval_weights and mention_gazetteer_entries cascade from sources). Every removal runs* them in the statement that deletes the sources row (gated on that delete), so the source and those rows go- * together, then again, ungated, in a later statement: that statement reads past a feedback write or eval capture- * that held one of the source's page rows while the delete waited for it (a later one finds no page).+ * together, then again in a later statement: that statement reads past a feedback write or eval capture that held one+ * of the source's page rows while the delete waited for it (a later one finds no page).*/export function removedSourceScrubCtes(gate: string): string {const deletes = ['retrieval_event_pages', 'retrieval_event_links', 'retrieval_feedback', 'context_volunteer_events']diff --git a/test/purge-scrub-followups-9068.test.ts b/test/purge-scrub-followups-9068.test.tsindex 317814ba75..e5bf5e4ef8 100644--- a/test/purge-scrub-followups-9068.test.ts+++ b/test/purge-scrub-followups-9068.test.ts@@ -8,10 +8,12 @@* retrieval logs, ratings, weights, the volunteer log, the gazetteer and the eval candidates' captured slugs for* that source in the delete's transaction, and so does the transcript stale-part delete, which now calls it;* (c) removing a source (unmanaged removeSource -> deleteSourceRow, and the managed lifecycle remove) drops every- * such row of the source, and strips its slugs from eval candidates no other covered source still answers.- * 2. Fails when: logEvalCandidate inserts slugs without probing the page row under a key-share lock, a hard delete or a- * source removal leaves any of those rows behind, or a scrub reaches another source's rows or a slug another- * source the candidate covers still holds.+ * such row of the source, and strips its slugs from eval candidates no other covered source still answers;+ * (d) a capture keeps a facts-arm result (facts/<id>) while its fact exists; (e) a source re-created with the same+ * id between deleteSourceRow's delete and its second scrub keeps its rows.+ * 2. Fails when: logEvalCandidate inserts slugs without probing the page row under a key-share lock or drops a live+ * fact's result, a hard delete or a source removal leaves any of those rows behind, or a scrub reaches another+ * source's rows, a re-created source's rows or a slug another source the candidate covers still holds.* 3. Existing coverage: purge-retrieval-feedback-8821 and purge-name-stores-9068 pin delete_page purge and the* tombstone purge only; nothing pinned the capture, the connector and stale-part deletes, or source removal.* 4. Real seams: PGLite, and Postgres when DATABASE_URL is set (test/postgres-unit-arms.txt); the feedback store's@@ -35,6 +37,7 @@ import { captureEvalCandidate } from '../src/core/eval-capture.ts';import { withResponseDeferral } from '../src/core/background-work.ts';import { deleteConnectorPage } from '../src/core/connectors/api-source.ts';import { removeSource } from '../src/core/sources-ops.ts';+import { deleteSourceRow } from '../src/core/source-delete.ts';import { runTranscriptsIngest } from '../src/core/transcripts/ingest.ts';import { buildTranscriptSlug } from '../src/core/transcripts/types.ts';import { registerLocalWriter } from '../src/core/persistence/identity.ts';@@ -145,6 +148,18 @@ describe('#9068 an eval capture does not write a purged slug back', () => {}}, 120_000);+ test('a capture keeps a facts-arm result while its fact exists', async () => {+ for (const engine of engines) {+ await page(engine, 'default', 'notes/live-page');+ const [fact] = await engine.executeRaw<{ id: number | string }>(+ `INSERT INTO facts (fact, source, source_id) VALUES ('alice-example joined acme-example', 'fixture', 'default') RETURNING id`);+ const query = `facts arm ${randomUUID()}`;+ await engine.logEvalCandidate(candidate(query, [`facts/${fact!.id}`, 'facts/999999999', 'notes/live-page'], ['default']));+ expect({ engine: engine.kind, slugs: await capturedSlugs(engine, query) })+ .toEqual({ engine: engine.kind, slugs: [[`facts/${fact!.id}`, 'notes/live-page']] });+ }+ }, 120_000);+const postgresTest = testBackends().includes('postgres') ? test : test.skip;postgresTest('on Postgres a capture during an open purge waits for it and drops the slug', async () => {const engine = engines.find(e => e.kind === 'postgres')!;@@ -240,6 +255,25 @@ describe('#9068 removing a source scrubs every row that names its pages', () =>}}), 120_000);+ test('a source re-created between the delete and its second scrub keeps its rows', () => withEnv({ GBRAIN_HOME: home }, async () => {+ for (const engine of engines) {+ const id = 'source-reused', slug = 'notes/reused-page', query = `reused ${randomUUID()}`;+ await traced(engine, id, slug);+ let calls = 0;+ const racing = { executeRaw: async (sql: string, params?: unknown[]) => {+ if (calls++ === 1) {+ await traced(engine, id, slug);+ await engine.logEvalCandidate(candidate(query, [slug, OTHER], [id]));+ }+ return engine.executeRaw(sql, params);+ } } as Pick<BrainEngine, 'executeRaw'>;+ expect(await deleteSourceRow(racing, id)).toBe(true);+ expect({ engine: engine.kind, residue: await residue(engine, id, slug) }).toEqual({ engine: engine.kind, residue: { retrieval_event_pages: 1,+ retrieval_event_links: 1, retrieval_feedback: 2, retrieval_weights: 2, context_volunteer_events: 1, mention_gazetteer_entries: 1 } });+ expect(await capturedSlugs(engine, query)).toEqual([[slug, OTHER]]);+ }+ }), 120_000);+test('the managed lifecycle remove does the same', async () => {for (const engine of engines) {const lifecycleHome = mkdtempSync(join(tmpdir(), 'gbrain-purge-followups-managed-'));