Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(purge): feedback writers skip purged pages; federated captures keep shared slugs

garrytan7d421a6b0faa
diff --git a/src/core/facts/purge-verify.ts b/src/core/facts/purge-verify.ts
index f080c3e56d..82c7a69931 100644
--- a/src/core/facts/purge-verify.ts
+++ b/src/core/facts/purge-verify.ts
@@ -619,7 +619,8 @@ export async function verifyPagePurge(engine: BrainEngine, input: { sourceId: st
await keyed('retrieval_event_links', `SELECT 1 FROM retrieval_event_links WHERE source_id=$1 AND (to_slug=$2 OR ${edgeNames('edge_key')})`, [src, slug]);
for (const table of ['retrieval_feedback', 'retrieval_weights']) await keyed(table, `SELECT 1 FROM ${table} WHERE source_id=$1
AND ((element_kind='page' AND element_key=$2) OR (element_kind='link' AND ${edgeNames('element_key')}))`, [src, slug]);
- await keyed('eval_candidates.retrieved_slugs', 'SELECT 1 FROM eval_candidates WHERE $1 = ANY(source_ids) AND $2 = ANY(retrieved_slugs)', [src, slug]);
+ await keyed('eval_candidates.retrieved_slugs', `SELECT 1 FROM eval_candidates WHERE $1 = ANY(source_ids) AND $2 = ANY(retrieved_slugs)
+ AND NOT EXISTS (SELECT 1 FROM pages po WHERE po.slug = $2 AND po.source_id = ANY(eval_candidates.source_ids) AND po.source_id <> $1)`, [src, slug]);
await keyed('files', 'SELECT 1 FROM files WHERE page_id IS NULL AND page_slug=$2 AND (source_id=$1 OR source_id IS NULL)', [src, slug]);
await keyed('write_gate_holds', 'SELECT 1 FROM write_gate_holds WHERE source_id=$1 AND slug=$2', [src, slug]);
if (!await relationExists(engine, 'query_cache')) stores.push(missingCache('query_cache'));
diff --git a/src/core/feedback/store.ts b/src/core/feedback/store.ts
index 6197175731..ab51f72b68 100644
--- a/src/core/feedback/store.ts
+++ b/src/core/feedback/store.ts
@@ -61,6 +61,10 @@ export interface AppliedRating {
newly_applied: boolean;
}
+const pageRowExists = (source: string, slug: string) => `EXISTS (SELECT 1 FROM pages pg WHERE pg.source_id = ${source} AND pg.slug = ${slug})`;
+const edgeEndsExist = (source: string, key: string) =>
+ `${pageRowExists(source, `split_part(${key}, '|', 1)`)} AND ${pageRowExists(source, `reverse(split_part(reverse(${key}), '|', 1))`)}`;
+
export async function insertRetrievalEvents(engine: BrainEngine, events: RetrievalEventInput[]): Promise<void> {
if (events.length === 0) return;
await engine.executeRaw(
@@ -74,6 +78,8 @@ export async function insertRetrievalEvents(engine: BrainEngine, events: Retriev
await engine.executeRaw(
`INSERT INTO retrieval_event_pages (event_id, source_id, slug, content_hash, rank, cited)
SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::int[], $6::text[]::boolean[])
+ AS t(event_id, source_id, slug, content_hash, rank, cited)
+ WHERE ${pageRowExists('t.source_id', 't.slug')}
ON CONFLICT DO NOTHING`,
[
pages.map(p => p.id), pages.map(p => p.source_id), pages.map(p => p.slug),
@@ -85,7 +91,8 @@ export async function insertRetrievalEvents(engine: BrainEngine, events: Retriev
if (links.length > 0) {
await engine.executeRaw(
`INSERT INTO retrieval_event_links (event_id, source_id, edge_key, to_slug)
- SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::text[])
+ SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::text[]) AS t(event_id, source_id, edge_key, to_slug)
+ WHERE ${pageRowExists('t.source_id', 't.to_slug')} AND ${edgeEndsExist('t.source_id', 't.edge_key')}
ON CONFLICT DO NOTHING`,
[links.map(l => l.id), links.map(l => l.source_id), links.map(l => l.edge_key), links.map(l => l.to_slug)],
);
@@ -163,6 +170,8 @@ export async function applyRatings(
`WITH input AS (
SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::int[], $5::text[])
AS t(element_kind, source_id, element_key, rating, content_hash)
+ WHERE CASE t.element_kind WHEN 'page' THEN ${pageRowExists('t.source_id', 't.element_key')}
+ ELSE ${edgeEndsExist('t.source_id', 't.element_key')} END
),
cur AS (
SELECT i.*, COALESCE(w.weight, ${NEUTRAL_WEIGHT})::real AS w0,
@@ -295,6 +304,10 @@ export async function pruneRetrievalFeedback(engine: BrainEngine, retentionDays:
* end of a `from|type|to` edge key) and the captured eval candidates whose results came from that source. Runs inside
* the purge's transaction; returns rows touched per store, keyed as the purge receipt names them.
*/
+/** A captured slug is the purged page's unless another source the candidate covers still has a page by that name. */
+const purgedCandidateSlug = (slug: string) =>
+ `${slug} = ANY($2::text[]) AND NOT EXISTS (SELECT 1 FROM pages po WHERE po.slug = ${slug} AND po.source_id = ANY(eval_candidates.source_ids) AND po.source_id <> $1)`;
+
export async function scrubPurgedSlugs(
engine: Pick<BrainEngine, 'executeRaw'>,
sourceId: string,
@@ -311,8 +324,8 @@ export async function scrubPurgedSlugs(
retrieval_weights: await count(`DELETE FROM retrieval_weights WHERE source_id = $1
AND ((element_kind = 'page' AND element_key = ANY($2::text[])) OR (element_kind = 'link' AND ${edgeNames('element_key')})) RETURNING 1`),
'eval_candidates.retrieved_slugs': await count(`UPDATE eval_candidates
- SET retrieved_slugs = ARRAY(SELECT s FROM unnest(retrieved_slugs) WITH ORDINALITY AS u(s, n) WHERE s <> ALL($2::text[]) ORDER BY n)
- WHERE $1 = ANY(source_ids) AND retrieved_slugs && $2::text[] RETURNING 1`),
+ SET retrieved_slugs = ARRAY(SELECT s FROM unnest(retrieved_slugs) WITH ORDINALITY AS u(s, n) WHERE NOT (${purgedCandidateSlug('u.s')}) ORDER BY n)
+ WHERE $1 = ANY(source_ids) AND EXISTS (SELECT 1 FROM unnest(retrieved_slugs) AS r(s) WHERE ${purgedCandidateSlug('r.s')}) RETURNING 1`),
};
}
diff --git a/test/purge-retrieval-feedback-8821.test.ts b/test/purge-retrieval-feedback-8821.test.ts
index 594055b148..56d57c4525 100644
--- a/test/purge-retrieval-feedback-8821.test.ts
+++ b/test/purge-retrieval-feedback-8821.test.ts
@@ -40,7 +40,9 @@ function context(): OperationContext {
async function seed() {
await engine.executeRaw(`INSERT INTO sources (id, name) VALUES ('source-b', 'source-b') ON CONFLICT DO NOTHING`);
- for (const slug of [SLUG, OTHER]) await importFromContent(engine, slug, `---\ntitle: ${slug}\ntype: note\n---\n# ${slug}\n\nBody text.\n`, { noEmbed: true, sourceId: 'default' });
+ for (const sourceId of ['default', 'source-b']) for (const slug of [SLUG, OTHER]) {
+ await importFromContent(engine, slug, `---\ntitle: ${slug}\ntype: note\n---\n# ${slug}\n\nBody text.\n`, { noEmbed: true, sourceId });
+ }
await insertRetrievalEvents(engine, [{ id: 'event-a', client_id: 'local', op: 'query', links: [
{ source_id: 'default', edge_key: EDGE, to_slug: OTHER }, { source_id: 'source-b', edge_key: EDGE, to_slug: OTHER }], pages: [
{ source_id: 'default', slug: SLUG, content_hash: null, rank: 1, cited: true },
@@ -105,15 +107,44 @@ describe('#8821 a page purge removes the slug from the retrieval logs and eval c
expect(await residue()).toEqual(ONLY_SOURCE_B);
await untouched();
}, 60_000);
+
+ test('a feedback write still queued when the purge commits does not bring the slug back', async () => {
+ const snapshot = await engine.readPageSnapshot(SLUG, { sourceId: 'default' });
+ await withEnv({ GBRAIN_HOME: home }, () => operations.find(o => o.name === 'delete_page')!.handler(context(),
+ { slug: SLUG, purge: true, request_id: randomUUID(), expected_revision: snapshot!.revision }));
+ await insertRetrievalEvents(engine, [{ id: 'event-late', client_id: 'local', op: 'query',
+ links: [{ source_id: 'default', edge_key: EDGE, to_slug: OTHER }],
+ pages: [{ source_id: 'default', slug: SLUG, content_hash: null, rank: 1, cited: true }, { source_id: 'default', slug: OTHER, content_hash: null, rank: 2, cited: false }] }]);
+ await applyRatings(engine, { eventId: 'event-late', clientId: 'local', signal: 'explicit', alpha: 0.2, targets: [
+ { kind: 'page', source_id: 'default', key: SLUG, rating: 5, content_hash: null }, { kind: 'link', source_id: 'default', key: EDGE, rating: 5, content_hash: null }] });
+ expect(await residue()).toEqual(ONLY_SOURCE_B);
+ expect(await engine.executeRaw(`SELECT 1 FROM retrieval_event_pages WHERE event_id='event-late' AND slug=$1`, [OTHER])).toHaveLength(1);
+ }, 60_000);
+
+ test('a federated capture keeps a slug that another of its sources still has a page for', async () => {
+ await engine.logEvalCandidate({ tool_name: 'query', query: 'federated', retrieved_slugs: [SLUG, OTHER], retrieved_chunk_ids: [],
+ source_ids: ['default', 'source-b'], expand_enabled: false, detail: null, detail_resolved: null, vector_enabled: false, expansion_applied: false,
+ latency_ms: 1, remote: false, job_id: null, subagent_id: null });
+ const snapshot = await engine.readPageSnapshot(SLUG, { sourceId: 'default' });
+ const result = await withEnv({ GBRAIN_HOME: home }, () => operations.find(o => o.name === 'delete_page')!.handler(context(),
+ { slug: SLUG, purge: true, request_id: randomUUID(), expected_revision: snapshot!.revision })) as Record<string, any>;
+ expect(await engine.executeRaw(`SELECT retrieved_slugs FROM eval_candidates WHERE query = 'federated'`)).toEqual([{ retrieved_slugs: [SLUG, OTHER] }]);
+ const stores = new Map((result.receipt.stores as Array<{ store: string; status: string }>).map(s => [s.store, s.status]));
+ expect(stores.get('eval_candidates.retrieved_slugs')).toBe('deleted');
+ }, 60_000);
});
test('#8821 a recorded tombstone the belief ledger expires scrubs the same rows', () => beliefBrain(async b => {
await b.put(SLUG, beliefPage({ title: 'Alice', prose: 'Body of the page.' }));
+ await b.put(OTHER, beliefPage({ title: 'Acme', prose: 'Body of the other page.' }));
await b.engine.executeRaw(`UPDATE pages SET deleted_at = now() - interval '100 hours' WHERE source_id = $1 AND slug = $2`, [b.sourceId, SLUG]);
await insertRetrievalEvents(b.engine, [{ id: 'event-b', client_id: 'local', op: 'query', links: [{ source_id: b.sourceId, edge_key: EDGE, to_slug: OTHER }],
pages: [{ source_id: b.sourceId, slug: SLUG, content_hash: null, rank: 1, cited: true }] }]);
await applyRatings(b.engine, { eventId: 'event-b', clientId: 'local', signal: 'explicit', alpha: 0.2,
targets: [{ kind: 'page', source_id: b.sourceId, key: SLUG, rating: 5, content_hash: null }] });
+ for (const table of ['retrieval_event_pages', 'retrieval_event_links', 'retrieval_feedback', 'retrieval_weights']) {
+ expect(await b.engine.executeRaw(`SELECT 1 FROM ${table} WHERE source_id = $1`, [b.sourceId])).toHaveLength(1);
+ }
const purged = await purgeDeletedPagesCoordinated(b.engine, 72);
expect(purged.slugs).toContain(SLUG);
for (const table of ['retrieval_event_pages', 'retrieval_event_links', 'retrieval_feedback', 'retrieval_weights']) {