Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(purge): close the Postgres race between a feedback write and an open purge
garrytan67730c736f20
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 50e94ca220..97fa6b3b81 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -30,7 +30,7 @@ After upgrading, `gbrain doctor` asks for a fresh `gbrain transcripts audit-secr- `src/core/core-memory.ts`, `src/core/entities/resolve.ts`, `src/core/context/retrieval-reflex.ts`, `src/core/context/proactive-brief.ts`: these reads filter with `privatePagesFilterForCaller` (and `privateTimelineEventFilterForCaller` for brief edits), the caller's member access plus the privacy rule with its grant and opt-out. Entity resolution and core memory take it only when asked (`forCaller`: the remote `remember` path, `get_brain_context` and the remote core budget message), so fence repair, subject inference and local core readers (`compile-context --include-core`, `gbrain core`, the session hook) keep the plain privacy fragment. Reported by @javieraldape (#8442).- `src/commands/access.ts`: `whoCanRead` asks the read path's own predicate (`accessAllowedSql` with each member's satisfied folder rules) per enabled member, and the note names folder rules. Reported by @Masashi-Ono0611 (#8743).-- `src/core/feedback/store.ts`: `scrubPurgedSlugs` deletes the source's `retrieval_event_pages`, `retrieval_event_links`, `retrieval_feedback` and `retrieval_weights` rows naming the slug (a page key or either end of a `from|type|to` edge key) and drops it from `eval_candidates.retrieved_slugs` where the results came from that source. Called in the purge transaction by `purgePageInTransaction`, the belief-ledger tombstone expiry and the bulk `purgeDeletedPages`. A federated capture keeps a slug that another of its sources still has a page for. The feedback writers insert rows only for pages that still exist, so a write queued before the purge cannot bring the slug back. `src/core/deletion-inventory.ts` moves the four tables to `swept`; `verifyPagePurge` reports each store. Reported by @Masashi-Ono0611 (#8822) and @alanshurafa (#8823).+- `src/core/feedback/store.ts`: `scrubPurgedSlugs` deletes the source's `retrieval_event_pages`, `retrieval_event_links`, `retrieval_feedback` and `retrieval_weights` rows naming the slug (a page key or either end of a `from|type|to` edge key) and drops it from `eval_candidates.retrieved_slugs` where the results came from that source. Called in the purge transaction by `purgePageInTransaction`, the belief-ledger tombstone expiry and the bulk `purgeDeletedPages`. A federated capture keeps a slug that another of its sources still has a page for. The feedback writers insert rows only for pages that still exist, checked under a key-share lock on the page row, and the page purge scrubs after it deletes the page, so a write that races the purge on Postgres cannot bring the slug back. `src/core/deletion-inventory.ts` moves the four tables to `swept`; `verifyPagePurge` reports each store. Reported by @Masashi-Ono0611 (#8822) and @alanshurafa (#8823).- `src/core/storage/s3.ts`: `getUrl` returns a SigV4 presigned GET with a 3600 s expiry. New dependency `@aws-sdk/s3-request-presigner` 3.1028.0 (the `client-s3` release), with `@aws-sdk/util-format-url` pinned to 3.972.9 from the same release. Reported by @liuwenyong1985 (#8793).- `src/core/repair/visibility.ts`: `planLineage` stamps `private` on a concept with no recoverable lineage and no visibility; an explicit value stays, and both still count in `concepts_without_lineage`. Reported by @Masashi-Ono0611 (#8856).- `src/mcp/http-transport.ts`, `src/commands/serve-http-oauth.ts`: `X-Gbrain-Client` joins the CORS allowed headers. Reported by @dajiaohuang (#8691).diff --git a/src/core/feedback/store.ts b/src/core/feedback/store.tsindex ab51f72b68..ecf244762b 100644--- a/src/core/feedback/store.ts+++ b/src/core/feedback/store.ts@@ -61,7 +61,7 @@ export interface AppliedRating {newly_applied: boolean;}-const pageRowExists = (source: string, slug: string) => `EXISTS (SELECT 1 FROM pages pg WHERE pg.source_id = ${source} AND pg.slug = ${slug})`;+const pageRowExists = (source: string, slug: string) => `EXISTS (SELECT 1 FROM pages pg WHERE pg.source_id = ${source} AND pg.slug = ${slug} FOR KEY SHARE)`;const edgeEndsExist = (source: string, key: string) =>`${pageRowExists(source, `split_part(${key}, '|', 1)`)} AND ${pageRowExists(source, `reverse(split_part(reverse(${key}), '|', 1))`)}`;diff --git a/src/core/persistence/page-purge.ts b/src/core/persistence/page-purge.tsindex 7aedeaec73..a09f0eb985 100644--- a/src/core/persistence/page-purge.ts+++ b/src/core/persistence/page-purge.ts@@ -325,7 +325,6 @@ export async function purgePageInTransaction(tx: BrainEngine, row: WriteRequest,removed.page_aliases = await count('DELETE FROM page_aliases WHERE source_id=$1 AND slug=$2 RETURNING 1', [sourceId, slug]);removed.slug_aliases = await count('DELETE FROM slug_aliases WHERE source_id=$1 AND canonical_slug=$2 RETURNING 1', [sourceId, slug]);removed.slug_rename_history = await count('DELETE FROM slug_rename_history WHERE source_id=$1 AND new_slug=$2 RETURNING 1', [sourceId, slug]);- Object.assign(removed, await scrubPurgedSlugs(tx, sourceId, [slug]));// #6511 [R2]: the preparation sweep's targets, plus the tail past its ceiling and the rows pending at scan time, re-tested here.const recheck = await lookup('tail_recheck', () => recheckRequestsCarrying(tx, sourceId, needles, prepared.sweep,{ exclude: row.id, beforePending: () => faultPoint('purge:pending_recheck', { requestId: row.request_id, sourceId, operation: row.operation }) }));@@ -358,6 +357,7 @@ export async function purgePageInTransaction(tx: BrainEngine, row: WriteRequest,await writeOriginDeletedMarker(tx, { sourceId, page: { id: page.id, slug, source_path: page.source_path } });const { shas, ...segmentRows } = await deletePageWithSegments(tx, sourceId, slug, page.id, segmentClaims);Object.assign(removed, segmentRows);+ Object.assign(removed, await scrubPurgedSlugs(tx, sourceId, [slug]));if (!retention) appendPurgeIntent({ request: row.id, request_id: row.request_id, source_id: sourceId, source_incarnation: row.source_incarnation,scope: 'page', slug, purge_epoch: epoch, content_hashes: [...hashes], facts: mirrorFacts, takes: mirrorTakes, lifetimes });const key = `${sourceId}:${row.request_id}`;diff --git a/test/e2e/purge-feedback-race-postgres.test.ts b/test/e2e/purge-feedback-race-postgres.test.tsnew file mode 100644index 0000000000..65b873910c--- /dev/null+++ b/test/e2e/purge-feedback-race-postgres.test.ts@@ -0,0 +1,98 @@+/**+ * #8821 on Postgres: a feedback write that runs while a page purge is still open cannot leave the purged slug behind.+ *+ * 1. Protects: a retrieval event or rating written (a) between the purge's journal reads and its page delete, or+ * (b) after the page delete but before the purge commits, leaves no row naming the purged slug once both commit.+ * 2. Fails when: the scrub runs before the page row is deleted, or the feedback writers' page probe does not wait on+ * the purge's row lock, so the late insert survives the purge's commit.+ * 3. Existing coverage: test/purge-retrieval-feedback-8821.test.ts (PGLite, one connection, no interleaving).+ * 4. Real seams: Postgres, two pool connections, the delete_page op with the purge's tail_recheck fault point, the+ * feedback store's own writers and scrub; repo-less brain.+ */+import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';+import { randomUUID } from 'node:crypto';+import { mkdtempSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import type { PostgresEngine } from '../../src/core/postgres-engine.ts';+import { operations, type OperationContext } from '../../src/core/operations.ts';+import { importFromContent } from '../../src/core/import-file.ts';+import { applyRatings, insertRetrievalEvents, scrubPurgedSlugs } from '../../src/core/feedback/store.ts';+import { installFaultHook } from '../../src/core/persistence/fault-points.ts';+import { disposePersistenceConsumer } from '../../src/core/persistence/service.ts';+import { isolatedPersistencePostgres } from '../helpers/persistence-postgres.ts';+import { hasDatabase } from './helpers.ts';+import { withEnv } from '../helpers/with-env.ts';++const describeE2E = hasDatabase() ? describe : describe.skip;+const SLUG = 'people/alice-example';+const OTHER = 'companies/acme-example';+const EDGE = `${SLUG}|works_at|${OTHER}`;+const home = mkdtempSync(join(tmpdir(), 'gbrain-purge-race-'));+let engine: PostgresEngine;+let close: () => Promise<void>;++const context = (): OperationContext => ({ engine, config: { engine: 'postgres', embedding_disabled: true, storage: { backend: 'local', bucket: 'b', localPath: join(home, 'storage') } } as never,+ logger: { info() {}, warn() {}, error() {} }, dryRun: false, remote: false, sourceId: 'default' });++async function lateFeedback(id: string) {+ await insertRetrievalEvents(engine, [{ id, client_id: 'local', op: 'query', links: [{ source_id: 'default', edge_key: EDGE, to_slug: OTHER }],+ pages: [{ source_id: 'default', slug: SLUG, content_hash: null, rank: 1, cited: true }] }]);+ await applyRatings(engine, { eventId: id, clientId: 'local', signal: 'explicit', alpha: 0.2, targets: [+ { kind: 'page', source_id: 'default', key: SLUG, rating: 5, content_hash: null }, { kind: 'link', source_id: 'default', key: EDGE, rating: 5, content_hash: null }] });+}++async function residue() {+ const n = async (sql: string) => (await engine.executeRaw(sql, [SLUG])).length;+ return {+ retrieval_event_pages: await n(`SELECT 1 FROM retrieval_event_pages WHERE source_id='default' AND slug=$1`),+ retrieval_event_links: await n(`SELECT 1 FROM retrieval_event_links WHERE source_id='default' AND strpos(edge_key, $1) > 0`),+ retrieval_feedback: await n(`SELECT 1 FROM retrieval_feedback WHERE source_id='default' AND strpos(element_key, $1) > 0`),+ retrieval_weights: await n(`SELECT 1 FROM retrieval_weights WHERE source_id='default' AND strpos(element_key, $1) > 0`),+ };+}+const NONE = { retrieval_event_pages: 0, retrieval_event_links: 0, retrieval_feedback: 0, retrieval_weights: 0 };++describeE2E('#8821 Postgres: feedback written during an open purge', () => {+ beforeAll(async () => {+ const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!);+ engine = pg.engine; close = pg.close;+ }, 120_000);+ afterAll(async () => { installFaultHook(undefined); await disposePersistenceConsumer(engine); await close?.(); rmSync(home, { recursive: true, force: true }); });+ beforeEach(async () => {+ for (const table of ['retrieval_event_pages', 'retrieval_event_links', 'retrieval_feedback', 'retrieval_weights', 'retrieval_events']) await engine.executeRaw(`DELETE FROM ${table}`);+ for (const slug of [SLUG, OTHER]) await importFromContent(engine, slug, `---\ntitle: ${slug}\ntype: note\n---\n# ${slug}\n\nBody text ${randomUUID()}.\n`, { noEmbed: true, sourceId: 'default' });+ });++ test('a write between the purge\'s journal reads and its page delete is scrubbed', async () => {+ let late: Promise<void> | null = null;+ installFaultHook(async point => {+ if (point !== 'purge:tail_recheck' || late) return;+ late = lateFeedback('event-mid');+ await Promise.race([late, new Promise(r => setTimeout(r, 3000))]);+ });+ try {+ const snapshot = await engine.readPageSnapshot(SLUG, { sourceId: 'default' });+ const result = await withEnv({ GBRAIN_HOME: home }, () => operations.find(o => o.name === 'delete_page')!.handler(context(),+ { slug: SLUG, purge: true, request_id: randomUUID(), expected_revision: snapshot!.revision })) as Record<string, any>;+ expect(result.status).toBe('purged');+ expect(late).not.toBeNull();+ await late;+ } finally {+ installFaultHook(undefined);+ }+ expect(await residue()).toEqual(NONE);+ }, 120_000);++ test('a write after the page delete waits for the purge and is dropped', async () => {+ let late: Promise<void> | null = null;+ await engine.transaction(async tx => {+ await tx.executeRaw(`DELETE FROM pages WHERE source_id='default' AND slug=$1`, [SLUG]);+ await scrubPurgedSlugs(tx, 'default', [SLUG]);+ late = lateFeedback('event-after');+ await new Promise(r => setTimeout(r, 500));+ });+ await late;+ expect(await residue()).toEqual(NONE);+ }, 120_000);+});