Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix: a retried remove under a replaced CLI registration can't delete a re-added source's pages
garrytan57b87f013769
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 59e8e8989d..8fb13dfee7 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -16,6 +16,7 @@ identifiers and attribution are available in the pre-removal Git revision### Itemized changes+- `src/core/persistence/source-lifecycle.ts`: a remove or purge retry also refuses `source_retire_superseded` when another CLI registration (one replaced since) admitted the same request id against a different incarnation of the source, and settles that admission `failed`; idempotency is per registration, so before this the retry ran as a new request and deleted the re-added source's pages. An admission bound to the current incarnation still resumes under the new registration (#8984).- `src/core/persistence/topology-receipts.ts`: `admitTopologyRetire` records a retire admission (a `persistence_topology_changes` row in state `recovering` with no recovery record, bound to the source incarnation, holding one lifetime ID and a full receipt reservation after the same quota check `assertTopologyChangeCapacity` made); `recordTopologyChange` settles a request's own admission `committed` and trues up its receipt bytes; `supersedeTopologyRetires` settles other requests' admissions `failed` when a remove or purge deletes their incarnation, or when any other lifecycle operation (archive, restore, rebind, claim) commits a change to it; it skips only the caller's own (principal, request id) pair, so a retry under a replaced CLI registration settles the old registration's admission too.- `src/core/persistence/source-lifecycle.ts`: a request id whose admission names another incarnation than the source registered now refuses `source_retire_superseded` before any lock or deletion (a dry run too) and settles the admission `failed`, so a later retry refuses the same way; the batches and the final transaction resume past their own admission (#8984).- `src/core/error-registry.ts`: new caller code `source_retire_superseded`, with a read-only writer-status fix; `sources writer status --json` gains `topology_requests` (recovering admissions and superseded requests).diff --git a/src/core/persistence/source-lifecycle.ts b/src/core/persistence/source-lifecycle.tsindex 4a0f9120a9..6c960d5e97 100644--- a/src/core/persistence/source-lifecycle.ts+++ b/src/core/persistence/source-lifecycle.ts@@ -160,9 +160,14 @@ export async function runManagedSourceLifecycle(engine:BrainEngine,input:SourceL`Pass the canonical directory for '${input.sourceId}' (the one the command in fix reports, or its verified successor), then rerun the ${input.operation}.`,{fix:ownerStatusFix(input.sourceId)});const [before]=await engine.executeRaw<SourceState>('SELECT id,incarnation,archived,local_path,config,name,last_commit FROM sources WHERE id=$1',[input.sourceId]);- if(prior&&prior.source_incarnation!==before?.incarnation){- if(!input.dryRun) await topologyTransaction(engine,async tx=>{await lockTopologyPrincipal(tx,principal);await settleTopologyRetire(tx,prior,'failed',{...prior.outcome,error:RETIRE_SUPERSEDED});});- throw retireSuperseded(prior);+ // #8984: idempotency is per principal, so a request_id another CLI registration admitted against an earlier incarnation+ // (the registration was replaced since) refuses too; one bound to this incarnation still resumes under the new registration.+ const [superseded]=prior?[prior]:['remove','purge'].includes(input.operation)?await engine.executeRaw<TopologyChange>(`SELECT * FROM persistence_topology_changes+ WHERE request_id=$1::uuid AND source_id=$2 AND principal_id<>$3::uuid AND operation IN ('remove','purge') AND source_incarnation IS DISTINCT FROM $4::uuid+ ORDER BY created_at LIMIT 1`,[requestId,input.sourceId,principal,before?.incarnation??null]):[];+ if(superseded&&superseded.source_incarnation!==before?.incarnation){+ if(!input.dryRun) await topologyTransaction(engine,async tx=>{await lockTopologyPrincipal(tx,principal);await settleTopologyRetire(tx,superseded,'failed',{...superseded.outcome,error:RETIRE_SUPERSEDED});});+ throw retireSuperseded(superseded);}if(input.operation!=='add'&&!before) throw opError('not_found','Source not found.',`No source '${input.sourceId}' is registered on this brain. Check the ID against the command in fix, which lists archived sources too.`,{fix:sourcesListFix()});diff --git a/test/e2e/source-retire-incarnation-postgres.test.ts b/test/e2e/source-retire-incarnation-postgres.test.tsindex f8af946a93..21c348e3d2 100644--- a/test/e2e/source-retire-incarnation-postgres.test.ts+++ b/test/e2e/source-retire-incarnation-postgres.test.ts@@ -111,4 +111,24 @@ describe('retrying a managed remove after its source was re-added (postgres)', (await expect(remove()).rejects.toMatchObject({ code: 'source_retire_superseded' });expect(await pages(engine)).toBe(3);}), 300_000);++ test('a retry under a replaced CLI registration after the source was removed and re-added refuses', () => fixture(async (engine, root) => {+ await engine.executeRaw('CREATE TABLE retire_fail(armed boolean NOT NULL, seen integer NOT NULL)');+ await engine.executeRaw(`CREATE FUNCTION retire_fail_fn() RETURNS trigger LANGUAGE plpgsql AS $$ DECLARE n integer; BEGIN+ UPDATE retire_fail SET seen=seen+1 WHERE armed RETURNING seen INTO n;+ IF n > ${SOURCE_RETIRE_BATCH_PAGES} THEN RAISE EXCEPTION 'injected retire batch failure'; END IF; RETURN OLD; END $$`);+ await engine.executeRaw('CREATE TRIGGER retire_fail BEFORE DELETE ON pages FOR EACH ROW EXECUTE FUNCTION retire_fail_fn()');+ await engine.executeRaw('INSERT INTO retire_fail VALUES(true,0)');+ const requestId = randomUUID();+ const remove = () => runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true, requestId });+ const first = await incarnation(engine);+ await expect(remove()).rejects.toThrow('injected retire batch failure');+ await engine.executeRaw('DROP TRIGGER retire_fail ON pages');+ expect(await change(engine, requestId)).toMatchObject({ state: 'recovering', source_incarnation: first });+ await registerLocalWriter(engine, 'cli', undefined, true);+ expect(await runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true })).toMatchObject({ state: 'committed' });+ await readd(engine, root, 2);+ for (let i = 0; i < 2; i++) await expect(remove()).rejects.toMatchObject({ code: 'source_retire_superseded' });+ expect(await pages(engine)).toBe(2);+ }), 300_000);});diff --git a/test/source-retire-incarnation.test.ts b/test/source-retire-incarnation.test.tsindex b73bae62a8..c48d0efbc1 100644--- a/test/source-retire-incarnation.test.ts+++ b/test/source-retire-incarnation.test.ts@@ -211,3 +211,37 @@ test('the superseded refusal points at a writer status that lists the stopped anexpect(refusal).toMatchObject({ code: 'source_retire_superseded', fix: { argv: ['gbrain', 'sources', 'writer', 'status', '--source', source, '--json'] } });expect(await status()).toEqual([expect.objectContaining({ request_id: requestId, operation: 'remove', state: 'failed', error: 'source_retire_superseded' })]);}), 60_000);++test('a retry under a replaced CLI registration after the source was removed and re-added refuses and keeps the new pages', () => fixture(SOURCE_RETIRE_BATCH_PAGES + 7, async root => {+ const requestId = randomUUID();+ const remove = () => runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true, requestId });+ const first = await incarnation();+ await expect(failAfterFirstBatch(remove)).rejects.toThrow('injected retire batch failure');+ expect(await change(requestId)).toMatchObject({ state: 'recovering', source_incarnation: first });+ await registerLocalWriter(engine, 'cli', undefined, true);+ expect(await runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true })).toMatchObject({ state: 'committed' });+ await readd(root, 2);+ expect(await incarnation()).not.toBe(first);+ for (let i = 0; i < 2; i++) await expect(remove()).rejects.toMatchObject({ code: 'source_retire_superseded' });+ await expect(runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true, requestId, dryRun: true }))+ .rejects.toMatchObject({ code: 'source_retire_superseded' });+ expect(await pages()).toBe(2);+ expect(await engine.executeRaw('SELECT state FROM persistence_topology_changes WHERE request_id=$1::uuid', [requestId])).toEqual([{ state: 'failed' }]);+}), 60_000);++test('a stopped remove whose source was replaced out of band refuses under a replaced registration and settles the old admission', () => fixture(SOURCE_RETIRE_BATCH_PAGES + 7, async root => {+ const requestId = randomUUID();+ const remove = () => runManagedSourceLifecycle(engine, { operation: 'remove', sourceId: source, confirmDestructive: true, requestId });+ await expect(failAfterFirstBatch(remove)).rejects.toThrow('injected retire batch failure');+ await registerLocalWriter(engine, 'cli', undefined, true);+ await unmanaged(async () => {+ await engine.executeRaw('DELETE FROM pages WHERE source_id=$1', [source]);+ await engine.executeRaw('DELETE FROM persistence_source_bindings WHERE source_id=$1', [source]);+ await engine.executeRaw('DELETE FROM sources WHERE id=$1', [source]);+ });+ await readd(root, 2);+ await expect(remove()).rejects.toMatchObject({ code: 'source_retire_superseded' });+ expect(await engine.executeRaw('SELECT state FROM persistence_topology_changes WHERE request_id=$1::uuid', [requestId])).toEqual([{ state: 'failed' }]);+ expect((await deactivationBlockers(engine)).filter(b => b.kind === 'topology_recovery')).toEqual([]);+ expect(await pages()).toBe(2);+}), 60_000);