Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: ingest keeps its ownership evidence in the attempt checkpoint as writes commit, so a resumed or failed attempt undoes everything it created

garrytan7b784f76310b
diff --git a/docs/guides/ingest.md b/docs/guides/ingest.md
index 15d7279d8b..9866afaab9 100644
--- a/docs/guides/ingest.md
+++ b/docs/guides/ingest.md
@@ -141,7 +141,9 @@ gbrain ingest-undo a3f9c2d1e0b4a5968 --yes --expect <plan_hash> # apply e
```
Undo retracts only what the attempt created, read from the report page's evidence (it outlives the write
-receipts' `persistence.receipt_retention_days` window, default 30) and cross-checked against the attempt record:
+receipts' `persistence.receipt_retention_days` window, default 30) and cross-checked against the attempt record,
+which saves the same evidence as each write commits, so an attempt that failed mid-commit, or was resumed after
+one, still owns everything it wrote:
- the fence rows it appended are struck (`valid_until` today) in the page's own fence, through the page door on
both storages, so a later edit, ingest or rebuild never brings them back; no withdrawal is recorded, so a later
diff --git a/src/core/ingest/checkpoint.ts b/src/core/ingest/checkpoint.ts
index 2790de62ab..c6e417147c 100644
--- a/src/core/ingest/checkpoint.ts
+++ b/src/core/ingest/checkpoint.ts
@@ -12,7 +12,7 @@ import { createHash, randomBytes } from 'node:crypto';
import type { BrainEngine } from '../engine.ts';
import type { Principal } from '../persistence/model.ts';
import type { ExtractedWindow } from './extract.ts';
-import type { IngestReport } from './report.ts';
+import type { IngestReport, ReportEvidence } from './report.ts';
export const INGEST_ATTEMPT_OP = 'ingest-attempt';
export const INGEST_IDENTITY_OP = 'ingest-identity';
@@ -56,6 +56,8 @@ export interface IngestAttemptState {
/** Fact ids this attempt counted as support for an existing claim: undo of another attempt skips them. */
support_fact_ids: number[];
report: IngestReport | null;
+ /** Ownership evidence, saved as each write commits, so a resume or undo after a failed commit still owns what it wrote. */
+ evidence?: ReportEvidence;
}
export function newAttemptId(): string {
diff --git a/src/core/ops/ingest.ts b/src/core/ops/ingest.ts
index cf8d4f5f0d..0a35c85dbf 100644
--- a/src/core/ops/ingest.ts
+++ b/src/core/ops/ingest.ts
@@ -324,7 +324,7 @@ async function commitPlan(ctx: OperationContext, state: IngestAttemptState, plan
report.source_page = { slug: state.source_slug, status: 'failed' };
throw opError('ingest_commit_failed', 'Page ' + state.source_slug + ' already exists with other content.', 'Nothing else was written. Pass another slug, or omit it to file the document under its content hash.');
}
- report.source_page = { slug: state.source_slug, status: 'existing' };
+ report.source_page = { slug: state.source_slug, status: evidence.source_created ? 'created' : 'existing' };
} else {
const r = await publish(ctx, sourceId, { slug: state.source_slug, content: sourcePageContent(state, title, text), ...(existing ? { expected_revision: existing.revision } : {}) });
if (!r.ok) {
@@ -334,6 +334,7 @@ async function commitPlan(ctx: OperationContext, state: IngestAttemptState, plan
report.source_page = { slug: state.source_slug, status: r.pending ? 'pending' : 'created', request_id: r.request_id };
report.write_requests.push({ slug: state.source_slug, request_id: r.request_id, state: r.pending ? 'pending' : 'committed' });
evidence.source_created = true;
+ await saveAttempt(ctx.engine, state);
}
await writeReport();
// New entity pages, each revalidated under the entity-create lock.
@@ -341,7 +342,11 @@ async function commitPlan(ctx: OperationContext, state: IngestAttemptState, plan
for (const page of plan.pages) {
const base: ReportPage = { name: page.name, slug: page.slug, type: page.type, status: 'skipped', ...(page.near_names ? { near_names: page.near_names } : {}) };
if (page.status === 'skipped') { report.pages.push({ ...base, reason: page.reason }); continue; }
- if (page.status === 'existing') { const row = { ...base, status: 'existing' as const }; report.pages.push(row); written.set(page.slug, row); continue; }
+ if (page.status === 'existing') {
+ const own = evidence.created_pages.find(c => c.slug === page.slug);
+ const row: ReportPage = own ? { ...base, status: own.proposed ? 'proposed' : 'created', request_id: own.request_id } : { ...base, status: 'existing' };
+ report.pages.push(row); written.set(page.slug, row); continue;
+ }
const create = (target: PlannedPage) => withEntityCreateLock(ctx, sourceId, async (): Promise<ReportPage | null> => {
const proposed = target.status === 'propose';
const at = { ...base, slug: target.slug };
@@ -352,6 +357,7 @@ async function commitPlan(ctx: OperationContext, state: IngestAttemptState, plan
report.write_requests.push({ slug: target.slug, request_id: r.request_id, state: r.pending ? 'pending' : 'committed' });
if (r.pending) return { ...at, status: 'pending', request_id: r.request_id };
evidence.created_pages.push({ slug: target.slug, request_id: r.request_id, revision: null, proposed });
+ await saveAttempt(ctx.engine, state);
return { ...at, status: proposed ? 'proposed' : 'created', request_id: r.request_id };
});
const conflict: ReportPage = { ...base, status: 'skipped', reason: 'create_conflict' };
@@ -385,6 +391,7 @@ async function commitPlan(ctx: OperationContext, state: IngestAttemptState, plan
}
const check = await checkFactAgainstSaved(ctx.engine, sourceId, { fact: f.fact, kind: f.kind, visibility: state.options.visibility, entity_slug: target.slug }, { excludePrivate });
if (check.verdict === 'withdrawn') { row.copies.push({ entity_slug: target.slug, status: 'skipped', reason: 'withdrawn' }); continue; }
+ if (check.verdict === 'duplicate' && evidence.created_facts.some(x => x.id === check.id)) { row.copies.push({ entity_slug: target.slug, status: 'created', fact_id: check.id }); continue; }
if (check.verdict === 'duplicate') {
row.copies.push({ entity_slug: target.slug, status: 'support', fact_id: check.id });
if (!evidence.support_fact_ids.includes(check.id)) evidence.support_fact_ids.push(check.id);
@@ -400,7 +407,7 @@ async function commitPlan(ctx: OperationContext, state: IngestAttemptState, plan
const pending = pendingOf(res);
row.copies.push({ entity_slug: target.slug, status: pending ? 'pending' : 'created', ...(Number.isFinite(id) ? { fact_id: id } : {}), request_id });
report.write_requests.push({ slug: target.slug, request_id, state: pending ? 'pending' : 'committed' });
- if (!pending && Number.isFinite(id)) evidence.created_facts.push({ id, entity_slug: target.slug, request_id });
+ if (!pending && Number.isFinite(id)) { evidence.created_facts.push({ id, entity_slug: target.slug, request_id }); await saveAttempt(ctx.engine, state); }
} catch (error) {
if (!(error instanceof OperationError)) throw error;
row.copies.push({ entity_slug: target.slug, status: 'failed', reason: error.code, request_id });
@@ -424,7 +431,7 @@ async function commitPlan(ctx: OperationContext, state: IngestAttemptState, plan
}
report.facts.push(row);
}
- for (const p of evidence.created_pages) p.revision = await snapshotRevision(ctx, sourceId, p.slug);
+ for (const p of evidence.created_pages) p.revision ??= await snapshotRevision(ctx, sourceId, p.slug);
}
async function runIngest(ctx: OperationContext, p: Params): Promise<Record<string, unknown>> {
@@ -513,7 +520,8 @@ async function runIngest(ctx: OperationContext, p: Params): Promise<Record<strin
if (now.storage !== state.topology.storage || now.source_incarnation !== state.topology.source_incarnation) {
throw opError('ingest_storage_unavailable', 'Source ' + sourceId + ' changed storage while attempt ' + state.attempt_id + ' was extracting.', 'Nothing was written. Run the same ingest again; finished windows are kept.', { reason: 'topology_changed', docs: DOCS + '#storage' });
}
- const evidence: ReportEvidence = { attempt_id: state.attempt_id, principal: principalKey(principal), source_id: sourceId, source_slug: state.source_slug, source_created: false,
+ // A resumed attempt keeps the evidence its earlier runs saved as their writes committed.
+ const evidence: ReportEvidence = state.evidence ??= { attempt_id: state.attempt_id, principal: principalKey(principal), source_id: sourceId, source_slug: state.source_slug, source_created: false,
storage: state.topology.storage, created_pages: [], created_facts: [], support_fact_ids: [], status: 'incomplete' };
try {
await commitPlan(ctx, state, plan, report, evidence, v.title, v.text);
@@ -562,6 +570,8 @@ async function readEvidence(ctx: OperationContext, state: IngestAttemptState): P
if (!fm || snap!.page.type !== INGEST_REPORT_TYPE) throw unverifiable(state.attempt_id, 'its report page is missing');
const ev = asRecord(fm.ingest) as unknown as ReportEvidence;
if (ev.attempt_id !== state.attempt_id || ev.principal !== principalKey(state.principal) || ev.source_id !== state.source_id) throw unverifiable(state.attempt_id, 'its report page names another attempt or principal');
+ // The checkpoint's evidence is saved as each write commits, so it also covers writes after the report page's last update.
+ if (state.evidence) return { ...ev, ...state.evidence };
return { ...ev, created_pages: Array.isArray(ev.created_pages) ? ev.created_pages : [], created_facts: Array.isArray(ev.created_facts) ? ev.created_facts : [], support_fact_ids: Array.isArray(ev.support_fact_ids) ? ev.support_fact_ids : [] };
}
diff --git a/test/e2e/ingest-postgres.test.ts b/test/e2e/ingest-postgres.test.ts
index 05c311cdd1..b3d358f011 100644
--- a/test/e2e/ingest-postgres.test.ts
+++ b/test/e2e/ingest-postgres.test.ts
@@ -83,6 +83,7 @@ d('ingest and ingest_undo (live Postgres)', () => {
test('undo keeps relied-on claims (' + label + ')', async () => { fx = await fixture(storage); await S.undoKeepsReliedOnScenario(engine); }, 90_000);
test('undo unverifiable (' + label + ')', async () => { fx = await fixture(storage); await S.undoUnverifiableScenario(engine); }, 60_000);
test('ambiguous namesake (' + label + ')', async () => { fx = await fixture(storage); await S.ambiguousNamesakeScenario(engine); }, 60_000);
+ test('failed commit resumed keeps ownership (' + label + ')', async () => { fx = await fixture(storage); await S.failedCommitResumeUndoScenario(engine); }, 90_000);
}
test('Git target unavailable refuses before writes', async () => { fx = await fixture('git'); await S.storageUnavailableScenario(engine); }, 60_000);
});
diff --git a/test/helpers/ingest-scenarios.ts b/test/helpers/ingest-scenarios.ts
index d20225a9ea..58cb7faf80 100644
--- a/test/helpers/ingest-scenarios.ts
+++ b/test/helpers/ingest-scenarios.ts
@@ -716,3 +716,41 @@ export async function ambiguousNamesakeScenario(engine: BrainEngine): Promise<vo
expect((await factsOn(engine, 'people/alice-example-b')).length).toBe(0);
} finally { stub.restore(); }
}
+
+/**
+ * Item C: a commit that dies after the first claim is written, then resumes: the resumed attempt still owns the source
+ * page, the entity pages and the claim the failed run committed, so undo retracts and deletes all of them.
+ */
+export async function failedCommitResumeUndoScenario(engine: BrainEngine): Promise<void> {
+ await seedEntities(engine);
+ const stub = stubChat();
+ const remember = operationsByName.remember!;
+ let calls = 0;
+ (operationsByName as Record<string, unknown>).remember = { ...remember, handler: async (ctx: OperationContext, p: Record<string, unknown>) => {
+ if (++calls === 2) throw new Error('Connection terminated unexpectedly');
+ return remember.handler(ctx, p);
+ } };
+ let first: Report;
+ try {
+ first = await ingest(engine, { content: goldenDocument(), title: 'Call notes' });
+ } finally { (operationsByName as Record<string, unknown>).remember = remember; }
+ try {
+ expect(first.status).toBe('failed');
+ expect(first.failure.stage).toBe('commit');
+ const committed = (await factsOn(engine, 'companies/zentrix')).map(f => f.fact);
+ expect(committed).toEqual(['Zentrix signed its first enterprise customer.']);
+ // Undo before any resume already owns the claim the failed run committed.
+ expect((await undo(engine, { attempt_id: first.attempt_id })).retract.map((r: Report) => r.entity_slug)).toEqual(['companies/zentrix']);
+ const resumed = await ingest(engine, { content: goldenDocument(), title: 'Call notes', attempt_id: first.attempt_id, resume: true });
+ expect(resumed.status).toBe('complete');
+ expect(resumed.facts.find((f: Report) => f.fact === 'Zentrix signed its first enterprise customer.').copies).toEqual([expect.objectContaining({ entity_slug: 'companies/zentrix', status: 'created' })]);
+ const preview = await undo(engine, { attempt_id: first.attempt_id });
+ expect(preview.delete.map((d: Report) => d.slug)).toEqual(expect.arrayContaining(['companies/zentrix', 'companies/acmx', first.source_page.slug]));
+ const applied = await undo(engine, { attempt_id: first.attempt_id, yes: true, expect: preview.plan_hash });
+ expect(applied.status).toBe('undone');
+ const live = await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM facts WHERE source = $1 AND valid_until IS NULL AND expired_at IS NULL', ['ingest:' + first.source_page.slug]);
+ expect(Number(live[0]!.n)).toBe(0);
+ expect(await engine.getPage(first.source_page.slug)).toBeNull();
+ expect(await engine.getPage('companies/zentrix')).toBeNull();
+ } finally { stub.restore(); }
+}
diff --git a/test/ingest.serial.test.ts b/test/ingest.serial.test.ts
index 76924257a4..151bfe1de5 100644
--- a/test/ingest.serial.test.ts
+++ b/test/ingest.serial.test.ts
@@ -132,6 +132,10 @@ for (const storage of ['git', 'database'] as ReviewStorage[]) {
fx = await reviewDoorFixture(engine, storage);
await S.ambiguousNamesakeScenario(engine);
}, 60_000);
+ test('a failed commit resumed keeps its ownership evidence, so undo retracts everything the attempt created', async () => {
+ fx = await reviewDoorFixture(engine, storage);
+ await S.failedCommitResumeUndoScenario(engine);
+ }, 90_000);
});
}