Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: an ingest left pending by a page write stays resumable, and its resume settles the pending receipts and writes the facts that waited

garrytan8532eb277e5b
diff --git a/docs/guides/ingest.md b/docs/guides/ingest.md
index 9866afaab9..bf8eab25b6 100644
--- a/docs/guides/ingest.md
+++ b/docs/guides/ingest.md
@@ -118,7 +118,9 @@ The attempt checkpoint is written before the first provider call and advanced af
and the cumulative spend), on both storages, so a resume after a budget stop, a kill switch or a crashed process
never pays for a finished window again. A resume needs the same document text (its hash must match). New options
(keyed after keyless, `--trusted-extraction`) or a re-ingest after undo start a new attempt on the same source
-page; claim dedup keeps it from saving a claim twice.
+page; claim dedup keeps it from saving a claim twice. A run that ends with writes still pending (exit 10) resumes the
+same way once they settle: the resume reads each pending write's receipt, counts a committed one as the attempt's
+own (undo retracts it), never submits one still pending twice, and writes the claims that waited on a pending page.
## Storage
diff --git a/src/core/ingest/report.ts b/src/core/ingest/report.ts
index a627a9bc55..f890375753 100644
--- a/src/core/ingest/report.ts
+++ b/src/core/ingest/report.ts
@@ -137,6 +137,10 @@ export interface ReportEvidence {
storage: IngestStorage;
created_pages: Array<{ slug: string; request_id: string; revision: string | null; proposed: boolean }>;
created_facts: Array<{ id: number; entity_slug: string; request_id: string }>;
+ /** Writes still pending when a run ended; a resume settles them from their receipts. */
+ pending_pages?: Array<{ slug: string; request_id: string; proposed: boolean }>;
+ pending_facts?: Array<{ entity_slug: string; fact: string; request_id: string }>;
+ source_request_id?: string;
support_fact_ids: number[];
status: ReportStatus;
undone_at?: string;
diff --git a/src/core/ops/ingest.ts b/src/core/ops/ingest.ts
index 0a35c85dbf..ca6dd41fcb 100644
--- a/src/core/ops/ingest.ts
+++ b/src/core/ops/ingest.ts
@@ -20,6 +20,7 @@ import { strikeFenceRow } from '../facts/forget.ts';
import { checkFactAgainstSaved } from '../facts/backstop.ts';
import { isUnverifiedExtraction, quarantineMarkers } from '../extraction-review.ts';
import { tryAcquireDbLock } from '../db-lock.ts';
+import { isTerminal } from '../persistence/model.ts';
import { withPublicationGuard } from '../persistence/publication-guard.ts';
import { resolveFactWriteTarget } from '../persistence/fact-write-target.ts';
import { privatePagesFilterForCaller, resolveExcludePrivatePages } from '../search/private-visibility.ts';
@@ -316,6 +317,25 @@ async function commitPlan(ctx: OperationContext, state: IngestAttemptState, plan
...(revision ? { expected_revision: revision } : {}) }, true);
if (!r.ok) throw opError('ingest_commit_failed', 'The ingest report page could not be written (' + r.code + ').', r.message);
};
+ // A resume settles the receipts an earlier run left pending before it writes anything again: a committed write is the
+ // attempt's own, one still pending is not submitted twice, and a refused one is planned and written again.
+ const { getWriteRequest } = await import('../persistence/journal.ts');
+ const receipt = (request_id: string) => getWriteRequest(ctx.engine, state.principal, request_id);
+ const pendingPages: NonNullable<ReportEvidence['pending_pages']> = [];
+ for (const pg of evidence.pending_pages ?? []) {
+ const row = await receipt(pg.request_id);
+ if (row?.state === 'committed') evidence.created_pages.push({ ...pg, revision: null });
+ else if (row && !isTerminal(row)) pendingPages.push(pg);
+ }
+ const pendingFacts: NonNullable<ReportEvidence['pending_facts']> = [];
+ for (const f of evidence.pending_facts ?? []) {
+ const row = await receipt(f.request_id);
+ const outcome = asRecord(row?.outcome);
+ if (row?.state === 'committed' && outcome.status !== 'duplicate' && Number.isFinite(Number(outcome.id))) evidence.created_facts.push({ id: Number(outcome.id), entity_slug: f.entity_slug, request_id: f.request_id });
+ else if (row && !isTerminal(row)) pendingFacts.push(f);
+ }
+ evidence.pending_pages = pendingPages;
+ evidence.pending_facts = pendingFacts;
// The source page: create-only, keyed by the document hash; an existing page with this document is reused.
const existing = await ctx.engine.readPageSnapshot(state.source_slug, { sourceId, includeDeleted: true });
if (existing && !existing.page.deleted_at) {
@@ -325,6 +345,8 @@ async function commitPlan(ctx: OperationContext, state: IngestAttemptState, plan
throw opError('ingest_commit_failed', 'Page ' + state.source_slug + ' already exists with other content.', 'Nothing else was written. Pass another slug, or omit it to file the document under its content hash.');
}
report.source_page = { slug: state.source_slug, status: evidence.source_created ? 'created' : 'existing' };
+ } else if (evidence.source_request_id && await receipt(evidence.source_request_id).then(row => !!row && !isTerminal(row))) {
+ report.source_page = { slug: state.source_slug, status: 'pending', request_id: evidence.source_request_id };
} else {
const r = await publish(ctx, sourceId, { slug: state.source_slug, content: sourcePageContent(state, title, text), ...(existing ? { expected_revision: existing.revision } : {}) });
if (!r.ok) {
@@ -334,6 +356,7 @@ async function commitPlan(ctx: OperationContext, state: IngestAttemptState, plan
report.source_page = { slug: state.source_slug, status: r.pending ? 'pending' : 'created', request_id: r.request_id };
report.write_requests.push({ slug: state.source_slug, request_id: r.request_id, state: r.pending ? 'pending' : 'committed' });
evidence.source_created = true;
+ evidence.source_request_id = r.request_id;
await saveAttempt(ctx.engine, state);
}
await writeReport();
@@ -352,10 +375,16 @@ async function commitPlan(ctx: OperationContext, state: IngestAttemptState, plan
const at = { ...base, slug: target.slug };
const snap = await ctx.engine.readPageSnapshot(target.slug, { sourceId, includeDeleted: true });
if (snap && !snap.page.deleted_at) return null;
+ const held = evidence.pending_pages!.find(x => x.slug === target.slug);
+ if (held) return { ...at, status: 'pending', request_id: held.request_id };
const r = await publish(ctx, sourceId, { slug: target.slug, content: entityPageContent(target, proposed, state), ...(snap ? { expected_revision: snap.revision } : {}) });
if (!r.ok) return { ...at, status: 'failed', reason: r.code };
report.write_requests.push({ slug: target.slug, request_id: r.request_id, state: r.pending ? 'pending' : 'committed' });
- if (r.pending) return { ...at, status: 'pending', request_id: r.request_id };
+ if (r.pending) {
+ evidence.pending_pages!.push({ slug: target.slug, request_id: r.request_id, proposed });
+ await saveAttempt(ctx.engine, state);
+ return { ...at, status: 'pending', request_id: r.request_id };
+ }
evidence.created_pages.push({ slug: target.slug, request_id: r.request_id, revision: null, proposed });
await saveAttempt(ctx.engine, state);
return { ...at, status: proposed ? 'proposed' : 'created', request_id: r.request_id };
@@ -389,6 +418,8 @@ async function commitPlan(ctx: OperationContext, state: IngestAttemptState, plan
row.copies.push({ entity_slug: c.entity_slug, status: pageRow?.status === 'pending' ? 'pending' : 'skipped', reason: pageRow?.status === 'pending' ? 'page_pending' : 'page_not_written' });
continue;
}
+ const held = evidence.pending_facts!.find(x => x.entity_slug === target.slug && x.fact === f.fact);
+ if (held) { row.copies.push({ entity_slug: target.slug, status: 'pending', request_id: held.request_id }); continue; }
const check = await checkFactAgainstSaved(ctx.engine, sourceId, { fact: f.fact, kind: f.kind, visibility: state.options.visibility, entity_slug: target.slug }, { excludePrivate });
if (check.verdict === 'withdrawn') { row.copies.push({ entity_slug: target.slug, status: 'skipped', reason: 'withdrawn' }); continue; }
if (check.verdict === 'duplicate' && evidence.created_facts.some(x => x.id === check.id)) { row.copies.push({ entity_slug: target.slug, status: 'created', fact_id: check.id }); continue; }
@@ -407,9 +438,19 @@ async function commitPlan(ctx: OperationContext, state: IngestAttemptState, plan
const pending = pendingOf(res);
row.copies.push({ entity_slug: target.slug, status: pending ? 'pending' : 'created', ...(Number.isFinite(id) ? { fact_id: id } : {}), request_id });
report.write_requests.push({ slug: target.slug, request_id, state: pending ? 'pending' : 'committed' });
- if (!pending && Number.isFinite(id)) { evidence.created_facts.push({ id, entity_slug: target.slug, request_id }); await saveAttempt(ctx.engine, state); }
+ if (!pending && Number.isFinite(id)) evidence.created_facts.push({ id, entity_slug: target.slug, request_id });
+ if (pending) evidence.pending_facts!.push({ entity_slug: target.slug, fact: f.fact, request_id });
+ await saveAttempt(ctx.engine, state);
} catch (error) {
if (!(error instanceof OperationError)) throw error;
+ // remember's frozen verb contract reports a pending write as unavailable with writeError write_pending.
+ if (error.writeError === 'write_pending') {
+ row.copies.push({ entity_slug: target.slug, status: 'pending', request_id });
+ report.write_requests.push({ slug: target.slug, request_id, state: 'pending' });
+ evidence.pending_facts!.push({ entity_slug: target.slug, fact: f.fact, request_id });
+ await saveAttempt(ctx.engine, state);
+ continue;
+ }
row.copies.push({ entity_slug: target.slug, status: 'failed', reason: error.code, request_id });
}
}
@@ -537,7 +578,8 @@ async function runIngest(ctx: OperationContext, p: Params): Promise<Record<strin
report.status = resolveStatus({ failed: items.includes('failed'), pending: items.includes('pending'), incomplete: false });
report.complete = report.status === 'complete';
state.support_fact_ids = evidence.support_fact_ids;
- state.phase = report.status === 'failed' ? 'failed' : 'complete';
+ // A write still pending keeps the attempt resumable: a resume settles its receipts and writes what waited on it.
+ state.phase = report.status === 'failed' ? 'failed' : report.status === 'pending' ? 'committing' : 'complete';
state.report = finish(ctx, state, report);
evidence.status = report.status;
const revision = await snapshotRevision(ctx, sourceId, state.report_slug);
diff --git a/test/e2e/ingest-postgres.test.ts b/test/e2e/ingest-postgres.test.ts
index b3d358f011..e62563e8a5 100644
--- a/test/e2e/ingest-postgres.test.ts
+++ b/test/e2e/ingest-postgres.test.ts
@@ -84,6 +84,7 @@ d('ingest and ingest_undo (live Postgres)', () => {
test('undo unverifiable (' + label + ')', async () => { fx = await fixture(storage); await S.undoUnverifiableScenario(engine); }, 60_000);
test('ambiguous namesake (' + label + ')', async () => { fx = await fixture(storage); await S.ambiguousNamesakeScenario(engine); }, 60_000);
test('failed commit resumed keeps ownership (' + label + ')', async () => { fx = await fixture(storage); await S.failedCommitResumeUndoScenario(engine); }, 90_000);
+ test('pending page write resumes (' + label + ')', async () => { fx = await fixture(storage); await S.pendingPageResumeScenario(engine); }, 90_000);
}
test('Git target unavailable refuses before writes', async () => { fx = await fixture('git'); await S.storageUnavailableScenario(engine); }, 60_000);
});
diff --git a/test/helpers/ingest-scenarios.ts b/test/helpers/ingest-scenarios.ts
index 58cb7faf80..ccba632795 100644
--- a/test/helpers/ingest-scenarios.ts
+++ b/test/helpers/ingest-scenarios.ts
@@ -717,6 +717,38 @@ export async function ambiguousNamesakeScenario(engine: BrainEngine): Promise<vo
} finally { stub.restore(); }
}
+/**
+ * Item B: a run whose entity page and claim are still pending (no write wait) ends pending, not complete; once publication settles,
+ * an explicit resume writes the claims it held back and counts the page it created as its own for undo.
+ */
+export async function pendingPageResumeScenario(engine: BrainEngine): Promise<void> {
+ await seedEntities(engine);
+ const stub = stubChat();
+ const content = 'Zentrix signed its first enterprise customer last week. Alice Example moved to Lisbon in March.';
+ try {
+ const first = await ingest(engine, { content, title: 'Call notes' }, { writeWaitMs: 0 } as never);
+ expect(first.status).toBe('pending');
+ expect(first.facts.find((f: Report) => f.fact === 'Zentrix signed its first enterprise customer.').copies).toEqual([expect.objectContaining({ entity_slug: 'companies/zentrix', status: 'pending', reason: 'page_pending' })]);
+ expect(first.facts.find((f: Report) => f.fact === 'Alice Example moved to Lisbon in March.').copies).toEqual([expect.objectContaining({ entity_slug: 'people/alice-example', status: 'pending' })]);
+ expect(first.actions.map((a: Report) => a.kind)).toContain('resume');
+ const unsettled = async () => Number((await engine.executeRaw<{ n: number }>("SELECT count(*)::int AS n FROM persistence_requests WHERE state NOT IN ('committed', 'conflict', 'failed', 'cancelled')"))[0]!.n);
+ for (let i = 0; i < 100 && await unsettled(); i++) {
+ await drain(engine);
+ await new Promise(resolve => setTimeout(resolve, 100));
+ }
+ expect(await engine.getPage('companies/zentrix')).not.toBeNull();
+ const resumed = await ingest(engine, { content, title: 'Call notes', attempt_id: first.attempt_id, resume: true });
+ expect(resumed.repeated).toBeUndefined();
+ expect(resumed.status).toBe('complete');
+ expect(resumed.facts.find((f: Report) => f.fact === 'Zentrix signed its first enterprise customer.').copies).toEqual([expect.objectContaining({ entity_slug: 'companies/zentrix', status: 'created' })]);
+ expect(resumed.facts.find((f: Report) => f.fact === 'Alice Example moved to Lisbon in March.').copies).toEqual([expect.objectContaining({ entity_slug: 'people/alice-example', status: 'created' })]);
+ expect((await factsOn(engine, 'companies/zentrix')).map(f => f.fact)).toEqual(['Zentrix signed its first enterprise customer.']);
+ const preview = await undo(engine, { attempt_id: first.attempt_id });
+ expect(preview.delete.map((d: Report) => d.slug)).toEqual(expect.arrayContaining(['companies/zentrix', first.source_page.slug]));
+ expect(preview.retract.map((r: Report) => r.entity_slug).sort()).toEqual(['companies/zentrix', 'people/alice-example']);
+ } finally { stub.restore(); }
+}
+
/**
* Item C: a commit that dies after the first claim is written, then resumes: the resumed attempt still owns the source
* page, the entity pages and the claim the failed run committed, so undo retracts and deletes all of them.
diff --git a/test/ingest.serial.test.ts b/test/ingest.serial.test.ts
index 151bfe1de5..07f901867e 100644
--- a/test/ingest.serial.test.ts
+++ b/test/ingest.serial.test.ts
@@ -132,6 +132,10 @@ for (const storage of ['git', 'database'] as ReviewStorage[]) {
fx = await reviewDoorFixture(engine, storage);
await S.ambiguousNamesakeScenario(engine);
}, 60_000);
+ test('a run left pending by a page write resumes and writes the claims it held back', async () => {
+ fx = await reviewDoorFixture(engine, storage);
+ await S.pendingPageResumeScenario(engine);
+ }, 90_000);
test('a failed commit resumed keeps its ownership evidence, so undo retracts everything the attempt created', async () => {
fx = await reviewDoorFixture(engine, storage);
await S.failedCommitResumeUndoScenario(engine);