Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix: ingest books a failed window's spend, writes facts with exact dedup only, and a staged upload cannot create an ingest-report page
garrytanfa2ac91ee3a2
diff --git a/src/core/ingest/extract.ts b/src/core/ingest/extract.tsindex 37eb4da42c..bbd5f38e39 100644--- a/src/core/ingest/extract.ts+++ b/src/core/ingest/extract.ts@@ -120,6 +120,8 @@ export function parseIngestExtraction(raw: string): { entities: ExtractedEntity[export type ProviderFailureReason = 'auth' | 'billing' | 'model_not_found' | 'content_block' | 'rate_limit' | 'transient';export class IngestProviderFailure extends Error {+ /** What the provider billed for the failed window (a blocked reply is billed in full); booked on the attempt. */+ spent_usd = 0;constructor(readonly reason: ProviderFailureReason, readonly model: string, readonly window: number, cause: string) {super(cause);}@@ -164,10 +166,12 @@ export async function extractWindow(input: WindowRunInput): Promise<ExtractedWin}text = result.text;} catch (error) {- if (error instanceof IngestProviderFailure || error instanceof BudgetExhausted) throw error;- if (input.signal?.aborted) throw error;- const reason = classifyGlobalLlmError(error) ?? 'transient';- throw new IngestProviderFailure(reason, input.model, input.window.index, sanitizeCause(error instanceof Error ? error.message : String(error)));+ if (error instanceof BudgetExhausted) throw Object.assign(error, { spent_usd: tracker.totalSpent });+ if (input.signal?.aborted && !(error instanceof IngestProviderFailure)) throw error;+ const failure = error instanceof IngestProviderFailure ? error+ : new IngestProviderFailure(classifyGlobalLlmError(error) ?? 'transient', input.model, input.window.index, sanitizeCause(error instanceof Error ? error.message : String(error)));+ failure.spent_usd = tracker.totalSpent;+ throw failure;}const parsed = parseIngestExtraction(text) ?? { entities: [], facts: [] };return { index: input.window.index, chars: input.window.text.length, entities: parsed.entities, facts: parsed.facts, spent_usd: tracker.totalSpent };diff --git a/src/core/ingest/namespace.ts b/src/core/ingest/namespace.tsindex cedf786d1e..25469b9d56 100644--- a/src/core/ingest/namespace.ts+++ b/src/core/ingest/namespace.ts@@ -56,6 +56,12 @@ function refused(slug: string, why: string): OperationError {{ docs: 'docs/guides/ingest.md#reserved-namespace', ...(REPORT_SLUG.test(slug) ? { fix: readFix('Shows the ingest attempt this report belongs to, read-only.', { argv: ['gbrain', 'ingest-show', attempt], mcp: { tool: 'ingest', arguments: { attempt_id: attempt } } }) } : {}) });}+/** A staged upload's page is in its staged body, not its admission params, so its frontmatter type is checked when it is prepared. */+export function assertStagedUploadNotReport(slug: string, page: { text: string } | { frontmatter: Record<string, unknown> | null | undefined }): void {+ const reserved = 'text' in page ? declaresReportType(page.text) : coerceFrontmatterString(page.frontmatter?.type) === INGEST_REPORT_TYPE;+ if (reserved) throw refused(slug, 'The ingest-report page type is reserved for ingest.');+}+/** The reserved-type and reserved-namespace check every generic page mutation passes at admission. */export async function assertIngestNamespaceWritable(engine: BrainEngine, input: { operation: string; slug: string; existing: Pick<PageSnapshot, 'page' | 'tags'> | null | undefined; params: Record<string, unknown> }): Promise<void> {const existing = input.existing && !input.existing.page.deleted_at ? input.existing : null;diff --git a/src/core/ops/ingest.ts b/src/core/ops/ingest.tsindex 2072b36d36..cf8d4f5f0d 100644--- a/src/core/ops/ingest.ts+++ b/src/core/ops/ingest.ts@@ -260,6 +260,7 @@ async function extractRemaining(ctx: OperationContext, state: IngestAttemptStateout = await extractWindow({ engine: ctx.engine, model: state.model!, window: w, windowsTotal: windows.length, remainingUsd: remaining });} catch (error) {state.in_flight = null;+ state.spent_usd += Number((error as { spent_usd?: unknown } | null)?.spent_usd) || 0;await saveAttempt(ctx.engine, state);if (error instanceof IngestProviderFailure) return { failure: error };if ((error as Error)?.name === 'BudgetExhausted') return { stop: 'budget' };diff --git a/src/core/persistence/memory-prepare.ts b/src/core/persistence/memory-prepare.tsindex 230443a6b4..148e65dc7a 100644--- a/src/core/persistence/memory-prepare.ts+++ b/src/core/persistence/memory-prepare.ts@@ -94,8 +94,9 @@ export async function prepareMemoryMutation(engine: BrainEngine, row: WriteRequeconst observedEmbeddingConfig = JSON.stringify(await engine.executeRaw(embeddingConfigSql));const { embedding, embedding_model, degraded } = await prepareFactEmbedding(input.fact, signal, await factEmbeddingDisabled(engine, config), clock?.deadlineAt);signal?.throwIfAborted();- // #5836: an inferred link dedups exact text only, so it never supersedes or drops a similar fact.- const dedupEmbedding = p.entity_inferred ? null : embedding;+ // #5836: an inferred link dedups exact text only, so it never supersedes or drops a similar fact. So does an ingest+ // write (provenance ingest:<source page>): a pasted document never retires a fact the user saved, which undo could not reinstate.+ const dedupEmbedding = p.entity_inferred || String(p.provenance ?? '').startsWith('ingest:') ? null : embedding;// `replaces`: the caller names the fact this one replaces (checked; the cosine rule does not apply).const replaces = p.replaces !== undefined && p.replaces !== null ? Number(p.replaces) : null;const replace = (e: BrainEngine, target: number, lock: boolean) =>diff --git a/src/core/persistence/page-upload.ts b/src/core/persistence/page-upload.tsindex 35774825d4..f34d0ab25c 100644--- a/src/core/persistence/page-upload.ts+++ b/src/core/persistence/page-upload.ts@@ -345,6 +345,7 @@ export async function prepareStagedUpload<T extends Publishable>(engine: BrainEnif (ref.bytes > PAGE_MAX_BYTES) return (await import('./upload-streamed.ts')).prepareStreamedUpload(engine, row, ref) as unknown as Promise<T>;const content = await stagedUploadText(engine, ref);if (content === null) throw bodyStagingGoneError(row.slug, ref.staging_id, null);+ (await import('../ingest/namespace.ts')).assertStagedUploadNotReport(row.slug, { text: content });const { body_ref: _ref, ...rest } = row.intent as Record<string, unknown>;return prepare({ ...row, intent: { ...rest, content,...(rest.kind === undefined && rest.timeline_section === undefined ? { timeline_section: timelineSectionOf(content, row.slug) } : {}) } });diff --git a/src/core/persistence/upload-streamed.ts b/src/core/persistence/upload-streamed.tsindex 2070d6d875..e58511c409 100644--- a/src/core/persistence/upload-streamed.ts+++ b/src/core/persistence/upload-streamed.ts@@ -133,6 +133,7 @@ export async function prepareStreamedUpload(engine: BrainEngine, row: WriteRequethrow opError('invalid_params', result.error ?? 'The uploaded page could not be prepared.', `Page ${row.slug} was not written. Fix what the message names, then upload it again with a new request_id.`);}const prepared: StreamedPreparedImport = ready;+ (await import('../ingest/namespace.ts')).assertStagedUploadNotReport(row.slug, { frontmatter: prepared.frontmatter });const privateRequest = privateRequestGuarded(row.operation, p.kind) ? readPrivateRequest('', () => prepared.frontmatter) : null;if (privateRequest) assertPrivateRequestKept(row, privateRequest.frontmatter);if (coreMarking(snapshot?.page.frontmatter).core || coreMarking(prepared.frontmatter).core) {diff --git a/test/helpers/ingest-scenarios.ts b/test/helpers/ingest-scenarios.tsindex 23624f2d35..2a0d2cd514 100644--- a/test/helpers/ingest-scenarios.ts+++ b/test/helpers/ingest-scenarios.ts@@ -11,7 +11,7 @@ import { localHostId } from '../../src/core/persistence/identity.ts';import { parseMarkdown } from '../../src/core/markdown.ts';import { __setChatTransportForTests, configureGateway, resetGateway } from '../../src/core/ai/gateway.ts';import { reviewCtx, committedFile, type ReviewDoorFixture } from './review-door-scenarios.ts';-import { randomUUID } from 'node:crypto';+import { createHash, randomUUID } from 'node:crypto';import { tryAcquireDbLock } from '../../src/core/db-lock.ts';export type Report = Record<string, any>;@@ -47,7 +47,7 @@ const GOLDEN_ROWS: StubRow[] = [export interface ChatStub { calls: number; restore: () => void }/** The ingest extractor's stub: each window gets the entities and claims whose marker sentence it contains. */-export function stubChat(opts: { rows?: StubRow[]; usage?: number; onCall?: (n: number) => Promise<void> | void } = {}): ChatStub {+export function stubChat(opts: { rows?: StubRow[]; usage?: number; onCall?: (n: number) => Promise<void> | void; stopReason?: (n: number) => string } = {}): ChatStub {const rows = opts.rows ?? GOLDEN_ROWS;configureGateway({ chat_model: 'anthropic:claude-haiku-4-5', env: { ANTHROPIC_API_KEY: 'test-key' } } as never);const stub: ChatStub = { calls: 0, restore: () => { __setChatTransportForTests(null); resetGateway(); } };@@ -58,7 +58,7 @@ export function stubChat(opts: { rows?: StubRow[]; usage?: number; onCall?: (n:const hit = rows.filter(r => text.includes(r.marker));return {text: JSON.stringify({ entities: hit.flatMap(r => r.entities), facts: hit.flatMap(r => r.facts.map(f => ({ ...f, notability: 'high' }))) }),- blocks: [], stopReason: 'end', model: 'anthropic:claude-haiku-4-5', providerId: 'anthropic',+ blocks: [], stopReason: opts.stopReason?.(stub.calls) ?? 'end', model: 'anthropic:claude-haiku-4-5', providerId: 'anthropic',usage: { input_tokens: opts.usage ?? 0, output_tokens: 0, cache_read_tokens: 0, cache_creation_tokens: 0 },} as never;});@@ -333,6 +333,13 @@ export async function namespaceScenario(engine: BrainEngine): Promise<void> {}const typed = await put.handler(reviewCtx(engine), { slug: 'notes/fake-report', content: '---\ntype: ingest-report\ntitle: x\n---\n\nmine\n' }).catch(e => e) as OperationError;expect(typed.code).toBe('ingest_namespace_reserved');+ const staged = '---\ntype: ingest-report\ntitle: x\n---\n\nstaged\n';+ const sha = (t: string) => createHash('sha256').update(t).digest('hex');+ const session = await operationsByName.begin_page_upload!.handler(reviewCtx(engine), { slug: 'notes/staged-fake-report', mode: 'create', total_bytes: Buffer.byteLength(staged), sha256: sha(staged), request_id: randomUUID() }) as Report;+ await operationsByName.upload_page_part!.handler(reviewCtx(engine), { upload_id: session.upload_id, part: 0, text: staged, part_sha256: sha(staged) });+ const committed = await operationsByName.commit_page_upload!.handler(reviewCtx(engine), { upload_id: session.upload_id, wait_ms: 30_000 }).catch(e => e) as Report;+ expect(JSON.stringify({ code: committed.code, error: committed.error, receipt: committed.receipt })).toContain('ingest_namespace_reserved');+ expect(await engine.getPage('notes/staged-fake-report')).toBeNull();const reportRev = (await engine.readPageSnapshot(report.report_page))!.revision;const editReport = await operationsByName.edit_page!.handler(reviewCtx(engine, { remote: true } as never), { slug: report.report_page, expected_revision: reportRev,edits: [{ old_text: 'ingest', new_text: 'mine' }] }).catch(e => e) as OperationError;@@ -664,3 +671,21 @@ export async function deletedPageScenario(engine: BrainEngine): Promise<void> {} finally { resetGateway(); }}++/** A window the provider blocks is still billed: its spend is booked on the attempt, so max_usd bounds the attempt across resumes. */+export async function failedWindowSpendScenario(engine: BrainEngine): Promise<void> {+ await seedEntities(engine);+ const stub = stubChat({ usage: 200_000, stopReason: n => (n === 2 ? 'content_filter' : 'end') });+ try {+ const first = await ingest(engine, { content: goldenDocument(), max_usd: 100 });+ expect(first.status).toBe('failed');+ expect(first.failure.reason).toBe('content_block');+ expect(stub.calls).toBe(2);+ const state = (await engine.executeRaw<{ spent: number | string; first: number | string }>(+ "SELECT (completed_keys->0->>'spent_usd')::float8 AS spent, (completed_keys->0->'windows'->'0'->>'spent_usd')::float8 AS first FROM op_checkpoints WHERE op = 'ingest-attempt' AND fingerprint = $1",+ [first.attempt_id]))[0]!;+ expect(Number(state.first)).toBeGreaterThan(0);+ expect(Number(state.spent)).toBeCloseTo(2 * Number(state.first), 6);+ expect(first.spent_usd).toBeCloseTo(2 * Number(state.first), 6);+ } finally { stub.restore(); }+}diff --git a/test/ingest.serial.test.ts b/test/ingest.serial.test.tsindex 525977e39d..388aea5c71 100644--- a/test/ingest.serial.test.ts+++ b/test/ingest.serial.test.ts@@ -56,6 +56,10 @@ for (const storage of ['git', 'database'] as ReviewStorage[]) {fx = await reviewDoorFixture(engine, storage);await S.deletedPageScenario(engine);}, 60_000);+ test('a blocked window is billed: its spend is booked on the attempt', async () => {+ fx = await reviewDoorFixture(engine, storage);+ await S.failedWindowSpendScenario(engine);+ }, 60_000);test('dry run makes no provider call and writes nothing', async () => {fx = await reviewDoorFixture(engine, storage);await S.dryRunScenario(engine);diff --git a/test/remember-entity-inference.test.ts b/test/remember-entity-inference.test.tsindex c4f09fe6d9..fab8bc5494 100644--- a/test/remember-entity-inference.test.ts+++ b/test/remember-entity-inference.test.ts@@ -175,6 +175,28 @@ describe('remember entity inference (#5836)', () => {}});+ test('an ingest write (provenance ingest:) dedups exact text only: it never supersedes a similar saved fact', async () => {+ configureGateway({ embedding_model: 'openai:text-embedding-3-small', embedding_dimensions: 1536, env: { OPENAI_API_KEY: 'sk-test-inference' } });+ __setEmbedTransportForTests((async (opts: { values: string[] }) => ({ embeddings: opts.values.map(() => [1, ...new Array(1535).fill(0)]) })) as never);+ await engine.setConfig('facts.supersession_thresholds', JSON.stringify({ 'openai:text-embedding-3-small@1536': 0.9 }));+ try {+ const saved = await remember({ fact: 'Ingest Example targets 40 percent gross margin', entity: 'companies/ingest-example' });+ expect(saved.status).toBe('inserted');+ const ingested = await remember({ fact: 'Ingest Example targets 45 percent gross margin', entity: 'companies/ingest-example', provenance: 'ingest:sources/doc-example' });+ expect(ingested.status).toBe('inserted');+ expect(ingested.superseded_fact_id).toBeUndefined();+ const kept = await factRow(saved.id);+ expect(kept.expired_at).toBeNull();+ expect(kept.superseded_by).toBeNull();+ const exact = await remember({ fact: 'Ingest Example targets 45 percent gross margin', entity: 'companies/ingest-example', provenance: 'ingest:sources/doc-example' });+ expect(exact).toMatchObject({ status: 'duplicate', id: ingested.id });+ } finally {+ await engine.unsetConfig('facts.supersession_thresholds');+ __setEmbedTransportForTests(null);+ configureGateway({ ...LEGACY_EMBEDDING_CONFIG, env: {} });+ }+ });+test('facts.entity_inference is a registered config key', async () => {const { KNOWN_CONFIG_KEYS } = await import('../src/core/config.ts');expect(KNOWN_CONFIG_KEYS).toContain('facts.entity_inference');