Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(remember): bound the fact embed inside the preparation budget so a slow embedder saves without a vector instead of preparation_stalled (#8135)
garrytan7754b90d8334
diff --git a/src/core/facts/single-prepare.ts b/src/core/facts/single-prepare.tsindex a532991e8c..d581cd5de6 100644--- a/src/core/facts/single-prepare.ts+++ b/src/core/facts/single-prepare.ts@@ -18,13 +18,23 @@ export interface SingleFactIntent {/** Speaker of the new claim; a fact another known speaker asserted is never its duplicate. */attributed_to?: FactAttribution | null;}-/** Provider work belongs to preparation, never to a page/source transaction. `disabled`: the brain opted out of embedding. */-export async function prepareFactEmbedding(fact: string, signal?: AbortSignal, disabled = false): Promise<{ embedding: Float32Array | null; embedding_model: string | null; degraded: boolean }> {+/** Preparation time kept back after the embed (a quarter of what is left, at most this): the dedup reads still run. */+const EMBED_RESERVE_MS = 5_000;+/**+ * Provider work belongs to preparation, never to a page/source transaction. `disabled`: the brain opted out of embedding.+ * `deadlineAt` (#8135): the preparation budget's end. The embed gets its own timeout short of it, and that timeout takes+ * the no-vector fallback like any embed failure; without it a slow embedder let the budget's abort win, the attempt was+ * charged, and the write ended `preparation_stalled` with nothing saved. The caller's own abort still propagates.+ */+export async function prepareFactEmbedding(fact: string, signal?: AbortSignal, disabled = false, deadlineAt?: number): Promise<{ embedding: Float32Array | null; embedding_model: string | null; degraded: boolean }> {signal?.throwIfAborted();- if (!disabled && isAvailable('embedding')) {+ const remaining = deadlineAt === undefined ? Infinity : deadlineAt - Date.now();+ const embedMs = remaining - Math.min(EMBED_RESERVE_MS, Math.floor(remaining / 4));+ if (!disabled && isAvailable('embedding') && embedMs > 0) {try {const model = getEmbeddingModel();- return { embedding: await embedOne(fact, { abortSignal: signal, embeddingModel: model, inputType: 'document' }), embedding_model: model, degraded: false };+ const abortSignal = Number.isFinite(embedMs) ? AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(embedMs)]) : signal;+ return { embedding: await embedOne(fact, { abortSignal, embeddingModel: model, inputType: 'document' }), embedding_model: model, degraded: false };} catch { signal?.throwIfAborted(); }}return { embedding: null, embedding_model: null, degraded: true };diff --git a/src/core/persistence/memory-prepare.ts b/src/core/persistence/memory-prepare.tsindex 82809d7579..62331e9e37 100644--- a/src/core/persistence/memory-prepare.ts+++ b/src/core/persistence/memory-prepare.ts@@ -9,6 +9,7 @@ import { factEmbeddingDisabled } from '../embedding-disabled.ts';import { engineMutationPrecondition, parseMutationPrecondition } from './preconditions.ts';import type { PreparedMutation } from './coordinator.ts';import type { WriteRequest } from './model.ts';+import type { ClaimPhaseClock } from './claim-phase.ts';import { appendContextNote, type InferredVia } from '../facts/subject-infer.ts';import { inferenceNote } from '../facts/subject-infer-write.ts';import { requestChannelTrust } from '../trust/channel.ts';@@ -59,7 +60,7 @@ async function activeUnlinkedCopy(engine: BrainEngine, sourceId: string, input:}/** Every retry renders the semantic append from the latest coherent snapshot. */-export async function prepareMemoryMutation(engine: BrainEngine, row: WriteRequest, config: GBrainConfig, signal?: AbortSignal): Promise<PreparedMutation> {+export async function prepareMemoryMutation(engine: BrainEngine, row: WriteRequest, config: GBrainConfig, signal?: AbortSignal, clock?: ClaimPhaseClock): Promise<PreparedMutation> {signal?.throwIfAborted();if (row.operation !== 'remember' || !row.intent) {throw opError('storage_error', 'Unknown memory mutation intent.',@@ -80,7 +81,7 @@ export async function prepareMemoryMutation(engine: BrainEngine, row: WriteRequesignal?.throwIfAborted();const embeddingConfigSql = "SELECT key,value FROM config WHERE key IN ('embedding_model','embedding_dimensions') ORDER BY key";const observedEmbeddingConfig = JSON.stringify(await engine.executeRaw(embeddingConfigSql));- const { embedding, embedding_model, degraded } = await prepareFactEmbedding(input.fact, signal, await factEmbeddingDisabled(engine, config));+ const { embedding, embedding_model, degraded } = await prepareFactEmbedding(input.fact, signal, await factEmbeddingDisabled(engine, config), clock?.deadlineAt);signal?.throwIfAborted();// #5836: an inferred link dedups exact text only, so it never supersedes or drops a similar fact.const dedupEmbedding = p.entity_inferred ? null : embedding;diff --git a/src/core/persistence/service.ts b/src/core/persistence/service.tsindex c456e90090..fe47c8467e 100644--- a/src/core/persistence/service.ts+++ b/src/core/persistence/service.ts@@ -101,7 +101,7 @@ export async function preparePersistedMutation(e: BrainEngine, row: WriteRequestif (row.operation === 'submit_job' && String(row.intent?.kind).startsWith('managed_maintenance_')) return (await import('./prepared-maintenance.ts')).prepareMaintenanceMutation(e, row, cfg, clock);if (row.operation === 'put_page' && row.intent?.kind === 'managed_file_import') return (await import('./import-prepare.ts')).prepareManagedImportMutation(e, row, cfg);if (row.operation === 'put_page' && row.intent?.kind === 'managed_file_repair') return (await import('./file-repair.ts')).prepareManagedFileRepairMutation(e, row, cfg);- if (row.operation === 'remember') return (await import('./memory-mutations.ts')).prepareMemoryMutation(e, row, cfg, signal);+ if (row.operation === 'remember') return (await import('./memory-mutations.ts')).prepareMemoryMutation(e, row, cfg, signal, clock);if (row.operation === 'loops_close' && row.intent?.kind === 'retire_loop_fact') return (await import('./loop-fact-retirement.ts')).prepareLoopFactRetirement(e, row, cfg);if (row.operation === 'decide_proposal') return (await import('../facts/proposal-supersede.ts')).prepareProposalMutation(e, row, cfg);if (row.operation === 'trust_owner_page') return (await import('../trust/page-handlers.ts')).prepareTrustOwnerPageMutation(e, row, cfg);diff --git a/test/remember-slow-embedder-8135.test.ts b/test/remember-slow-embedder-8135.test.tsnew file mode 100644index 0000000000..c8395d5775--- /dev/null+++ b/test/remember-slow-embedder-8135.test.ts@@ -0,0 +1,93 @@+/**+ * #8135: a slow but working embedder lost every remember. Its preparation embeds the fact under the consumer's 30 s+ * foreground budget while the embed call itself may take up to 60 s, so an embed slower than the budget let the budget's+ * abort win: `prepareFactEmbedding`'s no-vector fallback (`degraded: true`, re-embedded later) only ran when the embed+ * failed with the signal still live, the attempt was charged, and the second one finished the request+ * `failed`/`preparation_stalled` with nothing saved. The embed is now bounded inside the budget and its own timeout+ * takes the fallback, so the fact is saved without a vector on the first attempt.+ *+ * Real consumer, real remember preparer and publication, real gateway with a slow embed transport; the foreground+ * budget is scaled down (the consumer's `preparationMs`), the shape is the production one.+ */+import { afterAll, beforeAll, expect, test } from 'bun:test';+import { randomUUID } from 'node:crypto';+import { mkdtempSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { PGLiteEngine } from '../src/core/pglite-engine.ts';+import { PersistenceConsumer } from '../src/core/persistence/consumer.ts';+import { admitWrite } from '../src/core/persistence/journal.ts';+import { resetWriteSwitches } from '../src/core/persistence/switches.ts';+import { awaitWrite, disposePersistenceConsumer, preparePersistedMutation } from '../src/core/persistence/service.ts';+import '../src/core/persistence/memory-mutations.ts';+import { configureGateway, resetGateway, __setEmbedTransportForTests } from '../src/core/ai/gateway.ts';+import { admission, fixtures, initializeFixtures, selectFixtureHost, type HarnessConfig } from '../scripts/persistence/harness.ts';+import { LEGACY_EMBEDDING_CONFIG } from './helpers/legacy-embedding-config.ts';+import { withEnv } from './helpers/with-env.ts';++const home = mkdtempSync(join(tmpdir(), 'gbrain-remember-slow-embed-'));+const config: HarnessConfig = {+ kind: 'pglite', root: home, dataDir: join(home, 'data'), hostId: randomUUID(),+ seed: 8135, schedules: 0, operations: 0, sourceIds: ['source-a'], principalIds: [randomUUID()],+};+const env = { GBRAIN_HOME: home, GBRAIN_PERSISTENCE_FIXTURE_HOME: home };+const BUDGET_MS = 1_500;+let engine: PGLiteEngine;+let embedCalls = 0;++/** An embedder slower than the budget that honours cancellation (the reporter's saturated local provider). */+function slowEmbedder(ms: number) {+ __setEmbedTransportForTests((async (opts: { values: string[]; abortSignal?: AbortSignal }) => {+ embedCalls++;+ await new Promise<void>((resolve, reject) => {+ const timer = setTimeout(resolve, ms);+ opts.abortSignal?.addEventListener('abort', () => { clearTimeout(timer); reject(opts.abortSignal!.reason); }, { once: true });+ });+ return { embeddings: opts.values.map(() => new Array(LEGACY_EMBEDDING_CONFIG.embedding_dimensions).fill(0.01)) };+ }) as never);+}++beforeAll(async () => withEnv(env, async () => {+ engine = new PGLiteEngine();+ await engine.connect({});+ await engine.initSchema();+ selectFixtureHost(config.hostId);+ await initializeFixtures(engine, config);+ resetWriteSwitches();+ configureGateway({ ...LEGACY_EMBEDDING_CONFIG, env: { OPENAI_API_KEY: 'sk-test' } });+}), 120_000);++afterAll(async () => {+ __setEmbedTransportForTests(null);+ resetGateway();+ await engine.disconnect();+ rmSync(home, { recursive: true, force: true });+});++function rememberRow(fact: string) {+ return admitWrite(engine, admission(config, sourceA(), 'memory/unattributed', '', 0, {+ operation: 'remember', worktreeId: null, topologyGeneration: null, callerIntent: { fact },+ intent: { fact, entity_slug: null, fence: false, valid_from: new Date().toISOString(), valid_until: null, visibility: 'world', session_id: null },+ }));+}+let sources: Awaited<ReturnType<typeof fixtures>>;+const sourceA = () => sources[0]!;++test('#8135 a remember whose embed outlasts the preparation budget is saved without a vector, not lost as preparation_stalled', async () => withEnv(env, async () => {+ sources = await fixtures(engine, config);+ slowEmbedder(BUDGET_MS * 4);+ const fact = 'Alice Example keeps the acme-example ledger in a fireproof safe.';+ const row = await rememberRow(fact);+ const consumer = new PersistenceConsumer(engine, { engine: 'pglite' }, preparePersistedMutation,+ { hostId: config.hostId, pollMs: 20, preparationMs: BUDGET_MS, preparationBudgets: { maxAttempts: 2 }, onError: () => {} });+ try {+ consumer.start();+ const waited = await awaitWrite(engine, row, { engine: 'pglite' }, { waitMs: BUDGET_MS * 6 });+ expect({ state: waited.row.state, error_code: waited.row.error_code }).toEqual({ state: 'committed', error_code: null });+ expect(waited.row.preparation_attempts ?? 0).toBe(0);+ const facts = await engine.executeRaw<{ fact: string; has_vector: boolean }>(+ 'SELECT fact, embedding IS NOT NULL AS has_vector FROM facts WHERE source_id=$1 AND expired_at IS NULL', [row.source_id]);+ expect(facts).toEqual([{ fact, has_vector: false }]);+ expect(embedCalls).toBeGreaterThan(0);+ } finally { await disposePersistenceConsumer(engine); await consumer.stop(); }+}), 30_000);