Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(remember): bound the fact embed inside the preparation budget so a slow embedder saves without a vector instead of preparation_stalled (#8135)

garrytan7754b90d8334
diff --git a/src/core/facts/single-prepare.ts b/src/core/facts/single-prepare.ts
index a532991e8c..d581cd5de6 100644
--- a/src/core/facts/single-prepare.ts
+++ b/src/core/facts/single-prepare.ts
@@ -18,13 +18,23 @@ export interface SingleFactIntent {
/** Speaker of the new claim; a fact another known speaker asserted is never its duplicate. */
attributed_to?: FactAttribution | null;
}
-/** Provider work belongs to preparation, never to a page/source transaction. `disabled`: the brain opted out of embedding. */
-export async function prepareFactEmbedding(fact: string, signal?: AbortSignal, disabled = false): Promise<{ embedding: Float32Array | null; embedding_model: string | null; degraded: boolean }> {
+/** Preparation time kept back after the embed (a quarter of what is left, at most this): the dedup reads still run. */
+const EMBED_RESERVE_MS = 5_000;
+/**
+ * Provider work belongs to preparation, never to a page/source transaction. `disabled`: the brain opted out of embedding.
+ * `deadlineAt` (#8135): the preparation budget's end. The embed gets its own timeout short of it, and that timeout takes
+ * the no-vector fallback like any embed failure; without it a slow embedder let the budget's abort win, the attempt was
+ * charged, and the write ended `preparation_stalled` with nothing saved. The caller's own abort still propagates.
+ */
+export async function prepareFactEmbedding(fact: string, signal?: AbortSignal, disabled = false, deadlineAt?: number): Promise<{ embedding: Float32Array | null; embedding_model: string | null; degraded: boolean }> {
signal?.throwIfAborted();
- if (!disabled && isAvailable('embedding')) {
+ const remaining = deadlineAt === undefined ? Infinity : deadlineAt - Date.now();
+ const embedMs = remaining - Math.min(EMBED_RESERVE_MS, Math.floor(remaining / 4));
+ if (!disabled && isAvailable('embedding') && embedMs > 0) {
try {
const model = getEmbeddingModel();
- return { embedding: await embedOne(fact, { abortSignal: signal, embeddingModel: model, inputType: 'document' }), embedding_model: model, degraded: false };
+ const abortSignal = Number.isFinite(embedMs) ? AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(embedMs)]) : signal;
+ return { embedding: await embedOne(fact, { abortSignal, embeddingModel: model, inputType: 'document' }), embedding_model: model, degraded: false };
} catch { signal?.throwIfAborted(); }
}
return { embedding: null, embedding_model: null, degraded: true };
diff --git a/src/core/persistence/memory-prepare.ts b/src/core/persistence/memory-prepare.ts
index 82809d7579..62331e9e37 100644
--- a/src/core/persistence/memory-prepare.ts
+++ b/src/core/persistence/memory-prepare.ts
@@ -9,6 +9,7 @@ import { factEmbeddingDisabled } from '../embedding-disabled.ts';
import { engineMutationPrecondition, parseMutationPrecondition } from './preconditions.ts';
import type { PreparedMutation } from './coordinator.ts';
import type { WriteRequest } from './model.ts';
+import type { ClaimPhaseClock } from './claim-phase.ts';
import { appendContextNote, type InferredVia } from '../facts/subject-infer.ts';
import { inferenceNote } from '../facts/subject-infer-write.ts';
import { requestChannelTrust } from '../trust/channel.ts';
@@ -59,7 +60,7 @@ async function activeUnlinkedCopy(engine: BrainEngine, sourceId: string, input:
}
/** Every retry renders the semantic append from the latest coherent snapshot. */
-export async function prepareMemoryMutation(engine: BrainEngine, row: WriteRequest, config: GBrainConfig, signal?: AbortSignal): Promise<PreparedMutation> {
+export async function prepareMemoryMutation(engine: BrainEngine, row: WriteRequest, config: GBrainConfig, signal?: AbortSignal, clock?: ClaimPhaseClock): Promise<PreparedMutation> {
signal?.throwIfAborted();
if (row.operation !== 'remember' || !row.intent) {
throw opError('storage_error', 'Unknown memory mutation intent.',
@@ -80,7 +81,7 @@ export async function prepareMemoryMutation(engine: BrainEngine, row: WriteReque
signal?.throwIfAborted();
const embeddingConfigSql = "SELECT key,value FROM config WHERE key IN ('embedding_model','embedding_dimensions') ORDER BY key";
const observedEmbeddingConfig = JSON.stringify(await engine.executeRaw(embeddingConfigSql));
- const { embedding, embedding_model, degraded } = await prepareFactEmbedding(input.fact, signal, await factEmbeddingDisabled(engine, config));
+ const { embedding, embedding_model, degraded } = await prepareFactEmbedding(input.fact, signal, await factEmbeddingDisabled(engine, config), clock?.deadlineAt);
signal?.throwIfAborted();
// #5836: an inferred link dedups exact text only, so it never supersedes or drops a similar fact.
const dedupEmbedding = p.entity_inferred ? null : embedding;
diff --git a/src/core/persistence/service.ts b/src/core/persistence/service.ts
index c456e90090..fe47c8467e 100644
--- a/src/core/persistence/service.ts
+++ b/src/core/persistence/service.ts
@@ -101,7 +101,7 @@ export async function preparePersistedMutation(e: BrainEngine, row: WriteRequest
if (row.operation === 'submit_job' && String(row.intent?.kind).startsWith('managed_maintenance_')) return (await import('./prepared-maintenance.ts')).prepareMaintenanceMutation(e, row, cfg, clock);
if (row.operation === 'put_page' && row.intent?.kind === 'managed_file_import') return (await import('./import-prepare.ts')).prepareManagedImportMutation(e, row, cfg);
if (row.operation === 'put_page' && row.intent?.kind === 'managed_file_repair') return (await import('./file-repair.ts')).prepareManagedFileRepairMutation(e, row, cfg);
- if (row.operation === 'remember') return (await import('./memory-mutations.ts')).prepareMemoryMutation(e, row, cfg, signal);
+ if (row.operation === 'remember') return (await import('./memory-mutations.ts')).prepareMemoryMutation(e, row, cfg, signal, clock);
if (row.operation === 'loops_close' && row.intent?.kind === 'retire_loop_fact') return (await import('./loop-fact-retirement.ts')).prepareLoopFactRetirement(e, row, cfg);
if (row.operation === 'decide_proposal') return (await import('../facts/proposal-supersede.ts')).prepareProposalMutation(e, row, cfg);
if (row.operation === 'trust_owner_page') return (await import('../trust/page-handlers.ts')).prepareTrustOwnerPageMutation(e, row, cfg);
diff --git a/test/remember-slow-embedder-8135.test.ts b/test/remember-slow-embedder-8135.test.ts
new file mode 100644
index 0000000000..c8395d5775
--- /dev/null
+++ b/test/remember-slow-embedder-8135.test.ts
@@ -0,0 +1,93 @@
+/**
+ * #8135: a slow but working embedder lost every remember. Its preparation embeds the fact under the consumer's 30 s
+ * foreground budget while the embed call itself may take up to 60 s, so an embed slower than the budget let the budget's
+ * abort win: `prepareFactEmbedding`'s no-vector fallback (`degraded: true`, re-embedded later) only ran when the embed
+ * failed with the signal still live, the attempt was charged, and the second one finished the request
+ * `failed`/`preparation_stalled` with nothing saved. The embed is now bounded inside the budget and its own timeout
+ * takes the fallback, so the fact is saved without a vector on the first attempt.
+ *
+ * Real consumer, real remember preparer and publication, real gateway with a slow embed transport; the foreground
+ * budget is scaled down (the consumer's `preparationMs`), the shape is the production one.
+ */
+import { afterAll, beforeAll, expect, test } from 'bun:test';
+import { randomUUID } from 'node:crypto';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { PersistenceConsumer } from '../src/core/persistence/consumer.ts';
+import { admitWrite } from '../src/core/persistence/journal.ts';
+import { resetWriteSwitches } from '../src/core/persistence/switches.ts';
+import { awaitWrite, disposePersistenceConsumer, preparePersistedMutation } from '../src/core/persistence/service.ts';
+import '../src/core/persistence/memory-mutations.ts';
+import { configureGateway, resetGateway, __setEmbedTransportForTests } from '../src/core/ai/gateway.ts';
+import { admission, fixtures, initializeFixtures, selectFixtureHost, type HarnessConfig } from '../scripts/persistence/harness.ts';
+import { LEGACY_EMBEDDING_CONFIG } from './helpers/legacy-embedding-config.ts';
+import { withEnv } from './helpers/with-env.ts';
+
+const home = mkdtempSync(join(tmpdir(), 'gbrain-remember-slow-embed-'));
+const config: HarnessConfig = {
+ kind: 'pglite', root: home, dataDir: join(home, 'data'), hostId: randomUUID(),
+ seed: 8135, schedules: 0, operations: 0, sourceIds: ['source-a'], principalIds: [randomUUID()],
+};
+const env = { GBRAIN_HOME: home, GBRAIN_PERSISTENCE_FIXTURE_HOME: home };
+const BUDGET_MS = 1_500;
+let engine: PGLiteEngine;
+let embedCalls = 0;
+
+/** An embedder slower than the budget that honours cancellation (the reporter's saturated local provider). */
+function slowEmbedder(ms: number) {
+ __setEmbedTransportForTests((async (opts: { values: string[]; abortSignal?: AbortSignal }) => {
+ embedCalls++;
+ await new Promise<void>((resolve, reject) => {
+ const timer = setTimeout(resolve, ms);
+ opts.abortSignal?.addEventListener('abort', () => { clearTimeout(timer); reject(opts.abortSignal!.reason); }, { once: true });
+ });
+ return { embeddings: opts.values.map(() => new Array(LEGACY_EMBEDDING_CONFIG.embedding_dimensions).fill(0.01)) };
+ }) as never);
+}
+
+beforeAll(async () => withEnv(env, async () => {
+ engine = new PGLiteEngine();
+ await engine.connect({});
+ await engine.initSchema();
+ selectFixtureHost(config.hostId);
+ await initializeFixtures(engine, config);
+ resetWriteSwitches();
+ configureGateway({ ...LEGACY_EMBEDDING_CONFIG, env: { OPENAI_API_KEY: 'sk-test' } });
+}), 120_000);
+
+afterAll(async () => {
+ __setEmbedTransportForTests(null);
+ resetGateway();
+ await engine.disconnect();
+ rmSync(home, { recursive: true, force: true });
+});
+
+function rememberRow(fact: string) {
+ return admitWrite(engine, admission(config, sourceA(), 'memory/unattributed', '', 0, {
+ operation: 'remember', worktreeId: null, topologyGeneration: null, callerIntent: { fact },
+ intent: { fact, entity_slug: null, fence: false, valid_from: new Date().toISOString(), valid_until: null, visibility: 'world', session_id: null },
+ }));
+}
+let sources: Awaited<ReturnType<typeof fixtures>>;
+const sourceA = () => sources[0]!;
+
+test('#8135 a remember whose embed outlasts the preparation budget is saved without a vector, not lost as preparation_stalled', async () => withEnv(env, async () => {
+ sources = await fixtures(engine, config);
+ slowEmbedder(BUDGET_MS * 4);
+ const fact = 'Alice Example keeps the acme-example ledger in a fireproof safe.';
+ const row = await rememberRow(fact);
+ const consumer = new PersistenceConsumer(engine, { engine: 'pglite' }, preparePersistedMutation,
+ { hostId: config.hostId, pollMs: 20, preparationMs: BUDGET_MS, preparationBudgets: { maxAttempts: 2 }, onError: () => {} });
+ try {
+ consumer.start();
+ const waited = await awaitWrite(engine, row, { engine: 'pglite' }, { waitMs: BUDGET_MS * 6 });
+ expect({ state: waited.row.state, error_code: waited.row.error_code }).toEqual({ state: 'committed', error_code: null });
+ expect(waited.row.preparation_attempts ?? 0).toBe(0);
+ const facts = await engine.executeRaw<{ fact: string; has_vector: boolean }>(
+ 'SELECT fact, embedding IS NOT NULL AS has_vector FROM facts WHERE source_id=$1 AND expired_at IS NULL', [row.source_id]);
+ expect(facts).toEqual([{ fact, has_vector: false }]);
+ expect(embedCalls).toBeGreaterThan(0);
+ } finally { await disposePersistenceConsumer(engine); await consumer.stop(); }
+}), 30_000);