Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Purge only the previewed pages, keep edited facts when upkeep defers their embedding, and serialize and budget the upkeep embed step

garrytan508d7d97b040
diff --git a/src/commands/maintain.ts b/src/commands/maintain.ts
index 531634a4e2..043374b93c 100644
--- a/src/commands/maintain.ts
+++ b/src/commands/maintain.ts
@@ -371,7 +371,9 @@ async function runPurgeCli(engine: BrainEngine, opts: MaintainOptions): Promise<
setCliExitVerdict(writeCliError(opError('preview_changed', why, 'Preview again: gbrain maintain --purge', { fix, detail: `current hash ${hash}` }), 'maintain', { json: !!opts.json }));
return;
}
- const result = await purgeDeletedPagesCoordinated(engine, retention, { automatic: true });
+ const previewed = new Set(pages.map((p) => p.source_id ? `${p.source_id}/${p.slug}` : p.slug));
+ const result = await purgeDeletedPagesCoordinated(engine, retention, {
+ automatic: true, only: (p) => previewed.has(selection.managed ? `${p.source_id}/${p.slug}` : p.slug) });
if (result.error) setCliExitVerdict(1);
if (opts.json) { console.log(JSON.stringify({ mode: 'applied', purged: result.count, slugs: result.slugs, blocked: result.blocked, deferred: result.deferred, ...(result.error ? { error: result.error } : {}) }, null, 2)); return; }
console.log(`Purged ${result.count} page(s).${result.blocked.length ? ` ${result.blocked.length} blocked; see --json.` : ''}`);
diff --git a/src/core/cycle/extract-facts.ts b/src/core/cycle/extract-facts.ts
index 9ddf0c7a38..a1a619b26e 100644
--- a/src/core/cycle/extract-facts.ts
+++ b/src/core/cycle/extract-facts.ts
@@ -505,8 +505,9 @@ async function planFenceReconcile(db: BrainEngine, page: PageFenceInput, slug: s
* their positions (stale rows whose claim is still active in the fence stay,
* others are expired in place, new rows wait). In-place updates still apply.
*/
-function reconcileWrites(plan: FenceReconcilePlan) {
- const deferInserts = plan.toInsert.some(f => !f.embedding) && plan.stale.some(f => f.has_embedding);
+/** `embeddingDeferred`: the caller skipped embedding on purpose (the stale-fact drain adds vectors), so a vectorless insert still replaces. */
+function reconcileWrites(plan: FenceReconcilePlan, embeddingDeferred = false) {
+ const deferInserts = !embeddingDeferred && plan.toInsert.some(f => !f.embedding) && plan.stale.some(f => f.has_embedding);
const activeClaims = new Set(plan.extracted.filter(f => f.expired_at == null).map(f => `${f.fact}\u0000${f.source}`));
return {
deferInserts,
@@ -583,17 +584,17 @@ function maintenanceRequestIdFor(parts: unknown[]): string {
* changed or the request is still pending (the next run retries).
*/
async function publishFenceReconcile(engine: BrainEngine, authority: MaintenanceAuthority, page: { id: number; knowledge_revision?: string | null },
- slug: string, toInsert: FenceExtractedFact[]): Promise<{ inserted: number; updated: number; deleted: number; deferred: boolean; warnings: string[]; write_gate?: GateTally } | null> {
+ slug: string, toInsert: FenceExtractedFact[], deferEmbedding = false): Promise<{ inserted: number; updated: number; deleted: number; deferred: boolean; warnings: string[]; write_gate?: GateTally } | null> {
const sourceId = authority.writer.sourceId;
const revision = page.knowledge_revision ?? REVISION_BACKFILL_PENDING;
const embeddings = toInsert.filter(f => f.embedding).map(f => ({ row_num: f.row_num, fact: f.fact, model: f.embedding_model ?? null, vector: Array.from(f.embedding!) }));
for (let attempt = 0; ; attempt++) {
- const requestId = maintenanceRequestIdFor(['extract-facts-fence-v1', authority.writer.sourceIncarnation, sourceId, slug, revision, digest(embeddings), attempt]);
+ const requestId = maintenanceRequestIdFor(['extract-facts-fence-v1', authority.writer.sourceIncarnation, sourceId, slug, revision, digest(embeddings), attempt, ...(deferEmbedding ? ['defer-embedding'] : [])]);
const prior = await getWriteRequest(engine, authority.writer.principal, requestId);
if (prior && isTerminal(prior) && prior.state !== 'committed') continue;
try {
const receipt = await submitDatabaseMaintenanceIntent(engine, authority, slug,
- { kind: FENCE_FACTS_INTENT, expected_revision: revision, page_id: page.id, embeddings }, requestId);
+ { kind: FENCE_FACTS_INTENT, expected_revision: revision, page_id: page.id, embeddings, ...(deferEmbedding ? { defer_embedding: true } : {}) }, requestId);
return { inserted: Number(receipt.inserted ?? 0), updated: Number(receipt.updated ?? 0), deleted: Number(receipt.deleted ?? 0),
deferred: receipt.deferred === true, warnings: Array.isArray(receipt.warnings) ? receipt.warnings.map(String) : [],
...(receipt.write_gate ? { write_gate: receipt.write_gate as GateTally } : {}) };
@@ -606,7 +607,7 @@ async function publishFenceReconcile(engine: BrainEngine, authority: Maintenance
/** Preparer for `managed_maintenance_fence_facts`: replans the page's reconcile under its key and applies it. */
export async function prepareFenceFactsReconcile(engine: BrainEngine, row: WriteRequest, config: GBrainConfig): Promise<PreparedMutation> {
- const intent = row.intent as { page_id?: unknown; expected_revision?: unknown; embeddings?: Array<{ row_num: number; fact: string; model: string | null; vector: number[] }> } | null;
+ const intent = row.intent as { page_id?: unknown; expected_revision?: unknown; defer_embedding?: unknown; embeddings?: Array<{ row_num: number; fact: string; model: string | null; vector: number[] }> } | null;
const snapshot = await engine.readPageSnapshot(row.slug, { sourceId: row.source_id });
if (!snapshot || snapshot.page.id !== Number(row.page_id) || Number(intent?.page_id) !== snapshot.page.id) {
throw opError('page_identity_changed', 'The page was deleted or replaced before its facts fence was reconciled.',
@@ -628,7 +629,7 @@ export async function prepareFenceFactsReconcile(engine: BrainEngine, row: Write
const vector = vectors.get(`${fact.row_num}\u0000${fact.fact}`);
if (vector) { fact.embedding = new Float32Array(vector.vector); fact.embedding_model = vector.model; }
}
- const writes = reconcileWrites(plan);
+ const writes = reconcileWrites(plan, intent?.defer_embedding === true);
const written = await writeFenceReconcile(tx, row.source_id, row.slug, plan, writes);
return { status: 'completed', ...written, deferred: writes.deferInserts, warnings: [...warnings, ...written.warnings] };
} };
@@ -1078,7 +1079,7 @@ export async function runExtractFacts(
return 'stop';
}
- const writes = reconcileWrites(plan);
+ const writes = reconcileWrites(plan, opts.deferEmbedding === true);
const { detach, expireInPlace, deferInserts } = writes;
if (deferInserts) {
result.warnings.push(`${slug}: destructive fact reconciliation deferred; existing vectors preserved until embedding succeeds`);
@@ -1087,7 +1088,7 @@ export async function runExtractFacts(
type Written = { inserted: number; updated: number; warnings: string[]; deleted?: number; deferred?: boolean; write_gate?: GateTally };
const apply = async (): Promise<Written | null> => {
const authority = await managedAuthority();
- if (authority) return publishFenceReconcile(engine, authority, page, slug, toInsert);
+ if (authority) return publishFenceReconcile(engine, authority, page, slug, toInsert, opts.deferEmbedding === true);
try {
return await withUnmanagedBeliefs(engine, sourceId, [slug], 'extract_facts', run => maintenanceTransaction(engine, tx => run(tx, async () => {
opts.signal?.throwIfAborted();
diff --git a/src/core/maintenance/run.ts b/src/core/maintenance/run.ts
index 0f3af02e01..0fe2161954 100644
--- a/src/core/maintenance/run.ts
+++ b/src/core/maintenance/run.ts
@@ -33,6 +33,7 @@ import { readDecideConfig } from '../ai/decide/config.ts';
import { hasTypesafeKey } from '../ai/decide/index.ts';
import { isEngineDegraded } from '../degraded-marker.ts';
import { hasPendingMigrations } from '../migrate-pending.ts';
+import { tryAcquireDbLock } from '../db-lock.ts';
export const MAINTENANCE_LAST_RUN_KEY = 'maintenance.last_run';
export const MAINTENANCE_AUTO_KEY = 'maintenance.auto';
@@ -273,7 +274,21 @@ function backedOff(failures: UpkeepFailure[], step: string, scope: string, now:
return !!entry && entry.count >= UPKEEP_BACKOFF_FAILURES && now.getTime() - Date.parse(entry.last_at) < UPKEEP_BACKOFF_MS;
}
+/** One embed step at a time per brain, so overlapping runs cannot each spend the remaining daily cap. */
async function runEmbedStep(engine: BrainEngine, now: Date, signal: AbortSignal | undefined): Promise<{ step: UpkeepStep; embed: { spent_today_usd: number; cap_usd: number } }> {
+ const lock = await tryAcquireDbLock(engine, 'gbrain-upkeep-embed', 60);
+ if (!lock) {
+ return { step: { name: 'embed', scope: 'global', status: 'skipped', reason: 'embed_running' },
+ embed: { spent_today_usd: await readEmbedSpentToday(engine, now), cap_usd: await readEmbedDailyCapUsd(engine) } };
+ }
+ try {
+ return await embedWithinDailyCap(engine, now, signal);
+ } finally {
+ await lock.release();
+ }
+}
+
+async function embedWithinDailyCap(engine: BrainEngine, now: Date, signal: AbortSignal | undefined): Promise<{ step: UpkeepStep; embed: { spent_today_usd: number; cap_usd: number } }> {
const cap = await readEmbedDailyCapUsd(engine);
const spentBefore = await readEmbedSpentToday(engine, now);
const embed = (spent: number) => ({ spent_today_usd: Math.round(spent * 1e6) / 1e6, cap_usd: cap });
@@ -310,7 +325,7 @@ async function runEmbedStep(engine: BrainEngine, now: Date, signal: AbortSignal
}
} catch (error) {
spent = Math.max(spent, costCap.spent());
- if (!costCap.hit()) {
+ if (!costCap.hit() && !combined.signal.aborted) {
status = 'fail';
code = (error as { code?: string } | null)?.code ?? 'embed_failed';
}
@@ -394,9 +409,19 @@ export async function runUnattendedMaintenance(engine: BrainEngine, opts: Upkeep
const finished = items.every((item) => done.includes(item.key));
let embed: UpkeepRecord['embed'];
if (finished && plan.embedding && !opts.signal?.aborted) {
- const result = await runEmbedStep(engine, now, opts.signal);
- steps.push(result.step);
- embed = result.embed;
+ if (Date.now() >= deadline) {
+ steps.push({ name: 'embed', scope: 'global', status: 'skipped', reason: 'budget_exhausted' });
+ } else {
+ const budget = Number.isFinite(deadline)
+ ? anyAbortSignal([...(opts.signal ? [opts.signal] : []), AbortSignal.timeout(Math.max(1, deadline - Date.now()))]) : null;
+ try {
+ const result = await runEmbedStep(engine, now, budget?.signal ?? opts.signal);
+ steps.push(result.step);
+ embed = result.embed;
+ } finally {
+ budget?.dispose();
+ }
+ }
}
const failed = steps.filter((step) => step.status === 'fail');
const planned = new Map<string, string[]>([
diff --git a/src/core/persistence/purge-deleted.ts b/src/core/persistence/purge-deleted.ts
index 06346d087e..0a5cd49698 100644
--- a/src/core/persistence/purge-deleted.ts
+++ b/src/core/persistence/purge-deleted.ts
@@ -122,7 +122,7 @@ interface RecordedExpiry { slugs: string[]; exclude: number[]; blocked: DeletedP
* delete (`exclude`). A content failure counts toward the shared purge park (purge-parks.ts), so after
* PURGE_PARK_LIMIT failures it is parked and skipped, like the managed purge's; a transient one is retried next run.
*/
-async function expireRecordedTombstones(engine: BrainEngine, olderThanHours: number, retryParked: boolean, held: ReadonlySet<number>): Promise<RecordedExpiry> {
+async function expireRecordedTombstones(engine: BrainEngine, olderThanHours: number, retryParked: boolean, held: ReadonlySet<number>, only?: PurgeFilter): Promise<RecordedExpiry> {
const hours = String(Math.max(0, Math.floor(olderThanHours)));
await prunePurgeParks(engine);
const run = new ItemRun('purge.recorded_tombstones');
@@ -130,6 +130,7 @@ async function expireRecordedTombstones(engine: BrainEngine, olderThanHours: num
const failures: Array<{ row: RecordedTombstone; failure: ItemFailure }> = [];
for (const row of await recordedTombstones(engine, hours)) {
if (held.has(row.id)) continue;
+ if (only && !only(row)) { out.exclude.push(row.id); continue; }
if (isParked(row.park) && !retryParked) {
out.parked++;
out.exclude.push(row.id);
@@ -313,13 +314,22 @@ export function purgePhaseNote(result: DeletedPagePurge): string {
* managed sync made unless a live page of its source has the same content (sync-delete-tombstones.ts); it is
* reported `held` (code `sync_delete_unreviewed`). The consented manual purge leaves `automatic` unset.
*/
-export async function purgeDeletedPagesCoordinated(engine: BrainEngine, olderThanHours: number, opts: { retryParked?: boolean; automatic?: boolean } = {}): Promise<DeletedPagePurge> {
+/** Limits a purge to pages the caller already showed the person (`gbrain maintain --purge --expect`). */
+export type PurgeFilter = (page: { source_id: string; slug: string }) => boolean;
+
+export async function purgeDeletedPagesCoordinated(engine: BrainEngine, olderThanHours: number, opts: { retryParked?: boolean; automatic?: boolean; only?: PurgeFilter } = {}): Promise<DeletedPagePurge> {
if (!await managedPersistenceEnabled(engine)) {
// #7130: a brain deactivated after managed syncs keeps their tombstones' records and receipts, so the hold applies here too.
const kept = opts.automatic ? await readUnreviewedSyncTombstones(engine, olderThanHours) : [];
- const recorded = await expireRecordedTombstones(engine, olderThanHours, opts.retryParked === true, new Set(kept.map(row => row.id)));
+ const recorded = await expireRecordedTombstones(engine, olderThanHours, opts.retryParked === true, new Set(kept.map(row => row.id)), opts.only);
const exclude = [...recorded.exclude, ...kept.map(row => row.id)];
- const result = await maintenanceTransaction(engine, tx => tx.purgeDeletedPages(olderThanHours, exclude.length ? { excludeIds: exclude } : undefined));
+ const result = await maintenanceTransaction(engine, async tx => {
+ const unlisted = opts.only ? (await tx.executeRaw<{ id: number; source_id: string; slug: string }>(
+ `SELECT id, source_id, slug FROM pages WHERE deleted_at IS NOT NULL AND deleted_at < now() - ($1 || ' hours')::interval`,
+ [String(Math.max(0, Math.floor(olderThanHours)))])).filter(page => !opts.only!(page)).map(page => Number(page.id)) : [];
+ const excludeIds = [...exclude, ...unlisted];
+ return tx.purgeDeletedPages(olderThanHours, excludeIds.length ? { excludeIds } : undefined);
+ });
const held = kept.filter(row => row.expired);
return { slugs: [...recorded.slugs, ...result.slugs], count: recorded.slugs.length + result.count, blocked: [...recorded.blocked, ...held.map(row => syncDeleteHeld(row, olderThanHours))],
deferred: 0, failed: recorded.failed, inactive: 0, parked: recorded.parked, capacity_deferred: 0, held: held.length };
@@ -359,6 +369,7 @@ export async function purgeDeletedPagesCoordinated(engine: BrainEngine, olderTha
}
// Reported above with its receipt's failure; the next run re-validates and submits it again.
if (reconciledFailures.has(`${row.source_id}/${row.slug}`)) continue;
+ if (opts.only && !opts.only(row)) continue;
try {
const ctx: OperationContext = { engine, config, remote: false, sourceId: row.source_id, dryRun: false,
logger: { info() {}, warn() {}, error() {} } };
diff --git a/test/maintenance-unattended.test.ts b/test/maintenance-unattended.test.ts
index 9024280ced..2ef970527c 100644
--- a/test/maintenance-unattended.test.ts
+++ b/test/maintenance-unattended.test.ts
@@ -334,6 +334,61 @@ for (const kind of testBackends()) {
});
});
+ describe('review fixes: the embed step and fact deferral', () => {
+ const quick: typeof runCycle = async () => ({ status: 'ok', phases: [] }) as never;
+
+ test('an edited fact on a keyed brain is replaced, not only expired, when upkeep defers its embedding', async () => {
+ await keyed();
+ await engine.putPage('people/alice-example', { type: 'person', title: 'Alice', compiled_truth: 'Alice works on search.\n\n' + FENCE('Alice leads the search team') });
+ let at = Date.now();
+ await upkeep(engine, { runner: 'test', providers: KEYED, now: () => new Date(at) });
+ expect(await engine.executeRaw('SELECT fact FROM facts WHERE expired_at IS NULL AND embedding IS NOT NULL')).toEqual([{ fact: 'Alice leads the search team' }]);
+ await engine.putPage('people/alice-example', { type: 'person', title: 'Alice', compiled_truth: 'Alice works on search.\n\n' + FENCE('Alice leads the ranking team') });
+ at += UPKEEP_INTERVAL_MS + HOUR;
+ await upkeep(engine, { runner: 'test', providers: KEYED, now: () => new Date(at) });
+ expect(await engine.executeRaw('SELECT fact FROM facts WHERE expired_at IS NULL')).toEqual([{ fact: 'Alice leads the ranking team' }]);
+ });
+
+ test('two overlapping runs embed once: the second skips the embed step instead of spending the cap again', async () => {
+ await keyed();
+ const [col] = await engine.executeRaw<{ dims: number }>("SELECT atttypmod AS dims FROM pg_attribute WHERE attrelid = 'content_chunks'::regclass AND attname = 'embedding'");
+ __setEmbedTransportForTests((async ({ values }: { values: string[] }) => {
+ await new Promise((resolve) => setTimeout(resolve, 300));
+ return { embeddings: values.map(() => new Array(Number(col!.dims)).fill(0.01)) };
+ }) as never);
+ await engine.putPage('notes/embed-me', { type: 'concept', title: 'Embed', compiled_truth: 'A page whose chunks need vectors.' });
+ const [a, b] = await Promise.all([1, 2].map(() => upkeep(engine, { runner: 'test', providers: KEYED, cycle: quick })));
+ const embedSteps = [a, b].map((o) => o.record!.steps.find((s) => s.name === 'embed'));
+ expect(embedSteps.filter((s) => s?.status === 'ok')).toHaveLength(1);
+ expect(embedSteps).toContainEqual({ name: 'embed', scope: 'global', status: 'skipped', reason: 'embed_running' });
+ });
+
+ test('the embed step does not start once the run budget is spent', async () => {
+ await keyed();
+ await engine.putPage('notes/embed-me', { type: 'concept', title: 'Embed', compiled_truth: 'A page whose chunks need vectors.' });
+ const slowSource: typeof runCycle = async (e, o) => {
+ if (o.sourceId) await new Promise((resolve) => setTimeout(resolve, 120));
+ return quick(e, o);
+ };
+ const outcome = await upkeep(engine, { runner: 'test', providers: KEYED, cycle: slowSource, budgetMs: 60 });
+ expect(outcome.record!.cursor).toBeNull();
+ expect(outcome.record!.steps).toContainEqual({ name: 'embed', scope: 'global', status: 'skipped', reason: 'budget_exhausted' });
+ expect(calls).toEqual([]);
+ });
+ });
+
+ test('a purge limited to the previewed pages leaves a page that became eligible afterwards', async () => {
+ const { purgeDeletedPagesCoordinated } = await import('../src/core/persistence/purge-deleted.ts');
+ for (const slug of ['notes/previewed', 'notes/late']) {
+ await engine.putPage(slug, { type: 'concept', title: slug, compiled_truth: 'Deleted long ago.' });
+ await engine.softDeletePage(slug);
+ }
+ await engine.executeRaw("UPDATE pages SET deleted_at = now() - interval '100 hours' WHERE slug IN ('notes/previewed', 'notes/late')");
+ const result = await purgeDeletedPagesCoordinated(engine, 72, { automatic: true, only: (page) => page.slug === 'notes/previewed' });
+ expect(result.slugs).toEqual(['notes/previewed']);
+ expect(await engine.executeRaw("SELECT slug FROM pages WHERE slug LIKE 'notes/%' ORDER BY slug")).toEqual([{ slug: 'notes/late' }]);
+ });
+
describe('doctor states with a fixed clock', () => {
const NOW = Date.parse('2026-11-12T12:00:00.000Z');
const record = (at: number, extra: Record<string, unknown> = {}) => engine.setConfig('maintenance.last_run', JSON.stringify({