Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(persistence): give back set-aside git effects when the effect pass loses its connection

garrytan69dde0534904
diff --git a/src/core/persistence/effects.ts b/src/core/persistence/effects.ts
index 0e80e90811..a961d18723 100644
--- a/src/core/persistence/effects.ts
+++ b/src/core/persistence/effects.ts
@@ -811,6 +811,25 @@ export async function runPersistenceEffects(engine: BrainEngine, config: GBrainC
}
}
+/**
+ * The effect-side twin of the consumer's #6712 release: a pass whose claim loop lost its database connection (a pooler
+ * drop, a failover) gives back the git effects it claimed and set aside for their root, as soon as the database answers
+ * again, so the next pass claims them at once instead of waiting out their 2-minute lease. Fenced on each claim's token,
+ * so an effect another worker took over is untouched. Gives up after 30 s, leaving the lease to lapse as before.
+ */
+async function releaseAfterConnectionLoss(engine: BrainEngine, effects: PersistenceEffect[], signal?: AbortSignal): Promise<void> {
+ const deadline = Date.now() + 30_000;
+ for (let attempt = 0; effects.length && !signal?.aborted; attempt++) {
+ try {
+ for (const effect of effects) await retryEffect(engine, effect, 'database_contention', 0);
+ return;
+ } catch (error) {
+ if (!transientDatabaseFailure(error) || Date.now() >= deadline) return;
+ }
+ await new Promise(resolve => setTimeout(resolve, Math.min(2000, 250 * 2 ** attempt)));
+ }
+}
+
async function runEffectPass(engine: BrainEngine, config: GBrainConfig, opts: EffectWorkerOptions, touched: Set<string>, derived: DerivedFailures,
waited: Set<string>): Promise<number> {
const limit = Math.max(1, Math.min(opts.limit ?? 2, 20));
@@ -1023,24 +1042,32 @@ async function runEffectPass(engine: BrainEngine, config: GBrainConfig, opts: Ef
// A deferred effect that requeues itself (a page walk advancing its cursor,
// or work unblocked by an earlier effect) is claimable again after the flush.
for (;;) {
- while (attempted < limit && !opts.signal?.aborted) {
- const effect = await claimPersistenceEffect(engine, opts.hostId, opts.deadOwners);
- if (!effect) break;
- attempted++;
- touched.add(effect.request_id);
- let binding: WorktreeBinding | null = null;
- try {
- if (effect.worktree_id && !['embedding', 'facts-backstop', 'links'].includes(effect.kind)) binding = await getWorktreeBinding(engine, effect.source_id, opts.hostId);
- } catch (error) { await recordFailure(engine, effect, error, opts.signal); continue; }
- if (effect.kind === 'git' && binding?.local_path) {
- const root = binding.local_path;
- // #5182: the binding's recorded setting decides; the legacy hook probe only when nothing is recorded (fail-closed).
- if (!probes.has(root)) probes.set(root, gitDurabilityPolicy(binding, root).then(policy => ({ durable: policy.durable }), error => ({ error })));
- const group = singleFileGitEffect(effect) && effect.worktree_id
- ? [effect, ...await claimCoalescedGitEffects(engine, opts.hostId, effect.worktree_id, GIT_GROUP_SIZE - 1, opts.deadOwners)] : [effect];
- for (const member of group) touched.add(member.request_id);
- deferred.push({ effects: group, binding, hardened: probes.get(root)! });
- } else await run(effect, binding);
+ let setAside: PersistenceEffect | null = null;
+ try {
+ while (attempted < limit && !opts.signal?.aborted) {
+ const effect = await claimPersistenceEffect(engine, opts.hostId, opts.deadOwners);
+ if (!effect) break;
+ attempted++;
+ touched.add(effect.request_id);
+ let binding: WorktreeBinding | null = null;
+ try {
+ if (effect.worktree_id && !['embedding', 'facts-backstop', 'links'].includes(effect.kind)) binding = await getWorktreeBinding(engine, effect.source_id, opts.hostId);
+ } catch (error) { await recordFailure(engine, effect, error, opts.signal); continue; }
+ if (effect.kind === 'git' && binding?.local_path) {
+ const root = binding.local_path;
+ // #5182: the binding's recorded setting decides; the legacy hook probe only when nothing is recorded (fail-closed).
+ if (!probes.has(root)) probes.set(root, gitDurabilityPolicy(binding, root).then(policy => ({ durable: policy.durable }), error => ({ error })));
+ setAside = effect;
+ const group = singleFileGitEffect(effect) && effect.worktree_id
+ ? [effect, ...await claimCoalescedGitEffects(engine, opts.hostId, effect.worktree_id, GIT_GROUP_SIZE - 1, opts.deadOwners)] : [effect];
+ for (const member of group) touched.add(member.request_id);
+ deferred.push({ effects: group, binding, hardened: probes.get(root)! });
+ setAside = null;
+ } else await run(effect, binding);
+ }
+ } catch (error) {
+ if (transientDatabaseFailure(error)) await releaseAfterConnectionLoss(engine, [...(setAside ? [setAside] : []), ...deferred.flatMap(d => d.effects)], opts.signal);
+ throw error;
}
if (!deferred.length) break;
for (const { effects, binding, hardened } of deferred.splice(0)) {
diff --git a/test/persistence-effect-claim-loss.test.ts b/test/persistence-effect-claim-loss.test.ts
new file mode 100644
index 0000000000..552afe5c74
--- /dev/null
+++ b/test/persistence-effect-claim-loss.test.ts
@@ -0,0 +1,67 @@
+/**
+ * A git effect the effect pass claimed and set aside for its root, followed by a claim that loses its connection (a
+ * pooler drop or failover), stayed `running` until its 2-minute claim lease lapsed: the pass threw and nothing gave the
+ * claim back. The crash robot's `pooler_disconnect` fault hit this on CI as a ~2-minute stall. The pass now gives back
+ * every effect it claimed and had not started, fenced on each claim's token, as soon as the database answers again.
+ */
+import { describe, expect, test } from 'bun:test';
+import { randomUUID } from 'node:crypto';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { runPersistenceEffects } from '../src/core/persistence/effects.ts';
+import { localHostId } from '../src/core/persistence/identity.ts';
+import { managedBrain } from './helpers/managed-brain.ts';
+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';
+
+const connectionLost = () => Object.assign(new Error('write CONNECTION_CLOSED 127.0.0.1:6543'), { code: 'CONNECTION_CLOSED' });
+
+/** The first transactionDirect after a git effect's claim returns fails with a dropped connection. */
+function dropAfterGitClaim(engine: BrainEngine): BrainEngine {
+ let armed = false; let fired = false;
+ return new Proxy(engine, {
+ get(target, prop) {
+ if (prop === 'transactionDirect') {
+ return async (fn: never, opts: never) => {
+ if (armed && !fired) { fired = true; throw connectionLost(); }
+ const out = await target.transactionDirect(fn, opts) as { kind?: string; state?: string } | null;
+ if (out && !Array.isArray(out) && out.kind === 'git' && out.state === 'running') armed = true;
+ return out;
+ };
+ }
+ const value = Reflect.get(target, prop, target);
+ return typeof value === 'function' ? value.bind(target) : value;
+ },
+ });
+}
+
+async function queueEffects(engine: BrainEngine, git: Record<string, unknown>): Promise<void> {
+ const [{ incarnation, worktree }] = await engine.executeRaw<{ incarnation: string; worktree: string }>(
+ "SELECT source_incarnation::text AS incarnation, worktree_id::text AS worktree FROM persistence_source_bindings WHERE source_id='default'");
+ await engine.transaction(async tx => {
+ await tx.executeRaw("SELECT set_config('gbrain.persistence_protocol','2',true)");
+ const [request] = await tx.executeRaw<{ id: string }>(`INSERT INTO persistence_requests(principal_kind,principal_id,request_id,operation,source_id,source_incarnation,slug,digest,authority,
+ intent_bytes,terminal_reservation,state,completed_at) VALUES ('local_cli','test',$1::uuid,'put_page','default',$2::uuid,'notes/alpha-example','x','{}'::jsonb,0,16384,'committed',now()) RETURNING id::text`,
+ [randomUUID(), incarnation]);
+ await tx.executeRaw(`INSERT INTO persistence_effects(request_id,kind,data,state,source_id,source_incarnation,worktree_id,next_attempt_at)
+ VALUES ($1::uuid,'git',$2::text::jsonb,'queued','default',$3::uuid,$4::uuid,now()-interval '1 minute'),
+ ($1::uuid,'links',$5::text::jsonb,'queued','default',$3::uuid,NULL,now())`,
+ [request!.id, JSON.stringify(git), incarnation, worktree, JSON.stringify({ slug: 'notes/alpha-example' })]);
+ });
+}
+
+for (const backend of testBackends()) describe(`[${backend}] an effect pass that loses its connection gives back the git effects it set aside`, () => {
+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;
+ const cases = [
+ ['the next claim', { slug: 'notes/alpha-example', page_id: 1, expected_hash: null, relative_path: 'notes/alpha-example.md', version: 1 }],
+ ['the claim of its coalesced siblings', { slug: 'notes/alpha-example', page_id: 1, expected_hash: null, relative_path: 'notes/alpha-example.md' }],
+ ] as const;
+ for (const [failing, git] of cases) {
+ test(`forced probe: ${failing} fails on a dropped connection; the set-aside git effect is queued again, not running under its lease`, () => managedBrain(async ({ engine }) => {
+ await queueEffects(engine, git);
+ await expect(runPersistenceEffects(dropAfterGitClaim(engine), { engine: engine.kind }, { hostId: localHostId(), limit: 5 }))
+ .rejects.toMatchObject({ code: 'CONNECTION_CLOSED' });
+ const [effect] = await engine.executeRaw<{ state: string; token: string | null; lease: string | null }>(
+ "SELECT state, execution_token::text AS token, claim_expires_at::text AS lease FROM persistence_effects WHERE kind='git'");
+ expect(effect).toEqual({ state: 'queued', token: null, lease: null });
+ }, { databaseUrl }));
+ }
+});