Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(persistence): give back set-aside git effects when the effect pass loses its connection
garrytan69dde0534904
diff --git a/src/core/persistence/effects.ts b/src/core/persistence/effects.tsindex 0e80e90811..a961d18723 100644--- a/src/core/persistence/effects.ts+++ b/src/core/persistence/effects.ts@@ -811,6 +811,25 @@ export async function runPersistenceEffects(engine: BrainEngine, config: GBrainC}}+/**+ * The effect-side twin of the consumer's #6712 release: a pass whose claim loop lost its database connection (a pooler+ * drop, a failover) gives back the git effects it claimed and set aside for their root, as soon as the database answers+ * again, so the next pass claims them at once instead of waiting out their 2-minute lease. Fenced on each claim's token,+ * so an effect another worker took over is untouched. Gives up after 30 s, leaving the lease to lapse as before.+ */+async function releaseAfterConnectionLoss(engine: BrainEngine, effects: PersistenceEffect[], signal?: AbortSignal): Promise<void> {+ const deadline = Date.now() + 30_000;+ for (let attempt = 0; effects.length && !signal?.aborted; attempt++) {+ try {+ for (const effect of effects) await retryEffect(engine, effect, 'database_contention', 0);+ return;+ } catch (error) {+ if (!transientDatabaseFailure(error) || Date.now() >= deadline) return;+ }+ await new Promise(resolve => setTimeout(resolve, Math.min(2000, 250 * 2 ** attempt)));+ }+}+async function runEffectPass(engine: BrainEngine, config: GBrainConfig, opts: EffectWorkerOptions, touched: Set<string>, derived: DerivedFailures,waited: Set<string>): Promise<number> {const limit = Math.max(1, Math.min(opts.limit ?? 2, 20));@@ -1023,24 +1042,32 @@ async function runEffectPass(engine: BrainEngine, config: GBrainConfig, opts: Ef// A deferred effect that requeues itself (a page walk advancing its cursor,// or work unblocked by an earlier effect) is claimable again after the flush.for (;;) {- while (attempted < limit && !opts.signal?.aborted) {- const effect = await claimPersistenceEffect(engine, opts.hostId, opts.deadOwners);- if (!effect) break;- attempted++;- touched.add(effect.request_id);- let binding: WorktreeBinding | null = null;- try {- if (effect.worktree_id && !['embedding', 'facts-backstop', 'links'].includes(effect.kind)) binding = await getWorktreeBinding(engine, effect.source_id, opts.hostId);- } catch (error) { await recordFailure(engine, effect, error, opts.signal); continue; }- if (effect.kind === 'git' && binding?.local_path) {- const root = binding.local_path;- // #5182: the binding's recorded setting decides; the legacy hook probe only when nothing is recorded (fail-closed).- if (!probes.has(root)) probes.set(root, gitDurabilityPolicy(binding, root).then(policy => ({ durable: policy.durable }), error => ({ error })));- const group = singleFileGitEffect(effect) && effect.worktree_id- ? [effect, ...await claimCoalescedGitEffects(engine, opts.hostId, effect.worktree_id, GIT_GROUP_SIZE - 1, opts.deadOwners)] : [effect];- for (const member of group) touched.add(member.request_id);- deferred.push({ effects: group, binding, hardened: probes.get(root)! });- } else await run(effect, binding);+ let setAside: PersistenceEffect | null = null;+ try {+ while (attempted < limit && !opts.signal?.aborted) {+ const effect = await claimPersistenceEffect(engine, opts.hostId, opts.deadOwners);+ if (!effect) break;+ attempted++;+ touched.add(effect.request_id);+ let binding: WorktreeBinding | null = null;+ try {+ if (effect.worktree_id && !['embedding', 'facts-backstop', 'links'].includes(effect.kind)) binding = await getWorktreeBinding(engine, effect.source_id, opts.hostId);+ } catch (error) { await recordFailure(engine, effect, error, opts.signal); continue; }+ if (effect.kind === 'git' && binding?.local_path) {+ const root = binding.local_path;+ // #5182: the binding's recorded setting decides; the legacy hook probe only when nothing is recorded (fail-closed).+ if (!probes.has(root)) probes.set(root, gitDurabilityPolicy(binding, root).then(policy => ({ durable: policy.durable }), error => ({ error })));+ setAside = effect;+ const group = singleFileGitEffect(effect) && effect.worktree_id+ ? [effect, ...await claimCoalescedGitEffects(engine, opts.hostId, effect.worktree_id, GIT_GROUP_SIZE - 1, opts.deadOwners)] : [effect];+ for (const member of group) touched.add(member.request_id);+ deferred.push({ effects: group, binding, hardened: probes.get(root)! });+ setAside = null;+ } else await run(effect, binding);+ }+ } catch (error) {+ if (transientDatabaseFailure(error)) await releaseAfterConnectionLoss(engine, [...(setAside ? [setAside] : []), ...deferred.flatMap(d => d.effects)], opts.signal);+ throw error;}if (!deferred.length) break;for (const { effects, binding, hardened } of deferred.splice(0)) {diff --git a/test/persistence-effect-claim-loss.test.ts b/test/persistence-effect-claim-loss.test.tsnew file mode 100644index 0000000000..552afe5c74--- /dev/null+++ b/test/persistence-effect-claim-loss.test.ts@@ -0,0 +1,67 @@+/**+ * A git effect the effect pass claimed and set aside for its root, followed by a claim that loses its connection (a+ * pooler drop or failover), stayed `running` until its 2-minute claim lease lapsed: the pass threw and nothing gave the+ * claim back. The crash robot's `pooler_disconnect` fault hit this on CI as a ~2-minute stall. The pass now gives back+ * every effect it claimed and had not started, fenced on each claim's token, as soon as the database answers again.+ */+import { describe, expect, test } from 'bun:test';+import { randomUUID } from 'node:crypto';+import type { BrainEngine } from '../src/core/engine.ts';+import { runPersistenceEffects } from '../src/core/persistence/effects.ts';+import { localHostId } from '../src/core/persistence/identity.ts';+import { managedBrain } from './helpers/managed-brain.ts';+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';++const connectionLost = () => Object.assign(new Error('write CONNECTION_CLOSED 127.0.0.1:6543'), { code: 'CONNECTION_CLOSED' });++/** The first transactionDirect after a git effect's claim returns fails with a dropped connection. */+function dropAfterGitClaim(engine: BrainEngine): BrainEngine {+ let armed = false; let fired = false;+ return new Proxy(engine, {+ get(target, prop) {+ if (prop === 'transactionDirect') {+ return async (fn: never, opts: never) => {+ if (armed && !fired) { fired = true; throw connectionLost(); }+ const out = await target.transactionDirect(fn, opts) as { kind?: string; state?: string } | null;+ if (out && !Array.isArray(out) && out.kind === 'git' && out.state === 'running') armed = true;+ return out;+ };+ }+ const value = Reflect.get(target, prop, target);+ return typeof value === 'function' ? value.bind(target) : value;+ },+ });+}++async function queueEffects(engine: BrainEngine, git: Record<string, unknown>): Promise<void> {+ const [{ incarnation, worktree }] = await engine.executeRaw<{ incarnation: string; worktree: string }>(+ "SELECT source_incarnation::text AS incarnation, worktree_id::text AS worktree FROM persistence_source_bindings WHERE source_id='default'");+ await engine.transaction(async tx => {+ await tx.executeRaw("SELECT set_config('gbrain.persistence_protocol','2',true)");+ const [request] = await tx.executeRaw<{ id: string }>(`INSERT INTO persistence_requests(principal_kind,principal_id,request_id,operation,source_id,source_incarnation,slug,digest,authority,+ intent_bytes,terminal_reservation,state,completed_at) VALUES ('local_cli','test',$1::uuid,'put_page','default',$2::uuid,'notes/alpha-example','x','{}'::jsonb,0,16384,'committed',now()) RETURNING id::text`,+ [randomUUID(), incarnation]);+ await tx.executeRaw(`INSERT INTO persistence_effects(request_id,kind,data,state,source_id,source_incarnation,worktree_id,next_attempt_at)+ VALUES ($1::uuid,'git',$2::text::jsonb,'queued','default',$3::uuid,$4::uuid,now()-interval '1 minute'),+ ($1::uuid,'links',$5::text::jsonb,'queued','default',$3::uuid,NULL,now())`,+ [request!.id, JSON.stringify(git), incarnation, worktree, JSON.stringify({ slug: 'notes/alpha-example' })]);+ });+}++for (const backend of testBackends()) describe(`[${backend}] an effect pass that loses its connection gives back the git effects it set aside`, () => {+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;+ const cases = [+ ['the next claim', { slug: 'notes/alpha-example', page_id: 1, expected_hash: null, relative_path: 'notes/alpha-example.md', version: 1 }],+ ['the claim of its coalesced siblings', { slug: 'notes/alpha-example', page_id: 1, expected_hash: null, relative_path: 'notes/alpha-example.md' }],+ ] as const;+ for (const [failing, git] of cases) {+ test(`forced probe: ${failing} fails on a dropped connection; the set-aside git effect is queued again, not running under its lease`, () => managedBrain(async ({ engine }) => {+ await queueEffects(engine, git);+ await expect(runPersistenceEffects(dropAfterGitClaim(engine), { engine: engine.kind }, { hostId: localHostId(), limit: 5 }))+ .rejects.toMatchObject({ code: 'CONNECTION_CLOSED' });+ const [effect] = await engine.executeRaw<{ state: string; token: string | null; lease: string | null }>(+ "SELECT state, execution_token::text AS token, claim_expires_at::text AS lease FROM persistence_effects WHERE kind='git'");+ expect(effect).toEqual({ state: 'queued', token: null, lease: null });+ }, { databaseUrl }));+ }+});