Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: second Step 11 review fixes

garrytan9e6ee8b2b5d9
diff --git a/scripts/lib/gh-issue.ts b/scripts/lib/gh-issue.ts
index fd590089e0..0d980333d7 100644
--- a/scripts/lib/gh-issue.ts
+++ b/scripts/lib/gh-issue.ts
@@ -263,15 +263,22 @@ export function readZipEntry(zip: Uint8Array, name: string): Uint8Array | undefi
/** One JSON file from a run's named artifact; `reason` says why it could not be read (missing, expired, unreadable), and `transient` marks a failed request that a later read may get past. */
export async function fetchArtifactJson(client: GitHubClient, runId: number, artifact: string, file: string): Promise<{ value?: unknown; reason?: string; transient?: boolean }> {
if (!client.download) return { reason: 'this client cannot download artifacts' };
+ const failed = (e: unknown) => `reading run ${runId}'s '${artifact}' artifact failed: ${e instanceof Error ? e.message : String(e)}`;
+ let zip: Uint8Array;
try {
const list = await client.request<{ artifacts: Array<{ id: number; name: string; expired: boolean }> }>('GET', `repos/{repo}/actions/runs/${runId}/artifacts?name=${encodeURIComponent(artifact)}&per_page=10`);
const found = list.artifacts.find(a => a.name === artifact);
if (!found) return { reason: `run ${runId} has no '${artifact}' artifact` };
if (found.expired) return { reason: `run ${runId}'s '${artifact}' artifact has expired` };
- const bytes = readZipEntry(await client.download(`repos/{repo}/actions/artifacts/${found.id}/zip`), file);
+ zip = await client.download(`repos/{repo}/actions/artifacts/${found.id}/zip`);
+ } catch (e) {
+ return { reason: failed(e), transient: true };
+ }
+ try {
+ const bytes = readZipEntry(zip, file);
if (!bytes) return { reason: `run ${runId}'s '${artifact}' artifact has no readable ${file}` };
return { value: JSON.parse(new TextDecoder().decode(bytes)) };
} catch (e) {
- return { reason: `reading run ${runId}'s '${artifact}' artifact failed: ${e instanceof Error ? e.message : String(e)}`, transient: true };
+ return { reason: failed(e) };
}
}
diff --git a/src/core/persistence/effects.ts b/src/core/persistence/effects.ts
index a961d18723..413d3224a6 100644
--- a/src/core/persistence/effects.ts
+++ b/src/core/persistence/effects.ts
@@ -812,19 +812,22 @@ export async function runPersistenceEffects(engine: BrainEngine, config: GBrainC
}
/**
- * The effect-side twin of the consumer's #6712 release: a pass whose claim loop lost its database connection (a pooler
- * drop, a failover) gives back the git effects it claimed and set aside for their root, as soon as the database answers
+ * The effect-side twin of the consumer's #6712 release: a pass whose claim loop or deferred flush lost its database
+ * connection (a pooler drop, a failover) gives back the git effects it claimed and had not finished, as soon as the database answers
* again, so the next pass claims them at once instead of waiting out their 2-minute lease. Fenced on each claim's token,
- * so an effect another worker took over is untouched. Gives up after 30 s, leaving the lease to lapse as before.
+ * so an effect another worker took over is untouched. Stops after 30 s or on abort, checked between claims, leaving the
+ * rest to their lease as before; a retry resumes after the last claim given back.
*/
async function releaseAfterConnectionLoss(engine: BrainEngine, effects: PersistenceEffect[], signal?: AbortSignal): Promise<void> {
const deadline = Date.now() + 30_000;
- for (let attempt = 0; effects.length && !signal?.aborted; attempt++) {
+ const live = () => !signal?.aborted && Date.now() < deadline;
+ let next = 0;
+ for (let attempt = 0; next < effects.length && live(); attempt++) {
try {
- for (const effect of effects) await retryEffect(engine, effect, 'database_contention', 0);
+ for (; next < effects.length && live(); next++) await retryEffect(engine, effects[next]!, 'database_contention', 0);
return;
} catch (error) {
- if (!transientDatabaseFailure(error) || Date.now() >= deadline) return;
+ if (!transientDatabaseFailure(error)) return;
}
await new Promise(resolve => setTimeout(resolve, Math.min(2000, 250 * 2 ** attempt)));
}
@@ -1045,8 +1048,10 @@ async function runEffectPass(engine: BrainEngine, config: GBrainConfig, opts: Ef
let setAside: PersistenceEffect | null = null;
try {
while (attempted < limit && !opts.signal?.aborted) {
+ setAside = null;
const effect = await claimPersistenceEffect(engine, opts.hostId, opts.deadOwners);
if (!effect) break;
+ setAside = effect;
attempted++;
touched.add(effect.request_id);
let binding: WorktreeBinding | null = null;
@@ -1070,21 +1075,28 @@ async function runEffectPass(engine: BrainEngine, config: GBrainConfig, opts: Ef
throw error;
}
if (!deferred.length) break;
- for (const { effects, binding, hardened } of deferred.splice(0)) {
- const probe = await hardened;
- if ('error' in probe) {
- for (const effect of effects) await recordFailure(engine, effect, probe.error, opts.signal);
- continue;
- }
- const durable = probe.durable;
- if (durable && singleFileGitEffect(effects[0]!)) await commitGroup(effects, { ...binding, local_path: binding.local_path! });
- else if (singleFileGitEffect(effects[0]!) && effects[0]!.worktree_id) await recordGroup(effects, binding);
- // Coalesced siblings run with their own source's binding (sources can share a worktree).
- else for (const effect of effects) {
- let own: WorktreeBinding | null;
- try { own = effect === effects[0] ? binding : await getWorktreeBinding(engine, effect.source_id, opts.hostId); }
- catch (error) { await recordFailure(engine, effect, error, opts.signal); continue; }
- await run(effect, own, durable);
+ const groups = deferred.splice(0);
+ for (let i = 0; i < groups.length; i++) {
+ const { effects, binding, hardened } = groups[i]!;
+ try {
+ const probe = await hardened;
+ if ('error' in probe) {
+ for (const effect of effects) await recordFailure(engine, effect, probe.error, opts.signal);
+ continue;
+ }
+ const durable = probe.durable;
+ if (durable && singleFileGitEffect(effects[0]!)) await commitGroup(effects, { ...binding, local_path: binding.local_path! });
+ else if (singleFileGitEffect(effects[0]!) && effects[0]!.worktree_id) await recordGroup(effects, binding);
+ // Coalesced siblings run with their own source's binding (sources can share a worktree).
+ else for (const effect of effects) {
+ let own: WorktreeBinding | null;
+ try { own = effect === effects[0] ? binding : await getWorktreeBinding(engine, effect.source_id, opts.hostId); }
+ catch (error) { await recordFailure(engine, effect, error, opts.signal); continue; }
+ await run(effect, own, durable);
+ }
+ } catch (error) {
+ if (transientDatabaseFailure(error)) await releaseAfterConnectionLoss(engine, groups.slice(i).flatMap(g => g.effects), opts.signal);
+ throw error;
}
}
}
diff --git a/test/persistence-effect-claim-loss.test.ts b/test/persistence-effect-claim-loss.test.ts
index 552afe5c74..6eec8c508f 100644
--- a/test/persistence-effect-claim-loss.test.ts
+++ b/test/persistence-effect-claim-loss.test.ts
@@ -64,4 +64,61 @@ for (const backend of testBackends()) describe(`[${backend}] an effect pass that
expect(effect).toEqual({ state: 'queued', token: null, lease: null });
}, { databaseUrl }));
}
+
+ const flushCases = [['looking up the binding of the effect it just claimed', 0, 2, true], ['flushing the deferred git groups', 2, 1, false]] as const;
+ for (const [where, skip, limit, versioned] of flushCases) {
+ test(`forced probe: the connection drops while ${where}; every effect claimed and not finished is queued again`, () => managedBrain(async ({ engine }) => {
+ for (const n of [1, 2]) await queueEffects(engine, { slug: `notes/alpha-example-${n}`, page_id: n, expected_hash: null, relative_path: `notes/alpha-example-${n}.md`, ...(versioned ? { version: n } : {}) });
+ let claims = 0; let passed = 0; let failures = 0;
+ const drop = (call: (...a: never[]) => Promise<unknown>) => async (...args: never[]) => {
+ if (claims === limit && passed++ >= skip && failures < 2) { failures++; throw connectionLost(); }
+ const out = await call(...args) as { kind?: string; state?: string } | null;
+ if (out && !Array.isArray(out) && out.kind === 'git' && out.state === 'running') claims++;
+ return out;
+ };
+ const flaky = new Proxy(engine, {
+ get(target, prop) {
+ const value = Reflect.get(target, prop, target);
+ if (prop === 'transactionDirect' || prop === 'executeRaw' || prop === 'transaction') return drop(value.bind(target));
+ return typeof value === 'function' ? value.bind(target) : value;
+ },
+ });
+ await expect(runPersistenceEffects(flaky, { engine: engine.kind }, { hostId: localHostId(), limit }))
+ .rejects.toMatchObject({ code: 'CONNECTION_CLOSED' });
+ expect(failures).toBe(2);
+ const states = await engine.executeRaw<{ state: string; token: string | null }>("SELECT state, execution_token::text AS token FROM persistence_effects WHERE kind='git'");
+ expect(states).toEqual([{ state: 'queued', token: null }, { state: 'queued', token: null }]);
+ }, { databaseUrl }));
+ }
+
+ test('forced probe: the give-back stops at once when the worker is aborted, leaving the rest to their lease', () => managedBrain(async ({ engine }) => {
+ for (const version of [1, 2, 3]) await queueEffects(engine, { slug: `notes/alpha-example-${version}`, page_id: version, expected_hash: null, relative_path: `notes/alpha-example-${version}.md`, version });
+ const abort = new AbortController();
+ let claims = 0; let fired = false;
+ const flaky = new Proxy(engine, {
+ get(target, prop) {
+ if (prop === 'transactionDirect') {
+ return async (fn: never, opts: never) => {
+ if (claims === 3 && !fired) { fired = true; throw connectionLost(); }
+ const out = await target.transactionDirect(fn, opts) as { kind?: string; state?: string } | null;
+ if (out && !Array.isArray(out) && out.kind === 'git' && out.state === 'running') claims++;
+ return out;
+ };
+ }
+ if (prop === 'executeRaw') {
+ return async (sql: string, params?: unknown[]) => {
+ const out = await target.executeRaw(sql, params);
+ if (fired && sql.startsWith("UPDATE persistence_effects SET state='queued'")) abort.abort();
+ return out;
+ };
+ }
+ const value = Reflect.get(target, prop, target);
+ return typeof value === 'function' ? value.bind(target) : value;
+ },
+ });
+ await expect(runPersistenceEffects(flaky, { engine: engine.kind }, { hostId: localHostId(), limit: 5, signal: abort.signal }))
+ .rejects.toMatchObject({ code: 'CONNECTION_CLOSED' });
+ const states = await engine.executeRaw<{ state: string }>("SELECT state FROM persistence_effects WHERE kind='git' ORDER BY state");
+ expect(states.map(s => s.state)).toEqual(['queued', 'running', 'running']);
+ }, { databaseUrl }));
});
diff --git a/test/scripts/ci-reuse.test.ts b/test/scripts/ci-reuse.test.ts
index 67cfbb16a8..0740cba86d 100644
--- a/test/scripts/ci-reuse.test.ts
+++ b/test/scripts/ci-reuse.test.ts
@@ -431,6 +431,11 @@ describe('ci-reuse remote-gate decision', () => {
expect(await gateDeps(client({ head: { identities: ids } })).executed(r600)).toEqual({ tests: 3, failed: 1 });
expect(await gateDeps(client({ head: {} })).executed(r600)).toEqual({ reason: "run 600's executed-identities.json has no head.identities list" });
expect(await gateDeps(client(null, false)).executed(r600)).toEqual({ reason: "run 600 has no 'executed-identities' artifact" });
+ const download = (bytes: () => Promise<Uint8Array>): GitHubClient => ({ ...client(null), download: bytes });
+ expect(await gateDeps(download(async () => { throw new Error('HTTP 503'); })).executed(r600))
+ .toEqual({ reason: "reading run 600's 'executed-identities' artifact failed: HTTP 503", transient: true });
+ expect(await gateDeps(download(async () => zipOf('executed-identities.json', '{"head":'))).executed(r600))
+ .toEqual({ reason: expect.stringMatching(/^reading run 600's 'executed-identities' artifact failed: /) });
});
/** A run's receipts as scripts/lib/test-env.sh writes them: one artifact directory per job, one test file each. */