Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Retry a stalled background sync on its tier without the 30 s first budget again (GST-2515)

garrytan6670bed12d2c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9bcc76ae1a..6062712670 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,6 +20,8 @@ Since v0.60.677.0, every unknown-side `file_drift` hold written under the earlie
- A pass stopped partway keeps the holds it re-screened, and the next pass reads only the rest.
- `gbrain sync status` reads `resumable` / `commits_waiting` when nothing else is pending (or only holds that need no person) but the checkout's HEAD is past the last synced commit, with a `next` that runs the sync. It used to read `synced` until the next sync happened to run. A hold that needs a person keeps its own reason, and a company-brain source, which syncs to its approved revision, never reads it.
- With `sync.on_commit` off, the notifier's fix now says so, instead of saying that no resident runs.
+- A large source that the background syncer could not finish no longer retries forever with the same short budget. A resident gives its first run 30 seconds, and every resident restart or change of the process holding the notifier counted as "first" again, so each retry stopped at the same point having saved nothing. That short budget now goes only to a source with no failed attempts on record, and a retry after a run that saved nothing gets the full sync budget.
+- A source the background syncer marked stalled (`notifier_stalled`, "retrying every 300 s") now really is retried every five minutes. With `sync.on_commit` off (the default), the retry only happened while the original sync request was still on record, and a run that stopped early could clear that request, so the source sat stalled with nothing retrying it and nothing in the log. The retry now continues until a run completes, also after a restart or when another resident takes over the notifier, which now reads the state the previous one recorded.
## [0.60.685.0] - 2026-11-12
diff --git a/docs/guides/on-commit-sync.md b/docs/guides/on-commit-sync.md
index b01cc51066..338b81e816 100644
--- a/docs/guides/on-commit-sync.md
+++ b/docs/guides/on-commit-sync.md
@@ -172,6 +172,13 @@ sync status names the last error and the trace file
gbrain sync --source <id> --no-pull to see the error in the foreground, fix it,
and the notifier resumes on its next retry.
+The retries keep running every five minutes with sync.on_commit off (a run a
+recorded request started is finished on the retry tier, even after the request
+itself is settled) and across a resident restart or a change of lease owner,
+which reads the attempt count the previous owner recorded. The short budget a
+resident gives its first run applies only to a source with no failed attempts,
+so a retry after a run that advanced nothing gets the full sync budget.
+
A run that committed pages counts as progress even before the source's last
synced commit moves. When the last error is drain_holder_stalled, another
process holds the source's drain lease and has made no commit or cursor save
diff --git a/src/core/commit-notifier.ts b/src/core/commit-notifier.ts
index 82b05bfadd..8c8e882a8f 100644
--- a/src/core/commit-notifier.ts
+++ b/src/core/commit-notifier.ts
@@ -232,7 +232,8 @@ function notifierKit(opts: CommitNotifierOptions) {
const closeWatch = (rt: Rt) => { if (rt.watchClose) { try { rt.watchClose(); } catch { /* best effort */ } rt.watchClose = null; } };
const mode = (rt: Rt): DetectionMode => rt.watchClose ? 'watch' : 'poll';
const markerKey = (rt: Rt) => `${rt.head ?? ''}|${rt.markers.map(m => `${m.intent}:${m.generation}`).join(',')}`;
-const due = (rt: Rt) => (rt.watched && !!rt.head && rt.head !== rt.lastCommit) || rt.markers.length > 0;
+// GST-2515: an unwatched source (sync.on_commit off) has no HEAD to compare, so a run it owes (retry, or due after a resume or restart) keeps it due after its marker is gone.
+const due = (rt: Rt) => (rt.watched && !!rt.head && rt.head !== rt.lastCommit) || rt.markers.length > 0 || (!rt.watched && (rt.state === 'retry' || rt.state === 'due'));
/** Merges a fresh source listing into the runtime map (a moved root starts over); returns the sources new to it. */
function mergeListed(sources: Map<string, Rt>, rows: NotifierSource[]): Rt[] {
@@ -255,12 +256,13 @@ function mergeListed(sources: Map<string, Rt>, rows: NotifierSource[]): Rt[] {
/** The intent a start asks for (full over pull over freshness) and its time slice. */
function startPlan(rt: Rt, candidates: number, opts: Pick<CommitNotifierOptions, 'firstSliceSeconds' | 'sliceSeconds'>): { intent: MarkerIntent; first: boolean; slice: number | null } {
const intent: MarkerIntent = rt.markers.some(m => m.intent === 'full') ? 'full' : rt.markers.some(m => m.intent === 'pull') ? 'pull' : 'freshness';
- const first = !rt.started;
+ // GST-2515: the durable attempt count, not this process's memory, says whether an earlier attempt advanced nothing.
+ const first = !rt.started && rt.attempts === 0;
const slice = first && opts.firstSliceSeconds !== undefined && rt.kind !== 'classic' ? opts.firstSliceSeconds : candidates > 1 ? (opts.sliceSeconds ?? null) : null;
return { intent, first, slice };
}
-/** The durable state survives the resident: a restart resumes attempts, the next retry and a blocked run. */
+/** The durable state survives the resident: a restart resumes attempts, the next retry and a blocked run; so does a standby that becomes the owner (GST-2515: its own states date from its start). */
function restoreStates(sources: Map<string, Rt>, states: NotifierSourceRow[], nowMs: number): void {
for (const row of states) {
const rt = sources.get(row.source_id);
@@ -501,7 +503,7 @@ export function startCommitNotifier(opts: CommitNotifierOptions): CommitNotifier
running.set(rt.sourceId, { id: start.jobId, sourceId: rt.sourceId, head: rt.head, lastCommit: rt.lastCommit, intent, markers });
await transition(rt, { state: 'running', firstDue: 'keep' });
decide(rt.sourceId, 'start', { detail: `${intent}${slice ? ` slice ${slice}s` : ''}` });
- log(`${rt.sourceId}: ${intent === 'freshness' ? `HEAD moved to ${short(rt.head)} (last synced ${short(rt.lastCommit)})` : `a ${intent} request is pending`}; started ${rt.kind === 'classic' ? 'classic' : 'managed'} sync job ${start.jobId}`);
+ log(`${rt.sourceId}: ${intent === 'freshness' && rt.head ? `HEAD moved to ${short(rt.head)} (last synced ${short(rt.lastCommit)})` : `a ${intent} request is pending`}; started ${rt.kind === 'classic' ? 'classic' : 'managed'} sync job ${start.jobId}`);
}
};
@@ -520,7 +522,7 @@ export function startCommitNotifier(opts: CommitNotifierOptions): CommitNotifier
}
}
if (stopped) return;
- if (t - leaseAt >= heartbeatMs) { leaseAt = t; await ownership(); }
+ if (t - leaseAt >= heartbeatMs) { leaseAt = t; const was = owner; await ownership(); if (owner && !was && sources.size) restoreStates(sources, await guard('reading the notifier states', () => opts.store.readStates([...sources.keys()]), []), now()); }
if (t - listedAt >= LIST_EVERY_MS) { listedAt = t; await refreshSources(); }
if (stopped) return;
for (const rt of sources.values()) {
diff --git a/test/notifier-retry-unwatched-gst2515.test.ts b/test/notifier-retry-unwatched-gst2515.test.ts
new file mode 100644
index 0000000000..69e959c7c3
--- /dev/null
+++ b/test/notifier-retry-unwatched-gst2515.test.ts
@@ -0,0 +1,170 @@
+/**
+ * GST-2515: retries of a large source that never finish and never come back.
+ * - A resident restart (or a new lease owner) made every retry the "first" start of that process, so each one got the
+ * short first slice and stopped at the same point having committed nothing. The first slice is now only for a
+ * source whose durable attempt count is zero: a retry after an attempt that advanced nothing never gets it again.
+ * - With sync.on_commit off nothing reads the source's HEAD, so a run was due only while a wanted marker existed. A
+ * run's own marker settle can clear that marker after a resumable stop, which left the source in retry /
+ * notifier_stalled with no retry ever starting and nothing logged. A retry (or a resume) the notifier owes now keeps
+ * an unwatched source due on its tier, with no marker and across restarts and owner changes.
+ * The in-memory store covers the scheduling rules; the real SQL store (PGLite always, Postgres when DATABASE_URL is
+ * set) covers the durable round trip a restarted resident reads. Synthetic content only, injected clock.
+ */
+import { describe, expect, test } from 'bun:test';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { RETRY_BACKOFF_MS, startCommitNotifier, type CommitNotifier, type JobStatus, type NotifierStore, type StartRequest } from '../src/core/commit-notifier.ts';
+import { readNotifierSources, settleMarkers, upsertMarker } from '../src/core/notifier/store.ts';
+import { sqlNotifierStore } from '../src/core/notifier/resident.ts';
+import { isolatedSharedSkillsEngine } from './helpers/shared-skills-engine.ts';
+import { memoryNotifierStore, type MemoryLeases } from './helpers/notifier-memory-store.ts';
+
+const SOURCE = 'source-a';
+const HOST = '00000000-0000-4000-8000-0000000025a5';
+const A = 'a'.repeat(40), B = 'b'.repeat(40);
+const QUIET = 400;
+
+interface World { t: number; starts: StartRequest[]; jobs: Map<string, JobStatus> }
+
+function resident(w: World, store: NotifierStore, kind: 'serve' | 'jobs-worker' = 'serve'): CommitNotifier {
+ return startCommitNotifier({
+ resident: kind, log: () => {}, hostId: HOST, store, now: () => w.t, pollMs: 3000, quietMs: QUIET, heartbeatMs: 10_000,
+ firstSliceSeconds: 30, sliceSeconds: 120, maxConcurrent: 1,
+ setInterval: () => 'interval', clearInterval: () => {}, setTimeout: () => 'timer', clearTimeout: () => {},
+ readFlag: async () => false,
+ listSources: async () => [{ sourceId: SOURCE, root: '/nonexistent/source-a', gitDir: null, lastCommit: A, kind: 'managed' }],
+ lastCommit: async () => A, revParse: async () => B, syncRunning: () => false, drainHeld: async () => false,
+ startSync: async (_s, req) => { w.starts.push(req); const jobId = 'job-' + w.starts.length; w.jobs.set(jobId, { state: 'running' }); return { jobId }; },
+ jobStatus: jobId => w.jobs.get(jobId) ?? null,
+ });
+}
+/** The last start's job stops resumable at its deadline having committed nothing; clear then drops the marker, as the run's own settle does. */
+async function stallLast(w: World, n: CommitNotifier, clear: () => Promise<void> | void = () => {}) {
+ w.jobs.set('job-' + w.starts.length, { state: 'done', outcome: 'resumable', reason: 'deadline', committed: false });
+ await clear();
+ w.t += QUIET; await n.tick();
+}
+const slices = (w: World) => w.starts.map(s => s.sliceSeconds);
+
+describe('GST-2515: an unwatched source keeps its retry tier (in-memory store)', () => {
+ test('a marker-driven run that stopped without advancing is retried on every tier after its marker is gone', async () => {
+ const w: World = { t: 1_000_000, starts: [], jobs: new Map() };
+ const store = memoryNotifierStore({ lastCommit: () => A });
+ store.upsert(SOURCE, 'freshness', B);
+ const n = resident(w, store);
+ await n.tick();
+ expect(w.starts).toHaveLength(1);
+ for (const wait of [...RETRY_BACKOFF_MS, RETRY_BACKOFF_MS.at(-1)!]) {
+ await stallLast(w, n, () => { store.markers.clear(); });
+ w.t += wait; await n.tick();
+ }
+ expect(w.starts).toHaveLength(1 + RETRY_BACKOFF_MS.length + 1);
+ expect(n.decisions.filter(d => d.decision === 'stalled').length).toBeGreaterThan(0);
+ expect(slices(w)).toEqual([30, ...Array(RETRY_BACKOFF_MS.length + 1).fill(null)]);
+ await n.stop();
+ });
+
+ test('a restarted resident retries a stalled source after next_retry_at, without the first slice', async () => {
+ const w: World = { t: 1_000_000, starts: [], jobs: new Map() };
+ const store = memoryNotifierStore({ lastCommit: () => A });
+ store.upsert(SOURCE, 'freshness', B);
+ const first = resident(w, store);
+ await first.tick();
+ await stallLast(w, first, () => { store.markers.clear(); });
+ await first.stop();
+ expect(store.states.get(SOURCE)).toMatchObject({ state: 'retry', attempts: 1 });
+ const next = resident(w, store);
+ await next.tick();
+ expect(w.starts).toHaveLength(1);
+ w.t += RETRY_BACKOFF_MS[0]; await next.tick();
+ expect(slices(w)).toEqual([30, null]);
+ await next.stop();
+ });
+
+ test('a standby that takes the lease over reads the attempts its predecessor recorded: no first slice', async () => {
+ const w: World = { t: 1_000_000, starts: [], jobs: new Map() };
+ const leases: MemoryLeases = { rows: new Map(), now: () => w.t };
+ const store = memoryNotifierStore({ lastCommit: () => A, leases });
+ store.upsert(SOURCE, 'freshness', B);
+ const owner = resident(w, store, 'serve');
+ await owner.tick();
+ const standby = resident(w, store, 'jobs-worker');
+ await standby.tick();
+ expect([owner.owner, standby.owner]).toEqual([true, false]);
+ await stallLast(w, owner);
+ await owner.stop();
+ w.t += RETRY_BACKOFF_MS[0] + 10_000; await standby.tick();
+ expect(standby.owner).toBe(true);
+ expect(slices(w)).toEqual([30, null]);
+ await standby.stop();
+ });
+
+ test('a run that completes clears the owed retry: the source goes idle and nothing more starts', async () => {
+ const w: World = { t: 1_000_000, starts: [], jobs: new Map() };
+ const store = memoryNotifierStore({ lastCommit: () => A });
+ store.upsert(SOURCE, 'freshness', B);
+ const n = resident(w, store);
+ await n.tick();
+ await stallLast(w, n, () => { store.markers.clear(); });
+ w.t += RETRY_BACKOFF_MS[0]; await n.tick();
+ w.jobs.set('job-2', { state: 'done', outcome: 'synced' });
+ w.t += QUIET; await n.tick();
+ w.t += RETRY_BACKOFF_MS.at(-1)!; await n.tick();
+ expect(w.starts).toHaveLength(2);
+ expect(store.states.get(SOURCE)).toMatchObject({ state: 'idle', attempts: 0 });
+ await n.stop();
+ });
+});
+
+const arms: Array<[string, string | undefined]> = [['pglite', undefined], ...(process.env.DATABASE_URL ? [['postgres', process.env.DATABASE_URL] as [string, string]] : [])];
+
+describe.each(arms)('GST-2515: a restarted resident and the durable notifier row (%s)', (_arm, databaseUrl) => {
+ async function brain(run: (engine: BrainEngine) => Promise<void>) {
+ const { engine, close } = await isolatedSharedSkillsEngine(databaseUrl);
+ try {
+ await engine.executeRaw('DELETE FROM sync_notifier_leases WHERE host_id = $1', [HOST]);
+ await engine.executeRaw('INSERT INTO sources(id,name,local_path,config,last_commit) VALUES($1,$1,$2,$3,$4) ON CONFLICT (id) DO UPDATE SET last_commit = EXCLUDED.last_commit', [SOURCE, '/nonexistent/source-a', '{}', A]);
+ await run(engine);
+ } finally {
+ await engine.executeRaw('DELETE FROM sync_notifier_leases WHERE host_id = $1', [HOST]).catch(() => undefined);
+ await engine.executeRaw('DELETE FROM sources WHERE id = $1', [SOURCE]).catch(() => undefined);
+ await close();
+ }
+ }
+
+ test('the marker still due: the retry in the next resident runs without the first slice', async () => {
+ await brain(async engine => {
+ const w: World = { t: 1_000_000, starts: [], jobs: new Map() };
+ await upsertMarker(engine, { sourceId: SOURCE, intent: 'freshness', wantedHead: B, requestedBy: 'sync_nudge' });
+ const first = resident(w, sqlNotifierStore(engine));
+ await first.tick();
+ await stallLast(w, first);
+ await first.stop();
+ expect((await readNotifierSources(engine, [SOURCE]))[0]).toMatchObject({ state: 'retry', attempts: 1 });
+ const next = resident(w, sqlNotifierStore(engine), 'jobs-worker');
+ await next.tick();
+ expect(w.starts).toHaveLength(1);
+ w.t += RETRY_BACKOFF_MS[0] + 1000; await next.tick();
+ expect(slices(w)).toEqual([30, null]);
+ await next.stop();
+ });
+ }, 120_000);
+
+ test('the marker settled by the stopped run: the next resident still retries the source', async () => {
+ await brain(async engine => {
+ const w: World = { t: 1_000_000, starts: [], jobs: new Map() };
+ const marker = await upsertMarker(engine, { sourceId: SOURCE, intent: 'freshness', wantedHead: B, requestedBy: 'sync_nudge' });
+ const first = resident(w, sqlNotifierStore(engine));
+ await first.tick();
+ // The run's settle: the checkout's HEAD equals last_commit, so the marker reads as satisfied though the run stopped resumable.
+ await stallLast(w, first, async () => { expect(await settleMarkers(engine, SOURCE, [marker], { pulled: false, full: false, checkoutHead: A })).toEqual([]); });
+ await first.stop();
+ expect((await readNotifierSources(engine, [SOURCE]))[0]).toMatchObject({ state: 'retry', attempts: 1 });
+ const next = resident(w, sqlNotifierStore(engine), 'jobs-worker');
+ await next.tick();
+ expect(w.starts).toHaveLength(1);
+ w.t += RETRY_BACKOFF_MS[0] + 1000; await next.tick();
+ expect(slices(w)).toEqual([30, null]);
+ await next.stop();
+ });
+ }, 120_000);
+});
diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txt
index 363771695b..1ed883c2d2 100644
--- a/test/postgres-unit-arms.txt
+++ b/test/postgres-unit-arms.txt
@@ -321,6 +321,7 @@ test/migration-persistence-foreground-committed-index.test.ts
test/migration-persistence-request-id-index.test.ts
test/migration-persistence-request-slug-index.test.ts
test/notifier-passed-freshness-marker-7941.test.ts
+test/notifier-retry-unwatched-gst2515.test.ts
test/on-commit-embedding.serial.test.ts
test/oneshot-consumer-direct-route.test.ts
test/oneshot-housekeeping-cli.test.ts