Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
Stop old file-drift holds from wedging sync, and report commits waiting past the last sync (GST-2515)
garrytand646af6c4799
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex b09b68828b..3a4ae138fa 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -10,6 +10,17 @@ credits are retained; no result has been reassigned to another provider. Originaidentifiers and attribution are available in the pre-removal Git revision`6040075c6cb95be5881cc2e1b76ef7d71f4e5d29` (retained on 2026-09-23).+## [Unreleased]++**Sync no longer wedges on a source with thousands of old file-drift holds, and `sync status` stops calling a source synced while new commits wait**++Since v0.60.677.0, every unknown-side `file_drift` hold written under the earlier drift screen is screened once more, and that screen reads each held path's earlier committed versions. It read them with one Git process per path and reported nothing while it ran. On a source with thousands of such holds, the pass stayed silent past the sync's 300-second no-progress check and was stopped (`drain_stalled`) before it stamped any hold, so every later pass repeated the same reads and the source never synced again.++- The re-screen now reads every held path's history through one `git cat-file --batch-check` per 2,000 paths, with no file bodies, and reports progress as each path's history arrives. A Git read that hangs still reports nothing, so the no-progress check still stops it. On a generated repository with 80,000 files and 3,000 commits, 10,000 holds took 6.2 s instead of 141 s, and 80,000 holds took 66 s instead of 17 minutes. The longest silence was 36 ms instead of the whole read.+- A pass stopped partway keeps the holds it re-screened, and the next pass reads only the rest.+- `gbrain sync status` reads `resumable` / `commits_waiting` when nothing else is pending but the checkout's HEAD is past the last synced commit, with a `next` that runs the sync. It used to read `synced` / `nothing_pending` until the next sync happened to run.+- With `sync.on_commit` off, the notifier's fix now says so, instead of saying that no resident runs.+## [0.60.685.0] - 2026-11-12**`remember` now tells the truth about a fact saved for something that has no page yet, a source that lives in Git keeps that fact in Git, and the brain owner's own agents save facts shared by default.** Three changes to how agents save and withdraw facts land together.diff --git a/src/commands/sync/operator.ts b/src/commands/sync/operator.tsindex 14bf326257..eb862c9252 100644--- a/src/commands/sync/operator.ts+++ b/src/commands/sync/operator.ts@@ -305,6 +305,7 @@ export function syncStatusHeadline(status: Pick<SyncStatus, 'source_id' | 'statecase 'deletes_withheld': return `Source ${status.source_id} is blocked: sync withheld a mass delete; the rest of the source is synced.`;case 'selection_scoped': return `Source ${status.source_id} is synced at a narrower selection; the plain sync runs the configured one.`;case 'admission_paused': return `Source ${status.source_id} is resumable: sync.admission pauses what sync admits.`;+ case 'commits_waiting': return `Source ${status.source_id} is resumable: its checkout has commits past the last synced commit; the next sync imports them.`;case 'writeback_failed': return `Source ${status.source_id} is blocked: the Git effect of a file sync rewrote failed and is parked; the checkout lags the brain.`;}}diff --git a/src/core/notifier/status.ts b/src/core/notifier/status.tsindex 7cf4cb6a48..9fb26b5cac 100644--- a/src/core/notifier/status.ts+++ b/src/core/notifier/status.ts@@ -156,11 +156,14 @@ export async function readNotifierBlock(engine: BrainEngine, sourceId: string, o// #7243: the resident the row last recorded (the one its trace rows name), whose notifier lease is no longer live.const recorded = !live && row.owner_kind ? { kind: row.owner_kind, pid: remote || row.owner_pid == null ? null : Number(row.owner_pid), host_id: remote ? null : row.owner_host_id,seen_at: iso(row.owner_seen_at), lease_expired_at: iso(row.lease_expires_at) } : null;+ // GST-2515: with sync.on_commit off no resident watches any source's HEAD, even while one runs this host's notifier.+ const onCommit = live || holder || recorded ? true : parseOnCommit(await engine.getConfig(SYNC_ON_COMMIT_KEY).catch(() => null));const fix = !live && !holder? syncNowFix(sourceId, recorded? `The ${recorded.kind} resident recorded as this source's notifier${recorded.pid ? ` (PID ${recorded.pid})` : ''} holds no live notifier lease${recorded.lease_expired_at ? ` (it lapsed at ${recorded.lease_expired_at})` : ''}, `+ `so nothing watches this source now; this syncs the checkout HEAD now. If that resident is still running, it is not renewing its lease: restart it, and it catches up on its own.`- : 'No resident (gbrain serve, gbrain jobs work or the autopilot daemon) watches this source; this syncs the checkout HEAD now. A resident started later catches up on its own.')+ : onCommit ? 'No resident (gbrain serve, gbrain jobs work or the autopilot daemon) watches this source; this syncs the checkout HEAD now. A resident started later catches up on its own.'+ : 'No resident watches this source\'s HEAD: sync.on_commit is off, so a running resident (gbrain serve, gbrain jobs work or the autopilot daemon) syncs only explicit requests; this syncs the checkout HEAD now. `gbrain config set sync.on_commit true` turns on watching.'): !live ? (holderStalled(holder, lease?.db_now) ? syncNowFix(sourceId, `No notifier watches this source, and the ${holder!.kind} process holding its drain lease has made no commit or cursor save for ${HOLDER_STALL_MS / 60_000} min or more; `+ 'a foreground sync follows it and takes over when its lease lapses.') : null): stalled ? syncNowFix(sourceId, `The notifier made no progress after ${Number(row.attempts ?? 0)} attempts (${row.last_error_code ?? row.reason ?? 'blocked'}); a foreground sync shows the error`diff --git a/src/core/persistence/applied-watermark.ts b/src/core/persistence/applied-watermark.tsindex 3c99bb5285..d38bbb30a6 100644--- a/src/core/persistence/applied-watermark.ts+++ b/src/core/persistence/applied-watermark.ts@@ -99,15 +99,21 @@ export async function readWriteWatermark(engine: Pick<BrainEngine, 'executeRaw'>};}+/** The source's incarnation and its checkout on this host (the host binding's path, else the source's), or nulls. */+export async function readCheckoutRoot(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string): Promise<{ incarnation: string | null; root: string | null } | undefined> {+ const [source] = await engine.executeRaw<{ incarnation: string | null; root: string | null }>(`SELECT s.incarnation::text AS incarnation,+ COALESCE((SELECT h.local_path FROM persistence_host_bindings h JOIN persistence_source_bindings b ON b.worktree_id=h.worktree_id+ WHERE b.source_id=s.id AND h.host_id::text=$2 LIMIT 1), s.local_path) AS root FROM sources s WHERE s.id=$1`,+ [sourceId, (await import('./identity.ts')).existingHostIdFor(engine as BrainEngine)]);+ return source;+}+/** The full watermark of one source for `gbrain sources status --json` and MCP `sources_status`. */export async function readAppliedWatermark(engine: BrainEngine, sourceId: string,opts: { remote?: boolean; now?: number; behind?: BehindReader; filesBehind?: FilesBehindReader } = {}): Promise<AppliedWatermark> {const now = opts.now ?? Date.now();const write = await readWriteWatermark(engine, sourceId, now);- const [source] = await engine.executeRaw<{ incarnation: string | null; root: string | null }>(`SELECT s.incarnation::text AS incarnation,- COALESCE((SELECT h.local_path FROM persistence_host_bindings h JOIN persistence_source_bindings b ON b.worktree_id=h.worktree_id- WHERE b.source_id=s.id AND h.host_id::text=$2 LIMIT 1), s.local_path) AS root FROM sources s WHERE s.id=$1`,- [sourceId, (await import('./identity.ts')).existingHostIdFor(engine)]);+ const source = await readCheckoutRoot(engine, sourceId);const commit = write.indexed_through.commit;const root = !opts.remote && source?.root && commit ? source.root : null;const lag = root ? await (opts.behind ?? readGitBehind)(root, commit!).catch(() => null) : null;diff --git a/src/core/persistence/sync-git-stream.ts b/src/core/persistence/sync-git-stream.tsindex 8bac17e23d..5080219355 100644--- a/src/core/persistence/sync-git-stream.ts+++ b/src/core/persistence/sync-git-stream.ts@@ -33,6 +33,8 @@ export class SyncTreeReadIncomplete extends Error {}const READ_TIMEOUT_MS = 120_000;+/** GST-2515: held paths per history read, so one read of a path's specs stays well inside READ_TIMEOUT_MS on a deep history. */+const HISTORY_GROUP_PATHS = 2_000;const PROBE_TIMEOUT_MS = 60_000;const LFS_POINTER_MAX_BYTES = 1024;const OID = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/;@@ -241,32 +243,41 @@ export async function readRenameHints(gitRoot: string, from: string, to: string)}/**- * #7199: the blob ids `path` had at `commit` and its first `depth - 1` first-parent ancestors, newest first, one- * per commit that has the path (one `cat-file --batch` of `<commit>~k:<path>` specs; bodies are read past, never kept).- * Empty for a path a spec line cannot carry.+ * #7199, GST-2515: the blob ids each path had at `commit` and its first `depth - 1` first-parent ancestors, newest first, one+ * per commit that has the path. One `cat-file --batch-check` per group of paths answers every `<commit>~k:<path>` spec (no+ * bodies are read); `onPath` runs as each path's last answer arrives, and a throw from it (an abort) kills the read and rejects.+ * A group whose read is incomplete leaves its unanswered paths out of the map; a path a spec line cannot carry has no entry.*/-export async function pathBlobHistory(gitRoot: string, commit: string, path: string, depth: number): Promise<string[]> {- if (!OID.test(commit) || /[\n\r]/.test(path) || depth < 1) return [];- const oids: string[] = [];- let header: Buffer[] = [], skip = 0, answered = 0;- const result = await gitStream(gitRoot, ['cat-file', '--batch'], { timeoutMs: READ_TIMEOUT_MS,- stdin: lines(Array.from({ length: depth }, (_, k) => `${commit}~${k}:${path}`), '\n'), onChunk: chunk => {- let at = 0;- while (at < chunk.length) {- if (skip) { const take = Math.min(skip, chunk.length - at); skip -= take; at += take; continue; }- const newline = chunk.indexOf(10, at);- if (newline < 0) { header.push(chunk.subarray(at)); break; }- const line = Buffer.concat([...header, chunk.subarray(at, newline)]).toString('utf8');- header = []; at = newline + 1; answered += 1;- if (line.endsWith(' missing') || line.endsWith(' ambiguous')) continue;- const [oid = '', type = '', size = ''] = line.split(' ');- if (!OID.test(oid) || !/^\d+$/.test(size)) throw malformed('cat-file', line);- if (type === 'blob') oids.push(oid);- skip = Number(size) + 1;- }- } });- if (!result.ok || skip || header.length || answered !== depth) throw new SyncTreeReadIncomplete('cat-file', result.ok ? `${answered} of ${depth} specs answered` : outcome(result));- return oids;+export async function pathBlobHistories(gitRoot: string, commit: string, paths: readonly string[], depth: number,+ onPath?: (path: string) => void): Promise<Map<string, string[]>> {+ const histories = new Map<string, string[]>();+ if (!OID.test(commit) || depth < 1) return histories;+ const carried = paths.filter(path => !/[\n\r]/.test(path));+ for (let start = 0; start < carried.length; start += HISTORY_GROUP_PATHS) {+ const group = carried.slice(start, start + HISTORY_GROUP_PATHS);+ let header: Buffer[] = [], answered = 0, oids: string[] = [];+ await gitStream(gitRoot, ['cat-file', '--batch-check'], { timeoutMs: READ_TIMEOUT_MS,+ stdin: lines(group.flatMap(path => Array.from({ length: depth }, (_, k) => `${commit}~${k}:${path}`)), '\n'), onChunk: chunk => {+ let at = 0;+ while (at < chunk.length) {+ const newline = chunk.indexOf(10, at);+ if (newline < 0) { header.push(chunk.subarray(at)); break; }+ const line = Buffer.concat([...header, chunk.subarray(at, newline)]).toString('utf8');+ header = []; at = newline + 1;+ if (!line.endsWith(' missing') && !line.endsWith(' ambiguous')) {+ const [oid = '', type = '', size = ''] = line.split(' ');+ if (!OID.test(oid) || !/^\d+$/.test(size)) throw malformed('cat-file', line);+ if (type === 'blob') oids.push(oid);+ }+ if (++answered % depth) continue;+ const path = group[answered / depth - 1]!;+ histories.set(path, oids);+ oids = [];+ onPath?.(path);+ }+ } }).catch((error: unknown) => { if (!(error instanceof SyncTreeReadIncomplete)) throw error; });+ }+ return histories;}/**diff --git a/src/core/persistence/sync-pass-run.ts b/src/core/persistence/sync-pass-run.tsindex 8111ed7a79..db1d6a4c17 100644--- a/src/core/persistence/sync-pass-run.ts+++ b/src/core/persistence/sync-pass-run.ts@@ -35,7 +35,7 @@ import { passFoldBase, passIdleProbe, readShadowLevel, workingIdleChange, type Pimport { ownWritebackCommit, type PassFold } from './sync-pass-fold.ts';import { openPassRequests, settleCarried } from './sync-pass-carried.ts';import { loopYield } from './sync-yield.ts';-import { pathBlobHistory, recallWorkingStatCache, rememberWorkingStatCache, SyncTreeReadIncomplete } from './sync-git-stream.ts';+import { pathBlobHistories, recallWorkingStatCache, rememberWorkingStatCache, SyncTreeReadIncomplete } from './sync-git-stream.ts';import { syncPassKind } from './sync-classify.ts';import { PASS_BATCH_KEY_PREFIX, readConvergedWatermark, SyncAdmissionPaused, type PassEntry } from './sync-pass-batch.ts';import { assertDrainLease, currentDrainLease } from './sync-lease.ts';@@ -151,7 +151,7 @@ function entryPaths(level: ShadowLevel, gitPath: string): { path: string; source* stays; one new origin whose bytes equal a page leaving at another origin is that page's Git rename (`renameFrom`, the* page moves to the new slug). Deletes first, then imports.*/-async function batchEntries(engine: BrainEngine, level: ShadowLevel, deletes: ShadowPath[], imports: ShadowPath[]):+async function batchEntries(engine: BrainEngine, level: ShadowLevel, deletes: ShadowPath[], imports: ShadowPath[], onHistoryRead?: (path: string) => void):Promise<{ entries: PassEntry[]; collisions: SyncSlugCollision[]; skipped: ShadowPath[]; downgrades: DowngradedRename[];identity: Array<{ entry: ShadowPath; origin: ShadowPath | null; slug: string; ids: { page: string; file: string } }> }> {await faultPoint('sync:batch_entries', { sourceId: level.sourceId });@@ -206,11 +206,11 @@ async function batchEntries(engine: BrainEngine, level: ShadowLevel, deletes: Sh: entry.result.side === 'unknown' ? entry.state.baseline ?? undefined : undefined;// #7199: a side-`unknown` hold an older gbrain wrote after the pass that held it moved the source's last commit past the one// the classic sync read; its re-screen also compares the page with the file's last few committed versions before the held one.- const history = new Map<string, string[]>();- for (const entry of imports) if (entry.result.side === 'unknown' && entry.pageId !== null && entry.state.hold?.unscreened) {- const earlier = await pathBlobHistory(level.gitRoot, level.baselineCommit ?? level.head, entry.path, DRIFT_HISTORY_DEPTH).catch(() => []);- history.set(entry.path, [...new Set(earlier)].filter(oid => oid !== entry.state.level && oid !== entry.state.baseline).slice(0, DRIFT_HISTORY_VERSIONS));- }+ // GST-2515: one batched read for all of them, reporting each path it finished, so thousands of such holds neither start a Git+ // process each nor read as a stalled pass.+ const unscreened = imports.filter(entry => entry.result.side === 'unknown' && entry.pageId !== null && entry.state.hold?.unscreened);+ const read = unscreened.length ? await pathBlobHistories(level.gitRoot, level.baselineCommit ?? level.head, unscreened.map(entry => entry.path), DRIFT_HISTORY_DEPTH, onHistoryRead) : null;+ const history = new Map(unscreened.map(entry => [entry.path, [...new Set(read?.get(entry.path) ?? [])].filter(oid => oid !== entry.state.level && oid !== entry.state.baseline).slice(0, DRIFT_HISTORY_VERSIONS)]));const entries: PassEntry[] = [], collisions: SyncSlugCollision[] = [], skipped: ShadowPath[] = [], identity: Array<{ entry: ShadowPath; origin: ShadowPath | null; slug: string; ids: { page: string; file: string } }> = [];const claims = new Map<string, ShadowPath[]>();for (const entry of imports) {@@ -959,7 +959,7 @@ export async function runSyncPass(engine: BrainEngine, input: SyncPassInput): Prreturn synced(false);}opts.onProgress?.({ phase: 'managed_sync.round_admit', round: { round, target: head, paths: level.paths.length, imports: imports.length, deletes: deletes.length, in_flight: inFlight.length } });- const built = await batchEntries(engine, level, deletes, imports).catch(async (error: unknown) => {+ const built = await batchEntries(engine, level, deletes, imports, () => { assertActive(); beat.report(); }).catch(async (error: unknown) => {if ((error as { code?: string } | null | undefined)?.code === 'page_identity_changed') await save('blocked', 'page_identity_changed');throw error;});diff --git a/src/core/persistence/sync-status.ts b/src/core/persistence/sync-status.tsindex 5c14652d89..c27c97342f 100644--- a/src/core/persistence/sync-status.ts+++ b/src/core/persistence/sync-status.ts@@ -38,6 +38,8 @@ import { laneFaultInMessage } from './sync-window.ts';import { cursorFenceCause, syncKnockOnSql } from './sync-knock-on.ts';import { listIncompleteSyncReceipts, type SyncSettledIndexes } from './checkpoint-validation.ts';import { drainEstimate, engineStallProbe } from './sync-drain.ts';+import { readCheckoutRoot } from './applied-watermark.ts';+import { readGitBehind } from './sync-freshness.ts';import { classifyRecordedFault, classifyTerminal, type TerminalCategory } from './terminal-classify.ts';import type { PublicCategory } from './public-category.ts';import { readSyncFile, resolveManagedSyncContext } from './sync-discovery.ts';@@ -246,7 +248,9 @@ export const SYNC_STATUS_REASONS = ['nothing_pending', 'held_by_design', 'held_n// #7355: passes keep stopping before they end.'pass_abandoned',// #7596: the newest pass's process stopped (killed or crashed) without ending it.- 'pass_interrupted'] as const;+ 'pass_interrupted',+ // GST-2515: nothing else is pending, but the checkout's HEAD is past the last synced commit.+ 'commits_waiting'] as const;export type SyncStatusReason = typeof SYNC_STATUS_REASONS[number];export type SyncStatusRead = { status: 'read' } | { status: 'not_read'; fix: Action };/** The bound of each default probe (the oldest blocker, the recovery records). */@@ -479,7 +483,12 @@ export async function readSyncStatus(engine: BrainEngine, sourceId: string, optsconst connectorLastError = await (await import('./connector-status.ts')).readConnectorLastError(engine, sourceId);const gitPushHolds = committed?.git_push_held ? await readGitPushHolds(engine, [sourceId]) : [];const fenceHolds = committed?.fence_held ? await (await import('../generations/fence.ts')).fenceHoldEntries(engine, [sourceId]) : [];- const [state, stateReason] = statusState({ human: !!(liveError?.needs_human), heldForHuman: holds.some(hold => hold.needs_human), oldest, recovery, liveJob: liveJob !== null, unfinished, lastError: !!liveError, actionable: actionable.length > 0, holds: holds.length > 0, pass: passState });+ const settled = statusState({ human: !!(liveError?.needs_human), heldForHuman: holds.some(hold => hold.needs_human), oldest, recovery, liveJob: liveJob !== null, unfinished, lastError: !!liveError, actionable: actionable.length > 0, holds: holds.length > 0, pass: passState });+ // GST-2515: a source with nothing else pending is not synced while its checkout has commits past the last synced commit.+ // A scoped convergence keeps last_commit by design, and a writeback still integrating is the source's own commit.+ const waiting = settled[0] === 'synced' && settled[1] !== 'selection_scoped' && !gitPushHolds.length && source.last_commit+ ? await commitsPastLastSync(engine, sourceId, source.last_commit) : null;+ const [state, stateReason]: [SyncStatusState, SyncStatusReason] = waiting ? ['resumable', 'commits_waiting'] : settled;const admission = await syncAdmissionStatus(engine, sourceId, source.incarnation,{ stored: source.admission_stored, row_value: source.admission_row_value, set_at: source.admission_set_at, pinned: source.admission_pinned, destructive: source.admission_destructive });const passNext = passRow && passState && passState[1] === stateReason ? passStatusNext(sourceId, passState[1], passRow, resume, source.writeback_failed, interrupted) : null;@@ -491,6 +500,8 @@ export async function readSyncStatus(engine: BrainEngine, sourceId: string, optsresume_argv: resume, ...blockers, cursors: detail ? await liveCursors(engine, sourceId, source.incarnation, selected?.key ?? null, failures) : [],terminal_failures: terminal, connector_last_error: connectorLastError, git_push_holds: gitPushHolds, fence_holds: fenceHolds, worktree_refresh: worktreeRefresh, next: next ?? recovery_next ?? passNext ?? (stateReason === 'request_pending' && oldest.status === 'ok' && oldest.request ? { argv: ['gbrain', 'sync', ...resume], consent: [], actor: 'agent', requires_exclusive: false, verify,why: `Write request ${oldest.request.request_id} of this source is still ${oldest.request.state}${oldest.request.step ? ` (step ${oldest.request.step})` : ''}; a running owner finishes it on its own, and the next sync of this source drains it.` } satisfies Action+ : stateReason === 'commits_waiting' ? { argv: ['gbrain', 'sync', ...resume], consent: [], actor: 'agent', requires_exclusive: false, verify,+ why: `The checkout's HEAD is ${waiting} commit(s) past the last synced commit ${source.last_commit?.slice(0, 12)}; this sync imports them.` } satisfies Action: stateReason === 'status_unknown' ? { argv: ['gbrain', 'sync', 'status', '--source', sourceId, '--json'], consent: [], actor: 'agent', requires_exclusive: false, verify,why: `The oldest-blocker or recovery probe did not answer within ${SYNC_STATUS_PROBE_MS / 1000} s, so this status does not know the state (it is not reported synced). Rerun it after a short wait; gbrain doctor --json checks the database if it keeps timing out.` } satisfies Action: gitPushHolds.find(hold => hold.conflict || hold.parked)?.fix ?? fenceHolds[0]?.fix ?? (worktreeRefresh?.liveness === 'unknown' ? worktreeRefresh.fix : null)),@@ -530,6 +541,13 @@ function statusState(s: { human: boolean; heldForHuman: boolean; oldest: SyncStareturn ['synced', s.heldForHuman ? 'held_needs_human' : s.holds ? 'held_by_design' : 'nothing_pending'];}+/** GST-2515: how many commits the source's checkout on this host has past `lastCommit`; null when none, or when Git cannot say. */+async function commitsPastLastSync(engine: BrainEngine, sourceId: string, lastCommit: string): Promise<number | null> {+ const root = (await readCheckoutRoot(engine, sourceId).catch(() => undefined))?.root;+ const lag = root ? await readGitBehind(root, lastCommit).catch(() => null) : null;+ return lag?.commits ? lag.commits : null;+}+/** #6712 PR 2: the source's oldest unfinished request, read through the drain's own bounded head probe. */async function oldestBlocker(engine: BrainEngine, sourceId: string): Promise<SyncStatus['oldest_blocker']> {const head = engineStallProbe(engine, sourceId).head!;diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txtindex d14e51f902..363771695b 100644--- a/test/postgres-unit-arms.txt+++ b/test/postgres-unit-arms.txt@@ -593,6 +593,7 @@ test/sync-drain-effect-recovery-blocked.test.tstest/sync-drain-head-blocked-6543.test.tstest/sync-drain-inpass-stall.test.tstest/sync-drain-write-capacity.test.ts+test/sync-drift-rescreen-progress-gst2515.test.tstest/sync-dry-run-drain-7167.test.tstest/sync-exits-l4-6712.test.tstest/sync-fenced-slices-gbra133.test.ts@@ -638,6 +639,7 @@ test/sync-prescreen-drift-6712.test.tstest/sync-restored-file-purge.test.tstest/sync-slug-root-mode-6694.test.tstest/sync-staged-catchup.test.ts+test/sync-status-commits-waiting-gst2515.test.tstest/sync-status-failure-detail-6513.test.tstest/sync-status-truthful-7243.test.tstest/sync-storage-invalid-8007.test.tsdiff --git a/test/sync-drift-rescreen-progress-gst2515.test.ts b/test/sync-drift-rescreen-progress-gst2515.test.tsnew file mode 100644index 0000000000..a0960e7791--- /dev/null+++ b/test/sync-drift-rescreen-progress-gst2515.test.ts@@ -0,0 +1,106 @@+/**+ * GST-2515: since the file-drift screen version moved to 2 (#7199), every unknown-side file_drift hold written under version 1+ * is screened once more, and that screen reads each held path's earlier committed versions. It read them one Git process per+ * path and reported nothing while it ran, so a source with thousands of such holds sat silent past the drain's no-progress+ * window: the pass was stopped (drain_stalled) before any hold was stamped, and every later pass repeated the same reads.+ * The reads now run batched and report each path they finish; a pass stopped partway keeps the holds it re-screened.+ */+import { expect, test } from 'bun:test';+import { execFileSync } from 'node:child_process';+import { mkdirSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import type { BrainEngine } from '../src/core/engine.ts';+import { hardenedGitStream } from '../src/core/hardened-git.ts';+import { RECOVERY_VERSION } from '../src/core/markdown.ts';+import { withCoordinatedWrite } from '../src/core/persistence/context.ts';+import { engineStallProbe, runDrain } from '../src/core/persistence/sync-drain.ts';+import { __setSyncGitStreamForTests } from '../src/core/persistence/sync-git-stream.ts';+import { FILE_DRIFT_VERSION, writeGitHold } from '../src/core/persistence/sync-holds.ts';+import { performManagedSync } from '../src/core/persistence/sync-run.ts';+import { managedBrain } from './helpers/managed-brain.ts';+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';+import { TEST_WRITE_ATTRIBUTION } from './helpers/write-attribution.ts';++const git = (root: string, ...args: string[]) => execFileSync('git', ['-C', root, '-c', 'user.name=Example', '-c', '[email protected]',+ '-c', 'commit.gpgsign=false', ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();+const note = (slug: string, body: string) => '---\ntype: concept\ntitle: ' + slug + '\n---\n\n' + body + '\n';+const OPTS = { sourceId: 'default', noPull: true, noEmbed: true, noExtract: true };+const HOLDS = 60;+const slugs = Array.from({ length: HOLDS }, (_, i) => 'notes/n' + String(i).padStart(3, '0'));++const versions = async (engine: BrainEngine) => (await engine.executeRaw<{ v: number | null }>(+ "SELECT (completed_keys->0->'meta'->>'drift_version')::int AS v FROM op_checkpoints WHERE op='sync-hold' AND completed_keys->0->>'code'='file_drift'"))+ .reduce((tally, { v }) => ({ ...tally, ['v' + (v ?? 0)]: (tally['v' + (v ?? 0)] ?? 0) + 1 }), {} as Record<string, number>);++/** Every page drifted on both sides with no import record, each held by the version-1 screen: the holds an older gbrain left. */+async function versionOneHolds(engine: BrainEngine, root: string): Promise<void> {+ await performManagedSync(engine, OPTS);+ const coordinated = (sql: string) => engine.transaction(tx => withCoordinatedWrite(tx, ['default'], () => tx.executeRaw(sql), TEST_WRITE_ATTRIBUTION));+ await coordinated("DELETE FROM op_checkpoints WHERE op='sync-import-provenance'");+ await coordinated("UPDATE pages SET compiled_truth = compiled_truth || ' Edited in the database.', knowledge_revision = gen_random_uuid(), revision_write_request_id = NULL WHERE source_id='default'");+ for (const slug of slugs) writeFileSync(join(root, slug + '.md'), note(slug, 'A Git edit of ' + slug + '.'));+ git(root, 'commit', '-qam', 'edit');+ const [{ incarnation }] = await engine.executeRaw<{ incarnation: string }>("SELECT incarnation::text AS incarnation FROM sources WHERE id='default'");+ for (const slug of slugs) {+ const path = slug + '.md', page = (await engine.readPageSnapshot(slug, { sourceId: 'default' }))!;+ await engine.transaction(tx => writeGitHold(tx, { source_id: 'default', incarnation, path, source_path: path, slug, page_id: Number(page.page.id), code: 'file_drift',+ message: path + ': page ' + slug + ' differs from its file and has no import record', upstream_version: null, observed_at: new Date().toISOString(), run_id: null,+ mode: 'managed', meta: { recovery_version: RECOVERY_VERSION, sides: 'unknown', side: 'unknown', drift_version: 1, blob_oid: git(root, 'rev-parse', 'HEAD:' + path), page_revision: page.revision } }));+ }+ expect(await versions(engine)).toEqual({ v1: HOLDS });+}++/** Each history read (a cat-file of <commit>~k:<path> specs) takes delayMs longer, as one does on a deep history; counts the paths read. */+function slowHistoryReads(delayMs: number): { paths: () => number } {+ let paths = 0;+ __setSyncGitStreamForTests(async (root, args, opts) => {+ if (args[0] !== 'cat-file' || opts.stdin === undefined || typeof opts.stdin === 'string' || Buffer.isBuffer(opts.stdin) || Symbol.asyncIterator in opts.stdin) return hardenedGitStream(root, args, opts);+ const stdin = [...opts.stdin as Iterable<Buffer | string>].map(line => line.toString());+ const specs = stdin.join('').split('\n').filter(Boolean);+ if (!specs.some(spec => /~\d+:/.test(spec))) return hardenedGitStream(root, args, { ...opts, stdin });+ paths += specs.filter(spec => /~0:/.test(spec)).length;+ await new Promise(resolve => setTimeout(resolve, delayMs));+ return hardenedGitStream(root, args, { ...opts, stdin });+ });+ return { paths: () => paths };+}++for (const backend of testBackends()) {+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;+ const setup = ({ root }: { root: string }) => {+ mkdirSync(join(root, 'notes'), { recursive: true }); git(root, 'init', '-q');+ for (const slug of slugs) writeFileSync(join(root, slug + '.md'), note(slug, 'The committed body of ' + slug + '.'));+ git(root, 'add', '-A'); git(root, 'commit', '-qm', 'notes');+ };++ test('[' + backend + '] the one-time re-screen of many version-1 drift holds reports progress and finishes under the no-progress window', () => managedBrain(async ({ engine, root }) => {+ await versionOneHolds(engine, root);+ const reads = slowHistoryReads(150);+ try {+ const result = await runDrain({ governMs: 25, noProgressMs: 3_000, publicationCeilingMs: 60_000, probe: engineStallProbe(engine, 'default'),+ pass: (signal, onProgress) => performManagedSync(engine, { ...OPTS, signal, onProgress }) });+ expect(result.drain?.stop_reason ?? null).toBeNull();+ expect(result.drain?.outcome).toBe('synced');+ expect(reads.paths()).toBe(HOLDS);+ } finally { __setSyncGitStreamForTests(); }+ expect(await versions(engine)).toEqual({ ['v' + FILE_DRIFT_VERSION]: HOLDS });+ }, { databaseUrl, setup }), 240_000);++ test('[' + backend + '] a re-screen stopped partway keeps the holds it stamped, and the next pass reads only the rest', () => managedBrain(async ({ engine, root }) => {+ await versionOneHolds(engine, root);+ const stop = new AbortController();+ let committed = 0;+ await performManagedSync(engine, { ...OPTS, signal: stop.signal, onProgress: (event: { phase?: string }) => {+ if (event.phase === 'managed_sync.page_committed' && ++committed === 20) stop.abort();+ } }).catch(() => undefined);+ const after = await versions(engine);+ const stamped = after['v' + FILE_DRIFT_VERSION] ?? 0;+ expect(stamped).toBeGreaterThanOrEqual(20);+ expect(stamped).toBeLessThan(HOLDS);+ expect(after.v1).toBe(HOLDS - stamped);+ const reads = slowHistoryReads(0);+ try { await performManagedSync(engine, OPTS); } finally { __setSyncGitStreamForTests(); }+ expect(reads.paths()).toBe(HOLDS - stamped);+ expect(await versions(engine)).toEqual({ ['v' + FILE_DRIFT_VERSION]: HOLDS });+ }, { databaseUrl, setup }), 240_000);+}diff --git a/test/sync-status-coherence-7189.test.ts b/test/sync-status-coherence-7189.test.tsindex 57f8a55994..0e794e811e 100644--- a/test/sync-status-coherence-7189.test.ts+++ b/test/sync-status-coherence-7189.test.ts@@ -79,6 +79,8 @@ test('the notifier names the process holding the drain lease instead of saying nexpect(idle!.resident).toBeNull();expect(idle!.drain_holder).toBeNull();expect(idle!.fix!.why).toContain('No resident');+ // GST-2515: with sync.on_commit off, the fix says why nothing watches the source's HEAD rather than implying no resident runs.+ expect(idle!.fix!.why).toContain('sync.on_commit is off');const seen: Array<Awaited<ReturnType<typeof readNotifierBlock>>> = [];await withDrainLease(engine, id, { holder: { kind: 'serve', id: 'serve-example' } }, async () => {seen.push(await readNotifierBlock(engine, id), await readNotifierBlock(engine, id, { remote: true }));diff --git a/test/sync-status-commits-waiting-gst2515.test.ts b/test/sync-status-commits-waiting-gst2515.test.tsnew file mode 100644index 0000000000..df3be3a1fe--- /dev/null+++ b/test/sync-status-commits-waiting-gst2515.test.ts@@ -0,0 +1,39 @@+/**+ * GST-2515: a source whose checkout has commits past its last synced commit read "synced, nothing pending" whenever no+ * write, pass or hold was open, because the state never compared HEAD with the last synced commit. It now reads+ * resumable / commits_waiting with a next that runs the sync, and synced again once that sync imports them.+ */+import { expect, test } from 'bun:test';+import { execFileSync } from 'node:child_process';+import { mkdirSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { performManagedSync } from '../src/core/persistence/sync-run.ts';+import { readSyncStatus } from '../src/core/persistence/sync-status.ts';+import { managedBrain } from './helpers/managed-brain.ts';+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';++const git = (root: string, ...args: string[]) => execFileSync('git', ['-C', root, '-c', 'user.name=Example', '-c', '[email protected]',+ '-c', 'commit.gpgsign=false', ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();+const OPTS = { sourceId: 'default', noPull: true, noEmbed: true, noExtract: true };++for (const backend of testBackends()) {+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;+ const setup = ({ root }: { root: string }) => {+ mkdirSync(join(root, 'notes'), { recursive: true }); git(root, 'init', '-q');+ writeFileSync(join(root, 'notes/one.md'), '---\ntitle: One\n---\n\nThe first note.\n');+ git(root, 'add', '-A'); git(root, 'commit', '-qm', 'one');+ };+ test('[' + backend + '] commits past the last synced commit read commits_waiting until a sync imports them', () => managedBrain(async ({ engine, root }) => {+ await performManagedSync(engine, OPTS);+ expect(await readSyncStatus(engine, 'default')).toMatchObject({ state: 'synced', reason: 'nothing_pending' });+ writeFileSync(join(root, 'notes/two.md'), '---\ntitle: Two\n---\n\nThe second note.\n');+ git(root, 'add', '-A'); git(root, 'commit', '-qm', 'two');+ const waiting = await readSyncStatus(engine, 'default');+ expect(waiting).toMatchObject({ state: 'resumable', reason: 'commits_waiting' });+ expect(waiting.next?.argv.slice(0, 2)).toEqual(['gbrain', 'sync']);+ expect(waiting.next?.argv).toContain('default');+ expect(waiting.next?.why).toContain('1 commit(s) past the last synced commit');+ await performManagedSync(engine, OPTS);+ expect(await readSyncStatus(engine, 'default')).toMatchObject({ state: 'synced', reason: 'nothing_pending', last_commit: git(root, 'rev-parse', 'HEAD') });+ }, { databaseUrl, setup }), 120_000);+}