Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/** `performSync`: the library entrypoint (dispatch, filesystem lock, writer lock). */
import { serr } from '../../core/console-prefix.ts';
import {
assertSyncDispatchActive,
resolveSyncPersistenceMode,
} from '../../core/persistence/sync-authority.ts';
import {
hasSourceFilesystemLock,
withSourceFilesystemLock,
currentSourceFilesystemSignal,
assertSourceFilesystemActive,
} from '../../core/minions/source-filesystem.ts';
import { currentJobSignal, currentSubmissionAuthority } from '../../core/minions/submission-authority.ts';
import { currentCompanyBrainSync, getCompanyBrainProfile } from '../../core/company-brain/profile.ts';
import { endSyncDispatchRead, syncDispatchRow } from '../../core/sync-dispatch-read.ts';
import type { BrainEngine } from '../../core/engine.ts';
import type { OperationContext } from '../../core/ops/contract.ts';
import { refreshProjectionStatistics } from '../../core/search/projection-statistics.ts';
import { withRefreshingLock, LockUnavailableError, LockStolenError, syncLockId } from '../../core/db-lock.ts';
import { readSyncAnchor } from '../../core/sync-anchor.ts';
import { recordUpstreamObservation } from '../../core/sync-upstream.ts';
import { assertSyncApplicable } from '../../core/sync-applicability.ts';
import { performPlainDirectorySync, plainDirectorySyncRoot } from './plain-directory.ts';
import { noteIdentityDowngrades } from '../../core/sync-identity-downgrades.ts';
import { SyncLockBusyError, formatLockBusyMessage, buildPartialResult } from '../../core/sync-lock.ts';
import type { SyncOpts, SyncResult } from '../sync.ts';
import type { IdleSkip } from '../../core/persistence/sync-idle-skips.ts';
import { runConnectorSync } from './connector.ts';
import { performSyncInner } from './incremental.ts';
import { isConnectorSourceKind } from '../../core/persistence/connector-kinds.ts';
import { ignoredPrivacyLabelRunNotice } from '../../core/search/ignored-privacy-labels.ts';
export async function performSync(engine: BrainEngine, opts: SyncOpts): Promise<SyncResult> {
const since = opts.dryRun ? null : await syncStartedAt(engine);
const result = await routeSync(engine, opts);
if (!since || result.privacy_label_notice || result.status === 'up_to_date' || result.status === 'dry_run') return result;
const notice = await ignoredPrivacyLabelRunNotice(engine, opts.sourceId ?? 'default', result.pagesAffected?.length ? { slugs: result.pagesAffected } : { since }, 'sync');
return notice ? { ...result, privacy_label_notice: notice } : result;
}
/** GST-2462: the database clock when a sync starts; a managed sync lists no pages, so its notice reads the pages updated since. */
async function syncStartedAt(engine: BrainEngine): Promise<string | null> {
try { return (await engine.executeRaw<{ at: string }>('SELECT clock_timestamp()::text AS at'))[0]?.at ?? null; } catch { return null; }
}
async function routeSync(engine: BrainEngine, opts: SyncOpts): Promise<SyncResult> {
assertSyncDispatchActive();
const inheritedSignal = currentSourceFilesystemSignal();
if (inheritedSignal) opts = { ...opts, signal: opts.signal ? AbortSignal.any([opts.signal, inheritedSignal]) : inheritedSignal };
await assertSyncApplicable(engine, opts);
const managed = await resolveSyncPersistenceMode(engine, opts);
const finish = async (result: SyncResult, refresh = false): Promise<SyncResult> => {
assertSyncDispatchActive();
if (refresh && (result.pagesAffected.length > 0 || result.deleted > 0)) {
await refreshProjectionStatistics(engine, result.pagesAffected.length + result.deleted);
}
if (refresh && !opts.dryRun) await recordUpstreamObservation(engine, opts.sourceId ?? 'default', opts.repoPath);
return refresh && !opts.dryRun ? noteIdentityDowngrades(engine, opts.sourceId ?? 'default', result, serr) : result;
};
const interruptedBeforeWork = async (): Promise<SyncResult> => {
assertSourceFilesystemActive(true);
const lastCommit = opts.full ? null : await readSyncAnchor(engine, opts.sourceId, 'last_commit');
return buildPartialResult({
fromCommit: lastCommit, toCommit: lastCommit ?? '', filesImported: 0,
pagesAffected: [], chunksCreated: 0, added: 0, modified: 0, deleted: 0, renamed: 0,
reason: 'timeout',
});
};
const company = !opts.signal?.aborted && opts.sourceId && !currentCompanyBrainSync(opts.sourceId) && await getCompanyBrainProfile(engine, opts.sourceId);
assertSyncDispatchActive();
const mayBeConnector = managed && await mayBeConnectorSource(engine, opts);
// GBRA-133: the route is chosen; the work reads live (core/sync-dispatch-read.ts).
endSyncDispatchRead();
if (company) {
if (opts.signal?.aborted) return finish(await interruptedBeforeWork());
return (await import('../../core/company-brain/runtime.ts')).performCompanyBrainSync(engine, opts);
}
if (mayBeConnector) {
const connector = await runConnectorSync(engine, opts, true);
if (connector) return connector;
}
if (opts.signal?.aborted) return finish(await interruptedBeforeWork());
const plainRoot = await plainDirectorySyncRoot(engine, opts, managed);
if (plainRoot) return finish(await performPlainDirectorySync(engine, opts, plainRoot, managed), true);
if (managed) return performManagedBranch(engine, opts);
const filesystemRoot = opts.repoPath || await readSyncAnchor(engine, opts.sourceId, 'repo_path');
if (filesystemRoot && !hasSourceFilesystemLock(filesystemRoot)) {
let entered = false;
let result: SyncResult | undefined;
try {
return finish(await withSourceFilesystemLock(engine, filesystemRoot, async () => {
entered = true;
return result = await performSync(engine, opts);
}, { signal: opts.signal }));
} catch (err) {
if (err instanceof LockStolenError) throw err;
const isCallerAbort = err === opts.signal?.reason || (err instanceof Error && err.name === 'AbortError');
if (opts.signal?.aborted && isCallerAbort && !currentJobSignal()?.aborted) {
if (result?.status === 'partial') return finish(result);
if (!entered) return finish(await interruptedBeforeWork());
}
if (err instanceof LockUnavailableError) throw new SyncLockBusyError(await formatLockBusyMessage(engine, err.lockId), err.lockId);
throw err;
}
}
// Per-source leases protect the commit/bookmark window. A caller may
// skip this lease only when its broader scope already serializes the work.
if (opts.skipLock) {
return finish(await performSyncInner(engine, opts), true);
}
const lockKey = opts.lockId ?? syncLockId(opts.sourceId ?? 'default');
// Renewal loss aborts the import loop and prevents a successful bookmark
// result, including callers that did not supply their own cancellation.
try {
return finish(await withRefreshingLock(engine, lockKey, signal => performSyncInner(engine, {
...opts, signal: opts.signal ? AbortSignal.any([opts.signal, signal]) : signal,
})), true);
} catch (err) {
if (err instanceof LockUnavailableError) {
throw new SyncLockBusyError(await formatLockBusyMessage(engine, lockKey), lockKey);
}
throw err;
}
}
/** Whether `runConnectorSync` may sync this source: false only when the routing row shows a source it returns null for. */
async function mayBeConnectorSource(engine: BrainEngine, opts: SyncOpts): Promise<boolean> {
const routing = opts.sourceId && !opts.githubItem && !opts.resetCheckpoint ? await syncDispatchRow(engine, opts.sourceId) : undefined;
if (!routing) return true;
if (routing.id === null) return false;
try {
const config = typeof routing.config === 'string' ? JSON.parse(routing.config) : routing.config ?? {};
return isConnectorSourceKind(config.kind);
} catch { return true; }
}
/**
* The managed branch: the drain (or one pass), its staged stages and the post-sync settles, all under the source's drain
* lease (#6712 PR 1, persistence/sync-lease.ts). A plain CLI sync follows a live holder within its budget; every other
* caller gets `SyncLockBusyError` on a busy lease, as the classic per-source lock does.
*/
async function performManagedBranch(engine: BrainEngine, outer: SyncOpts): Promise<SyncResult> {
const sourceId = outer.sourceId ?? 'default';
const { currentDrainLease, withDrainLease } = await import('../../core/persistence/sync-lease.ts');
const lockKey = syncLockId(sourceId);
// GBRA-133: the settles an idle-clear tick leaves nothing to do (persistence/sync-idle-skips.ts), read inside the lease.
let skips: ReadonlySet<IdleSkip> = new Set();
const work = async (leaseSignal: AbortSignal | undefined): Promise<SyncResult> => {
const opts: SyncOpts = leaseSignal ? { ...outer, signal: outer.signal ? AbortSignal.any([outer.signal, leaseSignal]) : leaseSignal } : outer;
// #7167: a dry run reads and previews once; the drain's stall governor (no commit in 5 min) would abandon a long read
// of a large source as a writer stall, and nothing it does is a write the drain needs to watch.
let result = opts.drain && !opts.dryRun
? await (await import('../../core/persistence/sync-drain.ts')).drainManagedSync(engine, opts, true)
: await (await import('../../core/persistence/sync-run.ts')).performManagedSync(engine, opts);
// #6349: a delta past the cursor bound syncs in stages; a drain takes the next stage from where the last one landed
// until it reaches HEAD (each stage freezes its own bounded manifest, and a stop resumes at the stored cursor).
const { deadlinePassed } = await import('../../core/persistence/sync-drain.ts');
while (opts.drain && result.staged && result.drain?.outcome === 'synced' && !opts.dryRun && !opts.signal?.aborted) {
// #6706: a caller deadline ends the catch-up between stages; the stage that landed is kept and the next run takes the next one.
if (deadlinePassed(opts.deadlineAtMs)) { result = { ...result, drain: { ...result.drain, outcome: 'resumable', stop_reason: 'deadline' } }; break; }
serr(result.staged.slice
? `[sync] sliced catch-up: committed the ${result.staged.slice.phase} slice of ${result.staged.target.slice(0, 12)} through ${result.staged.slice.after ?? '(start)'} (the target exceeds the cursor bound); taking the next slice.`
: `[sync] staged catch-up: synced to ${result.staged.target.slice(0, 12)}, short of HEAD ${result.staged.head.slice(0, 12)} (the delta exceeds the cursor bound); taking the next stage.`);
const next = await (await import('../../core/persistence/sync-drain.ts')).drainManagedSync(engine, opts, true);
const stages = (result.staged.stages ?? 1) + 1, d = result.drain, n = next.drain;
result = { ...next, fromCommit: result.fromCommit, added: result.added + next.added, modified: result.modified + next.modified, deleted: result.deleted + next.deleted,
renamed: result.renamed + next.renamed, chunksCreated: result.chunksCreated + next.chunksCreated,
...(result.identityDowngrades?.length || next.identityDowngrades?.length ? { identityDowngrades: [...result.identityDowngrades ?? [], ...next.identityDowngrades ?? []] } : {}),
...(next.staged ? { staged: { ...next.staged, stages } } : {}),
...(n ? { drain: { ...n, passes: d.passes + n.passes, processed: d.processed + n.processed, written: d.written + n.written, waived: d.waived + n.waived } } : {}) };
if (!next.staged) serr(`[sync] staged catch-up reached HEAD after ${stages} stages.`);
}
printInheritedOptions(result);
if (opts.dryRun) return result;
skips = (await import('../../core/persistence/sync-idle-skips.ts')).idleSkips(sourceId);
// GBRA-133: the ghost-page scan reads only pages and the source's config, so it runs beside the settles before it; its line still prints after theirs.
const ghostScan = countSyncGhosts(engine, sourceId);
await recordUpstreamObservation(engine, sourceId, opts.repoPath);
if (['synced', 'first_sync', 'up_to_date'].includes(result.status) && !skips.has('settled_failures')) await clearSettledFailures(engine, sourceId);
const replayed = await replayImportedRemembers(engine, sourceId);
if (replayed) result = { ...result, replayedRemembers: replayed };
const ghosts = reportSyncGhosts(await ghostScan);
if (ghosts) result = { ...result, ghostPages: ghosts };
// #6712 PR 2: holds that turned needs_human (or file_drift content decisions) since the last count, for the soak counters.
if (!skips.has('needs_human_sweep')) await (await import('../../core/persistence/soak-counters.ts')).sweepNeedsHumanHolds(engine, sourceId).catch(() => undefined);
return result;
};
// A dry run reads and previews only; it never waits for, or keeps out, a drainer.
if (outer.dryRun) return work(undefined);
const { withSyncTail } = await import('../../core/persistence/sync-tail.ts');
try {
const { healOriginTwinsAfterManagedSync } = await import('../../core/repair/origin-twins-heal.ts');
const { healAfterManagedSync } = await import('../../core/repair/slug-autoheal.ts');
const drained = await withDrainLease(engine, sourceId, {
holder: outer.drainHolder ?? { kind: 'cli', id: null }, remote: currentSubmissionAuthority()?.kind === 'remote_generic',
...(outer.followDrainLease ? { follow: { signal: outer.signal, deadlineAtMs: outer.deadlineAtMs } } : {}),
tty: process.stderr.isTTY === true,
announce: (holder, phase) => {
const who = holder ? `a ${holder.kind} drainer (pid ${holder.pid ?? '?'} on ${holder.host ?? '?'}${holder.version ? `, gbrain ${holder.version}` : ''})` : 'another drainer';
const done = holder?.progress ? `${holder.progress.committed} page(s) committed since it started${holder.progress.last_committed_at ? `, last at ${holder.progress.last_committed_at}` : ''}` : 'progress unknown';
serr(phase === 'start'
? `[sync] source ${sourceId} is being drained by ${who}; following it until it finishes or this run's budget ends (one drainer per source; it resumes the same cursor). Ctrl-C stops following and leaves it running.`
: `[sync] following ${who}: ${done}.`);
},
}, work);
// GST-2492 (B2): a rule this sync applied moves the held facts it narrows now. It runs once the lease is released only so
// a follower sync does not wait on the move: the lease is re-entrant and the move's maintenance writes are not sync rows
// (no claim waits on the lease), so running it inside would not deadlock either.
const moves = await import('../../core/facts/held-fact-move.ts');
const held = currentDrainLease(sourceId) ? null : await moves.moveRuledSharedHeld(engine, sourceId)
.catch((error: unknown) => { serr(`[sync] held facts under a new folder rule wait for the next maintenance cycle: ${error instanceof Error ? error.message : String(error)}`); return null; });
for (const w of held ? moves.heldMoveIncomplete(held) : []) serr(`[sync] held facts moved under a new folder rule, incomplete: ${w.text}`);
if (held?.unmovable.length) serr(`[sync] ${held.unmovable.length} group(s) of held facts could not move yet; the next gbrain dream retries (gbrain repair facts-home --source ${sourceId} lists them).`);
const tailed = skips.has('sync_tail') ? drained : await withSyncTail(engine, sourceId, drained, { signal: outer.signal, deadlineAtMs: outer.deadlineAtMs });
const healed = skips.has('slug_heal') ? tailed : await healAfterManagedSync(engine, outer, tailed);
return skips.has('origin_heal') ? healed : await healOriginTwinsAfterManagedSync(engine, outer, healed);
} catch (err) {
if (err instanceof LockUnavailableError && err.lockId === lockKey) throw new SyncLockBusyError(await formatLockBusyMessage(engine, lockKey), lockKey);
throw err;
}
}
/** #6712 PR 1: the one-line notice of a kept opt-out, naming the command that settles what it skipped. */
function printInheritedOptions(result: SyncResult): void {
const kept = result.inherited_options ?? [];
if (!kept.length) return;
const flags = { noEmbed: '--no-embed', noExtract: '--no-extract', noSchemaPack: '--no-schema-pack' } as const;
const settle = [...(kept.includes('noEmbed') ? ['gbrain embed --stale'] : []), ...(kept.includes('noExtract') ? ['gbrain extract --stale'] : [])];
serr(`[sync] kept the unfinished run's ${kept.map(key => flags[key]).join(' ')} for the pages it imports${settle.length ? `; ${settle.join(' and ')} settle${settle.length === 1 ? 's' : ''} them afterwards` : ''}.`);
}
/**
* #6693: after a managed sync (any outcome: a partial run can still import the file), replay the caller remembers refused while
* their page's file awaited import or was held, whose page this run brought back in line with its file. Best effort: a failure
* is one stderr line and never fails the sync. Returns null when nothing was examined.
*/
async function replayImportedRemembers(engine: BrainEngine, sourceId: string): Promise<SyncResult['replayedRemembers'] | null> {
try {
const { loadConfig } = await import('../../core/config.ts');
const { replayImportedRemembers: replay } = await import('../../core/repair/failed-writes.ts');
const logger = { info: () => {}, warn: (m: string) => serr(m), error: (m: string) => serr(m) };
const report = await replay({ engine, config: loadConfig() ?? { engine: engine.kind }, logger, remote: false, dryRun: false, sourceId } as OperationContext, sourceId);
if (!report.examined) return null;
serr(`[sync] replayed ${report.replayed} of ${report.examined} remember(s) refused while their page's file awaited import`
+ `${report.waiting ? `; ${report.waiting} still waiting` : ''}${report.skipped ? `; ${report.skipped} not replayable (gbrain repair failed-writes --source ${sourceId} lists them)` : ''}`
+ `${report.remaining ? `; ${report.remaining} left for the next sync` : ''}.`);
return report;
} catch (error) {
serr(`[sync] could not replay remembers refused while their page's file awaited import: ${error instanceof Error ? error.message : String(error)} (gbrain repair failed-writes --source ${sourceId} replays them).`);
return null;
}
}
/** #6712 PR 2 (L4): the source's ghost pages (live pages whose file is gone), counted and named, never deleted by sync. */
async function countSyncGhosts(engine: BrainEngine, sourceId: string): Promise<{ sourceId: string; ghosts: SyncResult['ghostPages'] | null }> {
return { sourceId, ghosts: await (await import('../../core/persistence/sync-ghosts.ts')).countSyncGhosts(engine, sourceId).catch(() => null) };
}
/** Prints the ghost-page line of a scan and returns its report. */
function reportSyncGhosts({ sourceId, ghosts }: Awaited<ReturnType<typeof countSyncGhosts>>): SyncResult['ghostPages'] | null {
if (ghosts) serr(`[sync] ${ghosts.count}${ghosts.truncated ? '+' : ''} live page(s) of ${sourceId} have no canonical file (ghost pages${ghosts.samples.length ? `, e.g. ${ghosts.samples.join(', ')}` : ''}); `
+ `sync leaves them in place. Preview with ${ghosts.fix.argv!.join(' ')} (read-only), or restore the files and sync.`);
return ghosts;
}
/** #6288: a successful managed sync clears this source's recorded failures its imported commit already settles (#6543 D5: legacy ledger rows too); best effort. */
async function clearSettledFailures(engine: BrainEngine, sourceId: string): Promise<void> {
try {
const { clearSettledManagedSyncFailures, readManagedSyncImported } = await import('../../core/persistence/sync-failures.ts');
const imported = await readManagedSyncImported(engine, sourceId);
if (imported) await clearSettledManagedSyncFailures(engine, imported);
if (imported) await (await import('../../core/persistence/legacy-sync-failures.ts')).settleLegacySyncFailures(engine, imported);
} catch { }
}