Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/** `gbrain sync` CLI body (`runSyncInner`): flag parsing, --all fan-out, watch mode. */
import { existsSync } from 'fs';
import { getCompanyBrainProfile } from '../../core/company-brain/profile.ts';
import { slog, withSourcePrefix } from '../../core/console-prefix.ts';
import type { BrainEngine } from '../../core/engine.ts';
import { isHostAbsolutePath, msysToNativePath } from '../../core/path-confine.ts';
import { syncFailureJsonFields, readManagedSyncFailures } from '../../core/persistence/sync-failures.ts';
import { syncHoldJsonFields } from '../../core/persistence/sync-holds.ts';
import { printHoldNotes } from '../sync-diagnostics.ts';
import { getDefaultSourcePath } from '../../core/source-resolver.ts';
import {
resolveSyncAllEmbedPlan,
syncProducedEmbeddableContent,
resolveSyncEmbedBackfill,
formatSyncEmbedBackfillOutcome,
resolveSingleSyncEmbedPlan,
buildSingleSyncJsonEnvelope,
} from '../../core/sync-embed-backfill.ts';
import type { SyncEmbedBackfillOutcome } from '../../core/sync-embed-backfill.ts';
import { runBreakLock } from '../../core/sync-lock.ts';
import { automaticSyncSkip, isSyncDisabledConfig } from '../../core/sync-policy.ts';
import { assertSyncAnchorAbsolute, syncAnchorRelativeError } from '../../core/sync-applicability.ts';
import { composeAbortSignals } from '../../core/sync-reconcile.ts';
import { acknowledgeFailures, unacknowledgedSyncFailures } from '../../core/sync.ts';
import type { SyncResult, SyncOpts } from '../sync.ts';
import { printSyncHelp, parseSyncFlags, parseSyncFanoutFlags, noteRetryFailedDeprecated } from './args.ts';
import { implyManagedNoPull, refuseManagedForbiddenFlags } from './managed-no-pull.ts';
import { legacyLedgerRows } from '../../core/persistence/legacy-sync-failures.ts';
import type { SyncFlags, SyncFanoutFlags } from './args.ts';
import { manageGitignoreAtGitRoot } from './gitignore.ts';
import { partitionMissingPathSources } from './missing-path.ts';
import { performSync } from './perform.ts';
import {
maybeExtractionNudge,
maybeBackupCoverageRefresh,
printSyncResult,
shouldNudgeAfterSync,
isFailedPartial,
} from './report.ts';
import { runSyncTrigger } from './trigger.ts';
import { drainJsonFields, formatDrainSummary, syncOutcome } from '../../core/persistence/sync-drain.ts';
import { readManagedMode } from '../../core/persistence/ownership.ts';
import { setCliExitVerdict, writeJsonDocument } from '../../core/cli-force-exit.ts';
import { cliRenderContext, renderNotice, type Notice } from '../../core/agent-output.ts';
/** D2: under the `--json` guard only writeStdoutFinal reaches fd 1 (writeJsonDocument). */
const emitJson = (text: string): void => { void writeJsonDocument(text); };
/** #7630: a stored root that is not absolute names the cwd, so post-sync .gitignore upkeep skips it. */
const absoluteStoredPath = (path: string | null): string | null => path && isHostAbsolutePath(path) ? path : null;
type SyncAllSourceRow = { id: string; name: string; local_path: string | null; config: Record<string, unknown>; last_commit: string | null; chunker_version: string | null };
// Per-source result accumulator for the optional --json envelope.
type PerSourceResult = {
sourceId: string;
sourceName: string;
status: 'ok' | 'error' | 'skipped_missing_path';
result?: SyncResult;
error?: string;
/** #7646: a refusal's agent contract (code, why, fix), so an --all consumer gets what a single-source sync exits 3 with. */
contract?: Record<string, unknown>;
localPath?: string;
};
/** A sync result's own notices: a degraded tail, the change-reason scan, and ignored privacy labels (GST-2462). */
function runNotices(result: SyncResult): Notice[] {
return [...(result.tail?.notice ? [result.tail.notice] : []), ...(result.reason_notices ?? []), ...(result.privacy_label_notice ? [result.privacy_label_notice] : [])];
}
const failureContract = (error: unknown): Record<string, unknown> | undefined => {
if (!(error instanceof Error) || error.name !== 'OperationError') return undefined;
const { code, suggestion, why, fix } = (error as Error & { toJSON(): Record<string, unknown> }).toJSON();
return { code, ...(suggestion ? { suggestion } : {}), ...(why ? { why } : {}), ...(fix ? { fix } : {}) };
};
/**
* #6696 (A4): a sync receipt carries the capacity notices (60 s snapshot, at most once an hour per process per
* scope): under `--json` as rendered `notices`, otherwise as stderr lines.
*/
async function syncCapacityNotices(engine: BrainEngine, json: boolean, result?: SyncResult): Promise<{ notices?: unknown[] }> {
const { capacityNotices, CAPACITY_READ_WAIT_MS } = await import('../../core/persistence/capacity-snapshot.ts');
// #6712 PR 2 (L5): a degraded post-checkpoint tail rides the same notices. #6803 C4: the snapshot wait is bounded.
const capacity = await capacityNotices(engine, { throttle: true, deadlineAt: Date.now() + CAPACITY_READ_WAIT_MS }).catch(() => []);
const notices = [...(result ? runNotices(result) : []), ...capacity].map(notice => renderNotice(notice, cliRenderContext()));
if (!json) for (const notice of notices) process.stderr.write(`[gbrain notice ${notice.code}] ${notice.why}\n`);
return notices.length ? { notices } : {};
}
/** #6712 PR 2 (L5): one source's post-checkpoint tail in a `sync --all` receipt: its report and its rendered degraded notice. */
function syncTailJsonFields(result: SyncResult): { tail?: Omit<NonNullable<SyncResult['tail']>, 'notice'>; notices?: unknown[] } {
const notices = runNotices(result).map(n => renderNotice(n, cliRenderContext()));
const rendered = notices.length ? { notices } : {};
if (!result.tail) return rendered;
const { notice: _notice, ...tail } = result.tail;
return { tail, ...rendered };
}
function printTailNotice(result: SyncResult): void {
for (const notice of runNotices(result)) process.stderr.write(`[gbrain notice ${notice.code}] ${notice.why}\n`);
}
export async function runSyncInner(engine: BrainEngine, args: string[], opts: { ownSignals?: boolean } = {}) {
// v0.40 Federated Sync v2: `gbrain sync trigger` subcommand
// Routes to runSyncTrigger which queues a 'sync' minion job with
// auto_embed_backfill=true. Falls through to the normal sync path
// if 'trigger' isn't the first arg.
if (args[0] === 'trigger') {
return runSyncTrigger(engine, args.slice(1));
}
// #6340: the operator contract (status / unblock) reads and schedules; it never runs a sync.
// #6803: `sync --status` is canonicalized to `sync status` by the same classifier the CLI preflight uses.
const { syncOperatorVerb, runSyncOperator } = await import('./operator.ts');
const operatorVerb = syncOperatorVerb(args);
if (operatorVerb) return runSyncOperator(engine, operatorVerb, args);
// v0.37 fix wave (Lane D.4 + CDX2-12): print usage when `--help`/`-h` is
// passed. Pre-fix this was unreachable because the dispatcher's generic
// CLI-only short-circuit fired first; sync is now in CLI_ONLY_SELF_HELP.
if (args.includes('--help') || args.includes('-h')) {
printSyncHelp();
return;
}
// #6712 PR 2: a run with a recovery flag is a soak intervention (soak-counters.ts); a scheduler's plain run is not.
const soak = await import('../../core/persistence/soak-counters.ts');
if (soak.nonPlainSyncFlags(args).length) await soak.recordSoakEvent(engine, 'sync_non_plain');
if (args.includes('--acknowledge-managed')) return runAcknowledgeManaged(engine, args);
const flags = parseSyncFlags(args);
noteRetryFailedDeprecated(args);
const { syncAll, jsonOut, breakLock, forceBreakLock, maxAgeSeconds } = flags;
// v0.41.13.0 (T4 + D1): handle --break-lock / --force-break-lock BEFORE
// the sync would otherwise contend on the lock. v3's plan dropped the
// --all refusal at the same point so cron can self-heal across every
// source in one call; runBreakLock now widens to iterate sources when
// --all is set and accept maxAgeSeconds for age-gated breaks.
if (breakLock || forceBreakLock) return await runSyncBreakLock(engine, args, { syncAll, jsonOut, forceBreakLock, maxAgeSeconds });
// GBRA-133: one single-source run reads its routing rows once (core/sync-dispatch-read.ts); --all and --watch read live.
if (syncAll || flags.watch) return runRoutedSync(engine, args, flags, opts);
const { syncDispatchRow, withSyncDispatchRead } = await import('../../core/sync-dispatch-read.ts');
return withSyncDispatchRead(engine, () => {
const explicit = args.find((a, i) => args[i - 1] === '--source');
if (explicit) void syncDispatchRow(engine, explicit);
return runRoutedSync(engine, args, flags, opts);
});
}
async function runRoutedSync(engine: BrainEngine, args: string[], flags: ReturnType<typeof parseSyncFlags>, opts: { ownSignals?: boolean } = {}): Promise<void> {
const { repoPath, dryRun, skipFailed, syncAll, jsonOut } = flags;
let { noEmbed } = flags;
const fanout = parseSyncFanoutFlags(args, syncAll);
await refuseManagedForbiddenFlags(engine, flags, args);
// --skip-failed: acknowledge pre-existing unacked failures BEFORE the sync
// runs, not only ones the current run produces. Without this, the common
// recovery flow — fix the YAML, re-run sync, then run --skip-failed to
// clear the log — fails to clear anything: when there are no NEW failures
// (because the files are now fixed), the inner ack path in performSync is
// never reached, and "Already up to date." leaves the log untouched. Both
// doctor and printSyncResult instruct users to run --skip-failed in
// exactly this case, so the flag has to handle stale entries up-front.
const cli = await resolveCliSyncSource(engine, args, { repoPath, syncAll, dryRun, jsonOut, noEmbed });
const { sourceId, companyPolicy, embeddingCredentialError } = cli;
noEmbed = cli.noEmbed;
await implyManagedNoPull(engine, flags, syncAll ? null : sourceId);
// --skip-failed: acknowledge pre-existing unacked failures BEFORE the sync
// runs, not only ones the current run produces. Without this, the common
// recovery flow — fix the YAML, re-run sync, then run --skip-failed to clear
// the log — fails to clear anything (no NEW failures → the inner ack path in
// performSync is never reached, and "Already up to date." leaves the log).
//
// v0.42.42.0 (#2139, D13C): scoped PER SOURCE. `--all` clears every source's
// open failures; single-source clears only its own (don't ack source B's
// failures when syncing source A). Safe under parallel — the ledger
// serializes writes via `withLedgerLock` and keys rows by `source_id`
// (#1939), which is why the old D15 "no --skip-failed under parallel"
// refusal is lifted below.
if (skipFailed) {
const acked = syncAll ? acknowledgeFailures() : acknowledgeFailures(sourceId);
if (acked.count > 0) slog(`Acknowledged ${acked.count} pre-existing failure(s).`);
}
// v0.19.0 — `sync --all` iterates all registered sources with a
// local_path. Sources are the canonical v0.18.0 abstraction: per-source
// last_commit, last_sync_at, config.federated flags. Per-source
// bookmarks live in the sources table (not ~/.gbrain/config.json),
// which is why this path replaced Garry's OpenClaw `multi-repo.ts` shim.
//
// Only sources with a non-null local_path participate. A GitHub-only
// source (no checkout) has nothing for `sync` to pull. Sources with
// syncEnabled=false in config.jsonb are skipped too.
const { syncResumeCommand } = await import('../../core/sync-reconcile.ts');
const { getCliOptions } = await import('../../core/cli-options.ts');
const resumeCommand = syncResumeCommand(args, getCliOptions().brain);
// #6543: --all never delegates, so its opt-out keeps this process's own consumer here (single-source runs: sync-delegate.ts).
if (syncAll && (args.includes('--no-delegate') || process.env.GBRAIN_SYNC_NO_DELEGATE === '1')) {
(await import('../../core/persistence/consumer-election.ts')).forceOwnConsumer('opted_out');
}
if (syncAll) return await runSyncAll(engine, { ...flags, ...fanout }, { noEmbed, embeddingCredentialError, resumeCommand, ownSignals: opts.ownSignals === true });
return await runSingleSourceSync(engine, { ...flags, ...fanout }, { sourceId, companyPolicy, noEmbed, resumeCommand, args, ownSignals: opts.ownSignals === true });
}
/** #6288: `gbrain sync --source <id> --acknowledge-managed <request_id|path>` removes only recorded failures the imported commit settles. */
async function runAcknowledgeManaged(engine: BrainEngine, args: string[]): Promise<void> {
const { opError } = await import('../../core/ops/contract.ts');
const selector = args[args.indexOf('--acknowledge-managed') + 1];
const sourceId = args.find((a, i) => args[i - 1] === '--source');
const status = (id: string) => ({ argv: ['gbrain', 'sync', 'status', '--source', id, '--json'], consent: [], actor: 'agent' as const, requires_exclusive: false,
why: 'Shows the recorded failure, its class and the exact sync that settles it, read-only.' });
if (!selector || selector.startsWith('--') || !sourceId) throw opError('invalid_params', '--acknowledge-managed needs --source <id> and a request id or path.',
`Name the source and one recorded failure's request id or path, for example gbrain sync --source ${sourceId ?? 'default'} --acknowledge-managed notes/a.md; gbrain sync status --source ${sourceId ?? 'default'} --json lists them.`,
sourceId ? { fix: status(sourceId) } : undefined);
const { acknowledgeManagedSyncFailures, readManagedSyncImported } = await import('../../core/persistence/sync-failures.ts');
const imported = await readManagedSyncImported(engine, sourceId);
if (!imported) throw opError('managed_failure_unsettled', `Source ${sourceId} has no successful managed sync yet, so nothing can be acknowledged.`,
`Run gbrain sync --no-pull --source ${sourceId} first; a recorded failure is acknowledged only when the imported commit settles it.`, { fix: status(sourceId) });
const result = await acknowledgeManagedSyncFailures(engine, imported, selector);
for (const failure of result.cleared) slog(`Acknowledged recorded failure ${failure.code} for ${failure.path} (request ${failure.request_id ?? '<not-admitted>'}): commit ${imported.target.slice(0, 12)} already imports it.`);
if (result.refused.length || !result.cleared.length) {
throw opError('managed_failure_unsettled', result.refused.length ? `${result.refused.length} recorded failure(s) for ${selector} are not settled by the imported state.` : `No recorded managed failure of source ${sourceId} matches ${selector}.`,
result.refused.length ? `Refused: ${result.refused.map(r => `${r.failure.path} (${r.reason})`).join(', ')}. The file still differs from what commit ${imported.target.slice(0, 12)} imported, its run is unfinished, or it is held; nothing was removed for it. Resume or retry that sync instead.`
: `Check the request id or path against gbrain sync status --source ${sourceId} --json; nothing was removed.`, { fix: status(sourceId) });
}
if (args.includes('--json')) await writeJsonDocument(JSON.stringify({ source_id: sourceId, imported_commit: imported.target, cleared: result.cleared }));
}
async function runSyncBreakLock(
engine: BrainEngine,
args: string[],
input: { syncAll: boolean; jsonOut: boolean; forceBreakLock: boolean; maxAgeSeconds: number | undefined },
): Promise<void> {
const { syncAll, jsonOut, forceBreakLock, maxAgeSeconds } = input;
if (syncAll) {
const { listSources } = await import('../../core/sources-ops.ts');
const sources = await listSources(engine);
// listSources omits archived sources by default. We also require
// local_path because the lock key is per-source; pure-DB sources
// (no local_path) don't hold sync locks.
const activeSources = sources.filter((s) => s.local_path);
if (activeSources.length === 0) {
if (jsonOut) emitJson(JSON.stringify({ status: 'no_sources' }));
else console.error('No active sources to break-lock against.');
process.exit(0);
}
let worstExit = 0;
for (const src of activeSources) {
const lockKey = `gbrain-sync:${src.id}`;
const exit = await runBreakLock(engine, lockKey, src.id, {
force: forceBreakLock,
json: jsonOut,
maxAgeSeconds,
});
if (exit > worstExit) worstExit = exit;
}
process.exit(worstExit);
}
const sourceArg = args.find((a, i) => args[i - 1] === '--source');
// #4412: this branch used to hardcode `sourceArg ?? 'default'` while the
// sync itself resolves through the full ambient chain (--source >
// GBRAIN_SOURCE > dotfile > cwd > sole-non-default). Under
// GBRAIN_SOURCE=<src>, `sync --force-break-lock` inspected
// gbrain-sync:default — absent — printed "nothing to break", exit 0, and
// left the dead holder's row on gbrain-sync:<src>; the follow-up sync
// then refused for the 60s takeover grace. Resolve the SAME source the
// sync would lock. Explicit --source keeps the resolver-free path (no
// assertSourceExists) so leftover locks of deleted sources stay breakable.
const { resolveSourceWithTier: resolveBreakSource } = await import('../../core/source-resolver.ts');
const sourceId = sourceArg ?? (await resolveBreakSource(engine, null)).source_id;
const lockKey = `gbrain-sync:${sourceId}`;
const exit = await runBreakLock(engine, lockKey, sourceId, {
force: forceBreakLock,
json: jsonOut,
maxAgeSeconds,
});
process.exit(exit);
}
/**
* Resolve the source a single-source (or `--all`) CLI sync writes to, refusing
* the ambiguous cases, and validate embedding credentials before any work.
*/
async function resolveCliSyncSource(
engine: BrainEngine,
args: string[],
input: { repoPath: string | undefined; syncAll: boolean; dryRun: boolean; jsonOut: boolean; noEmbed: boolean },
): Promise<{ sourceId: string; companyPolicy: Awaited<ReturnType<typeof getCompanyBrainProfile>> | null; noEmbed: boolean; embeddingCredentialError: Error | undefined }> {
const { repoPath, syncAll, dryRun, jsonOut } = input;
let { noEmbed } = input;
// v0.18.0 Step 5: --source resolves to a sources(id) row. Falls back
// to pre-v0.17 global config (sync.repo_path + sync.last_commit) when
// no flag, no env, no dotfile is present.
//
// v0.41.13 (#1434): always call the resolver, not just when explicit/env
// is set. Pre-fix, `gbrain sync` without --source skipped resolution and
// left sourceId undefined — which the engine treated as the seeded
// 'default' source. Users with a single non-default registered source
// (studiovault, etc.) silently routed every write to a source holding
// 0 pages, then createVersion threw on the slug lookup.
//
// The resolver's new `sole_non_default` tier (5.5) routes those
// single-source brains to the right place automatically; the nudge
// surfaces the auto-route to stderr so the user knows what happened
// and can pass --source to override if needed.
const explicitSource = args.find((a, i) => args[i - 1] === '--source') || null;
const { resolveSourceWithTier, resolveSourceForRepoPath, formatSoleNonDefaultNudge, defaultWriteAllowedByEnv } =
await import('../../core/source-resolver.ts');
// #3765: an explicit --repo anchors source resolution at the REPO dir, not
// the caller's cwd. Pre-fix, `gbrain sync --repo ~/other-vault` parsed the
// path but resolved the source from cwd — anchors, page writes, and the
// per-source lock (`syncLockId(sourceId)`) all followed the WRONG source.
// Precedence: --source flag > repo-derived (dotfile/local_path at the repo
// dir) > the ambient chain. A conflicting GBRAIN_SOURCE refuses loudly.
let resolved: { source_id: string; tier: string; detail?: string } | null = null;
if (!explicitSource && repoPath) {
const derived = await resolveSourceForRepoPath(engine, repoPath);
if (derived) {
const envSource = process.env.GBRAIN_SOURCE;
if (envSource && envSource !== derived.source_id) {
console.error(
`--repo resolves to source '${derived.source_id}' (via ${derived.tier}) but ` +
`GBRAIN_SOURCE='${envSource}' is set. Pass --source <id> to disambiguate.`,
);
process.exit(1);
}
resolved = derived;
process.stderr.write(
`[gbrain] routing sync to source '${derived.source_id}' (resolved from --repo via ${derived.tier}).\n`,
);
}
}
if (!resolved) resolved = await resolveSourceWithTier(engine, explicitSource);
const sourceId: string = resolved.source_id;
const companyPolicy = !syncAll ? await getCompanyBrainProfile(engine, sourceId) : null;
if (companyPolicy) noEmbed = true;
let embeddingCredentialError: Error | undefined;
if (!noEmbed && !dryRun) {
const { validateEmbeddingCreds, EmbeddingCredentialError } = await import('../../core/embed-preflight.ts');
try { validateEmbeddingCreds(); }
catch (error) {
if (!syncAll) {
if (!(error instanceof EmbeddingCredentialError)) throw error;
if (jsonOut) emitJson(JSON.stringify({ status: 'embedding_credentials_missing', diagnosis: error.diagnosis }));
else console.error(`\n${error.userMessage}\n`);
process.exit(1);
}
embeddingCredentialError = error instanceof Error ? error : new Error(String(error));
}
}
if (resolved.tier === 'sole_non_default') {
const nudge = formatSoleNonDefaultNudge(sourceId);
if (nudge) process.stderr.write(nudge + '\n');
}
// #4583 (fixes #4564's misrouted-write symptom): refuse an unscoped
// single-source sync that would silently land in 'default' on a
// bulk-non-default brain. Exempt: `--all` (iterates every source, not an
// unscoped-to-default write) and `--dry-run` (writes nothing — the preview
// runs and the guard only WARNS that a real run would be refused). Escape:
// `--source default` (tier 'flag', never seed_default) or
// GBRAIN_ALLOW_DEFAULT_WRITE=1. Fail-open: a query error never blocks a sync.
if (resolved.tier === 'seed_default' && !syncAll && !defaultWriteAllowedByEnv()) {
const { assessDefaultWriteGuard, formatDefaultWriteRefusal } = await import('../../core/source-resolver.ts');
const assessment = await assessDefaultWriteGuard(engine);
if (assessment.shouldGuard) {
console.error((dryRun ? '[dry-run] a real run would be refused:\n' : '') + formatDefaultWriteRefusal('sync', assessment));
if (!dryRun) process.exit(1);
}
}
return { sourceId, companyPolicy, noEmbed, embeddingCredentialError };
}
/** `sync --all`: every registered source with a local_path, fanned out or serial. */
async function runSyncAll(
engine: BrainEngine,
flags: SyncFlags & SyncFanoutFlags,
input: { noEmbed: boolean; embeddingCredentialError: Error | undefined; resumeCommand: string; ownSignals: boolean },
): Promise<void> {
const {
dryRun, full, noPull, noBulk, lanes, noExtract, skipFailed, retryFailed, noSchemaPack, includeGitignored,
workingTree, missingPathMode, jsonOut, yesFlag, serialFlag, noAutoEmbed, maxSources, concurrency, timeoutSeconds,
} = flags;
const { noEmbed, embeddingCredentialError, resumeCommand, ownSignals } = input;
// v0.41.31: SELECT carries last_commit + chunker_version so the inline
// cost preview's "unchanged source → 0" short-circuit can mirror sync's
// own "do work?" gate (sync.ts:1057+1075) + doctor's sync_freshness.
// Both columns predate v0.41 (writeSyncAnchor / writeChunkerVersion); no
// schema migration needed.
// #3880: archived sources must not re-enter `sync --all`. The archived
// column is v34+ — fall back to the unfiltered query on older brains
// (house style per pickSoleNonDefaultSource).
let sources: SyncAllSourceRow[];
try {
sources = await engine.executeRaw<SyncAllSourceRow>(
`SELECT id, name, local_path, config, last_commit, chunker_version FROM sources WHERE local_path IS NOT NULL AND archived IS NOT TRUE`,
);
} catch {
sources = await engine.executeRaw<SyncAllSourceRow>(
`SELECT id, name, local_path, config, last_commit, chunker_version FROM sources WHERE local_path IS NOT NULL`,
);
}
if (!sources || sources.length === 0) {
slog('No sources with local_path configured. Use `gbrain sources add <id> --path <path>` first.');
return;
}
// v0.41.31 — mode-aware cost gate. Resolve federated_v2 ONCE here so both
// the gate (below) and the fan-out (further down) share it.
const { isFederatedV2Enabled } = await import('../../core/feature-flags.ts');
const v2Enabled = await isFederatedV2Enabled(engine);
// v0.40.5.0 Federated Sync v2 (master) + v0.40.6.0 layering (this branch):
// master added parallel fan-out via pMapAllSettled, embed-backfill auto-
// submit, --serial / --max-sources / --no-auto-embed flags, and feature-
// flagged the whole thing behind sync.federated_v2. This branch layers
// additive improvements on top:
// - humanSink swap so `--json` keeps stdout clean (D4)
// - --skip-failed / --retry-failed reject under parallel>1 (D15 — the
// sync-failures.jsonl is brain-global, parallel acks race)
// - connection-budget stderr warning at parallel × workers × 2 > 16 (D10)
// - withSourcePrefix wrap inside runOne so slog/serr lines from
// performSync get the [<source-id>] prefix under parallel mode (D6)
// - stable JSON envelope {schema_version:1, sources, ...} when --json
// v0.41.31: v2Enabled resolved once above (cost gate). Reused here.
// P4.2: the same predicate the autopilot dispatchers use decides which sources this bulk run skips.
const notApplicable: string[] = [];
for (const s of sources) {
if ((await automaticSyncSkip(engine, s))?.reason === 'sync_not_applicable') notApplicable.push(s.id);
}
const activeSources = sources.filter((s) => !isSyncDisabledConfig(s.config) && !notApplicable.includes(s.id));
const disabledCount = sources.filter((s) => isSyncDisabledConfig(s.config)).length;
const humanSink: NodeJS.WriteStream = jsonOut ? process.stderr : process.stdout;
const writeHuman = (line: string) => humanSink.write(line + '\n');
if (disabledCount > 0) {
writeHuman(`Skipping ${disabledCount} disabled source(s).`);
}
if (notApplicable.length > 0) {
writeHuman(`Skipping ${notApplicable.length} source(s) where sync does not apply (gbrain-owned content directory, not a Git checkout): ${notApplicable.join(', ')}. Add markdown files with \`gbrain import <dir> --source <id>\`.`);
}
// --missing-path skip: classify sources whose checkout is not on this
// machine instead of failing them (see parseMissingPathMode's rationale).
// Under the default 'fail' this is a no-op and behavior is unchanged.
let skippedMissingPath: typeof activeSources = [];
let runnableSources = activeSources;
if (missingPathMode === 'skip') {
const parts = partitionMissingPathSources(activeSources, existsSync);
runnableSources = parts.runnable;
skippedMissingPath = parts.missing;
for (const src of skippedMissingPath) {
writeHuman(` ⊘ ${src.name}: skipped — local_path not present on this host (${src.local_path})`);
}
if (skippedMissingPath.length > 0) {
writeHuman(`Skipped ${skippedMissingPath.length} source(s) whose local_path is not present on this host (--missing-path skip).`);
}
}
if (runnableSources.length === 0) {
if (jsonOut) {
emitJson(JSON.stringify({
schema_version: 1,
sources: skippedMissingPath
.slice()
.sort((a, b) => a.id.localeCompare(b.id))
.map((s) => ({
source_id: s.id,
name: s.name,
status: 'skipped_missing_path',
local_path: s.local_path,
})),
parallel: 0,
ok_count: 0,
error_count: 0,
skipped_count: skippedMissingPath.length,
}));
}
return;
}
// v0.42.42.0 (#2139) cost gate — shared with single-source sync. Above
// the floor, non-interactive runs keep importing (never exit 2), but the
// delivery statement is capability-aware: background queue only when a
// worker can drain it, otherwise an exact manual command.
const companyPolicies = new Map<string, Awaited<ReturnType<typeof getCompanyBrainProfile>>>();
const policyFailures = new Map<string, unknown>();
for (const source of runnableSources) {
// #7630: a relative local_path fails its source before the cost gate walks the directory.
if (source.local_path && !isHostAbsolutePath(source.local_path)) { policyFailures.set(source.id, syncAnchorRelativeError(source.id, source.local_path)); continue; }
try { companyPolicies.set(source.id, await getCompanyBrainProfile(engine, source.id)); }
catch (error) { policyFailures.set(source.id, error); }
}
const embedPlan = await resolveSyncAllEmbedPlan(engine, runnableSources.filter(src => !policyFailures.has(src.id) && !companyPolicies.get(src.id)), {
v2Enabled, serialFlag, noEmbed: noEmbed || !!embeddingCredentialError, noAutoEmbed, dryRun, jsonOut, yesFlag, full, includeGitignored, noPull,
});
if (embedPlan.stop) return;
const {
workerSurface: backfillSurface, fanOutEligible, effectiveNoEmbed, shouldBackfill,
} = embedPlan;
const embedBackfillBySource = new Map<string, SyncEmbedBackfillOutcome>();
const perSourceResults: PerSourceResult[] = [];
for (const src of skippedMissingPath) {
perSourceResults.push({
sourceId: src.id,
sourceName: src.name,
status: 'skipped_missing_path',
localPath: src.local_path ?? undefined,
});
}
// #1633 (Part B): one shared SIGINT controller for the whole --all fan-out.
// process-cleanup.ts doesn't own SIGINT, so without this Ctrl-C hard-cuts the
// run and can leak per-source locks; here it aborts every in-flight source so
// each performSync returns `partial` + releases its lock cleanly.
const allInterrupt = new AbortController();
const onAllSigint = () => { try { allInterrupt.abort(new Error('SIGINT')); } catch { /* */ } };
const runOne = async (src: typeof sources[number]): Promise<SyncResult> => {
if (policyFailures.has(src.id)) throw policyFailures.get(src.id);
const companyPolicy = companyPolicies.get(src.id);
if (!companyPolicy && embeddingCredentialError) throw embeddingCredentialError;
const cfg = (src.config || {}) as { strategy?: 'markdown' | 'code' | 'auto' };
// D18/#2139: planned fan-out or a cost-gate auto-defer skips inline
// embedding; the post-run delivery below reports/queues each source.
// v0.41.13.0 (T6 / D-V3-3 / D-V4-mech-6) — per-source AbortController.
//
// When the user passes --timeout, each source gets its OWN
// AbortController + countdown that starts when THIS runOne invocation
// starts. NOT a shared global controller — codex pass 2 caught that
// shared shape would starve later sources of their fair budget.
//
// try/finally + timer.unref() (D-V4-mech-6):
// - finally clearTimeout guarantees cleanup even when performSync
// throws (which pMapAllSettled catches outside this closure).
// Without finally, a throw would leak the timer and keep the CLI
// alive past `setTimeout(..., timeoutMs)`.
// - timer.unref() (Node-specific; the optional-chain handles
// environments without it) tells the event loop NOT to keep the
// process alive solely for this timer. Belt-and-suspenders with
// finally — even on a missed clearTimeout, the process can exit
// once all real work resolves.
const controller = timeoutSeconds !== undefined ? new AbortController() : undefined;
const timer = timeoutSeconds !== undefined
? setTimeout(() => controller!.abort(), timeoutSeconds * 1000)
: undefined;
timer?.unref?.();
const repoOpts: SyncOpts = {
repoPath: msysToNativePath(src.local_path!), // #2955: heal MSYS /c/... before joins
dryRun, full, noPull,
noEmbed: effectiveNoEmbed,
// #6696: a planned or cost-gate deferral (not the user's --no-embed) survives cursor adoption.
embedDeferred: effectiveNoEmbed && !noEmbed,
noExtract,
skipFailed, retryFailed, noSchemaPack,
includeGitignored,
workingTree,
sourceId: src.id,
strategy: cfg.strategy,
concurrency,
signal: composeAbortSignals(allInterrupt.signal, controller?.signal),
drain: true, noBulk, lanes, followDrainLease: true, drainHolder: { kind: 'cli', id: null },
};
// v0.40.6.0 (D6): wrap performSync in withSourcePrefix so every slog /
// serr line emitted from inside the sync code path gets prefixed with
// `[<source-id>] `. Under master's pMapAllSettled fan-out, this is
// what makes `grep '\[media-corpus\]'` against parallel output work.
//
// v0.41.13.0 (T6): wrap the performSync call in try/finally so the
// per-source timer is always cleared, even on throw.
let result: SyncResult;
try {
result = await withSourcePrefix(src.id, () => performSync(engine, repoOpts));
} finally {
if (timer !== undefined) clearTimeout(timer);
}
// v0.41.13.0 (T7 / D-V3-5): partial joins dry_run + blocked_by_failures
// in the conservative posture — defer gitignore management to the next
// clean sync. A partial sync's set of db_only paths isn't fully
// reconciled, so writing .gitignore entries based on it could leave
// stale or missing entries.
if (
!companyPolicy && result.status !== 'dry_run' &&
result.status !== 'blocked_by_failures' &&
result.status !== 'partial'
) {
manageGitignoreAtGitRoot(src.local_path!, engine.kind);
}
// Deliver planned or intrinsic >100-file deferrals. Intrinsic delivery
// is v2-only on worker-backed engines; no-worker engines still need a
// manual outcome. This preserves the worker-backed v2-off rollback.
if (
!companyPolicy && (shouldBackfill || (
result.embedDeferralReason === 'large_sync' &&
(v2Enabled || backfillSurface.status === 'no_worker_surface')
)) &&
!dryRun &&
result.status !== 'dry_run' &&
result.status !== 'up_to_date' &&
result.status !== 'partial' && syncProducedEmbeddableContent(result)
) {
try {
const outcome = await resolveSyncEmbedBackfill(engine, src.id, {
reason: 'sync_all', autoSubmitDisabled: noAutoEmbed,
});
embedBackfillBySource.set(src.id, outcome);
writeHuman(` → ${formatSyncEmbedBackfillOutcome(outcome, src.name)}`);
} catch (e) {
process.stderr.write(` → embed-backfill submission failed for ${src.name}: ${e instanceof Error ? e.message : String(e)}\n`);
}
}
return result;
};
// v0.42.42.0 (#2139, D13C): the v0.40.6.0 (D15) refusal of --skip-failed /
// --retry-failed under parallel sync is LIFTED. It existed because the
// failure ledger was once brain-global with racing acks; #1939 made the
// ledger per-(source_id, path) and serialized every write through
// `withLedgerLock`, so parallel per-source acks no longer race. Lifting it
// also removes the forcing-function that pushed recovery syncs to --serial
// (and thus armed the inline cost gate) — the root cause behind #2139.
// Effective parallelism — surfaced in the --json envelope so consumers
// know how the run was actually dispatched. 1 in the serial fallback,
// capped at min(sourceCount, --max-sources, 8) in the parallel path.
// #5984 (ENG-A14): managed sources drain one at a time. Parallel drains contend on the brain-wide
// publication counters (55P03) and, at high RTT, made no progress at all.
// #6482: no catch-to-unmanaged; an unknown persistence state refuses before any source is dispatched.
const managedSerial = fanOutEligible && runnableSources.length > 1 && await readManagedMode(engine);
if (managedSerial) writeHuman('Managed brain: sources drain one at a time (parallel drains contend on the shared publication counters).');
const effectiveParallel = fanOutEligible && !managedSerial
? Math.min(runnableSources.length, maxSources ?? 8)
: 1;
// GST-2869: a termination signal aborts the shared controller, so sources not yet dispatched return before any work.
await withTerminationOwner(ownSignals, allInterrupt, () => dispatchSyncAll({ fanOutEligible: fanOutEligible && !managedSerial, effectiveParallel, concurrency, runnableSources, runOne, writeHuman, humanSink, perSourceResults, onAllSigint }));
const terminated = await terminatedExit(allInterrupt);
const okCount = perSourceResults.filter((r) => r.status === 'ok').length;
const errCount = perSourceResults.filter((r) => r.status === 'error').length;
if (jsonOut) {
// #7190: each synced source's applied-through watermark, read after the run.
const { readAppliedWatermark } = await import('../../core/persistence/applied-watermark.ts');
const watermarks = new Map(await Promise.all(perSourceResults.filter(r => r.status !== 'skipped_missing_path')
.map(async r => [r.sourceId, await readAppliedWatermark(engine, r.sourceId).catch(() => null)] as const)));
emitSyncAllEnvelope({ perSourceResults, embedBackfillBySource, effectiveParallel, okCount, errCount, costGate: embedPlan.costGate, resumeCommand, watermarks });
}
// v0.42.7 (#1696): brain-wide extraction-lag nudge after the --all wave.
// Best-effort, stderr-only; skipped on dry-run.
if (!dryRun && terminated === null) await maybeExtractionNudge(engine);
// Monthly backup-coverage stale-only refresh (trusted local engine holder).
if (!dryRun && terminated === null) await maybeBackupCoverageRefresh(engine);
// #3068: any source wedged on a failed pull (partial/pull_failed) makes
// the whole --all run non-zero — it will not self-heal on retry, so a
// green exit would hide it from cron/monitoring. Timeout-class partials
// keep the pre-existing exit-0 behavior (they converge on retry).
const pullFailedCount = perSourceResults.filter(
(r) => r.status === 'ok' && r.result && isFailedPartial(r.result),
).length;
if (terminated !== null) { setCliExitVerdict(terminated); return; }
if (errCount > 0 || pullFailedCount > 0) process.exit(1);
return;
}
/** Runs every source (parallel or serial) and prints the aggregate; #5988: green sources still print their holds. */
export async function dispatchSyncAll(input: {
fanOutEligible: boolean;
effectiveParallel: number;
concurrency: number | undefined;
runnableSources: SyncAllSourceRow[];
runOne: (src: SyncAllSourceRow) => Promise<SyncResult>;
writeHuman: (line: string) => void;
humanSink: NodeJS.WriteStream;
perSourceResults: PerSourceResult[];
onAllSigint: () => void;
}): Promise<void> {
const { fanOutEligible, effectiveParallel, concurrency, runnableSources, runOne, writeHuman, humanSink, perSourceResults, onAllSigint } = input;
process.on('SIGINT', onAllSigint);
try {
if (fanOutEligible) {
const { pMapAllSettled } = await import('../../core/parallel.ts');
const cap = effectiveParallel;
// v0.40.6.0 (D10): connection-budget stderr warning. Each per-file
// worker opens its own PostgresEngine with poolSize=2, so the real
// live-connection ceiling is `cap × workers × 2` per wave plus the
// parent pool. The original PR understated by 2× — fix the math.
const effectiveWorkers = concurrency ?? 4;
const budget = cap * effectiveWorkers * 2;
if (budget > 16) {
process.stderr.write(
`[sync --all] Connection budget: parallel=${cap} × workers=${effectiveWorkers} × 2 ` +
`(per-file pool) = ${budget} concurrent connections per fan-out wave (+ parent pool). ` +
`Check pgbouncer/Postgres max_connections (SELECT count(*) FROM pg_stat_activity); ` +
`raise the cap or lower --max-sources/--workers if you see "too many clients" errors.\n`,
);
}
writeHuman(`\nParallel sync: ${runnableSources.length} sources, ${cap} concurrent workers.\n`);
const results = await pMapAllSettled(runnableSources, cap, async (src) => {
const r = await runOne(src);
return { name: src.name, result: r };
});
// Print per-source aggregate at the end. humanSink so --json stays clean.
writeHuman('\n--- sync --all aggregate ---');
for (let i = 0; i < results.length; i++) {
const r = results[i];
const src = runnableSources[i];
if (r.status === 'fulfilled') {
writeHuman(` ${syncOutcome(r.value.result) === 'blocked' ? '✗' : '✓'} ${src.name}: ${r.value.result.status} (added=${r.value.result.added}, modified=${r.value.result.modified}, deleted=${r.value.result.deleted})`);
if (syncOutcome(r.value.result) === 'blocked') printSyncResult(r.value.result, humanSink);
else printHoldNotes(r.value.result, writeHuman);
printTailNotice(r.value.result);
perSourceResults.push({
sourceId: src.id,
sourceName: src.name,
status: syncOutcome(r.value.result) === 'blocked' ? 'error' : 'ok',
result: r.value.result,
});
} else {
const msg = r.reason instanceof Error ? r.reason.message : String(r.reason);
process.stderr.write(` ✗ ${src.name}: ${msg}\n`);
perSourceResults.push({
sourceId: src.id,
sourceName: src.name,
status: 'error',
error: msg,
contract: failureContract(r.reason),
});
}
}
} else {
for (const src of runnableSources) {
writeHuman(`\n--- Syncing source: ${src.name} ---`);
try {
const result = await runOne(src);
printSyncResult(result, humanSink);
printTailNotice(result);
perSourceResults.push({
sourceId: src.id,
sourceName: src.name,
status: syncOutcome(result) === 'blocked' ? 'error' : 'ok',
result,
});
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
process.stderr.write(`Error syncing ${src.name}: ${msg}\n`);
perSourceResults.push({
sourceId: src.id,
sourceName: src.name,
status: 'error',
error: msg,
contract: failureContract(e),
});
}
}
}
} finally {
process.off('SIGINT', onAllSigint);
}
}
function emitSyncAllEnvelope(input: {
perSourceResults: PerSourceResult[];
embedBackfillBySource: Map<string, SyncEmbedBackfillOutcome>;
effectiveParallel: number;
okCount: number;
errCount: number;
costGate: Record<string, unknown> | undefined;
resumeCommand: string;
watermarks: Map<string, unknown>;
}): void {
const { perSourceResults, embedBackfillBySource, effectiveParallel, okCount, errCount, costGate, resumeCommand, watermarks } = input;
// Sort by source_id at emit time so the envelope is deterministic
// even though completion order is not (pMapAllSettled semantics).
const sortedSources = perSourceResults
.slice()
.sort((a, b) => a.sourceId.localeCompare(b.sourceId))
.map((r) => ({
source_id: r.sourceId,
name: r.sourceName,
status: r.status,
...(r.localPath ? { local_path: r.localPath } : {}),
...(r.result ? {
...syncFailureJsonFields(r.result),
...syncHoldJsonFields(r.result),
sync_status: r.result.status,
// #3068: surface the partial reason (e.g. pull_failed) so JSON
// consumers can distinguish a self-healing timeout from a wedge.
...(r.result.reason ? { reason: r.result.reason } : {}),
...(r.result.managedWrite ? { managed_write: r.result.managedWrite } : {}),
...drainJsonFields(r.result, resumeCommand, r.sourceId),
...syncTailJsonFields(r.result),
added: r.result.added,
modified: r.result.modified,
deleted: r.result.deleted,
chunks_created: r.result.chunksCreated,
embedded: r.result.embedded,
// Warning aggregates (malformed filenames, alias/undeclared
// types) — the whole point of the result-field plumbing is that
// JSON/worker consumers can see them (codex re-review).
...(r.result.malformedSkipped ? { malformed_skipped: r.result.malformedSkipped } : {}),
...(r.result.type_warnings ? { type_warnings: r.result.type_warnings } : {}),
} : {}),
...(r.error ? { error: r.error } : {}),
...r.contract,
...(embedBackfillBySource.has(r.sourceId)
? { embed_backfill: embedBackfillBySource.get(r.sourceId) }
: {}),
...(watermarks.get(r.sourceId) ? { watermark: watermarks.get(r.sourceId) } : {}),
}));
emitJson(JSON.stringify({
schema_version: 1,
sources: sortedSources,
parallel: effectiveParallel,
ok_count: okCount,
error_count: errCount,
skipped_count: perSourceResults.filter((r) => r.status === 'skipped_missing_path').length,
// #4684: the cost-gate status object rides inside the ONE envelope.
...(costGate ? { cost_gate: costGate } : {}),
}));
}
/** Single-source sync (one run, or `--watch`). */
/** GST-2869: 128 + the signal number, the exit a supervisor that sent the signal expects. */
const terminationExit = (signal: NodeJS.Signals): number => signal === 'SIGHUP' ? 129 : 143;
/**
* GST-2869: while a managed pass this CLI runs is live, the first SIGTERM or SIGHUP aborts `interrupt` with `SyncTerminated`,
* so the pass records `resumable` / `terminated` and the run prints its result, then exits 128 + n. A second signal, a signal
* with no live pass, or a run whose save or result outlasts process-cleanup's window takes today's cleanup-and-exit. Never under another
* owner (a worker's sync job): the newest owner would take that owner's signals.
*/
async function withTerminationOwner<T>(ownSignals: boolean, interrupt: AbortController, run: () => Promise<T>): Promise<T> {
const cleanup = await import('../../core/process-cleanup.ts');
if (!ownSignals || cleanup.hasSignalOwner()) return run();
const { SyncTerminated } = await import('../../core/persistence/sync-stop-class.ts');
const { managedPassLive } = await import('../../core/persistence/sync-pass-episode.ts');
const release = cleanup.registerSignalOwner('sync', signal => {
if (interrupt.signal.aborted || !managedPassLive()) { release(); process.kill(process.pid, signal); return; }
process.stderr.write(`[sync] ${signal}: saving the pass and stopping; send it again to exit now\n`);
interrupt.abort(new SyncTerminated(signal));
terminationFallback = setTimeout(() => { release(); void cleanup.triggerCleanupAndExit(terminationExit(signal)); }, cleanup.CLEANUP_DEADLINE_MS);
terminationFallback.unref?.();
}, ['SIGTERM', 'SIGHUP']);
try { return await run(); } finally { release(); }
}
let terminationFallback: ReturnType<typeof setTimeout> | undefined;
/** GST-2869: drops the fallback a termination armed once the whole run is done, so the result and its reads stay inside the window. */
export function clearTerminationFallback(): void {
clearTimeout(terminationFallback);
terminationFallback = undefined;
}
/** GST-2869: the exit of a run `withTerminationOwner` stopped, or null. */
async function terminatedExit(interrupt: AbortController): Promise<number | null> {
const { SyncTerminated } = await import('../../core/persistence/sync-stop-class.ts');
const reason: unknown = interrupt.signal.reason;
return reason instanceof SyncTerminated ? terminationExit(reason.signal) : null;
}
async function runSingleSourceSync(
engine: BrainEngine,
flags: SyncFlags & SyncFanoutFlags,
input: { sourceId: string; companyPolicy: Awaited<ReturnType<typeof getCompanyBrainProfile>> | null; noEmbed: boolean; resumeCommand: string; args: string[]; ownSignals: boolean },
): Promise<void> {
const {
repoPath, watch, interval, dryRun, full, noPull, noBulk, lanes, noExtract, skipFailed, retryFailed, resetCheckpoint, noSchemaPack,
includeGitignored, workingTree, jsonOut, redactPaths, yesFlag, noAutoEmbed, strategyArg, srcSubpath,
excludePatterns, includeHiddenPatterns, concurrency, timeoutSeconds,
} = flags;
const { sourceId, companyPolicy, noEmbed, resumeCommand, args, ownSignals } = input;
// #7630: before the cost gate walks the directory.
await assertSyncAnchorAbsolute(engine, { sourceId, repoPath });
// v0.41.13.0 (T6) — single-source --timeout: same per-source AbortController
// shape as the --all runOne closure. Timer scoped to this CLI invocation;
// try/finally clears it after performSync resolves (or throws).
const singleSourceController = timeoutSeconds !== undefined ? new AbortController() : undefined;
const singleSourceTimer = timeoutSeconds !== undefined
? setTimeout(() => singleSourceController!.abort(), timeoutSeconds * 1000)
: undefined;
singleSourceTimer?.unref?.();
// #1633 (Part B): graceful SIGINT cancel. process-cleanup.ts owns SIGTERM
// (lock release + exit) and the watchdog owns the hard deadline; SIGINT is
// deliberately left to callers, so Ctrl-C during a long sync aborts the
// in-flight import cleanly (performSync returns `partial`, bookmark unadvanced,
// lock released by its own finally) instead of a hard cut.
const singleSourceInterrupt = new AbortController();
const onSingleSourceSigint = () => { try { singleSourceInterrupt.abort(new Error('SIGINT')); } catch { /* */ } };
const opts: SyncOpts = {
repoPath, dryRun, full, noPull, noEmbed, noExtract, skipFailed, retryFailed, resetCheckpoint, noSchemaPack, includeGitignored, workingTree, sourceId,
strategy: strategyArg, concurrency,
srcSubpath,
exclude: excludePatterns.length > 0 ? excludePatterns : undefined,
includeHidden: includeHiddenPatterns.length > 0 ? includeHiddenPatterns : undefined,
signal: composeAbortSignals(singleSourceInterrupt.signal, singleSourceController?.signal),
drain: true, noBulk, lanes, followDrainLease: true, drainHolder: { kind: 'cli', id: null },
...(yesFlag ? { costApproved: true } : {}),
};
// v0.42.42.0 (#2139, Step 4b): single-source `gbrain sync` gets the SAME
// inline cost gate as `--all`. Previously single-source embedded inline with
// NO gate (only rail: the ≤100-file inline cap). Single-source always embeds
// INLINE (not the parallel-deferred fan-out), so mode is forced 'inline'.
// Skipped on --no-embed, --dry-run (performSync's own dry-run previews and
// spends nothing), and watch. Non-TTY above floor AUTO-DEFERS — so adding
// the gate can never wedge an existing cron; it converts silent ungated
// inline spend into informed inline-or-deferred spend.
let singleSourceAutoDefer = false;
let singleSourceNoWorkerSurface = false;
let singleCostGate: Record<string, unknown> | undefined;
if (!noEmbed && !dryRun && !watch) {
const gateRows = await engine.executeRaw<{ local_path: string | null; config: Record<string, unknown>; last_commit: string | null; chunker_version: string | null }>(
`SELECT local_path, config, last_commit, chunker_version FROM sources WHERE id = $1`,
[sourceId],
);
if (gateRows.length > 0) {
const gate = await resolveSingleSyncEmbedPlan(engine, {
sourceId,
local_path: absoluteStoredPath(gateRows[0].local_path) ?? repoPath ?? null,
config: gateRows[0].config ?? {},
last_commit: gateRows[0].last_commit,
chunker_version: gateRows[0].chunker_version,
}, {
dryRun: false,
jsonOut, yesFlag, full, includeGitignored, noAutoEmbed, noPull,
});
singleSourceNoWorkerSurface = gate.workerSurface.status === 'no_worker_surface';
singleCostGate = gate.costGate;
if (gate.stop) return;
if (gate.autoDeferEmbeds) {
opts.noEmbed = true;
// #6696: a runtime constraint, not an explicit flag: adopting an unfinished cursor's options must not lift it.
opts.embedDeferred = true;
singleSourceAutoDefer = true;
}
}
}
if (retryFailed) {
// v0.42.42.0 (#2139, D13C): scope the retry count to THIS source — rows
// carry source_id (#1939), so a single-source retry shouldn't report
// another source's failures.
// #6712 PR 1: a managed source has one cursor, whatever options a run passes, so every recorded failure is this retry's.
const managedBrain = await readManagedMode(engine);
let failures: Array<{ source_id: string }> = unacknowledgedSyncFailures().filter(f => f.source_id === sourceId);
if (managedBrain) failures = await readManagedSyncFailures(engine, [sourceId]);
const legacyRows = managedBrain ? legacyLedgerRows(sourceId).length : 0;
if (legacyRows) slog(`${legacyRows} legacy ledger row(s) of this source were recorded by the pre-managed sync and are not retried by a managed sync; this run settles every one whose file it imports.`);
if (failures.length === 0) {
slog(legacyRows ? 'No managed sync failures recorded; checking the durable sync cursor for unfinished or failed writes.' : 'No local ledger entries; checking the durable sync cursor for unfinished or failed writes.');
} else {
slog(`Retrying ${failures.length} previously-failed file(s)...`);
// #6712 PR 1: a failure whose run no cursor holds any more (an older gbrain's, under its own key) is not settled by
// a sync; the orphan repair retires it.
const orphaned = managedBrain ? (await (await import('../../core/persistence/managed-sync-orphans.ts')).classifyManagedSyncOrphans(engine, [sourceId]))
.orphans.filter(orphan => orphan.kind === 'failure_row').length : 0;
if (orphaned) slog(`${orphaned} of them belong to no sync run any more (an older gbrain's run under another cursor key); no sync settles them: preview and retire them with gbrain repair managed-sync-orphans --source ${sourceId}.`);
}
}
if (!watch) {
// v0.41.13.0 (T6): try/finally clears the single-source timer so it
// doesn't fire after performSync resolves OR throws.
let result: SyncResult;
// #6712 PR 4a: a managed dry run also reports the read-only convergence shadow (sync-convergence.ts).
const readConvergence = async () => (await import('../../core/persistence/sync-pass.ts')).dryRunConvergence(engine,
{ sourceId, full: !!full, workingTree, redactPaths: !!redactPaths, noEmbed, noExtract: !!noExtract, noSchemaPack: !!noSchemaPack });
process.on('SIGINT', onSingleSourceSigint);
try {
// #6317 (D1 a′): on a managed Postgres brain whose host a live serve owns, the drain runs inside that serve as this
// CLI's writer and returns the same SyncResult; every other case keeps this process's own consumer.
const delegated = companyPolicy ? null : await (await import('../sync-delegate.ts')).maybeDelegateManagedSyncToServe(engine, args, opts);
result = delegated?.kind === 'delegated' ? delegated.result : await withTerminationOwner(ownSignals, singleSourceInterrupt, () => performSync(engine, opts));
} catch (error) {
// A refused managed dry run (a case- or separator-only spelling pair) still carries its convergence receipt in the
// refusal's envelope; the refusal and its exit code are unchanged.
const { OperationError } = await import('../../core/ops/contract.ts');
if (!(dryRun && jsonOut && !companyPolicy && error instanceof OperationError)) throw error;
const convergence = await readConvergence();
if (!convergence) throw error;
const [{ writeCliError }, { setCliExitVerdict }] = await Promise.all([import('../../cli/cli-error.ts'), import('../../core/cli-force-exit.ts')]);
setCliExitVerdict(writeCliError(error, 'sync', { json: true, extra: { convergence }, emit: emitJson }));
return;
} finally {
if (singleSourceTimer !== undefined) clearTimeout(singleSourceTimer);
process.off('SIGINT', onSingleSourceSigint);
}
printSyncResult(result, jsonOut ? process.stderr : process.stdout);
for (const line of formatDrainSummary(result, resumeCommand, sourceId)) (jsonOut ? process.stderr : process.stdout).write(line + '\n');
// #3068: a pull_failed partial is NOT a success — unlike timeout-class
// partials (which converge on retry), a failing pull will not self-heal.
// Exit non-zero so cron/monitoring sees the wedge instead of a green run.
// Routed through the owned verdict channel (NOT bare `process.exitCode`,
// which PGLite's Emscripten runtime clobbers mid-run — see
// src/core/cli-force-exit.ts).
// #5984: the managed drain's verdict decides; a drain that stopped at its deadline with a pending write is resumable (exit 0).
if (syncOutcome(result) === 'blocked' || isFailedPartial(result)) setCliExitVerdict(1);
// GST-2869: a run a termination signal stopped exits 128 + n, ahead of every other verdict, and skips the post-pass extras.
const terminated = await terminatedExit(singleSourceInterrupt);
if (terminated !== null) setCliExitVerdict(terminated);
// v0.42.7 (#1696, D5): extraction-lag nudge after a completed single-source
// sync. Fire on every non-error completion (synced | first_sync | up_to_date)
// — NOT just 'synced'; a fresh/--full import (`first_sync`) is the biggest
// un-extracted backlog. Scoped to this source; best-effort, stderr-only.
if (terminated === null && shouldNudgeAfterSync(result.status)) await maybeExtractionNudge(engine, sourceId);
// Monthly backup-coverage: the sync CLI legitimately holds the engine
// (trusted local), so the stale-only compute piggybacks here — covering
// active CLI users without any serve involvement. Dry-run stays pure.
if (terminated === null && result.status !== 'dry_run') await maybeBackupCoverageRefresh(engine);
// Issue #2 + eng-review pass-2 finding #1 + Codex P1: manage .gitignore ONLY
// on successful sync. Skip on dry-run (don't mutate disk in preview mode)
// and blocked_by_failures (sync state is inconsistent — defer .gitignore
// until next clean run). v0.41.13.0 (T7 / D-V3-5): partial also skips —
// conservative posture matches blocked_by_failures. Resolve the effective
// repo path so the wire-up fires in the common case where the user runs
// `gbrain sync` without passing --repo every time.
if (
!companyPolicy && result.status !== 'dry_run' &&
result.status !== 'blocked_by_failures' &&
result.status !== 'partial'
) {
const effectiveRepoPath = opts.repoPath ?? absoluteStoredPath(await getDefaultSourcePath(engine));
if (effectiveRepoPath && !companyPolicy) {
manageGitignoreAtGitRoot(effectiveRepoPath, engine.kind);
}
}
// v0.42.42.0 (#2139, Step 4b): the inline gate auto-deferred this run's
// embeds (non-TTY, above floor) — enqueue a capped backfill job so the
// NULL-embedded chunks get embedded out of band instead of being stranded.
// Mirror --all: an intrinsic >100-file deferral is delivered on a worker-backed
// engine when federated v2 is on, and as a manual drain when no worker exists.
// An explicit --no-embed never submits a backfill.
let singleV2Enabled = false;
if (result.embedDeferralReason === 'large_sync' && !noEmbed && !singleSourceNoWorkerSurface) {
const { isFederatedV2Enabled } = await import('../../core/feature-flags.ts');
singleV2Enabled = await isFederatedV2Enabled(engine);
}
let singleEmbedBackfill: SyncEmbedBackfillOutcome | undefined;
if (
!companyPolicy && (singleSourceAutoDefer || (
result.embedDeferralReason === 'large_sync' && (singleSourceNoWorkerSurface || singleV2Enabled)
)) &&
result.status !== 'dry_run' &&
result.status !== 'up_to_date' &&
result.status !== 'partial' && syncProducedEmbeddableContent(result)
) {
try {
singleEmbedBackfill = await resolveSyncEmbedBackfill(engine, sourceId, {
reason: 'sync_autodefer', autoSubmitDisabled: noAutoEmbed,
});
process.stderr.write(` → ${formatSyncEmbedBackfillOutcome(singleEmbedBackfill)}.\n`);
} catch (e) {
process.stderr.write(` → embed-backfill submission failed: ${e instanceof Error ? e.message : String(e)}\n`);
}
}
const capacity = await syncCapacityNotices(engine, jsonOut, result);
const convergence = jsonOut && result.status === 'dry_run' && !companyPolicy ? await readConvergence() : undefined;
if (jsonOut) {
const watermark = await (await import('../../core/persistence/applied-watermark.ts')).readAppliedWatermark(engine, sourceId).catch(() => null);
emitJson(JSON.stringify({ ...buildSingleSyncJsonEnvelope(sourceId, result, singleEmbedBackfill, singleCostGate), ...(convergence ? { convergence } : {}), ...(watermark ? { watermark } : {}),
...(result.managedWrite ? { managed_write: result.managedWrite } : {}), ...drainJsonFields(result, resumeCommand, sourceId),
...(result.tail ? { tail: syncTailJsonFields(result).tail } : {}), ...capacity }));
}
return;
}
// Watch mode
let consecutiveErrors = 0;
slog(`Watching for changes every ${interval}s... (Ctrl+C to stop)`);
while (true) {
try {
const result = await performSync(engine, { ...opts, full: false });
consecutiveErrors = 0;
const ts = new Date().toISOString().slice(11, 19);
if (result.status === 'synced') {
slog(`[${ts}] Synced: +${result.added} ~${result.modified} -${result.deleted} R${result.renamed}`);
} else if (result.drain && result.drain.outcome !== 'synced') {
for (const line of formatDrainSummary(result, resumeCommand, sourceId)) slog(`[${ts}] ${line}`);
}
// Same gate as non-watch: only manage .gitignore on successful sync.
// v0.41.13.0 (T7 / D-V3-5): partial joins the deferred posture.
// Same repo-resolution path so watch mode catches the implicit-resolved case.
if (
!companyPolicy && result.status !== 'dry_run' &&
result.status !== 'blocked_by_failures' &&
result.status !== 'partial'
) {
const effectiveRepoPath = opts.repoPath ?? absoluteStoredPath(await getDefaultSourcePath(engine));
if (effectiveRepoPath) {
manageGitignoreAtGitRoot(effectiveRepoPath, engine.kind);
}
}
} catch (e: unknown) {
consecutiveErrors++;
const msg = e instanceof Error ? e.message : String(e);
console.error(`[${new Date().toISOString().slice(11, 19)}] Sync error (${consecutiveErrors}/5): ${msg}`);
if (consecutiveErrors >= 5) {
console.error(`5 consecutive sync failures. Stopping watch.`);
process.exit(1);
}
}
await new Promise(r => setTimeout(r, interval * 1000));
}
}