Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/** Full-tree sync: `performFullSync` (first sync, fallbacks, --full). */
import { assertSourceFilesystemActive } from '../../core/minions/source-filesystem.ts';
import { currentJobSignal } from '../../core/minions/submission-authority.ts';
import { currentCompanyBrainSync, softDeleteSyncPages } from '../../core/company-brain/profile.ts';
import { relative } from 'path';
import type { BrainEngine } from '../../core/engine.ts';
import { DELETE_BATCH_SIZE } from '../../core/engine-constants.ts';
import { collectSyncableFiles } from '../import.ts';
import {
isSyncable,
isPoisonedPath,
sanitizePathForDisplay,
unsyncableReason,
matchesAnyGlob,
loadSyncFailures,
formatCodeBreakdown,
formatFailedFileList,
applySyncFailureGate,
isSkippablePath,
resolveAutoSkipThreshold,
summarizeFailuresByCode,
STRICT_REQUIRES_MANAGED,
strictRequiresManagedAdvice,
isEmbeddingInfraCode,
DEFAULT_SOURCE_ID,
RENAME_SENTINEL_PREFIX,
readMetafilePagesMode,
type MetafilePagesMode,
} from '../../core/sync.ts';
import { diskMetafileHeads, firstMetafileHead, gitMetafileHeads, metafilePagePredicate, recordMetafilePagesApplied } from '../../core/sync-metafile-pages.ts';
import { chunkerStamp } from '../../core/chunkers/code.ts';
import { autoConcurrency } from '../../core/sync-concurrency.ts';
import { slog, serr } from '../../core/console-prefix.ts';
import { newestCommitMs } from '../../core/source-health.ts';
import { gitRelativePath } from '../../core/sync-git.ts';
import {
readSyncAnchor,
writeSyncAnchor,
writeChunkerVersion,
type SlugRootMode,
} from '../../core/sync-anchor.ts';
import { buildPartialResult } from '../../core/sync-lock.ts';
import { noteCommittedProgress } from '../../core/forward-progress.ts';
import {
MASS_RECONCILE_RATIO,
planReconcileDeletes,
listEverCommittedPaths,
massReconcileAllowed,
readSyncDeleteCaps,
syncDeleteTrip,
syncDeleteTripText,
} from '../../core/sync-reconcile.ts';
import type { SyncOpts, SyncResult } from '../sync.ts';
import { heldRenameOrigins, holdRefusedImport, moveHeldRenames, noteScreenedImport, settleFullSyncHolds, wouldHoldFields, type LegacyHolds } from './holds.ts';
import { sweepOrphanedRenameSentinels } from './rename-reconcile.ts';
import { classifyItemFailure } from '../../core/item-failure.ts';
import { fileHistory, scanClassicReasons, withReasonNotices } from './reasons.ts';
import { OperationError } from '../../core/ops/contract.ts';
/** GBRA-287: one DB-only page a full sync kept but could not re-export: its source path, the failure and what to do. */
export interface ReexportFailure { slug: string; path: string; code: string; message: string; fix: string }
export async function performFullSync(
engine: BrainEngine,
// #753/#774: the three roots resolved once at the top of performSyncInner.
// gitContextRoot — git repo root (git ops, slug base for scoped syncs)
// syncScopeRoot — where files are walked/imported (== gitContextRoot
// when no subpath scope is active)
// anchorPath — what gets written back to sync.repo_path/local_path
// slugRootMode — #4342 sticky namespace anchor (git-root|source-root)
roots: { gitContextRoot: string; syncScopeRoot: string; anchorPath: string; slugRootMode: SlugRootMode },
headCommit: string,
opts: SyncOpts,
holds: LegacyHolds | null = null,
): Promise<SyncResult> {
const { gitContextRoot, syncScopeRoot, anchorPath, slugRootMode } = roots;
const company = currentCompanyBrainSync(opts.sourceId);
// Scoped 'git-root' sync → slugs/source_path are git-root-relative (matches
// the incremental path's git-diff paths). Unscoped OR pinned 'source-root'
// (#4342) → undefined (dir-relative — local_path IS the slug base).
const slugRoot =
syncScopeRoot !== gitContextRoot && slugRootMode === 'git-root' ? gitContextRoot : undefined;
// Dry-run: walk the scope, count syncable files, return without writing.
// Fixes the silent-write-on-dry-run bug where performFullSync called
// runImport unconditionally regardless of opts.dryRun.
//
// v0.31.2 (codex C6): use the strategy-aware walker. Pre-fix this
// hardcoded `collectMarkdownFiles(repoPath)` and filtered with
// default-markdown `isSyncable(rel)`, so `gbrain sync --strategy
// code --dry-run` always reported zero files even when ~1500 code
// files were waiting.
// #5988: paths are relative to the slug base, the same keys holds and the ledger use.
const holdRoot = slugRoot ?? syncScopeRoot;
// GBRA-150: one sync.metafile_pages read for every walk below; the commit a gone file was last synced at is read
// before the bookmark advances, so the reconcile judges a deleted typed metafile by its last synced bytes.
const metafilePages = await readMetafilePagesMode(engine);
const previousCommit = await readSyncAnchor(engine, opts.sourceId, 'last_commit');
if (opts.dryRun) return fullSyncDryRun(engine, syncScopeRoot, headCommit, { ...opts, metafilePages }, { holds, holdRoot });
// v0.22.13 (PR #490 A1 + Q5): full sync is always "large" by definition
// (entire working tree). Auto-concurrency fires unconditionally for Postgres;
// PGLite stays serial because its engine is single-connection. Routes the
// policy through autoConcurrency() so it stays consistent with incremental
// sync and the jobs handler.
const FULL_SYNC_LARGE_MARKER = Number.MAX_SAFE_INTEGER;
const fullConcurrency = autoConcurrency(engine, FULL_SYNC_LARGE_MARKER, opts.concurrency);
slog(`Running full import of ${syncScopeRoot}${fullConcurrency > 1 ? ` (${fullConcurrency} workers)` : ''}...`);
const { runImport, ImportAbortError } = await import('../import.ts');
const importArgs = [syncScopeRoot];
if (opts.noEmbed) importArgs.push('--no-embed');
if (opts.includeGitignored) importArgs.push('--include-gitignored');
if (fullConcurrency > 1) importArgs.push('--workers', String(fullConcurrency));
// v0.31.2: thread strategy through so code-strategy first sync
// actually enumerates code files (closes bug 1).
// v0.30.x: thread sourceId so performFullSync routes pages to the named
// source (incremental path already does this).
// #753/#774: thread exclude (--exclude CLI) + slugRoot (monorepo subdir).
const _fullImportT0 = Date.now();
serr(`[gbrain phase] sync.fullsync.import start strategy=${opts.strategy ?? 'markdown'}`);
opts.onProgress?.({ phase: 'full_import' });
await moveHeldRenames(engine, holds, holdRoot);
// GBRA-217: a re-import of a source that already has pages reads its reasons from the whole history (a first sync creates
// every page, so it archives no snapshot to annotate and runs no scan).
const [known] = await engine.executeRaw<{ any: boolean }>('SELECT EXISTS (SELECT 1 FROM pages WHERE source_id = $1) AS any', [opts.sourceId ?? 'default']);
const reasons = await scanClassicReasons(engine, { sourceId: opts.sourceId, company, anchorPath, gitRoot: gitContextRoot, from: null, to: headCommit,
imports: known?.any === true ? 1 : 0 });
let result: import('../import.ts').RunImportResult;
try {
result = await runImport(engine, importArgs, {
signal: opts.signal,
commit: headCommit,
history: filePath => fileHistory(reasons, filePath),
strategy: opts.strategy,
sourceId: opts.sourceId,
exclude: opts.exclude,
includeHidden: opts.includeHidden,
includeGitignored: opts.includeGitignored,
slugRoot,
// issue #1939: performFullSync owns the failure ledger + bookmark via the
// shared gate below; don't let runImport double-record or write its own.
managedBookmark: true,
noticeRun: 'sync',
// #5988: a held rename destination whose old page changed is held, never imported as a new page.
heldPaths: new Set(holds?.heldPaths),
// #5988: content refusals are held, not failed (never gating the bookmark).
onFileResult: async (path, filePath, fileResult) => {
noteScreenedImport(holds, path, fileResult);
if (await holdRefusedImport(engine, holds, path, filePath, fileResult)) return 'held';
// GBRA-133 PR 3a: an imported page is committed work, so the sync deadline extends while a full import lands pages.
if (fileResult.status === 'imported') noteCommittedProgress();
return undefined;
},
});
if (opts.signal?.aborted) throw new ImportAbortError('interrupted', 1, result);
} catch (error) {
assertSourceFilesystemActive(true);
if (!(error instanceof ImportAbortError) || !error.partialResult || !opts.signal?.aborted || currentJobSignal()?.aborted) throw error;
const partial = error.partialResult;
return buildPartialResult({
fromCommit: await readSyncAnchor(engine, opts.sourceId, 'last_commit'), toCommit: headCommit,
filesImported: partial.imported, pagesAffected: [], chunksCreated: partial.chunksCreated,
added: partial.imported, modified: 0, deleted: 0, renamed: 0, reason: 'timeout',
});
}
serr(
`[gbrain phase] sync.fullsync.import done ${Date.now() - _fullImportT0}ms ` +
`imported=${result.imported} skipped=${result.skipped} errors=${result.errors}`,
);
// issue #1939 — gate the full-sync bookmark through the SAME shared ledger as
// the incremental path (Codex #6: a wedge here on first/forced sync was
// previously unreachable by the valve). A full re-import is authoritative for
// the whole tree, so any previously-tracked failing path that ISN'T failing
// now has been resolved → clear it (resets its auto-skip streak); current
// failures still climb their attempts.
const fullSourceId = opts.sourceId ?? DEFAULT_SOURCE_ID;
const fullFailureSet = new Set(result.failures.map(f => f.path));
if (holds?.existing.size || holds?.retryPaths.size) {
const walked = new Set(collectSyncableFiles(syncScopeRoot, { strategy: opts.strategy ?? 'markdown', includeGitignored: opts.includeGitignored,
includeHidden: opts.includeHidden, metafilePages }).map(abs => relative(holdRoot, abs)));
await settleFullSyncHolds(engine, holds, { root: holdRoot, walked, failed: fullFailureSet });
}
const fullSucceeded = loadSyncFailures()
.filter(e => e.source_id === fullSourceId && isSkippablePath(e.path) && !fullFailureSet.has(e.path))
.map(e => e.path);
const advanceFull = async (): Promise<void> => {
// Persist sync state so the next sync is incremental. Routed through
// writeSyncAnchor so --source pins the right sources row.
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', headCommit, newestCommitMs(gitContextRoot), gitContextRoot);
await engine.setConfig('sync.last_run', new Date().toISOString());
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
await writeChunkerVersion(engine, opts.sourceId, chunkerStamp());
await recordMetafilePagesApplied(engine, fullSourceId, metafilePages);
};
const fullGate = await applySyncFailureGate({
sourceId: fullSourceId,
failedFiles: result.failures,
succeededPaths: fullSucceeded,
commit: headCommit,
skipFailed: !company && opts.skipFailed === true,
...(company ? { threshold: 0 } : {}),
advance: advanceFull,
});
// #3479 blocker 2 — the same orphaned-`<rename:…>`-sentinel self-heal the
// incremental path applies (a full sync is often exactly the operator's
// reset move after a wedge). AFTER and OUTSIDE the gate (#3583): a gate
// that throws never clears anything, and the sweep carries the full
// clear-then-verify-restore semantics.
await sweepOrphanedRenameSentinels(engine, fullSourceId, fullFailureSet);
if (!fullGate.advanced) return withReasonNotices(await reportBlockedFullSync(engine, opts, anchorPath, headCommit, result, fullGate), reasons, opts.sourceId);
if (fullGate.acknowledged > 0) {
serr(` Acknowledged ${fullGate.acknowledged} failure(s) and advanced past them.`);
}
if (fullGate.autoSkipped.length > 0) {
serr(
`\n Auto-skipped ${fullGate.autoSkipped.length} file(s) that failed >= ` +
`${resolveAutoSkipThreshold()} consecutive syncs. These pages are NOT indexed; ` +
`'gbrain doctor' will warn until they're fixed.`,
);
}
const { deleted: reconciledDeletes, reexportFailures } = await reconcileFullSyncDeletes(engine, opts, { company, gitContextRoot, syncScopeRoot, slugRoot, metafilePages, previousCommit });
// #3479 blocker 2 — the post-gate sweep above ran BEFORE this reconcile,
// so a `<rename:…>` sentinel whose stale row the reconcile just removed
// would stay open until the NEXT run. Sweep again afterwards: a full sync
// is the operator's usual reset move, and it should converge in one run.
await sweepOrphanedRenameSentinels(engine, fullSourceId, fullFailureSet);
// Full sync doesn't track pagesAffected, so fall back to embed --stale.
// v0.37 fix wave (Lane D.3 + CDX2-8): switched to runEmbedCore for the
// same reason as the incremental path — surface dim-mismatch via hint
// instead of silently swallowing or killing the process.
let embedded = 0;
if (!opts.noEmbed) {
try {
const { runEmbedCore } = await import('../embed.ts');
await runEmbedCore(engine, { stale: true });
embedded = result.imported;
} catch (e: unknown) {
const { EmbeddingDimMismatchError } = await import('../embed.ts');
if (e instanceof EmbeddingDimMismatchError) {
serr('\n' + e.recipeMessage + '\n');
serr(`Tip: pass --no-embed to sync without embedding, then`);
serr(`run 'gbrain embed --stale' after fixing the schema.\n`);
}
// Other errors stay best-effort.
}
}
// #6722: a full sync walked every page of the source, so its large and embed_skip pages are all reported.
const { readPageSizeStates, addSizeState } = await import('../../core/page-embedding-budget.ts');
const sizeStates = (await readPageSizeStates(engine, fullSourceId, null))
.reduce<import('../../core/page-embedding-budget.ts').SyncSizeStates | undefined>((tally, state) => addSizeState(tally, state), undefined);
return withReasonNotices({
status: 'first_sync',
fromCommit: null,
toCommit: headCommit,
added: result.imported,
modified: 0,
deleted: reconciledDeletes,
renamed: 0,
chunksCreated: result.chunksCreated,
embedded,
pagesAffected: [],
...(sizeStates ? { size_states: sizeStates } : {}),
// Warning aggregates ride the result for worker/JSON consumers — a full
// sync that only prints to a daemon's stderr hides them from cron
// topologies (codex re-review; same rationale as the incremental path).
...(result.malformedSkipped ? { malformedSkipped: result.malformedSkipped } : {}),
...(result.type_warnings ? { type_warnings: result.type_warnings } : {}),
...(result.resealed ? { resealed: result.resealed } : {}),
...(result.privacy_label_notice ? { privacy_label_notice: result.privacy_label_notice } : {}),
...(reexportFailures.length ? { reexport_failures: reexportFailures } : {}),
}, reasons, opts.sourceId);
}
async function fullSyncDryRun(engine: BrainEngine, syncScopeRoot: string, headCommit: string, opts: SyncOpts & { metafilePages: MetafilePagesMode },
hold: { holds: LegacyHolds | null; holdRoot: string }): Promise<SyncResult> {
const dryRunMalformed: string[] = [];
let allFiles = collectSyncableFiles(syncScopeRoot, {
strategy: opts.strategy ?? 'markdown',
includeGitignored: opts.includeGitignored,
onExcluded: (rel) => { dryRunMalformed.push(rel); },
includeHidden: opts.includeHidden,
metafilePages: opts.metafilePages,
});
if (opts.exclude && opts.exclude.length > 0) {
allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(syncScopeRoot, abs), opts.exclude));
}
slog(
`Full-sync dry run (strategy=${opts.strategy ?? 'markdown'}): ` +
`${allFiles.length} file(s) would be imported ` +
`from ${syncScopeRoot} @ ${headCommit.slice(0, 8)}.`,
);
if (dryRunMalformed.length > 0) {
slog(
` ${dryRunMalformed.length} file(s) would be skipped: malformed filename ` +
`(brackets/control chars; rename to import): ` +
dryRunMalformed.slice(0, 20).map(sanitizePathForDisplay).join(', ') +
(dryRunMalformed.length > 20 ? `, … (+${dryRunMalformed.length - 20} more)` : ''),
);
}
return {
status: 'dry_run',
fromCommit: null,
toCommit: headCommit,
added: allFiles.length,
modified: 0,
deleted: 0,
renamed: 0,
chunksCreated: 0,
embedded: 0,
pagesAffected: [],
...(await wouldHoldFields(engine, hold.holds, hold.holdRoot, allFiles.map(abs => relative(hold.holdRoot, abs)))),
};
}
async function reportBlockedFullSync(
engine: BrainEngine,
opts: SyncOpts,
anchorPath: string,
headCommit: string,
result: import('../import.ts').RunImportResult,
fullGate: Awaited<ReturnType<typeof applySyncFailureGate>>,
): Promise<SyncResult> {
const codeBreakdown = formatCodeBreakdown(result.failures);
const strictRefused = summarizeFailuresByCode(result.failures).some(c => c.code === STRICT_REQUIRES_MANAGED);
if (fullGate.sentinelBlocked) {
// #3479 review — say WHICH sentinel fired: a `<rename:…>` block here
// used to print the history-changed message, pointing the operator at
// a force-push that never happened.
const fullRenameRows = result.failures.filter(f => f.path.startsWith(RENAME_SENTINEL_PREFIX));
if (fullRenameRows.length > 0) {
serr(
`\nFull sync blocked: a rename left a stale duplicate that could not be removed:\n` +
`${fullRenameRows.map(f => ` ${f.path}: ${f.error}`).join('\n')}\n\n` +
`If the delete keeps failing in your environment, remove the stale row ` +
`yourself: 'gbrain delete <stale-slug>' with the stale slug named above ` +
`(only rows whose backing file is gone are ever named — never a live page). ` +
`A sentinel reading 'stale row ?' names nothing on purpose: that run could not ` +
`prove ANY row stale — fix the unreadable tracked file it reports instead. ` +
`The sentinel then clears on the next sync.`,
);
} else {
serr(`\nFull sync blocked: repository history changed during sync.\n${codeBreakdown}`);
}
} else {
const fileFailCount = result.failures.filter(f => isSkippablePath(f.path)).length;
// #3875: code-aware copy — provider-infra failures must not be routed
// to --skip-failed (same rationale as the incremental gate above).
const infraCodes = summarizeFailuresByCode(result.failures).filter(c => isEmbeddingInfraCode(c.code));
if (strictRefused) {
serr(`\nFull sync blocked: ${fileFailCount} file(s) failed (${STRICT_REQUIRES_MANAGED}):\n${codeBreakdown}\n${formatFailedFileList(result.failures)}\n\n${strictRequiresManagedAdvice(opts.sourceId)}`);
} else if (infraCodes.length > 0) {
serr(
`\nFull sync blocked: ${fileFailCount} file(s) failed — embedding provider errors:\n` +
`${codeBreakdown}\n\n` +
`These are provider-health failures (timeout / rate limit / quota), not bad ` +
`files — do NOT use --skip-failed for them. Check the embedding provider, ` +
`then re-run 'gbrain sync --full'.`,
);
} else {
serr(
`\nFull sync blocked: ${fileFailCount} file(s) failed:\n` +
`${codeBreakdown}\n${formatFailedFileList(result.failures)}\n\n` +
`Pinpoint a file with 'gbrain frontmatter validate <path>' (--fix auto-repairs), ` +
`fix the YAML and re-run, or use '--skip-failed'. A file ` +
`that keeps failing auto-skips after ${resolveAutoSkipThreshold()} consecutive syncs.`,
);
}
}
await engine.setConfig('sync.last_run', new Date().toISOString());
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
return {
status: 'blocked_by_failures',
fromCommit: null,
toCommit: headCommit,
added: 0, modified: 0, deleted: 0, renamed: 0,
chunksCreated: result.chunksCreated,
embedded: 0,
pagesAffected: [],
failedFiles: result.failures.length,
failureCodes: summarizeFailuresByCode(result.failures),
...(strictRefused ? { failureAdvice: strictRequiresManagedAdvice(opts.sourceId) } : {}),
};
}
/**
* Soft-delete file-backed pages whose source file is gone (advancing full
* syncs only). Returns the number of pages that transitioned.
*/
async function reconcileFullSyncDeletes(
engine: BrainEngine,
opts: SyncOpts,
input: { company: ReturnType<typeof currentCompanyBrainSync>; gitContextRoot: string; syncScopeRoot: string; slugRoot: string | undefined;
metafilePages: MetafilePagesMode; previousCommit: string | null },
): Promise<{ deleted: number; reexportFailures: ReexportFailure[] }> {
const { company, gitContextRoot, syncScopeRoot, slugRoot, metafilePages, previousCommit } = input;
// #1970 (F-A): runImport is import-only — it never purges pages whose backing
// file was deleted since the last sync. A full re-import is authoritative for
// the whole tree, so reconcile deletes here too (this is what makes the
// object-absent fallback at performSyncInner correct for deletes, not just
// imports). Runs only on an advancing full sync (we're past the
// !fullGate.advanced early-return).
//
// SAFETY — must NOT re-introduce the #1433 stale-page data loss. A page is
// deleted ONLY when ALL three hold:
// 1. source_path != null → file-backed pages only; put_page/manual
// pages (null source_path) are never swept.
// 2. isSyncable(source_path) → excludes metafiles (README/log.md, the
// #1433 class) AND the wrong strategy (a markdown sync can't delete a
// code page, and vice versa).
// 3. source_path ∉ current → the backing file is genuinely gone from the
// working tree (collectSyncableFiles == the same enumeration runImport
// used, so paths are in the identical relative form as source_path).
// Skipped on the legacy no-sourceId path (the batch delete primitives require
// a sourceId; matches every other source-scoped feature).
let reconciledDeletes = 0;
const reexportFailures: ReexportFailure[] = [];
if (opts.sourceId) {
const sid = opts.sourceId;
// GBRA-150: a metafile-named row is reconcilable exactly when the walk would collect its file: its bytes on disk, or,
// for a file gone since the last sync, its blob at that commit. A bare or root one keeps its page (#1433).
const scopeGitPath = gitRelativePath(gitContextRoot, syncScopeRoot);
const rowRoot = slugRoot ?? syncScopeRoot;
const lastSynced = gitMetafileHeads(gitContextRoot, [previousCommit]);
const metafilePage = metafilePagePredicate(metafilePages,
firstMetafileHead(diskMetafileHeads(rowRoot), path => lastSynced.read(slugRoot || !scopeGitPath ? path : `${scopeGitPath}/${path}`)),
path => slugRoot && scopeGitPath && path.startsWith(`${scopeGitPath}/`) ? path.slice(scopeGitPath.length + 1) : path);
const reconcileSyncOpts = { ...(opts.strategy ? { strategy: opts.strategy } : {}), metafilePage };
// collectSyncableFiles returns ABSOLUTE paths; source_path is stored
// repo-relative (importFile uses `relative(dir, filePath)`), so relativize
// to the same form before membership-testing — otherwise every page looks
// stale and the reconcile would wrongly delete live pages.
//
// #2828: planReconcileDeletes ALSO normalizes path separators on both sides
// of the membership test. On a Windows checkout `path.relative` yields
// backslash paths while a stored source_path can hold git-derived forward
// slashes; without normalization every file-backed page mismatches, looks
// stale, and the reconcile wipes the whole source.
// #774: scoped syncs store git-root-relative source_paths (slugRoot), so
// relativize the walk to the same base — otherwise every page mismatches
// and the mass-delete valve trips on a perfectly healthy scoped source.
// includeHidden MUST be threaded here too: if it isn't, any page a
// --include-hidden full sync just imported would look "gone" on the
// very next reconcile pass (its file was never in this collection) and
// the mass-delete valve would remove it.
const currentFiles = company ? company.plan.manifest.filter(entry => entry.disposition === 'included').map(entry => entry.path) : collectSyncableFiles(syncScopeRoot, {
strategy: opts.strategy ?? 'markdown',
includeGitignored: opts.includeGitignored,
includeHidden: opts.includeHidden,
metafilePages,
})
.map(abs => relative(rowRoot, abs));
// #5988 (E13): the old page of a held rename keeps its origin until the
// renamed file imports; its file being gone does not make it stale.
const heldOrigins = await heldRenameOrigins(engine, sid);
const rows = (await engine.executeRaw<{ slug: string; source_path: string | null }>(
`SELECT slug, source_path FROM pages WHERE source_id = $1 AND source_path IS NOT NULL AND deleted_at IS NULL`,
[sid],
)).filter(row => !heldOrigins.has(row.source_path!));
// #774: a scoped full sync is authoritative ONLY for its scope — pages
// whose source_path lives outside the subpath (e.g. from an earlier
// root-level sync of this source) are out of this walk's sight and must
// not be treated as stale.
const scopePrefix = slugRoot ? gitRelativePath(gitContextRoot, syncScopeRoot) + '/' : '';
// 'malformed-path' rows ARE reconcile-eligible: junk filenames (bracket /
// control-char paths minted by misbehaving producers) can never be
// re-imported, so their rows are permanent search pollution unless the
// reconcile can sweep them. Strategy safety is preserved by classifier
// ordering — a path that fails the strategy check classifies as
// 'strategy', never 'malformed-path', so a markdown sync still can't
// delete code pages. The #1433 metafile protection is likewise untouched.
const reconcileEligible = (p: string): boolean =>
isSyncable(p, reconcileSyncOpts) ||
// Only the poison signature is sweepable; bare-bracket markdown rows
// from pre-gate releases survive reconcile (their file still exists —
// deleting the row would be silent data loss; cross-model finding).
(unsyncableReason(p, reconcileSyncOpts) === 'malformed-path' && isPoisonedPath(p));
const plan = planReconcileDeletes(
rows,
currentFiles,
p => (scopePrefix === '' || p.startsWith(scopePrefix)) && reconcileEligible(p),
);
if (plan.staleSlugs.length > 0 && plan.massDelete && !massReconcileAllowed()) {
// #2828 mass-delete safety valve: a reconcile that would sweep more than
// half of the pages this strategy manages, on a source with a non-trivial
// number of them, is almost always a path-comparison bug or the wrong repo
// path — NOT a genuine bulk deletion. Skip the delete and warn loudly
// instead of silently wiping the brain.
serr(
`\n WARNING: refusing to reconcile-delete ${plan.staleSlugs.length} of ` +
`${plan.reconcilableCount} file-backed page(s) for source '${sid}' ` +
`(> ${Math.round(MASS_RECONCILE_RATIO * 100)}% of them).\n` +
` A full sync removes pages only when their backing file is gone. Deleting\n` +
` this many at once almost always means the paths were compared wrong (e.g.\n` +
` a path-separator mismatch) or the WRONG repo path was synced — not that\n` +
` you actually deleted that many files. No pages were deleted.\n` +
` If this bulk removal is genuinely intended, re-run with ` +
`GBRAIN_ALLOW_MASS_RECONCILE=1 to restore the old behavior.`,
);
} else if (plan.staleSlugs.length > 0) {
// #2426: a stale page whose source_path was NEVER committed to git is
// DB-only write-through (the file was written into the clone but never
// committed/pushed, then lost — e.g. a fresh clone). "Absent from git"
// is the SYMPTOM of that bug, not evidence the content is disposable.
// Keep those pages and re-export their markdown to the working tree so
// they're file-backed again; only pages whose file once existed in git
// history (i.e. was genuinely deleted) are reconcile-deleted.
const everCommitted = company ? null : listEverCommittedPaths(gitContextRoot);
const pathBySlug = new Map(rows.map(r => [r.slug, r.source_path]));
let deletableSlugs = plan.staleSlugs;
const dbOnlySlugs: string[] = [];
if (everCommitted) {
deletableSlugs = [];
for (const slug of plan.staleSlugs) {
const sp = pathBySlug.get(slug);
if (sp && !everCommitted.has(sp.replace(/\\/g, '/'))) dbOnlySlugs.push(slug);
else deletableSlugs.push(slug);
}
} else if (!company) {
// #7130: Git could not list the history (a dangling ref fails `git log --all`), so no stale page is proven
// committed; deleting them would treat a never-committed page's absence as a deletion. Keep them all.
serr(`\n WARNING: kept ${plan.staleSlugs.length} stale page(s) of source '${sid}': Git could not list the history that proves their files were committed and removed (run git -C ${gitContextRoot} log --all to see why). No pages were deleted.`);
deletableSlugs = [];
}
// #7130: the absolute caps (sync.max_deletes, sync.max_delete_ratio) brake a full reconcile below the 50% valve too.
const caps = await readSyncDeleteCaps(engine, sid);
const trip = syncDeleteTrip(deletableSlugs.length, plan.reconcilableCount, caps);
if (trip && !massReconcileAllowed()) {
serr(`\n WARNING: refusing to reconcile-delete ${deletableSlugs.length} of ${plan.reconcilableCount} file-backed page(s) for source '${sid}' (${syncDeleteTripText(trip, caps)}). ` +
`No pages were deleted. Check the checkout first; if the removal is intended, raise the cap (gbrain config set ${trip === 'max_deletes' ? 'sync.max_deletes' : 'sync.max_delete_ratio'}.source.${sid} <value>) or re-run with GBRAIN_ALLOW_MASS_RECONCILE=1.`);
deletableSlugs = [];
}
if (dbOnlySlugs.length > 0) {
let reExported = 0;
let reExportFailures = 0;
const { writePageThrough } = await import('../../core/write-through.ts');
for (const slug of dbOnlySlugs) {
try {
const r = await writePageThrough(engine, slug, { sourceId: sid, expectedFileBytes: null });
if (r.written) { reExported++; continue; }
if (r.skipped === 'disabled_by_config' || r.skipped === 'mirror_read_only' || r.skipped === 'url_clone') continue;
if (r.skipped === 'file_changed') throw new OperationError('file_database_drift', 'The checkout file exists; re-export only restores absent files.', `If the file holds the newer edit, run gbrain sync --source ${sid} to import it; otherwise ask the user which version to keep.`);
if (r.error) throw new Error(r.error);
if (r.skipped === 'over_push_scan_cap') {
reExportFailures++;
const fix = 'Kept the database page. Its file would pass the pre-push secret-scan cap, so it stays database-only and no sync re-exports it; read it with gbrain get.';
reexportFailures.push({ slug, path: pathBySlug.get(slug)!, code: 'over_push_scan_cap', message: 'The rendered file is over the pre-push secret-scan cap.', fix });
serr(` ${slug} (over_push_scan_cap) ${fix}`);
continue;
}
throw new OperationError('storage_error', `The page was not re-exported (${r.skipped}).`, 'Run gbrain doctor --json for the write-through target, then sync again.');
} catch (error) {
const failure = classifyItemFailure(error);
if (failure.scope === 'run') throw error;
reExportFailures++;
const fix = `Kept the database page and any existing file. If the file is newer, import it with gbrain sync --source ${sid} --include-gitignored; ` +
`if the database is newer, fix the cause above and run gbrain sync --source ${sid} --full --no-pull to re-export it; when both changed, ask the user which edit wins.`;
reexportFailures.push({ slug, path: pathBySlug.get(slug)!, code: failure.code, message: failure.message, fix });
serr(` ${slug} (${failure.code}: ${failure.message}) ${fix}`);
}
}
serr(
`\n Kept ${dbOnlySlugs.length} page(s) whose markdown was never committed to git ` +
`(DB-only write-through — not deleting).` +
(reExported > 0 ? ` Re-exported ${reExported} of them to the working tree.` : '') +
(reExportFailures > 0 ? ` ${reExportFailures} re-export failure(s), listed above.` : '') +
`\n Commit + push them (e.g. scripts/brain-commit-push.sh, or 'gbrain sources harden') ` +
`so the next sync sees them as file-backed.`,
);
}
const deleteScopedOpts = { sourceId: sid };
// Malformed-path rows get their own line: unlike genuinely-deleted
// files, THEIR backing file is usually still on disk (the walker
// excludes it), so "source file was removed" would be a lie and the
// rename-to-rescue path must be stated at the moment of removal, not
// only in a doctor check the operator may see later (red-team catch).
const malformedDeleted = deletableSlugs.filter(slug => {
const sp = pathBySlug.get(slug);
return sp != null && unsyncableReason(sp, reconcileSyncOpts) === 'malformed-path';
}).length;
for (let i = 0; i < deletableSlugs.length; i += DELETE_BATCH_SIZE) {
const batch = deletableSlugs.slice(i, i + DELETE_BATCH_SIZE);
try {
// #4587: reconcile soft-deletes (72h recovery). Already-soft-
// deleted rows are excluded by the primitive's predicate, so the
// count only reports real transitions.
const deleted = await softDeleteSyncPages(engine, batch, deleteScopedOpts);
reconciledDeletes += deleted.length;
} catch {
// Per-slug fallback on a batch blip (mirrors the incremental delete
// loop's decompose). A stale page that won't delete is best-effort,
// not fatal — the run continues.
for (const slug of batch) {
try { reconciledDeletes += (await softDeleteSyncPages(engine, [slug], deleteScopedOpts)).length; }
catch { /* best-effort */ }
}
}
}
if (reconciledDeletes > 0) {
slog(` Reconciled ${reconciledDeletes} stale page(s) whose source file was removed (soft-deleted, recoverable 72h).`);
if (malformedDeleted > 0) {
slog(
` (${malformedDeleted} of them had malformed bracket/control-char filenames — ` +
`their files may still exist on disk; rename a file to re-import its content.)`,
);
}
}
}
}
return { deleted: reconciledDeletes, reexportFailures };
}