Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Incremental sync finalize (refactor wave 1, W4 sync): the zero-change exit,
* then after the drains the final checkpoint flush, the pin-reachability
* gate, the shared failure-ledger bookmark gate, and the post-advance
* extract / facts / embed passes.
*/
import { chunkerStamp } from '../../core/chunkers/code.ts';
import { serr, slog } from '../../core/console-prefix.ts';
import type { BrainEngine } from '../../core/engine.ts';
import { clearOpCheckpoint } from '../../core/op-checkpoint.ts';
import { commitTimeMsAsync } from '../../core/source-health.ts';
import { writeSyncAnchor, writeChunkerVersion } from '../../core/sync-anchor.ts';
import { recordMetafilePagesApplied } from '../../core/sync-metafile-pages.ts';
import { gitAsync } from '../../core/sync-git.ts';
import { buildPartialResult } from '../../core/sync-lock.ts';
import {
sanitizePathForDisplay,
DEFAULT_SOURCE_ID,
resolveAutoSkipThreshold,
applySyncFailureGate,
formatCodeBreakdown,
RENAME_SENTINEL_PREFIX,
isSkippablePath,
summarizeFailuresByCode,
STRICT_REQUIRES_MANAGED,
strictRequiresManagedAdvice,
isEmbeddingInfraCode,
formatFailedFileList,
readMetafilePagesMode,
} from '../../core/sync.ts';
import { shouldLogIngest } from '../import.ts';
import type { SyncResult } from '../sync.ts';
import { settleLegacyHolds } from './holds.ts';
import { gitlinkFields } from './unsupported-paths.ts';
import { sweepOrphanedRenameSentinels } from './rename-reconcile.ts';
import { partial, flushCheckpoint } from './sync-run.ts';
import type { SyncPlan, SyncRun } from './sync-run.ts';
export async function finishWithoutChanges(
engine: BrainEngine,
plan: Pick<SyncPlan, 'opts' | 'company' | 'gitContextRoot' | 'lastCommit' | 'pin' | 'pullFailed' | 'ckpt' | 'malformedSkipped' | 'totalChanges' | 'holds' | 'gitlinks' | 'metafilePages'>,
swept: number,
): Promise<SyncResult | undefined> {
const { opts, company, gitContextRoot, lastCommit, pin, pullFailed, ckpt, malformedSkipped, totalChanges, metafilePages } = plan;
if (totalChanges === 0) {
await settleLegacyHolds(engine, plan.holds);
// #3068: same guard as the git-HEAD-equality gate above — a failed pull
// plus zero imports must not produce a clean `up_to_date` (and must not
// advance the anchor past commits this run never looked at remotely).
// Reached when local-only commits landed with no syncable content while
// the pull kept failing. Nothing is imported (the #4786 sweep above may
// have soft-deleted pages — report it); the next sync re-diffs the same
// trivial range and retries the pull.
if (pullFailed) {
serr(
`[sync] git pull failed and no syncable changes imported — reporting partial ` +
`(not up_to_date); sync anchor unchanged at ${lastCommit.slice(0, 8)}.`,
);
return buildPartialResult({
fromCommit: lastCommit,
toCommit: lastCommit,
filesImported: 0,
pagesAffected: [],
chunksCreated: 0,
added: 0, modified: 0, deleted: swept, renamed: 0,
reason: 'pull_failed',
});
}
// Update sync state even with no syncable changes (git advanced). v0.42.x
// (#1794): advance to the PINNED target, and clear any checkpoint (a resume
// whose remaining range turned out to have no syncable changes still
// completes cleanly here).
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', pin, await commitTimeMsAsync(gitContextRoot, pin), gitContextRoot);
await engine.setConfig('sync.last_run', new Date().toISOString());
await writeChunkerVersion(engine, opts.sourceId, chunkerStamp());
if (!company) await recordMetafilePagesApplied(engine, opts.sourceId ?? DEFAULT_SOURCE_ID, metafilePages);
if (!company) { await clearOpCheckpoint(engine, ckpt.paths); await clearOpCheckpoint(engine, ckpt.target); }
// A commit whose ONLY changes are malformed filenames lands here with
// totalChanges === 0 — the anchor advances past those files forever, so
// this early return must surface the skips too (structured-review P2).
if (malformedSkipped.length > 0) {
serr(
` ${malformedSkipped.length} file(s) skipped: malformed filename ` +
`(brackets/control chars; rename to import): ` +
malformedSkipped.map(sanitizePathForDisplay).join(', '),
);
}
// #3479 blocker 2: this early return also bypasses the failure gate —
// sweep orphaned `<rename:…>` sentinels here too.
await sweepOrphanedRenameSentinels(engine, opts.sourceId ?? DEFAULT_SOURCE_ID);
return {
status: swept > 0 ? 'synced' : 'up_to_date',
fromCommit: lastCommit,
toCommit: pin,
added: 0, modified: 0, deleted: swept, renamed: 0,
chunksCreated: 0,
embedded: 0,
pagesAffected: [],
...(malformedSkipped.length > 0 ? { malformedSkipped: malformedSkipped.length } : {}),
...gitlinkFields(plan),
};
}
return undefined;
}
export async function finalizeIncrementalSync(
run: SyncRun,
plan: SyncPlan,
input: { noEmbed: boolean; typeWarningsEnabled: boolean; start: number },
): Promise<SyncResult> {
const { engine, pagesAffected, typeWarningCounts } = run;
const { opts, repoPath, headCommit, lastCommit, pin, filtered, malformedSkipped, totalChanges, uncommittedDrift } = plan;
const { noEmbed, typeWarningsEnabled, start } = input;
// v0.42.x (#1794): if checkpoint persistence died mid-run (pool dead through
// the whole retry budget), do NOT advance last_commit — return a
// checkpoint_unavailable partial so the next run re-drains (content_hash
// short-circuits the re-import). partial() overrides the reason when
// checkpointDead is set.
if (run.checkpointDead) {
return await partial(run, plan, 'timeout');
}
// v0.42.x (#1794): bank the final completed set before the gate so a block /
// rewrite still persists everything drained this run (the next run resumes).
await flushCheckpoint(run);
// Past the final flush we're on a terminal path (blocked or success); both
// either leave the checkpoint in place (blocked) or clear it (success), so the
// SIGTERM one-shot flush has nothing left to add. Deregister so a SIGTERM
// during the success-path git/anchor writes doesn't fire a stale flush.
run.deregisterCheckpointCleanup();
const headVerificationSucceeded = await verifyPinnedHead(run, plan);
// #5988: a deleted or renamed-away held file clears its hold; so does one
// whose file is gone or no longer synced. A failed delete keeps it.
const failedPaths = new Set(run.failedFiles.map(f => f.path));
await settleLegacyHolds(engine, plan.holds, [...filtered.deleted, ...filtered.renamed.map(r => r.from)].filter(p => !failedPaths.has(p)));
const elapsed = Date.now() - start;
const gated = await applyBookmarkGate(run, plan, headVerificationSucceeded);
if ('done' in gated) return gated.done;
const { gate } = gated;
// Advanced. Surface what the gate did past the failures.
if (gate.acknowledged > 0) {
serr(` Acknowledged ${gate.acknowledged} failure(s) and advanced past them.`);
}
if (gate.autoSkipped.length > 0) {
serr(
`\n Auto-skipped ${gate.autoSkipped.length} file(s) that failed >= ` +
`${resolveAutoSkipThreshold()} consecutive syncs:\n` +
gate.autoSkipped.map(p => ` ${p}`).join('\n') + '\n' +
` Bookmark advanced; these pages are NOT indexed and remain in ` +
`sync-failures.jsonl. 'gbrain doctor' will warn until they're fixed.`,
);
}
// Log ingest. #3969: mirror runImport's shouldLogIngest gate — a run that
// landed nothing (no pages written, no chunks, no failures acknowledged or
// auto-skipped) is a poll, not an ingest event; skip the row unless
// opts.logNoop opts back in.
if (shouldLogIngest(
{
imported: pagesAffected.length,
errors: gate.acknowledged + gate.autoSkipped.length,
chunksCreated: run.chunksCreated,
},
opts.logNoop === true,
)) {
await engine.logIngest({
// #3242 (attribution sub-bug): credit the sync to the source it wrote
// to, not the shared 'default' bucket.
...(opts.sourceId ? { source_id: opts.sourceId } : {}),
source_type: 'git_sync',
source_ref: `${repoPath} @ ${headCommit.slice(0, 8)}`,
pages_updated: pagesAffected,
summary: `Sync: +${filtered.added.length} ~${filtered.modified.length} -${filtered.deleted.length} R${filtered.renamed.length}, ${run.chunksCreated} chunks, ${elapsed}ms`,
});
}
const extractError = await extractAffectedPages(run, plan);
await backstopAffectedPageFacts(run, plan);
const { embedded, embedSlugs } = await embedAffectedPages(run, plan, noEmbed);
// #6722: the pages this run left large or embed_skip, read once after the imports.
const { readPageSizeStates, addSizeState } = await import('../../core/page-embedding-budget.ts');
const sizeStates = (await readPageSizeStates(engine, opts.sourceId ?? 'default', pagesAffected.filter(slug => !run.deletedSlugs.has(slug))))
.reduce<import('../../core/page-embedding-budget.ts').SyncSizeStates | undefined>((tally, state) => addSizeState(tally, state), undefined);
if (malformedSkipped.length > 0) {
serr(
`\n ${malformedSkipped.length} file(s) skipped: malformed filename ` +
`(brackets/control chars) — rename to import. Not counted as failures.`,
);
}
const typeWarnings = [...typeWarningCounts.values()];
if (typeWarningsEnabled && typeWarnings.length > 0) {
const { renderTypeWarningSummary } = await import('../../core/schema-pack/type-usage.ts');
for (const line of renderTypeWarningSummary(typeWarnings)) serr(` ${line}`);
serr(` (silence with: gbrain config set schema.type_warnings false)`);
}
// A run that only re-screened held files (and imported none) changed nothing.
const onlyRescreened = totalChanges === (plan.holds?.rescreen.length ?? 0) && pagesAffected.length === 0 && run.swept === 0;
return {
status: onlyRescreened ? 'up_to_date' : 'synced',
fromCommit: lastCommit,
toCommit: pin,
added: filtered.added.length,
modified: filtered.modified.length,
deleted: filtered.deleted.length + run.swept,
renamed: filtered.renamed.length,
chunksCreated: run.chunksCreated,
embedded,
pagesAffected,
...(sizeStates ? { size_states: sizeStates } : {}),
...(totalChanges > 100 && embedSlugs.length > 0 ? { embedDeferralReason: 'large_sync' as const } : {}),
malformedSkipped: malformedSkipped.length,
...(typeWarningsEnabled && typeWarnings.length > 0 ? { type_warnings: typeWarnings } : {}),
...(uncommittedDrift ? { uncommitted: uncommittedDrift } : {}),
...(extractError ? { extract_error: extractError } : {}),
...identityDowngradesOf(plan),
...gitlinkFields(plan),
};
}
/** #6729: the inexact renames this run split into a delete plus an add, in the source's origin form (recorded by `performSync`). */
function identityDowngradesOf(plan: Pick<SyncPlan, 'manifest' | 'inScope' | 'isSelectedForRun' | 'syncOpts' | 'modePath'>): Pick<SyncResult, 'identityDowngrades'> {
const selected = (path: string) => plan.inScope(path) && plan.isSelectedForRun(path, plan.syncOpts);
const pairs = (plan.manifest.downgradedRenames ?? []).filter(pair => selected(pair.from) || selected(pair.to))
.map(pair => ({ ...pair, from: plan.modePath(pair.from), to: plan.modePath(pair.to) }));
return pairs.length ? { identityDowngrades: pairs } : {};
}
/** v0.42.x (#1794, T3): pin-reachability gate; records a `<head>` sentinel on a rewrite. */
async function verifyPinnedHead(run: SyncRun, plan: Pick<SyncPlan, 'company' | 'gitContextRoot' | 'pin'>): Promise<boolean> {
const { failedFiles } = run;
const { company, gitContextRoot, pin } = plan;
// v0.42.x (#1794, T3): pin-reachability gate, replacing the pre-v0.42 strict
// "HEAD == captured" head-drift gate. CODEX-3 originally blocked on ANY HEAD
// movement to catch external `git checkout`/`reset` that would make the
// imported chunks reflect a different tree. But the #1794 repro has an enrich
// process committing to the SAME repo every ~2 min, so the strict gate
// blocked every run — a co-equal cause of non-convergence. Under pinning we
// drain a FIXED lastCommit..pin range, so:
// - HEAD == pin → nothing moved; advance.
// - HEAD is descendant of pin (forward progress) → SAFE. The new commits
// are outside this run's range and get picked up by the next sync's
// pin..HEAD diff. Advance to pin.
// - pin NOT an ancestor of HEAD (history REWRITE / reset / force-push) →
// the tree we imported against is gone. Block; do not advance.
let headVerificationSucceeded = false;
try {
const currentHead = company ? company.plan.revision!.commit : await gitAsync(gitContextRoot, ['rev-parse', 'HEAD']);
if (currentHead !== pin) {
let pinStillReachable = false;
try {
await gitAsync(gitContextRoot, ['merge-base', '--is-ancestor', pin, currentHead]);
pinStillReachable = true;
} catch {
pinStillReachable = false;
}
if (!pinStillReachable) {
failedFiles.push({
path: '<head>',
error: `git history rewritten during sync: pinned target ${pin.slice(0, 8)} is no longer an ancestor of HEAD ${currentHead.slice(0, 8)}`,
});
} else {
headVerificationSucceeded = true;
}
// else: forward progress (enrich committed on top) — safe, advance to pin.
} else {
headVerificationSucceeded = true;
}
} catch (e) {
// rev-parse failure is itself a drift signal (worktree disappeared).
failedFiles.push({
path: '<head>',
error: `git HEAD verification failed: ${e instanceof Error ? e.message : String(e)}`,
});
}
return headVerificationSucceeded;
}
async function applyBookmarkGate(
run: SyncRun,
plan: Pick<SyncPlan, 'opts' | 'company' | 'gitContextRoot' | 'anchorPath' | 'lastCommit' | 'pin' | 'ckpt' | 'filtered' | 'holds' | 'metafilePages'>,
headVerificationSucceeded: boolean,
): Promise<{ done: SyncResult } | { gate: Awaited<ReturnType<typeof applySyncFailureGate>> }> {
const { engine, succeededPaths, failedFiles, pagesAffected } = run;
const { opts, company, gitContextRoot, anchorPath, lastCommit, pin, ckpt, filtered, metafilePages } = plan;
// issue #1939 — gate the bookmark through the shared failure ledger.
// • Fresh failures still BLOCK (fail-closed): the next sync re-walks the
// diff and re-attempts. Escape hatch: --skip-failed.
// • A file that fails >= threshold consecutive syncs AUTO-SKIPS so a poison
// file can't wedge all indexing forever (recorded, surfaced by doctor).
// • A `<head>` SENTINEL (history rewrite) HARD-BLOCKS even with
// --skip-failed — advancing would record a commit that no longer matches
// the indexed tree.
// `advance` is the bookmark write; the gate runs it ONLY when advancing, and
// ALWAYS before marking anything auto-skipped/acknowledged (crash-atomic).
const advance = async (): Promise<void> => {
// v0.42.x (#1794): advance to the PINNED target (not live HEAD) — commits
// past the pin are the next sync's pin..HEAD diff. `commitTimeMs(pin)` stamps
// newest_content_at against the commit we drained to. `last_sync_at` is bumped
// HERE and ONLY here so the autopilot scheduler never sees a stuck source as
// "fresh". The checkpoint rows clear here — CONVERGENCE CONTRACT: sync
// convergence == IMPORT convergence; downstream extract/facts/embed is
// decoupled (its own resumable stale sweeps).
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', pin, await commitTimeMsAsync(gitContextRoot, pin), gitContextRoot);
await engine.setConfig('sync.last_run', new Date().toISOString());
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
await writeChunkerVersion(engine, opts.sourceId, chunkerStamp());
if (!company) await recordMetafilePagesApplied(engine, opts.sourceId ?? DEFAULT_SOURCE_ID, metafilePages);
if (!company) { await clearOpCheckpoint(engine, ckpt.paths); await clearOpCheckpoint(engine, ckpt.target); }
};
// issue #1939 adversarial finding #1: a file that failed to parse (open ledger
// row) and is then deleted/renamed-away never re-enters failedFiles and never
// imports, so its row would never clear and would age doctor to a permanent
// FAIL. Treat removed paths as resolved so the ledger self-heals.
const resolvedPaths = [
...succeededPaths,
// #5988: a held file is accounted for by its hold, so its ledger rows resolve.
...(plan.holds?.heldPaths ?? []),
...filtered.deleted,
...filtered.renamed.map(r => r.from),
// A prior transient rev-parse timeout records a hard-blocking sentinel that
// operators cannot acknowledge manually. Once pin ancestry is verified on
// a later run, clear that stale sentinel through the ordinary success path.
...(headVerificationSucceeded ? ['<head>'] : []),
];
const gate = await applySyncFailureGate({
sourceId: opts.sourceId ?? DEFAULT_SOURCE_ID,
failedFiles,
succeededPaths: resolvedPaths,
commit: pin,
skipFailed: !company && opts.skipFailed === true,
...(company ? { threshold: 0 } : {}),
// GBRA-130 A-18: files of this drain imported this run or banked by an
// earlier run of it (an infra code on one file while others import goes
// chronic, also across resumes) and the import breaker's verdict (a run
// where most files failed counts no attempt and auto-skips nothing).
imported: run.filesImported + run.bankedImports,
systemic: run.itemRun.systemic() !== null,
advance,
});
// #3479 blocker 2 — self-heal for orphaned `<rename:…>` sentinels: a
// force-push that invalidates the pinned target means the rename never
// re-enters the diff, so the ordinary convergence path above can never
// clear the row and doctor ages it to a permanent FAIL no CLI can fix.
// Deliberately AFTER the gate and OUTSIDE it (#3583): the gate used to
// clear these via succeededPaths, which cleared BEFORE advance() — a
// throwing advance then lost the sentinel with the verify never reached.
// Out here, a gate that throws never clears anything (fail-closed), and
// the sweep carries the full clear-then-verify-restore semantics.
await sweepOrphanedRenameSentinels(
engine, opts.sourceId ?? DEFAULT_SOURCE_ID, new Set(failedFiles.map(f => f.path)),
);
if (!gate.advanced) {
const codeBreakdown = formatCodeBreakdown(failedFiles);
const strictRefused = summarizeFailuresByCode(failedFiles).some(c => c.code === STRICT_REQUIRES_MANAGED);
// Two sentinel classes block here: `<head>` (pin ancestry broken) and
// `<rename:…>` (#3056 — a rename-reconcile delete failed and advancing
// would permanently bank the duplicate). Pick the message by which fired —
// and when BOTH fired, the rename detail is appended to the head message
// rather than silently losing to it (#3479 review).
const renameRows = failedFiles.filter(f => f.path.startsWith(RENAME_SENTINEL_PREFIX));
// The failing rows verbatim (path + error): the error names the stale
// slug and the old path, which the operator remedy below points at —
// a code-count breakdown alone can't tell them which row to delete.
const renameDetail = renameRows.map(f => ` ${f.path}: ${f.error}`).join('\n');
// #3479 blocker 1 — the sentinel hard-blocks even --skip-failed by
// design, so an environment where the DELETE can never succeed (RLS
// denying DELETE, an FK RESTRICT) needs a documented exit or a cosmetic
// duplicate becomes a total sync outage. The remedy is deliberately NOT
// pitched at a fully read-only database: 'gbrain delete' soft-deletes
// via UPDATE, so it unwedges exactly the environments where writes work
// but this DELETE does not.
const renameRemedy =
`The next 'gbrain sync' retries the reconcile from the same diff. If the delete ` +
`keeps failing in your environment (RLS denying DELETE, an FK RESTRICT — anywhere ` +
`UPDATE still works), remove the stale row yourself: 'gbrain delete <stale-slug>' ` +
`with the stale slug named above (the reconcile only names rows whose backing file ` +
`is gone from the working tree — never a live page). A sentinel reading 'stale row ?' ` +
`names nothing on purpose: that run could not prove ANY row stale, usually because a ` +
`tracked file could not be read — fix or remove that file instead of deleting a page. ` +
`The reconcile then finds nothing left to delete and the sentinel clears on the next run.`;
if (gate.sentinelBlocked && failedFiles.some(f => f.path === '<head>')) {
serr(
`\nSync blocked: repository history changed during sync (force-push / reset).\n` +
`${codeBreakdown}\n\n` +
`The pinned target is no longer an ancestor of HEAD; advancing would record ` +
`a commit that doesn't match the indexed tree. Re-run sync to re-pin against ` +
`current HEAD.` +
(renameRows.length > 0
? `\n\nA rename also left a stale duplicate that could not be removed:\n` +
`${renameDetail}\n\n${renameRemedy}`
: ''),
);
} else if (gate.sentinelBlocked) {
serr(
`\nSync blocked: a rename left a stale duplicate that could not be removed:\n` +
`${renameDetail || codeBreakdown}\n\n` +
renameRemedy,
);
} else if (strictRefused) {
serr(`\nSync blocked: ${failedFiles.filter(f => isSkippablePath(f.path)).length} file(s) failed (${STRICT_REQUIRES_MANAGED}):\n${codeBreakdown}\n${formatFailedFileList(failedFiles)}\n\n${strictRequiresManagedAdvice(opts.sourceId)}`);
} else if (run.itemRun.systemic()) {
const fileFailCount = failedFiles.filter(f => isSkippablePath(f.path)).length;
serr(
`\nSync blocked: ${fileFailCount} file(s) failed, most of the files this run tried; this looks like ` +
`a shared cause, so no attempt was counted and nothing auto-skips:\n` +
`${codeBreakdown}\n${formatFailedFileList(failedFiles)}\n\n` +
`Fix the shared cause (permissions, the embedding provider, the database) and re-run 'gbrain sync', ` +
`or acknowledge these files with 'gbrain sync --skip-failed${opts.sourceId ? ` --source ${opts.sourceId}` : ''}'.`,
);
} else {
const fileFailCount = failedFiles.filter(f => isSkippablePath(f.path)).length;
// #3875: code-aware copy. Provider-infra failures (embed timeout /
// rate limit / quota) are NOT bad files — suggesting --skip-failed for
// them acknowledges away perfectly good content. Point at provider
// health + a plain re-run (or --full to rebuild) instead.
const infraCodes = summarizeFailuresByCode(failedFiles).filter(c => isEmbeddingInfraCode(c.code));
if (infraCodes.length > 0) {
serr(
`\nSync blocked: ${fileFailCount} file(s) failed — embedding provider errors:\n` +
`${codeBreakdown}\n\n` +
`These are provider-health failures (timeout / rate limit / quota), not bad ` +
`files — do NOT use --skip-failed for them. Check the embedding provider ` +
`(is it running? out of quota?), then re-run 'gbrain sync' (only the failed ` +
`files are re-attempted), or 'gbrain sync --full' to rebuild.`,
);
} else {
serr(
`\nSync blocked: ${fileFailCount} file(s) failed to parse:\n` +
`${codeBreakdown}\n${formatFailedFileList(failedFiles)}\n\n` +
`Pinpoint a file with 'gbrain frontmatter validate <path>' (--fix auto-repairs), ` +
`fix the frontmatter and re-run, or use 'gbrain sync --skip-failed' to ` +
`acknowledge and move on. A file that keeps failing auto-skips after ` +
`${resolveAutoSkipThreshold()} consecutive syncs.`,
);
}
}
// Update last_run + repo_path (progress on infra) but NOT last_commit. The
// checkpoint is INTENTIONALLY left in place — the banked completed set lets
// the next run skip the drained files and re-attempt only the failures.
await engine.setConfig('sync.last_run', new Date().toISOString());
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
// v0.42.x (#1794): surface banked progress so a blocked run doesn't read as
// total loss (last_commit is unchanged by design; the checkpoint is banked).
serr(
`[sync] banked ${run.bankedFiles} file(s) this run; next 'gbrain sync' resumes ` +
`from the checkpoint (last_commit unchanged at ${(lastCommit ?? '').slice(0, 8)}).`,
);
return { done: {
status: 'blocked_by_failures',
fromCommit: lastCommit,
toCommit: pin,
added: filtered.added.length,
modified: filtered.modified.length,
deleted: filtered.deleted.length + run.swept,
renamed: filtered.renamed.length,
chunksCreated: run.chunksCreated,
embedded: 0,
pagesAffected,
failedFiles: failedFiles.length,
failureCodes: summarizeFailuresByCode(failedFiles),
...(strictRefused ? { failureAdvice: strictRequiresManagedAdvice(opts.sourceId) } : {}),
bankedFiles: run.bankedFiles,
} };
}
return { gate };
}
async function extractAffectedPages(run: SyncRun, plan: Pick<SyncPlan, 'opts' | 'gitContextRoot' | 'pin' | 'totalChanges'>): Promise<string | undefined> {
const { engine, pagesAffected } = run;
const { opts, gitContextRoot, pin, totalChanges } = plan;
// Auto-extract links + timeline (cheap CPU, but skip-inline for LARGE syncs).
// Thread opts.sourceId so the extract phase reconciles edges + timeline
// entries against the right source — pre-fix (Data R1 HIGH 1) this phase
// bypassed sourceId entirely and the bare-slug subquery in addTimelineEntry
// (Data R1 HIGH 2) crashed with 21000 in multi-source brains.
//
// v0.42.x (#1794, T4): size-gate inline extract on `totalChanges <= 100`
// (same threshold as noEmbed). A large sync (the #1794 case) would otherwise
// run links+timeline extraction over tens of thousands of pages inline,
// re-coupling a slow pass into the just-decoupled convergence path. Instead we
// leave `links_extracted_at` UNSTAMPED so the resumable `extract --stale`
// watermark sweep (run by the autopilot cycle / on demand) picks the pages
// up. For resumed large syncs, pagesAffected holds only THIS run's slugs, but
// the stale sweep scans the whole source, so banked-across-runs pages are
// covered regardless.
const extractOpts = opts.sourceId ? { sourceId: opts.sourceId } : undefined;
if (!opts.noExtract && totalChanges > 100 && pagesAffected.length > 0) {
// #2849: above the size gate the deferred extraction must be DURABLY
// QUEUED, not just hinted. The autopilot cycle's extract phase is
// slug-scoped (an up_to_date follow-up sync hands it an empty
// pagesAffected), so a webhook-driven large sync left
// `links_extracted_at` unstamped FOREVER unless an operator ran
// `gbrain extract --stale` by hand. Submit a source-scoped stale-sweep
// job bound to the consumed commit (idempotency key) so repeated
// webhook deliveries / sync retries of the same commit coalesce onto
// one job. The sweep itself is the watermark scan — it picks up the
// pages this run imported AND any banked across resumed runs.
// Best-effort: queue submission failure falls back to the hint-only
// behavior (the pages stay stale + visible to doctor, never mis-stamped).
let queuedJobId: number | string | null = null;
try {
const { MinionQueue } = await import('../../core/minions/queue.ts');
const { STALE_TIME_BUDGET_MS } = await import('../extract.ts');
const queue = new MinionQueue(engine);
const payload = {
stale: true,
...(opts.sourceId ? { sourceId: opts.sourceId } : {}),
reason: 'sync_size_gate',
// Bound to the PIN this run drained to (== headCommit unless resuming
// a stored target), not live HEAD — the sweep covers what we imported.
deferred_commit: pin,
};
// The stale sweep has its own internal wall-clock budget
// (GBRAIN_EXTRACT_TIME_BUDGET_MS-derived); without an explicit
// timeout_ms the job would inherit the tight null-default and get
// wall-clock-killed mid-sweep (#1737 class). 5-min headroom.
const timeoutMs = STALE_TIME_BUDGET_MS + 5 * 60 * 1000;
// NO maxWaiting here: with an unscoped (NULL-sourceId) payload the
// queue's coalesce filter matches ANY waiting 'extract' job (e.g. a
// remediation-submitted {mode:'links'} row) and returns THAT job —
// silently dropping the sweep while we log "queued". The idempotency
// key alone is the dedup for repeat submissions toward the same pin.
const key = `extract-stale:${opts.sourceId ?? 'default'}:${pin}`;
const isLiveSweep = (j: { status: string; data: Record<string, unknown> }): boolean =>
j.data?.stale === true && ['waiting', 'delayed', 'active'].includes(j.status);
let job = await queue.add('extract', payload, { idempotency_key: key, timeout_ms: timeoutMs });
if (!isLiveSweep(job)) {
// The key slot holds a FINISHED row: a prior sweep toward this pin
// that completed BEFORE this run's pages landed (checkpoint-resume /
// blocked-advance re-sync of the same target). Those pages went
// stale after that sweep's watermark pass, so coalescing onto the
// finished row would strand them — queue a fresh sweep under a
// run-unique key. (An 'active' sweep is safe to coalesce onto: its
// end-of-run staleRemaining re-count chains a continuation.)
job = await queue.add('extract', payload, {
idempotency_key: `${key}:${Date.now()}`,
timeout_ms: timeoutMs,
});
}
// Only claim "queued" once we verified the returned row IS a live
// stale sweep — never trust queue.add's row blind.
if (isLiveSweep(job)) queuedJobId = job.id;
} catch { /* best-effort — hint below still tells the operator */ }
slog(
` Large sync: deferring link/timeline extraction` +
(queuedJobId != null
? ` — queued stale-sweep job #${queuedJobId} (source: ${opts.sourceId ?? 'default'}); a running jobs worker will consume it.`
: `.`) +
` Run 'gbrain extract --stale${opts.sourceId ? ` --source-id ${opts.sourceId}` : ''}' to extract now.`,
);
}
let extractError: string | undefined;
if (!opts.noExtract && totalChanges <= 100 && pagesAffected.length > 0) {
try {
const { extractLinksForSlugs, extractTimelineForSlugs, stampExtracted, slugsSafeToStamp } = await import('../extract.ts');
const { indexMarkdownTree } = await import('../extract-tree.ts');
// #774: pages' source_path is git-root-relative, so extract resolves
// files from gitContextRoot (== repoPath realpath when unscoped).
// GBRA-133 2b.1: one tree walk per run, shared by both hooks.
const tree = await indexMarkdownTree(gitContextRoot, await readMetafilePagesMode(engine));
const linksResult = await extractLinksForSlugs(engine, gitContextRoot, pagesAffected, { ...extractOpts, tree });
const timelineResult = await extractTimelineForSlugs(engine, gitContextRoot, pagesAffected, { ...extractOpts, tree });
if (linksResult.created > 0 || timelineResult.created > 0) {
slog(` Extracted: ${linksResult.created} links, ${timelineResult.created} timeline entries`);
}
// v0.42.7 (#1696, CDX-6): stamp the links_extracted_at watermark for the
// pages we just extracted, AFTER the import set their updated_at, so
// links_extracted_at >= updated_at (page is now fresh, not flagged stale).
// Source-correct via opts.sourceId. Stamp at the CALL SITE (not inside
// extractLinksForSlugs) so we use the per-source sourceId the sync owns.
// Best-effort: a stamp miss just means extract --stale re-sweeps later.
// Only the slugs both hooks actually read — a page the extractor
// skipped must stay stale so the sweep still owes it.
await stampExtracted(
engine,
slugsSafeToStamp(linksResult, timelineResult)
.map((slug) => ({ slug, source_id: opts.sourceId ?? 'default' })),
);
const failed = [...(linksResult.errors ?? []), ...(timelineResult.errors ?? [])];
if (failed.length > 0) extractError = `${failed.length} page(s) not extracted, e.g. ${failed[0]!.slug}: ${failed[0]!.error}`;
} catch (e) {
extractError = e instanceof Error ? e.message : String(e);
}
// A15: best-effort (the import stands and failed pages stay stale for
// `extract --stale`), but never silent.
if (extractError) serr(` Link/timeline extraction failed: ${extractError}. Run 'gbrain extract --stale${opts.sourceId ? ` --source-id ${opts.sourceId}` : ''}' after fixing it.`);
}
return extractError;
}
async function backstopAffectedPageFacts(run: SyncRun, plan: Pick<SyncPlan, 'opts'>): Promise<void> {
const { engine, pagesAffected } = run;
const { opts } = plan;
// v0.31.2: facts extraction now routes through the shared
// src/core/facts/backstop.ts helper (PR1 commit 6). Sync uses
// queue mode (fire-and-forget) + 'high-only' filter so a 50-page
// sync doesn't block on N sequential Sonnet calls. The pre-fix
// inline loop is gone — it carried (a) a dead-code type filter
// ('conversation'/'transcript'/'therapy'/'call' aren't real
// PageTypes), (b) a divergent eligibility shape from put_page,
// and (c) raw extract→insert without dedup/supersede.
if (!opts.noExtract && pagesAffected.length > 0 && pagesAffected.length <= 50) {
const { runFactsBackstop } = await import('../../core/facts/backstop.ts');
const factsSourceId = opts.sourceId ?? 'default';
for (const slug of pagesAffected) {
try {
// v0.40 D21: source-scoped getPage. Pre-v0.40 this called
// engine.getPage(slug) WITHOUT sourceId, then wrote facts under
// factsSourceId. On a federated brain with the same slug in two
// sources (e.g. people/garry-tan in default + zion-brain), this
// would attribute facts to the wrong source. Codex outside-voice
// catch on the v0.40 plan review.
const page = await engine.getPage(slug, { sourceId: factsSourceId });
if (!page) continue;
await runFactsBackstop(
{
slug,
type: page.type,
compiled_truth: page.compiled_truth ?? '',
frontmatter: page.frontmatter ?? {},
},
{
engine,
sourceId: factsSourceId,
sessionId: `sync:${slug}`,
source: 'sync:import',
mode: 'queue',
notabilityFilter: 'high-only',
},
);
} catch { /* per-page enqueue is best-effort */ }
}
}
}
async function embedAffectedPages(
run: SyncRun,
plan: Pick<SyncPlan, 'opts' | 'totalChanges'>,
noEmbed: boolean,
): Promise<{ embedded: number; embedSlugs: string[] }> {
const { engine, pagesAffected, deletedSlugs } = run;
const { opts, totalChanges } = plan;
// Auto-embed (skip for large syncs — embedding calls OpenAI).
// Thread sourceId so incremental source syncs embed the page row they just
// imported instead of falling back to the default source.
//
// v0.37 fix wave (Lane D.3 + CDX2-8): switched from `runEmbed` (which
// does its own process.exit) to `runEmbedCore` so sync can detect the
// dim-mismatch class and surface a stderr hint without killing the
// sync. Non-mismatch errors stay best-effort (rate limits, transient
// network) — those shouldn't break sync.
let embedded = 0;
// #1284: never hand deleted slugs to the embedder — embedPage throws
// 'Page not found' per deleted slug and logs one error line each. Filter
// against this run's confirmed-deleted set (slugs re-imported later in the
// run were removed from it at their push sites).
const embedSlugs = pagesAffected.filter((s) => !deletedSlugs.has(s));
if (!noEmbed && embedSlugs.length > 0 && pagesAffected.length <= 100) {
try {
const { runEmbedCore } = await import('../embed.ts');
const embedOpts = opts.sourceId
? { slugs: embedSlugs, sourceId: opts.sourceId }
: { slugs: embedSlugs };
await runEmbedCore(engine, embedOpts);
embedded = embedSlugs.length;
} catch (e: unknown) {
const { EmbeddingDimMismatchError } = await import('../embed.ts');
if (e instanceof EmbeddingDimMismatchError) {
serr('\n' + e.recipeMessage + '\n');
serr(`Tip: pass --no-embed to sync without embedding, then`);
serr(`run 'gbrain embed --stale' after fixing the schema.\n`);
}
// Other errors stay best-effort — rate limits, transient network.
}
} else if (noEmbed || totalChanges > 100) {
slog(`Text imported. Run 'gbrain embed --stale' to generate embeddings.`);
}
return { embedded, embedSlugs };
}