Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { lookupRefsForSlugs } from './link-reconciliation.ts';
import { collectWantedLinks, isWantedPagesEnabled, type WantedLinkInput } from './wanted-links.ts';
import { readLineGrammarSettings } from './line-grammar.ts';
/**
* Serve-resident maintenance sweep [CX-P0.1, CX-P0.3, CX2-4].
*
* Nothing previously ingested the transcript corpus into a live brain
* (dream is disabled by default; the CLI can't open PGLite under a live
* serve), and remote `put_page` deliberately skips auto link/timeline
* extraction — so the graph never compounded from harness writes. The
* sweep is the serve process's (the lock owner's) bounded, spend-gated
* answer. Three passes, each individually fail-soft:
*
* 1. FACTS-FENCE RECONCILIATION [CX2-4] — zero-LLM. Recently-modified
* pages carrying a `## Facts` fence get reconciled into the facts DB
* index by the SAME cycle extractor the dream cycle uses
* (src/core/cycle/extract-facts.ts:runExtractFacts, scoped via
* opts.slugs). Fence rows carry explicit per-row visibility; the
* corpus pass below resolves unset visibility through
* resolveDefaultVisibility inside the shared pipeline (backstop.ts).
*
* 2. LINK/TIMELINE EXTRACTION [CX-P0.3] — zero-LLM, deterministic. The
* same per-page cores `gbrain extract links|timeline --source db`
* runs: extractPageLinks + parseTimelineEntries, endpoint-validated
* through resolveCandidateSources and batch-written via
* addLinksBatch / addTimelineEntriesBatch, then watermark-stamped
* (stampExtracted) since both kinds ran.
*
* 3. CORPUS INGEST [CX-P0.1] — LLM-backed, spend-gated. Unprocessed
* `.txt` files in the dream corpus dir run through the narrowest
* one-transcript entry (runFactsPipeline: extract → resolve → dedup
* → insert). KEYLESS RULE [CX-P0.5]: no extraction provider ⇒ skip
* with {reason:'keyless'} — agent-authored fences cover it. A
* `<file>.ingested` sidecar (written AFTER success — crash-safe,
* exactly-once) marks completion; a `<file>.in-progress` claim
* sidecar (O_EXCL) fences concurrent sweeps off the same file so
* two processes never double-pay one transcript's LLM call.
*
* Budget: a wall-clock budget stops the sweep BETWEEN items (and threads an
* AbortSignal into the zero-LLM fence pass only; a corpus extraction in
* flight finishes and writes its sidecar, E-N2); the report carries the
* partial counts. runMaintenanceSweep NEVER throws.
*
* Heavy dependencies (cycle extractor, extract command cores, facts
* pipeline, gateway) are lazy-imported inside each pass — the
* backstop.ts:335 precedent — so importing this module at serve boot is
* cheap and a broken optional dep degrades to a skip, never a crash.
*/
import { join } from 'node:path';
import { readdir, readFile, writeFile, rm, stat } from 'node:fs/promises';
import type { CorpusFactTime } from './context/corpus-windows.ts';
import type { BrainEngine, LinkBatchInput, TimelineBatchInput } from './engine.ts';
import type { FactsBackstopCtx } from './facts/backstop.ts';
import type { CapabilityReport } from './capability.ts';
import { maintenanceTransaction } from './persistence/attribution.ts';
import { managedPersistenceEnabled } from './persistence/ownership.ts';
import { isSegmentedPage } from './page-body/page-shape.ts';
import { speakerOf } from './context/session-speaker.ts';
/** Delay before the serve-startup sweep fires (post-connect settle). */
export const STARTUP_SWEEP_DELAY_MS = 3_000;
/** Fence begin marker — duplicated from facts-fence.ts to keep this module light. */
const FACTS_FENCE_BEGIN_MARKER = 'gbrain:facts:begin';
/**
* Cap on the candidate set the pass-1 leading-wildcard LIKE scans: the fence
* marker can't use an index, so the LIKE runs over a recency-bounded subquery
* (newest N rows in the window) instead of every matching-window row — a
* bulk-sync day can't turn pass 1 into a whole-table scan.
*/
export const FENCE_LIKE_SCAN_CAP = 500;
/** Sidecar suffix marking a corpus file as processed. */
export const CORPUS_INGESTED_SUFFIX = '.ingested';
/**
* Claim sidecar marking a corpus file as being ingested RIGHT NOW. Created
* with O_EXCL (`wx`) before the LLM call so a manual `gbrain sweep --once`
* and the serve-idle sweep (separate processes on a Postgres brain) can't
* both pay for the same transcript. Replaced by the `.ingested` sidecar on
* success; removed on failure so the next sweep retries.
*/
export const CORPUS_CLAIM_SUFFIX = '.in-progress';
/** Claims older than this belong to dead sweeps and are reclaimable. */
export const CORPUS_CLAIM_STALE_MS = 60 * 60 * 1000;
/** #5887 skip reason per unfinished window run (the file waits for a later sweep). */
const CORPUS_WINDOW_SKIP = {
aborted: 'budget_exhausted:corpus',
contended: 'corpus_in_progress',
partial: 'corpus_windows_pending',
changed: 'corpus_changed',
source_changed: 'corpus_source_changed',
unavailable: 'corpus_chat_unavailable',
backoff: 'corpus_window_backoff',
dead_lettered: 'corpus_window_dead_lettered',
} as const;
export interface SweepOpts {
/** Source to sweep. Default 'default' (the serve's registered source). */
sourceId?: string;
/** Max pages per pass / corpus files per sweep. Default 20. */
batchLimit?: number;
/** Wall-clock budget; the sweep stops between items when exceeded. Default 5000. */
budgetMs?: number;
/** The clock the budget reads (test seam, so a test's timing does not depend on machine load). Default Date.now. */
now?: () => number;
/** Recency window (days) for "recently-modified pages". Default 7. */
recentDays?: number;
/** Diagnostic sink (stderr in serve contexts). Default: silent. */
log?: (msg: string) => void;
/**
* Capability report override (test seam / caller already computed one).
* Default: the engine-resolved extraction model checked against the
* configured gateway (facts/extraction-availability.ts) — no network.
*/
capabilities?: CapabilityReport;
/**
* #6786: false skips the corpus pass (pass 3). `bootstrap verify` passes
* false: its probes need only passes 1 and 2, and its process may not have
* the gateway the extraction would use.
*/
corpus?: boolean;
}
export interface SweepSkip {
reason: string;
count: number;
}
export interface SweepReport {
corpusIngested: number;
factsReconciled: number;
linksExtracted: number;
/** Stale sweep-owned edges reconciled away (#4196). */
linksRemoved: number;
timelineExtracted: number;
/** #5887: per corpus file windowed this sweep — windows extracted now and still remaining. */
corpus_files: Array<{ file: string; windows_done: number; windows_remaining: number }>;
skipped: SweepSkip[];
durationMs: number;
}
/**
* #6159: the session file's write time as its facts' time (`turnAt`), or
* undefined when the file time is not trusted (logged; the facts are then
* dated at extraction).
*/
function corpusTurnAt(name: string, factTime: CorpusFactTime, log: (line: string) => void): Date | undefined {
if ('at' in factTime) return factTime.at;
log(`[sweep] ${name}: file time not trusted (${factTime.rejected}); its facts are dated at extraction time.`);
return undefined;
}
/**
* Run one bounded maintenance sweep. Never throws — failures land in
* report.skipped with a per-pass reason.
*/
export async function runMaintenanceSweep(
engine: BrainEngine,
opts: SweepOpts = {},
): Promise<SweepReport> {
const now = opts.now ?? Date.now;
const started = now();
const sourceId = opts.sourceId ?? 'default';
const batchLimit = Math.max(1, opts.batchLimit ?? 20);
const budgetMs = Math.max(0, opts.budgetMs ?? 5_000);
const recentDays = Math.max(1, opts.recentDays ?? 7);
const log = opts.log ?? (() => {});
const deadline = started + budgetMs;
const report: SweepReport = {
corpusIngested: 0,
factsReconciled: 0,
linksExtracted: 0,
linksRemoved: 0,
timelineExtracted: 0,
corpus_files: [],
skipped: [],
durationMs: 0,
};
const skip = (reason: string, count = 1): void => {
if (count <= 0) return;
const existing = report.skipped.find(s => s.reason === reason);
if (existing) existing.count += count;
else report.skipped.push({ reason, count });
};
const overBudget = () => now() >= deadline;
const budgetLeft = () => budgetMs > 0 ? Math.max(0, deadline - now()) / budgetMs : 0;
// Budget abort signal: threads into the fence pass's per-page loop so a
// long item can be interrupted at its own checkpoints. unref'd — the sweep must never hold the
// process open (the serve unref convention).
const budgetController = new AbortController();
const budgetTimer = setTimeout(
() => budgetController.abort(),
Math.max(0, deadline - now()),
);
budgetTimer.unref?.();
const cutoffIso = new Date(started - recentDays * 86_400_000).toISOString();
try {
// ── Pass 1: facts-fence reconciliation [CX2-4] — zero-LLM ─────────
try {
if (overBudget()) {
skip('budget_exhausted:facts_fence');
} else {
// The leading-wildcard LIKE can't use an index, so it scans only the
// newest FENCE_LIKE_SCAN_CAP rows in the recency window (inner
// subquery) rather than every row a bulk-sync day may have touched.
const rows = await engine.executeRaw<{ slug: string }>(
`SELECT slug FROM (
SELECT id, slug, compiled_truth, body_storage, updated_at FROM pages
WHERE source_id = $1
AND deleted_at IS NULL
AND updated_at >= $2::timestamptz
ORDER BY updated_at DESC
LIMIT $4
) AS recent
WHERE compiled_truth LIKE $3
OR (body_storage = 'segmented' AND EXISTS (SELECT 1 FROM page_body_manifest m
WHERE m.page_id = recent.id AND m.gen_to IS NULL AND m.staging_id IS NULL AND m.facts_slots <> '[]'::jsonb))
ORDER BY updated_at DESC
LIMIT $5`,
[sourceId, cutoffIso, `%${FACTS_FENCE_BEGIN_MARKER}%`, FENCE_LIKE_SCAN_CAP, batchLimit],
);
if (rows.length > 0) {
const { runExtractFacts } = await import('./cycle/extract-facts.ts');
const r = await runExtractFacts(engine, {
slugs: rows.map(row => row.slug),
sourceId,
signal: budgetController.signal,
});
report.factsReconciled = r.factsInserted;
// The reconcile's refusals (FACTS_PAGE_CACHE_STALE, LOCK_TIMEOUT, …)
// ride r.warnings; dropped, a refused pass reads as "nothing to do".
for (const w of r.warnings) log(`[sweep] facts reconcile: ${w}`);
if (r.guardTriggered) skip('facts_fence_guard');
if (budgetController.signal.aborted) skip('budget_exhausted:facts_fence');
}
}
} catch (e) {
skip('facts_fence_error');
log(`[sweep] facts-fence pass failed: ${e instanceof Error ? e.message : String(e)}`);
}
// ── Pass 2: link/timeline extraction [CX-P0.3] — zero-LLM ─────────
try {
if (overBudget()) {
skip('budget_exhausted:links_timeline');
} else {
await runLinksTimelinePass(engine, {
sourceId,
batchLimit,
cutoffIso,
overBudget,
budgetLeft,
report,
skip,
log,
});
}
} catch (e) {
skip('links_timeline_error');
log(`[sweep] links/timeline pass failed: ${e instanceof Error ? e.message : String(e)}`);
}
// ── Pass 3: corpus ingest [CX-P0.1] — spend-gated [CX-P0.5] ───────
try {
if (opts.corpus === false) {
// #6786: the caller asked for passes 1 and 2 only.
} else if (overBudget()) {
skip('budget_exhausted:corpus');
} else {
// E-N2: the budget stops the corpus pass BETWEEN files and windows
// (overBudget), never mid-call. An extraction slower than the budget
// aborted at the deadline wrote no sidecar, so every scheduled
// `sweep --once` paid for the same call and finished nothing. The
// gateway's own timeout still bounds a hung call.
await runCorpusIngestPass(engine, {
sourceId,
batchLimit,
overBudget,
signal: new AbortController().signal,
capabilities: opts.capabilities,
now,
report,
skip,
log,
});
}
} catch (e) {
skip('corpus_error');
log(`[sweep] corpus pass failed: ${e instanceof Error ? e.message : String(e)}`);
}
} catch (e) {
// Structural failure outside every per-pass catch (should be
// unreachable). The never-throw contract holds regardless.
skip('sweep_error');
log(`[sweep] sweep failed: ${e instanceof Error ? e.message : String(e)}`);
} finally {
clearTimeout(budgetTimer);
}
report.durationMs = now() - started;
return report;
}
interface PassCtx {
sourceId: string;
batchLimit: number;
overBudget: () => boolean;
report: SweepReport;
skip: (reason: string, count?: number) => void;
}
/** The share of the sweep's budget still left (1 at its start, 0 once it is spent). */
type FreshBudget = { budgetLeft: () => number };
/**
* Segmented pages whose links/timeline pass did not fit a whole sweep budget, `source\0slug` to the page's updated_at
* then: later sweeps in this process leave them out of the pass's page query until the page changes, so one page too
* large for any budget cannot take every sweep's budget, nor the batch slots of the pages behind it.
* `gbrain extract --stale` extracts them.
*/
const deferredSegmentedPages = new Map<string, string>();
/** Test seam: forget deferred segmented pages. */
export function _resetDeferredSegmentedPagesForTests(): void {
deferredSegmentedPages.clear();
}
/**
* Pass 2 body. Calls the SAME per-page cores as `gbrain extract
* links|timeline --source db` (extract.ts:extractLinksFromDB /
* extractTimelineFromDB): extractPageLinks + parseTimelineEntries, with
* resolveCandidateSources doing the multi-source endpoint validation and
* stampExtracted advancing the links_extracted_at watermark (both kinds
* run here, so stamping is correct per extract.ts's C3/D6 rule).
*/
async function runLinksTimelinePass(
engine: BrainEngine,
ctx: PassCtx & FreshBudget & { cutoffIso: string; log: (msg: string) => void },
): Promise<void> {
const { sourceId, batchLimit, cutoffIso, overBudget, report, skip, log } = ctx;
const {
extractPageLinks,
resolvedLinkCandidate,
parseTimelineEntries,
makeResolver,
isGlobalBasenameEnabled,
isAutoLinkEnabled,
isAutoTimelineEnabled,
isCrossSourceLinksEnabled,
} = await import('./link-extraction.ts');
// Respect the same operator kill switches put_page's inline hooks honor.
const [linksEnabled, timelineEnabled] = await Promise.all([
isAutoLinkEnabled(engine),
isAutoTimelineEnabled(engine),
]);
if (!linksEnabled) skip('auto_link_disabled');
if (!timelineEnabled) skip('auto_timeline_disabled');
if (!linksEnabled && !timelineEnabled) return;
// #4196: honor the watermark this pass stamps, or repeated bounded sweeps
// re-select the same newest batchLimit rows forever and page batchLimit+1
// is never reached. Same predicate as engine-sql/pages.ts stalePagesWhere
// (no versionTs branch — extractor-version catch-up is `extract --stale`'s
// job; the sweep is a recency back-stop). The µs to_char projection is the
// #1768 stamp discipline: stamp the row's READ updated_at, not now(), so an
// edit between SELECT and stamp stays stale and re-sweeps. Deferred pages are left out in SQL, before the LIMIT.
const deferred = await liveDeferredPages(engine, sourceId, cutoffIso);
skip('segmented_page_deferred', deferred.length);
const recent = await engine.executeRaw<{ slug: string; updated_at_iso: string; body_storage: string | null; body_bytes: number | null; read_at: string }>(
`SELECT slug, to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS read_at,
to_char(updated_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS updated_at_iso,
body_storage, body_bytes
FROM pages
WHERE source_id = $1
AND deleted_at IS NULL
AND updated_at >= $2::timestamptz
AND (links_extracted_at IS NULL OR updated_at > links_extracted_at)
AND NOT (slug = ANY($4::text[]))
ORDER BY updated_at DESC
LIMIT $3`,
[sourceId, cutoffIso, batchLimit, deferred],
);
const readAt = recent[0]?.read_at; if (!readAt) return;
// resolveCandidateSources + stampExtracted are the shared helpers the
// extract command exports precisely so sibling walkers can't drift from
// its F10 multi-source resolution (see extract.ts:114).
const { resolveCandidateSources, stampExtracted, resolveLinkFallbackDefault } = await import('../commands/extract.ts');
const resolver = makeResolver(engine, { mode: 'batch', sourceId });
const globalBasename = await isGlobalBasenameEnabled(engine);
// #3190: pack-aware verbs — the sweep must type edges the same way the
// extract command does or reconciliation flip-flops the link_type.
const { loadActivePackForLocalEngine } = await import('./schema-pack/best-effort.ts');
const pack = (await loadActivePackForLocalEngine(engine, { sourceId }))?.manifest ?? null;
if (linksEnabled && !pack) throw new Error('Cannot extract links: active schema pack is unavailable.');
type Extracted = Awaited<ReturnType<typeof extractPageLinks>>;
type Snapshot = NonNullable<Awaited<ReturnType<BrainEngine['readPageSnapshot']>>>;
const { extractSegmentedPageLinks, segmentedRoleText } = await import('./page-body/segment-links.ts');
const { BodyReadStopped, bodyBlocks, detachedValue } = await import('./page-body/segment-blocks.ts');
/**
* One batch through every phase. A segmented page's reads ask `stop` before each segment, so one that does not fit
* the budget throws BodyReadStopped between segments.
*/
const runBatch = async (recent: Array<{ slug: string; updated_at_iso: string }>, stop?: () => boolean): Promise<void> => {
// A segmented body is read block by block (page-body/segment-links.ts), never as one string: its role phrases once
// for both extractions, and its timeline entries from the first extraction's blocks.
const roleTexts = new Map<string, string>();
const linksOf = async (snapshot: Snapshot, opts: Parameters<typeof extractPageLinks>[5] & { temporal?: boolean; onBlock?: (block: { text: string }) => void }): ReturnType<typeof extractSegmentedPageLinks> => {
const { page } = snapshot;
if (!isSegmentedPage(page)) return extractPageLinks(page.slug, `${page.compiled_truth}\n${page.timeline}`, page.frontmatter, page.type, resolver, opts);
const ref = { id: Number(page.id), slug: page.slug, source_id: sourceId, type: page.type, frontmatter: page.frontmatter };
const roleText = roleTexts.get(page.slug) ?? await segmentedRoleText(engine, ref, { stop });
roleTexts.set(page.slug, roleText);
return extractSegmentedPageLinks(engine, ref, resolver, { ...opts, roleText, stop });
};
const temporalBodies = new Map<string, NonNullable<Awaited<ReturnType<typeof extractSegmentedPageLinks>>['temporalBody']>>();
const tlBatch: TimelineBatchInput[] = [];
const processedRefs: Array<{ slug: string; source_id: string; extractedAt: string }> = [];
const pageCandidates: Array<{ slug: string; candidates: Extracted['candidates'] }> = [];
const snapshots = new Map<string, Snapshot>();
// Phase 1: per-page extraction. The per-slug getPage loop stays a loop —
// BrainEngine has no batch read-by-slug-list primitive (resolveSlugsByPaths
// is path→slug only), and the loop is bounded by batchLimit (default 20).
for (let i = 0; i < recent.length; i++) {
if (overBudget()) {
skip('budget_exhausted:links_timeline', recent.length - i);
break;
}
const slug = recent[i].slug;
const page = await engine.getPage(slug, { sourceId });
if (!page) continue;
const snapshot = await engine.readPageSnapshot(slug, { sourceId });
const segmented = isSegmentedPage(page);
if (!snapshot || segmented !== isSegmentedPage(snapshot.page)
|| (!segmented && (snapshot.page.compiled_truth !== page.compiled_truth || snapshot.page.timeline !== page.timeline))
|| snapshot.page.type !== page.type || JSON.stringify(snapshot.page.frontmatter) !== JSON.stringify(page.frontmatter)) continue;
snapshots.set(slug, snapshot);
// A segmented page's entries are copied off their block (detachedValue), or each would keep its whole block alive.
const entries: ReturnType<typeof parseTimelineEntries> = [];
const timelineOf = (block: { text: string }) => { for (const entry of parseTimelineEntries(block.text)) entries.push(detachedValue(entry)); };
if (linksEnabled) {
// skipFrontmatter matches user-invoked `gbrain extract links`
// (frontmatter backfill stays a migration-orchestrator concern).
const extracted = await linksOf(snapshot, { skipFrontmatter: true, globalBasename, pack, temporal: segmented,
...(segmented && timelineEnabled ? { onBlock: timelineOf } : {}) });
if (extracted.temporalBody) temporalBodies.set(slug, extracted.temporalBody);
if (extracted.candidates.length > 0) {
pageCandidates.push({ slug, candidates: extracted.candidates });
}
}
if (timelineEnabled) {
if (!segmented) entries.push(...parseTimelineEntries(page.compiled_truth + '\n' + page.timeline));
else if (!linksEnabled) for await (const block of bodyBlocks(engine, snapshot.page, { stop })) timelineOf(block);
for (const entry of entries) {
// Same row shape as extractTimelineFromDB's batch push (extract.ts):
// #3957 — parsed source label threaded so FS- and DB-extracted rows
// share one dedup shape; detail '' when empty.
tlBatch.push({
slug,
date: entry.date,
source: entry.source,
summary: entry.summary,
detail: entry.detail || '',
source_id: sourceId,
});
}
}
processedRefs.push({ slug, source_id: sourceId, extractedAt: recent[i].updated_at_iso });
}
// Phase 2: endpoint validation is scoped to the slugs the candidates
// actually name — NOT engine.listAllPageRefs() (the full (slug, source_id)
// map, O(all pages) per sweep; the extract command amortizes that cost over
// a whole-brain run, a recurring bounded sweep must not). The targeted
// lookup keeps listAllPageRefs' visibility semantics (deleted_at IS NULL),
// so resolveCandidateSources' F10 resolution is unchanged — it just sees
// only the rows it can possibly use. Zero candidates ⇒ zero queries.
const linkBatch: LinkBatchInput[] = [];
const endpointMetadata = new Map<string, { slug: string; source_id: string; type: string; knowledge_revision: string }>();
const incomplete = new Set<string>();
const wantedBySlug = new Map<string, WantedLinkInput[]>();
const wantedEnabled = await isWantedPagesEnabled(engine);
// One settings snapshot per sweep; publication refuses a page if the settings move meanwhile (it stays unstamped).
const grammar = await readLineGrammarSettings(engine);
const lineGrammar = { enabled: grammar.enabled, allowUndeclaredTypes: grammar.allowUndeclaredTypes };
if (pageCandidates.length > 0) {
const needed = new Set<string>();
for (const { slug, candidates } of pageCandidates) {
needed.add(slug);
for (const c of candidates) {
needed.add(c.targetSlug);
if (c.fromSlug) needed.add(c.fromSlug);
}
}
const { allSlugs, slugToSources, metadata } = await lookupRefsForSlugs(engine, [...needed]);
for (const row of metadata) endpointMetadata.set(`${row.source_id}\0${row.slug}`, row);
// #3478: the 'default' fallback is a federation feature — a sweep over an
// isolated source must not push cross-source edges. Single-row fetchSource
// (not loadAllSources) keeps the sweep's bounded-cost discipline; a missing
// sources row fails closed to isolated.
const { fetchSource, isSourceFederated } = await import('./sources-load.ts');
const sourceRow = await fetchSource(engine, sourceId);
const allowCrossSource = sourceRow !== null && isSourceFederated(sourceRow.config);
// #3757: thread the same `link_resolution.cross_source` opt-in (#2589) and
// configured default source (#4611) the CLI extract lanes pass. Without
// them the sweep dropped every edge into a non-default source AND its
// reconcile below deleted the ones `extract links --source db` created.
const [crossSource, linkDefaultSourceId] = await Promise.all([
isCrossSourceLinksEnabled(engine),
resolveLinkFallbackDefault(engine),
]);
let crossSourceDrops = 0;
for (const { slug } of pageCandidates) {
const { candidates, attendanceComplete } = await linksOf(snapshots.get(slug)!,
{ skipFrontmatter: true, globalBasename, pack, lineGrammar, targetType: (targetSlug, targetSourceId) => {
const resolved = resolveCandidateSources({ targetSlug, targetSourceId, linkType: '', context: '' }, slug,
sourceId, allSlugs, slugToSources, allowCrossSource, { crossSource, defaultSourceId: linkDefaultSourceId });
return resolved.ok ? endpointMetadata.get(`${resolved.toSourceId}\0${targetSlug}`)?.type : undefined;
} });
if (!attendanceComplete) {
incomplete.add(slug);
skip('attendance_resolution_incomplete');
continue;
}
if (wantedEnabled) {
wantedBySlug.set(slug, collectWantedLinks({ candidates, originSourceId: sourceId,
crossSourceAllowed: allowCrossSource || crossSource, resolve: c => resolveCandidateSources(c, slug, sourceId,
allSlugs, slugToSources, allowCrossSource, { crossSource, defaultSourceId: linkDefaultSourceId }) }));
}
for (const c of candidates) {
// #2589: a cross_source drop here means the target exists only in
// other sources and cross-source links are off — counted in the
// skip ledger exactly like the extract paths count it.
const resolved = resolveCandidateSources(
c, slug, sourceId, allSlugs, slugToSources, allowCrossSource,
{ crossSource, defaultSourceId: linkDefaultSourceId },
);
if (!resolved.ok) {
if (resolved.reason === 'cross_source') crossSourceDrops++;
continue;
}
linkBatch.push(resolvedLinkCandidate(c, slug, sourceId, resolved));
}
}
if (crossSourceDrops > 0) skip('cross_source_link', crossSourceDrops);
}
// Engine batch primitives self-retry; default auditSite labels apply
// (BATCH_AUDIT_SITES is a closed enum owned by retry.ts).
// A managed brain refuses raw timeline inserts (managed_writer_guard): each page publishes through the coordinator.
const timelineUnsettled = new Set<string>();
if (tlBatch.length > 0 && await managedPersistenceEnabled(engine)) {
const { publishManagedPageTimeline } = await import('../commands/extract-timeline-db.ts');
for (const slug of new Set(tlBatch.map(row => row.slug))) {
if (overBudget() && stop) throw new BodyReadStopped(slug); // a segmented page's batch is that page alone
if (overBudget()) { skip('budget_exhausted:timeline'); timelineUnsettled.add(slug); continue; }
try {
const added = await publishManagedPageTimeline(engine, slug, sourceId, stop);
if (added === 'unsettled') timelineUnsettled.add(slug);
else report.timelineExtracted += added;
} catch (e) {
if (e instanceof BodyReadStopped) throw e;
skip('timeline_publish_failed');
timelineUnsettled.add(slug);
}
}
} else if (tlBatch.length > 0) {
report.timelineExtracted += await maintenanceTransaction(engine, tx => tx.addTimelineEntriesBatch(tlBatch)); // gbrain-allow-direct-insert: same extract-path rationale as addLinksBatch above [CX-P0.3]
}
// #4196: reconcile removals. The sweep is the ONLY link extraction remote
// put_page pages ever get (runAutoLink is skipped by design), so add-only
// inserts leave a phantom edge forever once a page drops a reference.
// Mirror runAutoLink's provenance-scoped removal (ops/pages.ts) — markdown /
// NULL-legacy / wikilink-resolved only — minus its own-frontmatter clause:
// the sweep extracts with skipFrontmatter, so its desired set can never
// contain frontmatter candidates and deleting them would clobber valid
// edges. 'manual' and 'mentions' are never touched. A page whose reconcile
// fails (or is cut by budget) is left unstamped so the next sweep retries.
const stampable = new Set(processedRefs.map(r => r.slug).filter(slug => !timelineUnsettled.has(slug)));
if (linksEnabled) {
const desiredBySlug = new Map<string, LinkBatchInput[]>(
processedRefs.map(r => [r.slug, []]),
);
for (const b of linkBatch) {
desiredBySlug.get(b.origin_slug ?? b.from_slug)?.push(b);
}
for (const ref of processedRefs) {
if (incomplete.has(ref.slug) || overBudget()) {
if (!incomplete.has(ref.slug)) skip('budget_exhausted:link_reconcile');
stampable.delete(ref.slug);
continue;
}
const desired = desiredBySlug.get(ref.slug)!;
try {
const snapshot = snapshots.get(ref.slug)!;
const result = await engine.replaceDerivedLinks({ slug: ref.slug, sourceId,
expectedRevision: snapshot.revision, sourceIncarnation: snapshot.sourceIncarnation }, desired,
{ includeFrontmatter: false, preserveExisting: true, lineGrammar: grammar,
wanted: { producers: ['body'], rows: wantedBySlug.get(ref.slug) ?? [], readAt }, temporalBody: temporalBodies.get(ref.slug), expectedEndpoints: [...new Set(desired.flatMap(row =>
[`${row.from_source_id}\0${row.from_slug}`, `${row.to_source_id}\0${row.to_slug}`]))].map(key => {
const endpoint = endpointMetadata.get(key)!;
return { slug: endpoint.slug, sourceId: endpoint.source_id, revision: endpoint.knowledge_revision };
}) });
report.linksRemoved += result.removed;
report.linksExtracted += result.created;
} catch {
skip('link_reconcile_failed');
stampable.delete(ref.slug);
}
}
}
// Stamp only when BOTH kinds ran for these pages (extract.ts C3/D6:
// links_extracted_at covers links AND timeline). Per-ref extractedAt is the
// page's read updated_at (#1768 µs discipline; D4 — an edit between SELECT
// and stamp stays stale).
if (linksEnabled && timelineEnabled) {
const toStamp = processedRefs.filter(r => stampable.has(r.slug));
if (toStamp.length > 0) await stampExtracted(engine, toStamp);
}
};
await runLinksTimelineBatches(recent, ctx, runBatch);
}
/**
* The slugs of this source's deferred segmented pages that are still stale and unchanged since they were deferred;
* an entry whose page changed, was stamped or left the recency window is forgotten.
*/
async function liveDeferredPages(engine: BrainEngine, sourceId: string, cutoffIso: string): Promise<string[]> {
const prefix = `${sourceId}\0`;
const entries = [...deferredSegmentedPages].filter(([key]) => key.startsWith(prefix));
if (entries.length === 0) return [];
const rows = await engine.executeRaw<{ slug: string; updated_at_iso: string }>(
`SELECT slug, to_char(updated_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS updated_at_iso
FROM pages
WHERE source_id = $1
AND deleted_at IS NULL
AND updated_at >= $2::timestamptz
AND (links_extracted_at IS NULL OR updated_at > links_extracted_at)
AND slug = ANY($3::text[])`,
[sourceId, cutoffIso, entries.map(([key]) => key.slice(prefix.length))],
);
const live = new Set(rows.filter(row => deferredSegmentedPages.get(prefix + row.slug) === row.updated_at_iso).map(row => row.slug));
for (const [key] of entries) if (!live.has(key.slice(prefix.length))) deferredSegmentedPages.delete(key);
return [...live];
}
/** A segmented page that does not fit is deferred only when it started with at least this share of the sweep's budget. */
const FRESH_BUDGET_SHARE = 0.5;
/**
* #C5: a segmented page is read several times per pass and can outlast any budget, so it never shares a batch: the
* inline pages run first as one batch, then each segmented page alone, smallest first, while budget remains. One
* that does not fit stops at a segment bound (BodyReadStopped) and is left unstamped. It is deferred until its page
* changes only when it started with at least FRESH_BUDGET_SHARE of the sweep's budget, so it cannot take the budget of
* later sweeps in this process; one that started after other work spent more is left for the next sweep.
*/
async function runLinksTimelineBatches(
recent: Array<{ slug: string; updated_at_iso: string; body_storage: string | null; body_bytes: number | null }>,
ctx: Pick<PassCtx, 'sourceId' | 'overBudget' | 'skip'> & FreshBudget & { log: (msg: string) => void },
runBatch: (recent: Array<{ slug: string; updated_at_iso: string }>, stop?: () => boolean) => Promise<void>,
): Promise<void> {
const { sourceId, overBudget, skip, log } = ctx;
const { BodyReadStopped } = await import('./page-body/segment-blocks.ts');
const inline = recent.filter(row => row.body_storage !== 'segmented');
const segmented = recent.filter(row => row.body_storage === 'segmented').sort((a, b) => Number(a.body_bytes ?? 0) - Number(b.body_bytes ?? 0));
if (inline.length > 0) await runBatch(inline);
for (let i = 0; i < segmented.length; i++) {
if (overBudget()) {
skip('budget_exhausted:links_timeline', segmented.length - i);
break;
}
const row = segmented[i]!;
const whole = ctx.budgetLeft() >= FRESH_BUDGET_SHARE;
try {
await runBatch([row], overBudget);
} catch (e) {
if (!(e instanceof BodyReadStopped)) throw e;
if (!whole) {
skip('budget_exhausted:links_timeline');
continue;
}
deferredSegmentedPages.set(`${sourceId}\0${row.slug}`, row.updated_at_iso);
skip('budget_deferred:segmented_page');
log(`[sweep] links/timeline: ${row.slug} (${row.body_bytes ?? '?'} bytes, segmented) does not fit the sweep budget; deferred until it changes. \`gbrain extract --stale --source-id ${sourceId}\` extracts it.`);
}
}
}
/**
* Pass 3 body. `runFactsPipeline` — the narrowest existing entry that takes
* raw transcript text through extract → resolve → dedup → insert — runs once
* per writeback turn file and once per turn-boundary window of every other
* corpus file (#5887, context/corpus-windows.ts: per-file and per-sweep
* window caps, `.progress` resume). Visibility left unset so the
* pipeline resolves the operator default via resolveDefaultVisibility
* (backstop.ts:359, [ENG-8]). `.ingested` written only after the file's last
* window succeeds.
*
* Concurrency: a `<file>.in-progress` claim sidecar (O_EXCL create) fences
* each file before its LLM call — a manual `gbrain sweep --once` racing the
* serve-idle sweep never double-spends on the same transcript. Success
* replaces the claim with the `.ingested` sidecar; failure removes the claim
* (next sweep retries); claims older than CORPUS_CLAIM_STALE_MS belong to
* dead sweeps and are reclaimed.
*/
async function runCorpusIngestPass(
engine: BrainEngine,
ctx: PassCtx & {
signal: AbortSignal;
capabilities?: CapabilityReport;
now: () => number;
log: (msg: string) => void;
},
): Promise<void> {
const { batchLimit, overBudget, signal, report, skip, log, now } = ctx;
// Corpus dir: dream's session corpus (transcripts.ts:66 precedent);
// default ~/.gbrain/transcripts/corpus (GBRAIN_HOME-aware via configDir).
let dir = await engine.getConfig('dream.synthesize.session_corpus_dir');
if (!dir) {
const { configDir } = await import('./config.ts');
dir = join(configDir(), 'transcripts', 'corpus');
}
// #6268: source-stamped files live in the spool subdirectory (listed
// first); top-level files are the legacy, unstamped corpus.
const { CORPUS_SPOOL_SUBDIR, corpusSpoolDir, parseWbFileName, selfCaptureSidecarJson, corpusFileSessionId, corpusTextForExtraction, corpusFileCompletion, corpusFileFinished, CORPUS_WINDOW_FAILURE_CAP } = await import('./context/corpus-segments.ts');
const spoolDir = corpusSpoolDir(dir);
const windows = await import('./context/corpus-windows.ts');
type Candidate = { base: string; name: string; location: 'spool' | 'legacy'; label: string };
const candidates: Candidate[] = [];
const deferred: Candidate[] = [];
// #6785: a capped window re-arms once when this fingerprint changed since its last failure.
const { extractionConfigFingerprint } = await import('./facts/extraction-availability.ts');
const fingerprint = await extractionConfigFingerprint(engine).catch(() => null);
const parkedAt = now();
for (const [base, location] of [[spoolDir, 'spool'], [dir, 'legacy']] as const) {
let entries: string[];
try {
entries = await readdir(base);
} catch {
continue; // no corpus dir = nothing to ingest (not an error)
}
// ONE readdir feeds both the .txt listing and the sidecar checks.
const entrySet = new Set(entries);
const txtFiles = entries.filter(n => n.endsWith('.txt')).sort();
if (txtFiles.length === 0) continue;
// #5887: finished files re-enter when changed since their `.progress`.
const picked = await windows.selectCorpusCandidates(base, txtFiles, entrySet, batchLimit - candidates.length, overBudget,
(full) => windows.corpusFileParked(full, fingerprint, parkedAt));
skip('already_ingested', picked.alreadyIngested);
const label = (name: string) => location === 'spool' ? join(CORPUS_SPOOL_SUBDIR, name) : name;
for (const name of picked.candidates) candidates.push({ base, name, location, label: label(name) });
for (const name of picked.deferred) deferred.push({ base, name, location, label: label(name) });
}
candidates.push(...deferred.slice(0, Math.max(0, batchLimit - candidates.length)));
if (candidates.length === 0) return;
// #6091: every corpus file answers to the capture gate, resolved once per
// pass from the AUTHORITATIVE DB plane ({gate:true}: never a last-known-good
// enabled bundle) with the file mirror for drift. `.wb-` turn files (the
// Stop-hook backstop when serve/IPC was away, OV2-11) retire on off or unset;
// `.seg-` segments and SessionEnd `<sid>.txt` transcripts retire on an
// explicit off and extract while unset; a read error, plane drift or an
// invalid mode holds them untouched. Retirement runs BEFORE the
// keyless/kill-switch short-circuits so an operator's OFF retires banked
// files even when the brain cannot extract — otherwise they linger eligible
// and a later re-enable would extract text the operator already revoked.
const { applyCaptureGate, captureLaneForFile, resolveCaptureGate } = await import('./context/capture-consent.ts');
const { resolveCorpusFileSource } = await import('./context/corpus-source.ts');
const gate = await resolveCaptureGate(engine);
const retireCandidatesIfOff = async (): Promise<Set<string>> => {
const retired = new Set<string>();
for (const { base, name, label } of candidates) {
const decision = gate[captureLaneForFile(name)];
if (decision.action !== 'retire') continue;
try {
if ((await applyCaptureGate(join(base, name), decision)).action !== 'retire') continue;
retired.add(label);
skip('writeback_off');
} catch { /* per-file best effort — the next sweep retries */ }
}
return retired;
};
// [CX-P0.5] Keyless rule: no extraction provider configured ⇒ skip the
// whole pass. Agent-authored fences (pass 1) carry keyless memory.
const { extractionAvailableForEngine } = await import('./facts/extraction-availability.ts');
if (!(await extractionAvailableForEngine(engine, ctx.capabilities))) {
const retired = await retireCandidatesIfOff();
skip('keyless', candidates.length - retired.size);
return;
}
// Existing spend gate: operators flip facts.extraction_enabled off to
// stop ALL fact extraction brain-wide (facts/extract.ts:43).
const { isFactsExtractionEnabled } = await import('./facts/extract.ts');
if (!(await isFactsExtractionEnabled(engine))) {
const retired = await retireCandidatesIfOff();
skip('extraction_disabled', candidates.length - retired.size);
return;
}
const writerRefused = await corpusWriterPreflight(engine, log);
const { runFactsPipeline } = await import('./facts/backstop.ts');
const { isDreamOutput } = await import('./cycle/transcript-discovery.ts');
const { claudeCliSelfProjectDirs, isClaudeCliSelfSessionId } = await import('./ai/providers/claude-cli-scratch.ts');
const selfProjectDirs = claudeCliSelfProjectDirs();
let windowsLeft = windows.resolveCorpusWindowsPerSweepTotal(process.env, log);
for (let i = 0; i < candidates.length; i++) {
if (overBudget()) {
skip('budget_exhausted:corpus', candidates.length - i);
break;
}
if (windowsLeft <= 0) {
skip('corpus_window_cap', candidates.length - i);
break;
}
const { base, name, location, label } = candidates[i];
const full = join(base, name);
// Atomic claim BEFORE any spend — the losing sweep skips, never re-pays.
const claimPath = full + CORPUS_CLAIM_SUFFIX;
if (!(await acquireCorpusClaim(claimPath))) {
skip('corpus_in_progress');
continue;
}
let abortLoop = false;
const attempted = { count: 0 };
try {
// Re-check under the claim: another sweep may have finished this file
// between our readdir and our claim (it releases its claim only after
// writing the .ingested sidecar, so this closes the double-spend gap).
let doneAlready = false;
try { doneAlready = corpusFileFinished(corpusFileCompletion(full, name)); } catch { /* unreadable: not done */ }
if (doneAlready) {
skip('already_ingested');
continue;
}
// #5413: a corpus file captured from gbrain's own claude-cli call, in
// any capture form. Extracting it spawns another claude-cli call; the
// classification is permanent, so the terminal sidecar stops the retry.
if (isClaudeCliSelfSessionId(corpusFileSessionId(name), selfProjectDirs)) {
await writeFile(full + CORPUS_INGESTED_SUFFIX, selfCaptureSidecarJson());
skip('self_capture');
continue;
}
// #6091: the capture gate, under the claim, before any provider call.
const applied = await applyCaptureGate(full, gate[captureLaneForFile(name)]);
if (applied.action !== 'extract') {
skip(applied.reason);
continue;
}
const wbMeta = parseWbFileName(name);
const fileStat = windows.corpusFileStat(await stat(full));
const raw = await readFile(full, 'utf-8');
// Anti-loop: never ingest dream-generated outputs. Marking them
// processed is safe — the classification is deterministic and
// permanent, and the sidecar stops the sweep re-reading them forever.
if (isDreamOutput(raw)) {
await writeFile(
full + CORPUS_INGESTED_SUFFIX,
JSON.stringify({ ingested_at: new Date().toISOString(), skipped: 'dream_output' }) + '\n',
);
skip('dream_output');
continue;
}
// #6268 source fidelity: every corpus file's facts, links and manifest
// go to the source its NAME (or its session's frozen record) carries,
// never the pass's source. A file with no evidence of its source is
// held: no sidecar, a typed skip, an operator mapping releases it.
const fileSource = await resolveCorpusFileSource(engine, dir, name, location);
if (!fileSource.ok) {
skip(fileSource.reason);
continue;
}
if (await writerRefused(fileSource.sourceId)) {
skip('writer_coordinator_required');
continue;
}
const pipelineCtx: FactsBackstopCtx = {
engine,
sourceId: fileSource.sourceId,
sessionId: wbMeta ? wbMeta.sessionId : `sweep:corpus:${name}`,
// Writeback turn files keep their lane's provenance + salient
// notability filter so batch-extracted turns are indistinguishable
// from prompt-harvested ones.
source: wbMeta ? 'hook:writeback' : 'sweep:corpus',
...speakerOf(full),
mode: 'inline',
remote: false,
abortSignal: signal,
// #5888/#6159: the file's write time anchors the dedup window and dates the facts (never validFrom: a batch-key input).
turnAt: corpusTurnAt(name, windows.corpusFactTime(fileStat.mtime_ms), log),
reAdmitFileRefusals: true,
...(wbMeta && gate.mode === 'salient' ? { notabilityFilter: 'medium-and-up' as const } : {}),
// visibility deliberately unset → resolveDefaultVisibility [ENG-8]
};
// #5812: pasted blocks never reach the extractor (file unchanged). A wb
// file is one gated turn: one call. Every other corpus file is windowed
// at turn boundaries (#5887) and may finish over several sweeps.
let r: Awaited<ReturnType<typeof runFactsPipeline>>;
if (wbMeta) {
const turn = await windows.runCorpusTurn({
full, signal, incarnation: fileSource.incarnation, failures: { fingerprint, now }, attempted,
extract: () => runFactsPipeline(corpusTextForExtraction(name, raw), pipelineCtx),
});
if (turn.status !== 'complete') {
abortLoop = turn.status === 'unavailable';
skip(CORPUS_WINDOW_SKIP[turn.status], abortLoop ? candidates.length - i : 1);
if (turn.status === 'dead_lettered') log(deadLetterLine(label, CORPUS_WINDOW_FAILURE_CAP));
continue;
}
r = turn.result;
} else {
const run = await windows.runCorpusWindows({
full, raw, fileStat, overBudget, signal,
source: { id: fileSource.sourceId, incarnation: fileSource.incarnation },
maxWindows: Math.min(windows.CORPUS_WINDOWS_PER_SWEEP, windowsLeft),
extract: text => runFactsPipeline(text, pipelineCtx),
failures: { fingerprint, now }, attempted,
});
report.corpus_files.push({ file: label, windows_done: run.windowsDone, windows_remaining: run.windowsRemaining });
if (run.status !== 'complete') {
abortLoop = run.status === 'aborted' || run.status === 'unavailable';
skip(CORPUS_WINDOW_SKIP[run.status], abortLoop ? candidates.length - i : 1);
if (run.status === 'dead_lettered') log(deadLetterLine(label, CORPUS_WINDOW_FAILURE_CAP));
continue;
}
r = run.result;
}
// POST-check (adversarial review, same class the harvest FIFO pins):
// runFactsPipeline returns NORMALLY with partial results when the
// budget signal aborts mid-file. Writing `.ingested` here would
// permanently mark a half-extracted file done — skip the sidecar,
// release the claim, and let the next sweep retry it.
if (signal.aborted) {
skip('budget_exhausted:corpus', candidates.length - i);
abortLoop = true;
continue;
}
// Cathedral 5 — best-effort checkpoint-link publish when the backstop
// ingests a compaction segment (queue overflow, serve down at compact
// time, keyless-then-keyed): without this, every segment that falls to
// the sweep loses its brain:// links forever. The harvest FIFO remains
// the guaranteed lane (receipt retry); here a publish failure is
// fail-open and `.ingested` is written regardless (facts are durable;
// re-extracting just to retry a link append is the worse trade).
const linksBanked = r.entity_slugs.length ? await bankSegmentLinks(engine, base, name, fileSource.sourceId, r.entity_slugs) : 0;
// Sidecar AFTER success — a crash before this line re-processes the
// file next sweep (dedup absorbs the repeats), never loses it.
await writeFile(
full + CORPUS_INGESTED_SUFFIX,
JSON.stringify({
ingested_at: new Date().toISOString(),
facts_inserted: r.inserted,
facts_duplicate: r.duplicate,
// Honesty: the sweep is the LAST attempt (the harvest lane already
// declined to sidecar on this), so a non-transport extraction skip
// is terminal HERE — record why instead of a silent zero-count.
...(r.skipped_reason ? { skipped: r.skipped_reason } : {}),
...(linksBanked ? { links_banked: linksBanked } : {}),
}) + '\n',
);
report.corpusIngested += 1;
} catch (e) {
if (e instanceof Error && e.name === 'AbortError') {
skip('budget_exhausted:corpus', candidates.length - i);
abortLoop = true;
} else if ((e as { code?: unknown }).code === 'ambient_capture_off') {
skip('writeback_off_inflight');
} else {
skip('corpus_file_error');
const code = (e as { code?: unknown }).code;
log(`[sweep] corpus ingest failed for ${name}${typeof code === 'string' ? ` (${code})` : ''}: ${e instanceof Error ? e.message : String(e)}`);
}
} finally {
// #6785: every attempted window counts against the total, a thrown one included.
windowsLeft -= attempted.count;
// Success (its .ingested sidecar now stands in) and failure (retry next
// sweep) both release the claim.
await rm(claimPath, { force: true }).catch(() => {});
}
if (abortLoop) break;
}
}
/** Cathedral 5: publish a compaction segment's verified entity links to its session manifest; the count banked, 0 on any failure. */
async function bankSegmentLinks(engine: BrainEngine, base: string, name: string, sourceId: string, slugs: string[]): Promise<number> {
try {
const segs = await import('./context/corpus-segments.ts');
const parsed = segs.parseSegmentFileName(name);
if (!parsed) return 0;
const verified: Array<{ slug: string; title: string }> = [];
for (const slug of slugs) {
try {
const page = await engine.getPage(slug, { sourceId });
if (page) verified.push({ slug, title: page.title || slug });
} catch { /* a non-resolvable link is never banked */ }
}
if (!verified.length) return 0;
const ss = await import('./context/session-state.ts');
const ledger = segs.readSegmentLedger(base, parsed.sessionId);
const n = Math.max(1, ledger.findIndex((e) => e.hash === parsed.hash) + 1);
const ok = await ss.appendCheckpointManifest(engine, sourceId, null, parsed.sessionId, verified, { seg: parsed.hash, n });
return ok ? verified.length : 0;
} catch {
return 0; /* link publish is best-effort on the backstop lane */
}
}
/**
* #6785: the facts-absorb owner-readiness preflight (#5362), once per source
* per pass and before any model call: a claimed but not activated canonical
* worktree refuses the fence write AFTER extraction, which every later sweep
* would pay for again. Returns whether the source is held.
*/
async function corpusWriterPreflight(engine: BrainEngine, log: (msg: string) => void): Promise<(sourceId: string) => Promise<boolean>> {
const { assertSourceLegacyWriter } = await import('./persistence/filesystem-guard.ts');
const held = new Map<string, boolean>();
return async (sourceId) => {
if (!held.has(sourceId)) {
try {
await assertSourceLegacyWriter(engine, sourceId);
held.set(sourceId, false);
} catch (e) {
if ((e as { code?: unknown }).code !== 'writer_coordinator_required') throw e;
held.set(sourceId, true);
log(`[sweep] corpus extraction into source ${sourceId} is held before any model call: writer_coordinator_required `
+ `(${e instanceof Error ? e.message : String(e)}). Check the writer: gbrain sources writer status ${sourceId}`);
}
}
return held.get(sourceId)!;
};
}
function deadLetterLine(label: string, cap: number): string {
return `[sweep] ${label}: corpus_window_dead_lettered — its next window failed ${cap} times after the model call, `
+ 'so the sweep stopped paying for it. Fix the cause, then preview a retry: gbrain sweep --retry-failed';
}
/**
* Try to claim a corpus file for ingestion. O_EXCL (`wx`) create is the
* atomic primitive; a live existing claim (< CORPUS_CLAIM_STALE_MS old)
* means another sweep owns the file. Stale claims are removed and re-raced
* (one contender wins the second `wx`). Never throws. EXPORTED (cathedral 5)
* so the serve-side checkpoint harvest shares the exact same fencing — a
* sweep and a harvest can never double-spend on one segment.
*/
export async function acquireCorpusClaim(claimPath: string): Promise<boolean> {
const body = JSON.stringify({ claimed_at: new Date().toISOString(), pid: process.pid }) + '\n';
const tryCreate = () =>
writeFile(claimPath, body, { flag: 'wx', mode: 0o600 }).then(() => true, () => false);
if (await tryCreate()) return true;
try {
const st = await stat(claimPath);
if (Date.now() - st.mtimeMs <= CORPUS_CLAIM_STALE_MS) return false; // live claim
} catch {
// Claim vanished between wx-create and stat — its owner just released.
// Treat as contended; a later sweep picks the file up if still needed.
return false;
}
await rm(claimPath, { force: true }).catch(() => {});
return tryCreate();
}
// ── Serve-startup arming [ENG-5] ─────────────────────────────────────────
export interface StartupSweepOpts {
/** Post-connect settle delay. Default STARTUP_SWEEP_DELAY_MS (3s). */
delayMs?: number;
/** Source to sweep. Default 'default'. */
sourceId?: string;
/** Env for the GBRAIN_SWEEP kill switch. Default process.env. */
env?: Record<string, string | undefined>;
/** Timer seams (tests). Defaults: global setTimeout/clearTimeout. */
setTimeoutFn?: (fn: () => void, ms: number) => unknown;
clearTimeoutFn?: (handle: unknown) => void;
/** Sweep body override (tests). Default: runMaintenanceSweep. */
sweep?: (engine: BrainEngine) => Promise<unknown>;
}
/**
* Arm the one-shot startup sweep for a serve process. Returns a cancel
* handle for shutdown, or null when the GBRAIN_SWEEP=0 kill switch is set.
* The timer is unref'd (never holds the process open) and the sweep body
* swallows every error — best-effort by construction, same posture as the
* resolve-IPC block in src/mcp/server.ts.
*/
export function armStartupSweep(
engine: BrainEngine,
opts: StartupSweepOpts = {},
): { cancel: () => void } | null {
const env = opts.env ?? process.env;
if (env.GBRAIN_SWEEP === '0') return null;
const setT = opts.setTimeoutFn ?? ((fn: () => void, ms: number) => setTimeout(fn, ms));
const clearT = opts.clearTimeoutFn
?? ((h: unknown) => clearTimeout(h as ReturnType<typeof setTimeout>));
const run = opts.sweep
?? ((e: BrainEngine) => runMaintenanceSweep(e, { sourceId: opts.sourceId }));
const handle = setT(() => {
Promise.resolve()
.then(() => run(engine))
.catch(() => { /* startup sweep is best-effort; never crash serve */ });
}, opts.delayMs ?? STARTUP_SWEEP_DELAY_MS);
(handle as { unref?: () => void } | null)?.unref?.();
return {
cancel: () => {
try { clearT(handle); } catch { /* noop */ }
},
};
}
// ── HTTP serve corpus drain [X8, D18] ─────────────────────────────────────
/** Cadence of the `serve --http` corpus drain. */
export const CORPUS_DRAIN_INTERVAL_MS = 10 * 60_000;
/** Per-drain budget. The corpus pass stops between items (E-N2), so this bounds how many files one drain starts. */
export const CORPUS_DRAIN_BUDGET_MS = 60_000;
export interface CorpusDrainOpts {
/** Source to sweep. Default 'default'. */
sourceId?: string;
/** Env for the GBRAIN_SWEEP kill switch. Default process.env. */
env?: Record<string, string | undefined>;
/** Timer seams (tests). Defaults: global setInterval/clearInterval. */
setIntervalFn?: (fn: () => void, ms: number) => unknown;
clearIntervalFn?: (handle: unknown) => void;
/** Sweep body override (tests). Default: runMaintenanceSweep with CORPUS_DRAIN_BUDGET_MS. */
sweep?: (engine: BrainEngine) => Promise<unknown>;
/** #6785: diagnostic sink. Default: stderr (serve.err). */
log?: (msg: string) => void;
}
/**
* True when the corpus dir holds a `.txt` the sweep would extract: not
* finished by the one classifier (#6786), and not stopped at the failure cap
* at the window it starts at now (#6785; a backing-off file still counts).
* Missing dir = false.
*/
async function corpusHasPendingFiles(engine: BrainEngine): Promise<boolean> {
let dir = await engine.getConfig('dream.synthesize.session_corpus_dir');
if (!dir) {
const { configDir } = await import('./config.ts');
dir = join(configDir(), 'transcripts', 'corpus');
}
const { CORPUS_FAILURE_SUFFIX, corpusFileAwaitsExtraction, corpusSpoolDir, readCorpusFailure } = await import('./context/corpus-segments.ts');
const { corpusFailureGate, corpusFrontierKey } = await import('./context/corpus-windows.ts');
const { extractionConfigFingerprint } = await import('./facts/extraction-availability.ts');
let fingerprint: Promise<string | null> | null = null;
const stopped = async (full: string, name: string): Promise<boolean> => {
const rec = readCorpusFailure(full);
// A record for a window the file no longer starts at (rewritten) is stale.
if (!rec || (await corpusFrontierKey(full, name)) !== rec.key) return false;
fingerprint ??= extractionConfigFingerprint(engine).catch(() => null);
return corpusFailureGate(rec, rec.key, rec.source_incarnation, await fingerprint, Date.now()) === 'dead_lettered';
};
for (const base of [corpusSpoolDir(dir), dir]) {
const names = await readdir(base).catch(() => [] as string[]);
const present = new Set(names);
for (const n of names) {
if (!n.endsWith('.txt') || !corpusFileAwaitsExtraction(join(base, n), n)) continue;
if (present.has(n + CORPUS_FAILURE_SUFFIX) && await stopped(join(base, n), n)) continue;
return true;
}
}
return false;
}
/** #6785: the drain's skip reasons worth an operator's eye (every reason but `already_ingested`), sorted. */
function drainSkipReasons(report: unknown): string[] {
const skipped = (report as Partial<SweepReport> | null)?.skipped;
if (!Array.isArray(skipped)) return [];
return skipped.filter(s => s.reason !== 'already_ingested').map(s => `${s.reason} x${s.count}`).sort();
}
/**
* X8 (D18): the corpus drain a `serve --http` process owns. A stdio serve
* sweeps at startup and on idle ticks; the HTTP serve never swept, so a turn
* the serve-side harvest refused (#5557) or a session-end corpus file waited
* for a hand-run `gbrain sweep --once` and then for retention GC. Every
* CORPUS_DRAIN_INTERVAL_MS, when a corpus file has no `.ingested` sidecar,
* one bounded sweep runs under the corpus pass's own gates (keyless skip,
* the writeback gate, claims, per-file and per-sweep window caps). Never
* overlaps itself; unref'd; never throws. `tick` runs one decision now.
* #6785: the sweep's diagnostics and a thrown tick reach `log` (stderr),
* each distinct line once per serve run, and the skip reasons whenever
* their set changes. Returns null when GBRAIN_SWEEP=0.
*/
export function armCorpusDrain(
engine: BrainEngine,
opts: CorpusDrainOpts = {},
): { cancel: () => void; tick: () => Promise<void> } | null {
const env = opts.env ?? process.env;
if (env.GBRAIN_SWEEP === '0') return null;
const setI = opts.setIntervalFn ?? ((fn: () => void, ms: number) => setInterval(fn, ms));
const clearI = opts.clearIntervalFn
?? ((h: unknown) => clearInterval(h as ReturnType<typeof setInterval>));
const sink = opts.log ?? ((msg: string) => { process.stderr.write(`${msg}\n`); });
const seen = new Set<string>();
const log = (msg: string): void => {
if (seen.has(msg)) return;
if (seen.size >= 1000) seen.clear();
seen.add(msg);
sink(msg);
};
const run = opts.sweep
?? ((e: BrainEngine) => runMaintenanceSweep(e, { sourceId: opts.sourceId, budgetMs: CORPUS_DRAIN_BUDGET_MS, log }));
let inFlight = false;
let cancelled = false;
let lastReasons = '';
const tick = async (): Promise<void> => {
if (inFlight || cancelled) return;
inFlight = true;
try {
if (await corpusHasPendingFiles(engine)) {
const reasons = drainSkipReasons(await run(engine));
const set = reasons.map(r => r.slice(0, r.lastIndexOf(' x'))).join(',');
if (set !== lastReasons && reasons.length) sink(`[sweep] corpus drain skipped: ${reasons.join(', ')}`);
lastReasons = set;
}
} catch (e) {
// The drain is best-effort and never kills serve, but it says why.
log(`[sweep] corpus drain failed: ${e instanceof Error ? e.message : String(e)}`);
} finally {
inFlight = false;
}
};
const handle = setI(() => { void tick(); }, CORPUS_DRAIN_INTERVAL_MS);
(handle as { unref?: () => void } | null)?.unref?.();
return {
tick,
cancel: () => {
cancelled = true;
try { clearI(handle); } catch { /* noop */ }
},
};
}