Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* corpus-windows.ts — #5887: windowed fact extraction of session-corpus files.
*
* The extractor reads at most MAX_TURN_TEXT_CHARS (8,000) characters per
* call, so a long transcript passed whole lost everything after its head.
* Here a corpus file (`toCorpusText` output: `[user]` / `[assistant]` blocks
* separated by blank lines) is cut into windows that each fit the extractor:
*
* - Turns are found only at the exact `\n\n[user]\n` / `\n\n[assistant]\n`
* markers `toCorpusText` emits (plus a marker at offset 0).
* - Each whole user turn is paste-stripped FIRST (#5812), then the stripped
* text is windowed, so a paste that crosses a window boundary never leaks.
* - Windows hold whole turns; a turn longer than a window is split, and every
* continuation repeats its `[role]` header, so assistant text is never read
* as the user's own words.
*
* Progress lives in `<file>.progress` (never deleted by the hook's resume
* rewrite; GC'd with the `.txt`): a generation counter, the hashes of finished
* turns, an optional continuation into a split turn, the file stat of the
* snapshot that finished, and a lease. A rewrite (append, compaction
* remainder, same-size or shorter replacement) resumes at the first turn
* whose hash is not recorded. Every write is compare-and-set under an O_EXCL
* lock file: a commit lands only when the generation, the last recorded turn
* hash and the lease owner still match and the progress grows. The lease stops
* a second sweep (whose claim the hook deleted on rewrite) from extracting a
* window the first sweep is still working on. Callers keep the existing
* `.in-progress` claim discipline; this module never touches claims.
*/
import { randomUUID } from 'node:crypto';
import { readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { MAX_TURN_TEXT_CHARS } from '../facts/extract.ts';
import type { runFactsPipeline } from '../facts/backstop.ts';
import { withAIInvocationGuard } from '../ai/invocation-guard.ts';
import { parseCorpusTurns, type CorpusTurn } from './corpus-turns.ts';
import { VERSION } from '../../version.ts';
import {
CORPUS_DETERMINISTIC_FAILURE_CODES, CORPUS_FAILURE_SUFFIX, corpusFailureHeld, CORPUS_NON_TERMINAL_SKIPS, CORPUS_PROGRESS_LOCK_SUFFIX, CORPUS_PROGRESS_SUFFIX, CORPUS_WINDOW_FAILURE_CAP,
corpusFileCompletion, corpusFileFinished, parseCorpusProgress, parseWbFileName, readCorpusFailure, type CorpusFailureRecord,
} from './corpus-segments.ts';
export { parseCorpusTurns, type CorpusTurn };
/** Per-file window cap per sweep; longer files converge over later sweeps. */
export const CORPUS_WINDOWS_PER_SWEEP = 8;
/** Cross-file window total per sweep (override: GBRAIN_CORPUS_WINDOWS_PER_SWEEP). */
export const CORPUS_WINDOWS_PER_SWEEP_TOTAL = 32;
/** A lease older than this belongs to a dead extractor (matches the claim's hour). */
export const CORPUS_PROGRESS_LEASE_STALE_MS = 60 * 60 * 1000;
const LOCK_STALE_MS = 30_000;
const LOCK_ATTEMPTS = 40;
const LOCK_RETRY_MS = 25;
const MAX_RECORDED_TURNS = 20_000;
const MAX_ENTITY_SLUGS = 500;
type PipelineResult = Awaited<ReturnType<typeof runFactsPipeline>>;
export interface CorpusWindow {
text: string;
/** Index of the last turn this window touches. */
endTurn: number;
/** Offset into that turn's body where the window stops; null when the turn finished. */
endOffset: number | null;
}
export interface CorpusFileStat {
size: number;
mtime_ms: number;
ino: number;
}
export interface CorpusProgress {
version: 1;
generation: number;
turns: Array<{ start: number; end: number; sha256: string }>;
continuation: { turn_sha256: string; stripped_offset: number } | null;
windows_done: number;
/** Stat of the snapshot whose every window finished; null while windows remain. */
finished: CorpusFileStat | null;
totals: { inserted: number; duplicate: number; superseded: number };
entity_slugs: string[];
skipped_reason?: string;
/** #6091: hashes of turns captured or retired under writeback off; never extracted wherever they sit. */
retired_turns?: string[];
/**
* #6268: the source (and its incarnation) this file's windows were
* extracted into, bound by the first window run. A resume under another
* source or a recreated source is refused (`source_changed`).
*/
source?: CorpusProgressSource | null;
lease: { owner: string; at: number } | null;
}
export interface CorpusProgressSource {
id: string;
incarnation: string;
}
export interface CorpusWindowRun {
/**
* #6786 `unavailable`: the extractor answered a non-terminal skip (no
* servable model); nothing was recorded for that window. #6785 `backoff` /
* `dead_lettered`: the next window failed after its model call before and
* waits out its retry delay, or reached CORPUS_WINDOW_FAILURE_CAP.
*/
status: 'complete' | 'partial' | 'aborted' | 'contended' | 'changed' | 'source_changed' | 'unavailable' | 'backoff' | 'dead_lettered';
windowsDone: number;
windowsRemaining: number;
/** Cumulative across every sweep that worked on this file. */
result: PipelineResult;
}
let invalidKnobWarned = false;
/**
* The cross-file window total for one sweep. `GBRAIN_CORPUS_WINDOWS_PER_SWEEP`
* overrides it with a positive integer; anything else falls back to the
* default with one warning per process.
*/
export function resolveCorpusWindowsPerSweepTotal(
env: Record<string, string | undefined> = process.env,
warn: (msg: string) => void = () => {},
): number {
const raw = env.GBRAIN_CORPUS_WINDOWS_PER_SWEEP;
if (raw === undefined || raw.trim() === '') return CORPUS_WINDOWS_PER_SWEEP_TOTAL;
const n = Number(raw.trim());
if (Number.isSafeInteger(n) && n > 0) return n;
if (!invalidKnobWarned) {
invalidKnobWarned = true;
warn(`[sweep] GBRAIN_CORPUS_WINDOWS_PER_SWEEP=${JSON.stringify(raw)} is not a positive integer; using ${CORPUS_WINDOWS_PER_SWEEP_TOTAL}`);
}
return CORPUS_WINDOWS_PER_SWEEP_TOTAL;
}
/** TEST SEAM: re-arm the once-per-process invalid-knob warning. */
export function __resetCorpusWindowsWarningForTests(): void {
invalidKnobWarned = false;
}
function isHigh(code: number): boolean { return code >= 0xd800 && code <= 0xdbff; }
function isLow(code: number): boolean { return code >= 0xdc00 && code <= 0xdfff; }
/** Cut point in (from, max]: a newline, else a space, in the window's back half; never inside a surrogate pair. */
function cutPoint(body: string, from: number, max: number): number {
if (max >= body.length) return body.length;
const floor = from + Math.floor((max - from) / 2);
const newline = body.lastIndexOf('\n', max - 1);
if (newline >= floor) return newline + 1;
const space = body.lastIndexOf(' ', max - 1);
if (space >= floor) return space + 1;
return isHigh(body.charCodeAt(max - 1)) && isLow(body.charCodeAt(max)) ? max - 1 : max;
}
/**
* Plan every remaining window from `from`. Whole turns are packed together;
* a turn that does not fit a fresh window is split with its header repeated.
* A window whose text is empty only records empty (paste-only) turns as done.
*/
export function planCorpusWindows(
turns: CorpusTurn[],
from: { turn: number; offset: number },
maxChars: number = MAX_TURN_TEXT_CHARS,
skip: ReadonlySet<string> = new Set(),
): CorpusWindow[] {
const windows: CorpusWindow[] = [];
let parts: string[] = [];
let len = 0;
let last: { turn: number; offset: number | null } = { turn: from.turn - 1, offset: null };
const flush = (): void => {
windows.push({ text: parts.join('\n\n'), endTurn: last.turn, endOffset: last.offset });
parts = [];
len = 0;
};
for (let t = from.turn; t < turns.length; t++) {
const { header, body } = turns[t];
let offset = t === from.turn ? Math.min(from.offset, body.length) : 0;
if (offset >= body.length || skip.has(turns[t].sha256)) {
last = { turn: t, offset: null };
continue;
}
while (offset < body.length) {
const sep = parts.length ? 2 : 0;
const room = maxChars - len - sep - header.length;
const rest = body.length - offset;
if (rest <= room) {
parts.push(header + body.slice(offset));
len += sep + header.length + rest;
last = { turn: t, offset: null };
break;
}
if (parts.length && (header.length + rest <= maxChars || room < maxChars / 4)) {
flush();
continue;
}
const cut = cutPoint(body, offset, offset + room);
parts.push(header + body.slice(offset, cut));
len += sep + header.length + (cut - offset);
offset = cut;
last = { turn: t, offset: cut < body.length ? cut : null };
flush();
}
}
if (parts.length) flush();
else if (last.turn >= from.turn && last.turn > (windows.at(-1)?.endTurn ?? from.turn - 1)) flush();
return windows;
}
/** Where extraction resumes: the first turn whose hash is not recorded. */
export function resumePoint(turns: CorpusTurn[], progress: CorpusProgress | null): { turn: number; offset: number } {
if (!progress) return { turn: 0, offset: 0 };
const done = new Set(progress.turns.map((t) => t.sha256));
let turn = 0;
while (turn < turns.length && done.has(turns[turn].sha256)) turn++;
const cont = progress.continuation;
const offset = cont && turn < turns.length && turns[turn].sha256 === cont.turn_sha256
? Math.min(cont.stripped_offset, turns[turn].body.length)
: 0;
return { turn, offset };
}
/** The progress sidecar, or null when absent or unparseable (replayed safely). I/O errors throw. */
export async function readCorpusProgress(full: string): Promise<CorpusProgress | null> {
try {
return parseCorpusProgress(await readFile(full + CORPUS_PROGRESS_SUFFIX, 'utf8'));
} catch (e) {
if ((e as NodeJS.ErrnoException).code === 'ENOENT') return null;
throw e;
}
}
/** The earliest file time trusted as when a session was written. */
const MIN_FACT_TIME_MS = Date.UTC(2000, 0, 1);
/** Clock skew a future file time may carry and still be trusted. */
const FACT_TIME_SKEW_MS = 60_000;
export type CorpusFactTime = { at: Date } | { rejected: 'invalid' | 'before_2000' | 'future' };
/**
* #6159: the session file's write time as the time its facts happened. A time
* that is not finite, before 2000 or more than a minute ahead of now is not
* trusted, and the caller dates the facts at extraction instead.
*/
export function corpusFactTime(mtimeMs: number, nowMs: number = Date.now()): CorpusFactTime {
if (!Number.isFinite(mtimeMs)) return { rejected: 'invalid' };
if (mtimeMs < MIN_FACT_TIME_MS) return { rejected: 'before_2000' };
if (mtimeMs > nowMs + FACT_TIME_SKEW_MS) return { rejected: 'future' };
return { at: new Date(mtimeMs) };
}
export function corpusFileStat(st: { size: number; mtimeMs: number; ino: number }): CorpusFileStat {
return { size: st.size, mtime_ms: st.mtimeMs, ino: st.ino };
}
export function sameCorpusFileStat(a: CorpusFileStat | null, b: CorpusFileStat): boolean {
return !!a && a.size === b.size && a.mtime_ms === b.mtime_ms && a.ino === b.ino;
}
async function acquireLock(lockPath: string): Promise<boolean> {
for (let attempt = 0; attempt < LOCK_ATTEMPTS; attempt++) {
try {
await writeFile(lockPath, `${process.pid}\n`, { flag: 'wx', mode: 0o600 });
return true;
} catch (e) {
if ((e as NodeJS.ErrnoException).code !== 'EEXIST') return false;
}
const age = await stat(lockPath).then((s) => Date.now() - s.mtimeMs, () => 0);
if (age > LOCK_STALE_MS) await rm(lockPath, { force: true }).catch(() => {});
else await new Promise((r) => setTimeout(r, LOCK_RETRY_MS));
}
return false;
}
/**
* Compare-and-set: under the lock, `next` sees the current progress and
* returns the replacement, or null to refuse. The generation always advances.
* Returns the written progress, or null when refused or the lock is busy.
*/
async function updateProgress(
full: string,
next: (cur: CorpusProgress | null) => Omit<CorpusProgress, 'generation'> | null,
): Promise<CorpusProgress | null> {
const lockPath = full + CORPUS_PROGRESS_LOCK_SUFFIX;
if (!(await acquireLock(lockPath))) return null;
try {
const cur = await readCorpusProgress(full);
const proposed = next(cur);
if (!proposed) return null;
const written: CorpusProgress = { ...proposed, generation: (cur?.generation ?? 0) + 1 };
const path = full + CORPUS_PROGRESS_SUFFIX;
const tmp = `${path}.tmp-${process.pid}`;
await writeFile(tmp, JSON.stringify(written) + '\n', { mode: 0o600 });
await rename(tmp, path);
return written;
} finally {
await rm(lockPath, { force: true }).catch(() => {});
}
}
function freshProgress(): Omit<CorpusProgress, 'generation'> {
return {
version: 1,
turns: [],
continuation: null,
windows_done: 0,
finished: null,
totals: { inserted: 0, duplicate: 0, superseded: 0 },
entity_slugs: [],
lease: null,
};
}
const lastSha = (p: CorpusProgress | null): string | null => p?.turns.at(-1)?.sha256 ?? null;
function resultOf(p: CorpusProgress): PipelineResult {
return {
...p.totals,
fact_ids: [],
entity_slugs: [...p.entity_slugs],
...(p.skipped_reason ? { skipped_reason: p.skipped_reason as PipelineResult['skipped_reason'] } : {}),
};
}
/**
* V6 — record a file finished before windowed extraction as fully extracted
* at its current turns, so later growth extracts only the appended turns and
* the head is never re-sent. Writes only when no progress exists yet.
*/
export async function adoptFinishedCorpusFile(full: string): Promise<boolean> {
const st = corpusFileStat(await stat(full));
const raw = await readFile(full, 'utf8');
const after = corpusFileStat(await stat(full));
if (!sameCorpusFileStat(st, after)) return false;
const turns = parseCorpusTurns(raw);
const written = await updateProgress(full, (cur) => cur ? null : {
...freshProgress(),
turns: turns.slice(-MAX_RECORDED_TURNS).map(({ start, end, sha256: h }) => ({ start, end, sha256: h })),
finished: st,
});
return written !== null;
}
/**
* #6091: record a corpus file as finished WITHOUT extracting it — the capture
* gate retired it (explicit `memory.auto_writeback off`). `.progress` is
* finished at the file's current stat with every current turn hash recorded
* and `skipped_reason`, then the `.ingested` sidecar names the reason. A
* retire overrides another extractor's lease (its next compare-and-set fails),
* and a later rewrite of the file (a resumed session) extracts only turns that
* are not recorded here. Returns false when the file changed mid-read or the
* progress lock stayed busy (the caller retries on its next pass).
*/
export async function retireCorpusFile(full: string, reason: string = 'writeback_off'): Promise<boolean> {
const st = corpusFileStat(await stat(full));
const raw = await readFile(full, 'utf8');
if (!sameCorpusFileStat(st, corpusFileStat(await stat(full)))) return false;
const written = await updateProgress(full, (cur) => ({
...(cur ?? freshProgress()),
turns: mergeTurns(cur?.turns ?? [], parseCorpusTurns(raw)),
retired_turns: mergeRetired(cur, parseCorpusTurns(raw).map((t) => t.sha256)),
continuation: null,
finished: st,
skipped_reason: reason,
lease: null,
}));
if (!written) return false;
await writeFile(full + '.ingested', JSON.stringify({ ingested_at: new Date().toISOString(), skipped: reason }) + '\n');
return true;
}
/**
* #6091: record turns captured under explicit off as done, so extraction of
* this file resumes past them. Leaves `finished` alone: turns the file holds
* beyond these still extract. False when the progress lock stayed busy.
*/
export async function retireCorpusTurns(full: string, hashes: readonly string[]): Promise<boolean> {
const written = await updateProgress(full, (cur) => {
const known = new Set(cur?.turns.map((t) => t.sha256) ?? []);
const added = hashes.filter((h) => !known.has(h));
const retired = new Set(cur?.retired_turns ?? []);
if (cur && added.length === 0 && hashes.every((h) => retired.has(h))) return null;
return { ...(cur ?? freshProgress()), turns: [...(cur?.turns ?? []), ...added.map((h) => ({ start: 0, end: 0, sha256: h }))].slice(-MAX_RECORDED_TURNS),
retired_turns: mergeRetired(cur, hashes) };
});
return written !== null || (await readCorpusProgress(full)) !== null;
}
function mergeRetired(cur: CorpusProgress | null, hashes: readonly string[]): string[] {
return [...new Set([...(cur?.retired_turns ?? []), ...hashes])].slice(-MAX_RECORDED_TURNS);
}
/** #6091: turns that must never reach extraction, wherever a rewrite puts them. */
export function retiredTurns(progress: CorpusProgress | null): ReadonlySet<string> {
return new Set(progress?.retired_turns ?? []);
}
function mergeTurns(recorded: CorpusProgress['turns'], turns: CorpusTurn[]): CorpusProgress['turns'] {
const known = new Set(recorded.map((t) => t.sha256));
const added = turns.filter((t) => !known.has(t.sha256)).map(({ start, end, sha256: h }) => ({ start, end, sha256: h }));
return [...recorded, ...added].slice(-MAX_RECORDED_TURNS);
}
/** A failed window is retried on the next sweep, then after this delay, doubling per failure (10, 20, 40 minutes). */
export const CORPUS_WINDOW_RETRY_BASE_MS = 10 * 60_000;
/** The failure key of a writeback turn file (one turn, one call). */
export const CORPUS_TURN_FAILURE_KEY = 'turn';
/** Where a window starts: its first turn's hash and the offset into it. Stable while the file only grows. */
function windowKey(turns: CorpusTurn[], at: { turn: number; offset: number }): string {
return `${turns[at.turn]?.sha256 ?? 'end'}@${at.offset}`;
}
/**
* #6785: whether the window at `key` may be paid for now. A record for
* another window or another source incarnation does not apply. A capped
* window stays stopped, except once when the extraction-config fingerprint
* changed since its last failure. #9279: a window stopped by a deterministic
* failure runs once on a gbrain build other than the one that recorded it
* (a record from before #9279 carries no build), so an upgrade that fixes the
* refusal extracts it with no command. That automatic retry happens once per
* record (upgrade_retried); after it only an operator retry runs the window.
*/
export function corpusFailureGate(
record: CorpusFailureRecord | null,
key: string,
incarnation: string | null,
fingerprint: string | null,
nowMs: number,
): 'run' | 'backoff' | 'dead_lettered' {
if (!record || record.key !== key || record.source_incarnation !== incarnation) return 'run';
if (record.count >= CORPUS_WINDOW_FAILURE_CAP) {
const rebuilt = CORPUS_DETERMINISTIC_FAILURE_CODES.includes(record.last_code) && record.build !== VERSION && record.upgrade_retried !== true;
return rebuilt || rearms(record, fingerprint) ? 'run' : 'dead_lettered';
}
if (record.count === 1) return 'run';
return nowMs < record.last_at + CORPUS_WINDOW_RETRY_BASE_MS * 2 ** (record.count - 2) ? 'backoff' : 'run';
}
function rearms(record: CorpusFailureRecord, fingerprint: string | null): boolean {
return !record.rearmed && record.fingerprint !== null && fingerprint !== null && record.fingerprint !== fingerprint;
}
/**
* #6785: whether the file's recorded failing window is backing off or
* stopped right now. Selection serves these files last, so stopped files
* never take every batch slot from healthy ones.
*/
export function corpusFileParked(full: string, fingerprint: string | null, nowMs: number): boolean {
const record = readCorpusFailure(full);
return !!record && corpusFailureGate(record, record.key, record.source_incarnation, fingerprint, nowMs) !== 'run';
}
/** Whether a failure record's window (`<turn sha256>@<offset>`) still lies at or after `cursor` in `turns`. */
function failureAhead(key: string, turns: CorpusTurn[], cursor: { turn: number; offset: number }): boolean {
const at = key.lastIndexOf('@');
const sha = key.slice(0, at);
const offset = Number(key.slice(at + 1));
return turns.some((t, i) => t.sha256 === sha && (i > cursor.turn || (i === cursor.turn && offset >= cursor.offset)));
}
/**
* The key of the window the next run of `full` starts at (the frontier a
* failure record names), or null when the file or its progress is
* unreadable. A writeback turn file's key is always CORPUS_TURN_FAILURE_KEY.
*/
export async function corpusFrontierKey(full: string, name: string): Promise<string | null> {
if (parseWbFileName(name)) return CORPUS_TURN_FAILURE_KEY;
try {
const turns = parseCorpusTurns(await readFile(full, 'utf-8'));
return windowKey(turns, resumePoint(turns, await readCorpusProgress(full)));
} catch {
return null;
}
}
/**
* #6785: run one extraction; when it throws after a chat model call was
* admitted (the spend), count a failure against the window at `key`, then
* rethrow, whatever ended it (a later call the spend guard refused
* included). A throw before any admitted chat call (a refused first call
* included), an abort, or capture switched off mid-call (the next sweep
* retires the file) costs nothing to retry and is not counted. `paid` says
* whether a chat call was admitted, for a caller that must count a failure
* after the extraction returned.
*/
export async function extractUnderFailureCount<T>(
failure: { full: string; record: CorpusFailureRecord | null; key: string; incarnation: string | null; fingerprint: string | null; nowMs: () => number; signal?: AbortSignal },
extract: () => Promise<T>,
): Promise<{ value: T; paid: boolean }> {
let paid = false;
try {
// Inherit: an outer guard admits first, so this one only sees admitted calls.
const value = await withAIInvocationGuard(async (call) => {
if (call.kind === 'chat') paid = true;
return { settle: async () => {} };
}, extract, { inherit: true });
return { value, paid };
} catch (e) {
const counts = paid && !failure.signal?.aborted && !(e instanceof Error && e.name === 'AbortError')
&& (e as { code?: unknown } | null)?.code !== 'ambient_capture_off';
if (counts) {
await recordCorpusFailure(failure.full, failure.record, failure.key, failure.incarnation, failure.fingerprint, failure.nowMs(), e).catch(() => {});
}
throw e;
}
}
function failureCode(e: unknown): string {
const { code, reason, name } = (e ?? {}) as { code?: unknown; reason?: unknown; name?: unknown };
for (const v of [code, reason, name]) if (typeof v === 'string' && /^[A-Za-z][A-Za-z0-9_:-]{0,63}$/.test(v)) return v;
return 'error';
}
/**
* #6785: count one failed attempt of the window at `key` (caller holds the
* file's `.in-progress` claim). Another window or incarnation starts a new
* count; the error label never resets it. A capped window re-armed by a
* fingerprint change starts over once. #9279: a deterministic failure
* (CORPUS_DETERMINISTIC_FAILURE_CODES) stops the window at once, stamped with
* this build, instead of paying for four more identical refusals. A failure on
* the automatic retry of a held window marks it upgrade_retried; a transient
* one there starts a fresh count, so it still gets the ordinary retries, and
* the window stays held (its file kept) until it commits or an operator retries.
* Returns the record written.
*/
export async function recordCorpusFailure(
full: string,
prev: CorpusFailureRecord | null,
key: string,
incarnation: string | null,
fingerprint: string | null,
nowMs: number,
error: unknown,
): Promise<CorpusFailureRecord> {
const same = !!prev && prev.key === key && prev.source_incarnation === incarnation;
const rearm = same && prev!.count >= CORPUS_WINDOW_FAILURE_CAP && rearms(prev!, fingerprint);
const code = failureCode(error);
const deterministic = CORPUS_DETERMINISTIC_FAILURE_CODES.includes(code);
const held = same && corpusFailureHeld(prev);
const retried = same && (prev!.upgrade_retried === true || (held && prev!.build !== VERSION));
const count = same && !rearm && !(held && !deterministic) ? prev!.count + 1 : 1;
const record: CorpusFailureRecord = {
version: 1,
key,
source_incarnation: incarnation,
count: deterministic ? Math.max(count, CORPUS_WINDOW_FAILURE_CAP) : count,
first_at: same ? prev!.first_at : nowMs,
last_at: nowMs,
last_code: code,
fingerprint,
rearmed: same ? prev!.rearmed || rearm : false,
...(deterministic ? { build: VERSION } : {}),
...(retried ? { upgrade_retried: true } : {}),
...(deterministic || held ? { held: true } : {}),
};
const path = full + CORPUS_FAILURE_SUFFIX;
const tmp = `${path}.tmp-${process.pid}`;
await writeFile(tmp, JSON.stringify(record) + '\n', { mode: 0o600 });
await rename(tmp, path);
return record;
}
/** Remove the failure record once its window committed. */
export async function clearCorpusFailure(full: string): Promise<void> {
await rm(full + CORPUS_FAILURE_SUFFIX, { force: true });
}
/**
* #6785/#6786: one writeback turn file (one call, no `.progress`) under the
* same failure cap and non-terminal rule as a window. `complete` carries the
* result (the caller writes `.ingested`); `unavailable` records nothing.
*/
export async function runCorpusTurn(opts: {
full: string;
signal?: AbortSignal;
incarnation: string | null;
failures?: { fingerprint: string | null; now?: () => number };
/** Incremented before each attempted extraction, thrown or not: the caller's window budget. */
attempted?: { count: number };
extract: () => Promise<PipelineResult>;
}): Promise<{ status: 'complete'; result: PipelineResult } | { status: 'unavailable' | 'backoff' | 'dead_lettered' }> {
const fingerprint = opts.failures?.fingerprint ?? null;
const clock = opts.failures?.now ?? Date.now;
const record = readCorpusFailure(opts.full);
const gate = corpusFailureGate(record, CORPUS_TURN_FAILURE_KEY, opts.incarnation, fingerprint, clock());
if (gate !== 'run') return { status: gate };
if (opts.attempted) opts.attempted.count++;
const { value: result } = await extractUnderFailureCount(
{ full: opts.full, record, key: CORPUS_TURN_FAILURE_KEY, incarnation: opts.incarnation, fingerprint, nowMs: clock, signal: opts.signal },
opts.extract,
);
if (result.skipped_reason && CORPUS_NON_TERMINAL_SKIPS.includes(result.skipped_reason)) return { status: 'unavailable' };
if (record) await clearCorpusFailure(opts.full).catch(() => {});
return { status: 'complete', result };
}
/**
* #6786: a progress whose windows answered a non-terminal skip (pre-#6786
* builds recorded those turns as done) is rewound to its retired turns, so
* every other turn is extracted again. The source binding, retired turns,
* totals and entity links stay.
*/
function rewindNonTerminal(cur: CorpusProgress): Omit<CorpusProgress, 'generation'> {
if (!cur.skipped_reason || !CORPUS_NON_TERMINAL_SKIPS.includes(cur.skipped_reason)) return cur;
const retired = retiredTurns(cur);
const { skipped_reason: _dropped, ...rest } = cur;
return { ...rest, turns: cur.turns.filter((t) => retired.has(t.sha256)), continuation: null, finished: null };
}
/**
* Extract up to `maxWindows` non-empty windows of `raw` (the snapshot whose
* stat is `fileStat`), committing `.progress` after each one. `complete`
* means every window of this snapshot is done (the caller may write
* `.ingested`); `partial` means the cap stopped it; `aborted` means the
* signal or budget fired (nothing uncommitted is recorded); `contended`
* means another extractor holds the lease or advanced the progress first;
* `unavailable` means the extractor had no servable model (#6786: nothing is
* recorded, and a progress a pre-#6786 build finished that way is rewound
* first). Transport and write errors throw after the lease is released; each
* one thrown after a model call counts against its window (#6785:
* `backoff`, then `dead_lettered` at CORPUS_WINDOW_FAILURE_CAP).
*/
export async function runCorpusWindows(opts: {
full: string;
raw: string;
fileStat: CorpusFileStat;
maxWindows: number;
extract: (text: string) => Promise<PipelineResult>;
overBudget: () => boolean;
signal: AbortSignal;
/** #6268: the file's resolved source; binds on the first run, refuses a resume under another. */
source?: CorpusProgressSource;
/** #6785: the extraction-config fingerprint that may re-arm a capped window once (null: never), and the clock. */
failures?: { fingerprint: string | null; now?: () => number };
/** Incremented before each attempted window, thrown or not: the caller's window budget. */
attempted?: { count: number };
}): Promise<CorpusWindowRun> {
const { full, fileStat, signal, source } = opts;
const incarnation = source?.incarnation ?? null;
const fingerprint = opts.failures?.fingerprint ?? null;
const clock = opts.failures?.now ?? Date.now;
const owner = `${process.pid}:${randomUUID()}`;
const now = Date.now();
let sourceChanged = false;
let state = await updateProgress(full, (cur) => {
if (source && cur?.source && (cur.source.id !== source.id || cur.source.incarnation !== source.incarnation)) {
sourceChanged = true;
return null;
}
if (cur?.lease && cur.lease.owner !== owner && now - cur.lease.at < CORPUS_PROGRESS_LEASE_STALE_MS) return null;
return { ...(cur ? rewindNonTerminal(cur) : freshProgress()), ...(source && !cur?.source ? { source } : {}), lease: { owner, at: now } };
});
const turns = parseCorpusTurns(opts.raw);
if (!state) {
const cur = await readCorpusProgress(full).catch(() => null);
const remaining = planCorpusWindows(turns, resumePoint(turns, cur), MAX_TURN_TEXT_CHARS, retiredTurns(cur)).filter((w) => w.text).length;
return { status: sourceChanged ? 'source_changed' : 'contended', windowsDone: 0, windowsRemaining: remaining, result: resultOf(cur ?? { ...freshProgress(), generation: 0 }) };
}
let cursor = resumePoint(turns, state);
const windows = planCorpusWindows(turns, cursor, MAX_TURN_TEXT_CHARS, retiredTurns(state));
let failure = readCorpusFailure(full);
let done = 0;
let status: CorpusWindowRun['status'] = 'complete';
let holding = true;
try {
for (let w = 0; w < windows.length; w++) {
const win = windows[w];
const key = windowKey(turns, cursor);
let r: PipelineResult | null = null;
let paid = false;
if (win.text) {
if (done >= opts.maxWindows) { status = 'partial'; break; }
if (opts.overBudget() || signal.aborted) { status = 'aborted'; break; }
const gate = corpusFailureGate(failure, key, incarnation, fingerprint, clock());
if (gate !== 'run') { status = gate; break; }
const text = win.text;
if (opts.attempted) opts.attempted.count++;
const attempt = await extractUnderFailureCount({ full, record: failure, key, incarnation, fingerprint, nowMs: clock, signal }, () => opts.extract(text));
r = attempt.value;
paid = attempt.paid;
if (r.skipped_reason && CORPUS_NON_TERMINAL_SKIPS.includes(r.skipped_reason)) { status = 'unavailable'; break; }
}
if (signal.aborted) { status = 'aborted'; break; }
const finishedTurns = turns.slice(cursor.turn, win.endOffset === null ? win.endTurn + 1 : win.endTurn);
const isLast = w === windows.length - 1;
const expectGen = state.generation;
const expectLast = lastSha(state);
const advances = finishedTurns.length > 0 || (win.endOffset ?? 0) > cursor.offset;
const commit = updateProgress(full, (cur) => {
if (!advances || !cur || cur.generation !== expectGen || lastSha(cur) !== expectLast || cur.lease?.owner !== owner) return null;
const slugs = r ? [...new Set([...cur.entity_slugs, ...r.entity_slugs])].slice(-MAX_ENTITY_SLUGS) : cur.entity_slugs;
return {
...cur,
turns: [...cur.turns, ...finishedTurns.map(({ start, end, sha256: h }) => ({ start, end, sha256: h }))].slice(-MAX_RECORDED_TURNS),
continuation: win.endOffset === null ? null : { turn_sha256: turns[win.endTurn].sha256, stripped_offset: win.endOffset },
windows_done: cur.windows_done + (r ? 1 : 0),
finished: isLast ? fileStat : null,
totals: r
? { inserted: cur.totals.inserted + r.inserted, duplicate: cur.totals.duplicate + r.duplicate, superseded: cur.totals.superseded + r.superseded }
: cur.totals,
entity_slugs: slugs,
...(r?.skipped_reason ? { skipped_reason: r.skipped_reason } : {}),
lease: isLast ? null : { owner, at: Date.now() },
};
});
// #6785: a paid window whose progress did not commit is paid again next time; it counts like a thrown one.
const countUncommitted = (e: unknown) => paid
? recordCorpusFailure(full, failure, key, incarnation, fingerprint, clock(), e).catch(() => failure)
: Promise.resolve(failure);
let next: CorpusProgress | null;
try {
next = await commit;
} catch (e) {
await countUncommitted(e);
throw e;
}
if (!next) {
failure = await countUncommitted({ code: 'corpus_progress_contended' });
status = 'contended';
break;
}
state = next;
if (r) done++;
if (isLast) holding = false;
cursor = win.endOffset === null ? { turn: win.endTurn + 1, offset: 0 } : { turn: win.endTurn, offset: win.endOffset };
// This window's record, or one whose window is gone (another incarnation, or its turn no longer ahead), is done.
if (failure && (failure.key === key || failure.source_incarnation !== incarnation || !failureAhead(failure.key, turns, cursor))) {
await clearCorpusFailure(full).catch(() => {});
failure = null;
}
}
if (windows.length === 0) {
const finished = await updateProgress(full, (cur) =>
cur && cur.lease?.owner === owner ? { ...cur, finished: fileStat, continuation: null, lease: null } : null);
if (finished) {
state = finished;
holding = false;
} else status = 'contended';
}
} finally {
if (holding) {
const released = await updateProgress(full, (cur) =>
cur && cur.lease?.owner === owner ? { ...cur, lease: null } : null).catch(() => null);
if (released) state = released;
}
}
const remaining = planCorpusWindows(turns, cursor, MAX_TURN_TEXT_CHARS, retiredTurns(state)).filter((w) => w.text).length;
// Rewritten while extracting: the snapshot's progress stands, but the file
// is not done; its new turns wait for the next run.
if (status === 'complete' && !sameCorpusFileStat(fileStat, corpusFileStat(await stat(full)))) status = 'changed';
return { status, windowsDone: done, windowsRemaining: status === 'complete' || status === 'changed' ? 0 : remaining, result: resultOf(state) };
}
/**
* The sweep's candidate list: files without a final `.ingested` (#6786: a
* `chat_unavailable` sidecar is not final), plus finished files that changed
* since `.progress` recorded them (stat compare, no rehash). A
* pre-window `.ingested` file is adopted at its current turns (budget
* permitting) so only later growth is extracted. Returns the candidates (at
* most `batchLimit`), the files `parked` says are backing off or stopped
* (#6785: at most `batchLimit`, served after every candidate so they never
* starve healthy files), and how many files are already done.
*/
export async function selectCorpusCandidates(
dir: string,
txtFiles: string[],
entrySet: Set<string>,
batchLimit: number,
overBudget: () => boolean,
parked: (full: string) => boolean = () => false,
): Promise<{ candidates: string[]; deferred: string[]; alreadyIngested: number }> {
const candidates: string[] = [];
const deferred: string[] = [];
let alreadyIngested = 0;
for (const name of txtFiles) {
const full = join(dir, name);
let state: ReturnType<typeof corpusFileCompletion> = 'pending';
if (entrySet.has(name + '.ingested')) {
try { state = corpusFileCompletion(full, name); } catch { state = 'changed'; }
if (state === 'legacy' && !overBudget()) await adoptFinishedCorpusFile(full).catch(() => false);
}
if (corpusFileFinished(state)) alreadyIngested++;
else if (entrySet.has(name + CORPUS_FAILURE_SUFFIX) && parked(full)) {
if (deferred.length < batchLimit) deferred.push(name);
} else if (candidates.length < batchLimit) candidates.push(name);
}
return { candidates, deferred, alreadyIngested };
}