Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* `extract` Minion job handler (built-in; registered by registerBuiltinHandlers in src/commands/jobs.ts).
*/
import type { BrainEngine } from '../../engine.ts';
import type { MinionHandler } from '../types.ts';
import { MinionQueue } from '../queue.ts';
export function makeExtractHandler(engine: BrainEngine): MinionHandler {
return async (job) => {
const { runExtractCore, extractStaleFromDB, STALE_TIME_BUDGET_MS } = await import('../../../commands/extract.ts');
const { staleExtractionCounts } = await import('../../extraction-lag.ts');
// #2849: stale mode — the durable follow-up for extraction deferred by
// performSync's size gate (totalChanges > 100). Runs the same DB-source
// watermark sweep as `gbrain extract --stale`, scoped to the source the
// sync that deferred it was scoped to (job.data.sourceId; absent =
// unscoped, matching what the CLI hint tells a default-brain operator
// to run). The sweep is checkout-less + idempotent, so retries and
// overlapping submissions converge.
if (job.data.stale === true) {
const sourceIdFilter = typeof job.data.sourceId === 'string' ? job.data.sourceId : undefined;
const r = await extractStaleFromDB(engine, {
dryRun: !!job.data.dryRun,
jsonMode: false,
sourceIdFilter,
catchUp: false,
});
// Internal 30-min budget hit with work remaining → chain a
// continuation job so a very large deferred backlog converges without
// waiting for the next sync. Forward-progress guard (pagesProcessed >
// 0) prevents an infinite chain if the sweep can't advance. #8672: the raw
// stale count includes attendance-blocked and held pages, which no rerun
// clears, so a sweep that left only those chains nothing. #9295: a continuation chains again only when it
// left fewer pages stale than the sweep before it, so a page no sweep settles cannot keep the chain alive.
const previous = typeof job.data.stale_remaining === 'number' ? job.data.stale_remaining : Infinity;
if (!job.data.dryRun && r.staleRemaining > 0 && r.staleRemaining < previous && (r.pagesProcessed > 0 || (r.mentions?.pages ?? 0) > 0)
&& !(await staleExtractionCounts(engine, sourceIdFilter)).complete) {
try {
const queue = new MinionQueue(engine);
// NO maxWaiting: with an unscoped (NULL-sourceId) payload the
// coalesce filter matches ANY waiting 'extract' job and would
// swallow the continuation. Each completed sweep chains at most
// one continuation and the sweep is an idempotent watermark scan,
// so there is no pile-up to guard against.
await queue.add(
'extract',
{ ...job.data, continuation_of: job.id, stale_remaining: r.staleRemaining },
{ timeout_ms: STALE_TIME_BUDGET_MS + 5 * 60 * 1000 },
);
} catch { /* best-effort: next sync/manual sweep picks up the rest */ }
}
return { stale: true, source_id: sourceIdFilter ?? null, ...r };
}
const mode = (typeof job.data.mode === 'string' && ['links', 'timeline', 'all'].includes(job.data.mode))
? (job.data.mode as 'links' | 'timeline' | 'all')
: 'all';
const dir = typeof job.data.dir === 'string'
? job.data.dir
: (await engine.getConfig('sync.repo_path')) ?? '.';
// #3957: thread the job's source id into the fs-walk extractors. Without
// it the batch rows default to source_id='default' and the pages JOIN
// drops every row on a non-'default' brain (silent "created 0"), and the
// full-walk watermark stamp targets the wrong source.
const sourceId = typeof job.data.sourceId === 'string' ? job.data.sourceId : undefined;
const result = await runExtractCore(engine, { mode, dir, dryRun: !!job.data.dryRun, sourceId });
// #5904: refused timeline writes fail the job instead of completing over rows that were never written.
if (result.timeline_refused) throw new Error(`extract: ${result.timeline_refused} timeline write(s) refused; nothing written for them. Run gbrain extract --stale.`);
return result;
};
}