Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix: stop the stale extract sweep chaining forever on frontmatter references
garrytan6331e9f60309
diff --git a/src/commands/extract.ts b/src/commands/extract.tsindex 155d8372cc..4994d0d5f3 100644--- a/src/commands/extract.ts+++ b/src/commands/extract.ts@@ -2152,7 +2152,10 @@ export async function extractStaleFromDB(const versionTs = linkExtractorWatermarkFor(grammar.generation);// Pre-flight count — cheap indexed COUNT. dry-run reports and returns.- const totalStale = await engine.countStalePagesForExtraction({ sourceId: sourceIdFilter, versionTs });+ // #9295: a body-only sweep counts only body wanted rows; it cannot settle a frontmatter row. The managed sweep re-resolves both.+ const managed = await managedPersistenceEnabled(engine);+ const wantedProducers = managed || includeFrontmatter ? undefined : ['body'];+ const totalStale = await engine.countStalePagesForExtraction({ sourceId: sourceIdFilter, versionTs, wantedProducers });if (dryRun) {const m = await previewMentionPass(engine, sourceIdFilter), counts = opts.quiet ? null : await staleExtractionCounts(engine, sourceIdFilter);if (jsonMode && !opts.quiet) {@@ -2165,7 +2168,7 @@ export async function extractStaleFromDB(// Managed brains: the writer guard owns canonical rows, so links, missing// canonical timeline rows and the watermark publish on the one managed path// (the mention pass runs inside it).- if (await managedPersistenceEnabled(engine)) {+ if (managed) {const { runManagedStaleExtraction, formatManagedStaleExtraction } = await import('../core/persistence/links-maintenance.ts');const r = await runManagedStaleExtraction(engine, { sourceId: sourceIdFilter, catchUp, timeBudgetMs, counts: !opts.quiet, retryHeld: opts.retryHeld });if (!jsonMode) log(formatManagedStaleExtraction(r, false, false));@@ -2240,7 +2243,7 @@ export async function extractStaleFromDB(for (;;) {await plannerTick(pagesProcessed);const rows = await engine.listStalePagesForExtraction({- batchSize: STALE_BATCH_SIZE, afterPageId, sourceId: sourceIdFilter, versionTs,+ batchSize: STALE_BATCH_SIZE, afterPageId, sourceId: sourceIdFilter, versionTs, wantedProducers,});if (rows.length === 0) break;await loadSourceLinkPacks(engine, rows.map(page => page.source_id), packs);@@ -2369,7 +2372,7 @@ export async function extractStaleFromDB(progress.finish();if (packUnavailable) throw new Error('Cannot extract links: active schema pack is unavailable.');const mentions = await runMentionPass(engine, { sourceId: sourceIdFilter, ...(catchUp ? {} : { deadline: startMs + timeBudgetMs }) });- const staleRemaining = await engine.countStalePagesForExtraction({ sourceId: sourceIdFilter, versionTs }) + mentions.remaining;+ const staleRemaining = await engine.countStalePagesForExtraction({ sourceId: sourceIdFilter, versionTs, wantedProducers }) + mentions.remaining;const counts = opts.quiet ? null : await staleExtractionCounts(engine, sourceIdFilter);if (!jsonMode) {diff --git a/src/core/engine-sql/pages.ts b/src/core/engine-sql/pages.tsindex 5f5a4b7092..f415c54e9b 100644--- a/src/core/engine-sql/pages.ts+++ b/src/core/engine-sql/pages.ts@@ -772,12 +772,20 @@ export async function resolveSlugs(* origin last resolved it. Their next extraction creates the edge* (src/core/wanted-links.ts).*/-const WANTED_ORIGIN_IS_STALE = sqlFragment`id IN (SELECT w.origin_page_id FROM wanted_links w+/**+ * #9295: `producers` limits the wanted rows to the producers the caller's sweep re-resolves. A body-only sweep cannot+ * settle a frontmatter row, so counting one made the page stale after every sweep; the row stays unchecked and a+ * frontmatter sweep still finds it.+ */+function wantedOriginIsStale(producers?: readonly string[]) {+ const producer = producers ? sqlFragment` AND w.producer = ANY(${[...producers]}::text[])` : sqlFragment``;+ return sqlFragment`id IN (SELECT w.origin_page_id FROM wanted_links wJOIN pages t ON t.source_id = w.target_source_id AND t.slug = w.target_ref AND t.deleted_at IS NULL- WHERE t.updated_at > w.checked_at+ WHERE t.updated_at > w.checked_at${producer}UNION SELECT w.origin_page_id FROM wanted_links wJOIN pages t ON t.source_id = w.target_source_id AND regexp_replace(t.slug, '^.*/', '') = w.target_ref AND t.deleted_at IS NULL- WHERE w.ref_kind = 'name' AND t.updated_at > w.checked_at)`;+ WHERE w.ref_kind = 'name' AND t.updated_at > w.checked_at${producer})`;+}/*** GBRA-130 A-1: a page whose link extraction is held: its failure record is `held` at the page's current knowledge@@ -788,6 +796,8 @@ function linkExtractionHeld(versionTs?: string) {return sqlFragment`COALESCE(links_extract_failure->>'state' = 'held' AND links_extract_failure->>'revision' = knowledge_revision::text${generation}, false)`;}+type StaleOpts = { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only'; wantedProducers?: readonly string[] };+/*** Shared stale-for-extraction predicate. `attendance` narrows it by the #5761* marker: a page is attendance-blocked while its marker equals its current@@ -795,10 +805,11 @@ function linkExtractionHeld(versionTs?: string) {* reconsidering blocked pages. `held` (GBRA-130 A-1) excludes or selects pages* whose link extraction is held (linkExtractionHeld); managed extraction excludes them.*/-function stalePagesWhere(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only' }) {+function stalePagesWhere(opts?: StaleOpts) {+ const wanted = wantedOriginIsStale(opts?.wantedProducers);const version = opts?.versionTs- ? sqlFragment`(links_extracted_at IS NULL OR links_extracted_at < ${opts.versionTs}::timestamptz OR updated_at > links_extracted_at OR ${WANTED_ORIGIN_IS_STALE})`- : sqlFragment`(links_extracted_at IS NULL OR updated_at > links_extracted_at OR ${WANTED_ORIGIN_IS_STALE})`;+ ? sqlFragment`(links_extracted_at IS NULL OR links_extracted_at < ${opts.versionTs}::timestamptz OR updated_at > links_extracted_at OR ${wanted})`+ : sqlFragment`(links_extracted_at IS NULL OR updated_at > links_extracted_at OR ${wanted})`;const source = opts?.sourceId ? sqlFragment` AND source_id = ${opts.sourceId}` : sqlFragment``;const attendance = opts?.attendance === 'exclude'? sqlFragment` AND links_attendance_blocked_revision IS DISTINCT FROM knowledge_revision`@@ -820,11 +831,12 @@ export function staleForExtractionSql(versionTs: string): SqlFragment {* `attendance: false` is the pre-v180 shape (no marker column: nothing is attendance-blocked). `held` (GBRA-130 A-1) also* excludes held pages from the stale count and counts them (`held: false` is the pre-v258 shape: nothing is held).*/-export function extractionLagStatement(opts: { sourceId?: string; versionTs: string; attendance: boolean; held?: boolean }): { text: string; params: unknown[] } {+export function extractionLagStatement(opts: { sourceId?: string; versionTs: string; attendance: boolean; held?: boolean; wantedProducers?: readonly string[] }): { text: string; params: unknown[] } {const source = opts.sourceId ? sqlFragment` AND source_id = ${opts.sourceId}` : sqlFragment``;- const stale = stalePagesWhere({ versionTs: opts.versionTs, ...(opts.attendance ? { attendance: 'exclude' as const } : {}), ...(opts.held ? { held: 'exclude' as const } : {}) });- const blocked = opts.attendance ? sqlFragment`count(*) FILTER (WHERE ${stalePagesWhere({ versionTs: opts.versionTs, attendance: 'blocked' })})::int` : sqlFragment`0`;- const held = opts.held ? sqlFragment`count(*) FILTER (WHERE ${stalePagesWhere({ versionTs: opts.versionTs, held: 'only' })})::int` : sqlFragment`0`;+ const base = { versionTs: opts.versionTs, wantedProducers: opts.wantedProducers };+ const stale = stalePagesWhere({ ...base, ...(opts.attendance ? { attendance: 'exclude' as const } : {}), ...(opts.held ? { held: 'exclude' as const } : {}) });+ const blocked = opts.attendance ? sqlFragment`count(*) FILTER (WHERE ${stalePagesWhere({ ...base, attendance: 'blocked' })})::int` : sqlFragment`0`;+ const held = opts.held ? sqlFragment`count(*) FILTER (WHERE ${stalePagesWhere({ ...base, held: 'only' })})::int` : sqlFragment`0`;return renderFragment(sqlFragment`SELECT count(*)::int AS total, count(*) FILTER (WHERE ${stale})::int AS stale, ${blocked} AS attendance_blocked, ${held} AS heldFROM pages WHERE deleted_at IS NULL AND ${trustedSql(quarantineFilterFragment('pages'))}${source}`);}@@ -835,7 +847,7 @@ export function heldLinkExtractionStatement(opts: { sourceId?: string; versionTsWHERE ${stalePagesWhere({ sourceId: opts.sourceId, versionTs: opts.versionTs, held: 'only' })} ORDER BY id LIMIT ${opts.limit}`);}-export async function countStalePagesForExtraction(exec: ScopedRead, opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only' }): Promise<number> {+export async function countStalePagesForExtraction(exec: ScopedRead, opts?: StaleOpts): Promise<number> {const { text, params } = renderFragment(sqlFragment`SELECT count(*)::int AS count FROM pages WHERE ${stalePagesWhere(opts)}`);const { rows } = await exec.unsafe<{ count?: number }>(text, params);return Number(rows[0]?.count ?? 0);@@ -847,6 +859,7 @@ export async function listStalePagesForExtraction(exec: ScopedRead, opts: {sourceId?: string;versionTs?: string;held?: 'exclude';+ wantedProducers?: readonly string[];}): Promise<StalePageRow[]> {const afterClause = opts.afterPageId != null ? sqlFragment` AND id > ${opts.afterPageId}` : sqlFragment``;// #1768: project a deterministic full-µs UTC string alongside updated_at.diff --git a/src/core/engine.ts b/src/core/engine.tsindex 1c7c846989..e66b6d4c21 100644--- a/src/core/engine.ts+++ b/src/core/engine.ts@@ -1435,7 +1435,7 @@ export interface BrainEngine {* `attendance` (#5761) excludes or selects pages whose attendance marker* equals their current knowledge revision.*/- countStalePagesForExtraction(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only' }): Promise<number>;+ countStalePagesForExtraction(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only'; wantedProducers?: readonly string[] }): Promise<number>;/*** List a keyset page (ordered by `id`, `id > afterPageId`) of stale pages* WITH their content so the caller extracts without an N+1 `getPage`. Same@@ -1449,6 +1449,7 @@ export interface BrainEngine {sourceId?: string;versionTs?: string;held?: 'exclude';+ wantedProducers?: readonly string[];}): Promise<StalePageRow[]>;/*** Stamp `links_extracted_at` for a batch of pages keyed on the uniquediff --git a/src/core/extraction-lag.ts b/src/core/extraction-lag.tsindex 337d165485..7245d3d162 100644--- a/src/core/extraction-lag.ts+++ b/src/core/extraction-lag.ts@@ -16,6 +16,8 @@ import { previewMentionPass } from './mentions/stale.ts';import { readMentionPolicy } from './mentions/policy.ts';import { resolveEnvNumber, warnOnceForEnv } from './env-number.ts';import { isMissingSchemaObject } from './utils.ts';+import { managedPersistenceEnabled } from './persistence/ownership.ts';+import { resolveIncludeFrontmatter } from './extract-frontmatter.ts';import { LINK_HOLD_THRESHOLD_RUNS, linkExtractionRetryFix } from './persistence/links-extraction-holds.ts';/** Warn threshold (percent of live pages actionable-stale); `GBRAIN_EXTRACTION_LAG_WARN_PCT` overrides it. */@@ -58,8 +60,10 @@ export async function readExtractionLag(engine: BrainEngine, opts: { sourceId?:async function readLagRow(engine: BrainEngine, opts: { sourceId?: string; timeoutMs?: number }) {const versionTs = await effectiveLinkExtractorWatermark(engine);+ // #9295: count the wanted rows the configured sweep re-resolves (a managed or frontmatter sweep: every producer).+ const wantedProducers = await managedPersistenceEnabled(engine) || await resolveIncludeFrontmatter(engine) ? undefined : ['body'];const read = async (attendance: boolean, held: boolean) => {- const { text, params } = extractionLagStatement({ sourceId: opts.sourceId, versionTs, attendance, held });+ const { text, params } = extractionLagStatement({ sourceId: opts.sourceId, versionTs, attendance, held, wantedProducers });return (await engine.executeRaw<{ total: number; stale: number; attendance_blocked: number; held: number }>(text, params, { timeoutMs: opts.timeoutMs ?? 10_000 }))[0];};let row;diff --git a/src/core/minions/handlers/extract.ts b/src/core/minions/handlers/extract.tsindex 330e57ab67..7ccd571ae7 100644--- a/src/core/minions/handlers/extract.ts+++ b/src/core/minions/handlers/extract.ts@@ -29,8 +29,10 @@ export function makeExtractHandler(engine: BrainEngine): MinionHandler {// waiting for the next sync. Forward-progress guard (pagesProcessed >// 0) prevents an infinite chain if the sweep can't advance. #8672: the raw// stale count includes attendance-blocked and held pages, which no rerun- // clears, so a sweep that left only those chains nothing.- if (!job.data.dryRun && r.staleRemaining > 0 && (r.pagesProcessed > 0 || (r.mentions?.pages ?? 0) > 0)+ // clears, so a sweep that left only those chains nothing. #9295: a continuation chains again only when it+ // left fewer pages stale than the sweep before it, so a page no sweep settles cannot keep the chain alive.+ const previous = typeof job.data.stale_remaining === 'number' ? job.data.stale_remaining : Infinity;+ if (!job.data.dryRun && r.staleRemaining > 0 && r.staleRemaining < previous && (r.pagesProcessed > 0 || (r.mentions?.pages ?? 0) > 0)&& !(await staleExtractionCounts(engine, sourceIdFilter)).complete) {try {const queue = new MinionQueue(engine);@@ -41,7 +43,7 @@ export function makeExtractHandler(engine: BrainEngine): MinionHandler {// so there is no pile-up to guard against.await queue.add('extract',- { ...job.data, continuation_of: job.id },+ { ...job.data, continuation_of: job.id, stale_remaining: r.staleRemaining },{ timeout_ms: STALE_TIME_BUDGET_MS + 5 * 60 * 1000 },);} catch { /* best-effort: next sync/manual sweep picks up the rest */ }diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.tsindex 1e424a3b99..a5294fbb3d 100644--- a/src/core/pglite-engine.ts+++ b/src/core/pglite-engine.ts@@ -1900,7 +1900,7 @@ export class PGLiteEngine implements BrainEngine {return chunksImpl.deleteChunks(this.engineSql, slug, opts);}- async countStalePagesForExtraction(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only' }): Promise<number> {+ async countStalePagesForExtraction(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only'; wantedProducers?: readonly string[] }): Promise<number> {return pagesImpl.countStalePagesForExtraction(scopedRead(this.engineSql), opts);}@@ -1910,6 +1910,7 @@ export class PGLiteEngine implements BrainEngine {sourceId?: string;versionTs?: string;held?: 'exclude';+ wantedProducers?: readonly string[];}): Promise<StalePageRow[]> {return pagesImpl.listStalePagesForExtraction(scopedRead(this.engineSql), opts);}diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.tsindex d9d95ac7d8..e5baee50ab 100644--- a/src/core/postgres-engine.ts+++ b/src/core/postgres-engine.ts@@ -1725,7 +1725,7 @@ export class PostgresEngine implements BrainEngine {return chunksImpl.deleteChunks(this.engineSql, slug, opts);}- async countStalePagesForExtraction(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only' }): Promise<number> {+ async countStalePagesForExtraction(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only'; wantedProducers?: readonly string[] }): Promise<number> {return this.withScopedReadTransaction(undefined, opts?.sourceId, tx => pagesImpl.countStalePagesForExtraction(scopedRead(this.engineSqlOn(tx)), opts));}@@ -1735,6 +1735,7 @@ export class PostgresEngine implements BrainEngine {sourceId?: string;versionTs?: string;held?: 'exclude';+ wantedProducers?: readonly string[];}): Promise<StalePageRow[]> {return this.withScopedReadTransaction(undefined, opts.sourceId, tx => pagesImpl.listStalePagesForExtraction(scopedRead(this.engineSqlOn(tx)), opts));}diff --git a/test/extract-stale-frontmatter-loop-9295.test.ts b/test/extract-stale-frontmatter-loop-9295.test.tsnew file mode 100644index 0000000000..8aed47d338--- /dev/null+++ b/test/extract-stale-frontmatter-loop-9295.test.ts@@ -0,0 +1,109 @@+/**+ * #9295: on the default config (`autopilot.incremental_extract_include_frontmatter` off) the stale sweep rewrites only+ * an origin's body wanted-link rows, but the staleness check also counted its frontmatter rows, so a page whose+ * frontmatter names a page written later stayed stale after every sweep and the `extract` job chained continuations+ * without end. Now a sweep counts only the producers it re-resolves (the frontmatter row stays unchecked, so turning+ * the frontmatter sweep on still finds it), and a continuation chains only while the remaining count falls.+ * PGLite always; PostgreSQL when DATABASE_URL is set (test/postgres-unit-arms.txt). A managed brain re-resolves both+ * producers, so its sweep settles the page.+ */+import { afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test';+import { randomUUID } from 'node:crypto';+import type { BrainEngine } from '../src/core/engine.ts';+import { PGLiteEngine } from '../src/core/pglite-engine.ts';+import { makeExtractHandler } from '../src/core/minions/handlers/extract.ts';+import { extractStaleFromDB } from '../src/commands/extract.ts';+import { readExtractionLag } from '../src/core/extraction-lag.ts';+import { submitPageMutation } from '../src/core/persistence/page-mutations.ts';+import { importFromContent } from '../src/core/import-file.ts';+import { BLOCK_MAX_BYTES } from '../src/core/page-body/segment-blocks.ts';+import { setCliOptions } from '../src/core/cli-options.ts';+import type { OperationContext } from '../src/core/ops/contract.ts';+import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';+import { managedBrain } from './helpers/managed-brain.ts';+import { testBackends } from './helpers/test-backends.ts';++const ORIGIN = 'people/alice-example';+const TARGET = 'companies/acme-toolkit';+const MAX_CHAIN = 6;++const ctxOf = (engine: BrainEngine): OperationContext => ({ engine, config: { engine: engine.kind, embedding_disabled: true }, sourceId: 'default',+ remote: false, dryRun: false, logger: { info() {}, warn() {}, error() {} } }) as OperationContext;+const put = (ctx: OperationContext, slug: string, type: string, frontmatter = '') => submitPageMutation(ctx, { operation: 'put_page',+ params: { slug, request_id: randomUUID(), content: `---\ntype: ${type}\ntitle: ${slug}\n${frontmatter}---\n\n${slug} notes.\n` } });+const wanted = (engine: BrainEngine) => engine.executeRaw<{ producer: string; target_ref: string }>('SELECT producer, target_ref FROM wanted_links ORDER BY producer');+const queued = async (engine: BrainEngine) => engine.executeRaw<{ id: number; data: Record<string, unknown> }>(+ "SELECT id, data FROM minion_jobs WHERE name = 'extract' AND status = 'waiting' ORDER BY id");++/** Runs the stale job, then each continuation it queues, as the worker would; returns how many continuations ran. */+async function runChain(engine: BrainEngine, between?: () => Promise<void>): Promise<number> {+ const handler = makeExtractHandler(engine);+ await handler({ id: 1, data: { stale: true } } as never);+ for (let n = 1; n <= MAX_CHAIN; n++) {+ const [next] = await queued(engine);+ if (!next) return n - 1;+ await engine.executeRaw("UPDATE minion_jobs SET status = 'completed' WHERE id = $1", [next.id]);+ await between?.();+ await handler({ id: Number(next.id), data: next.data } as never);+ }+ return MAX_CHAIN + 1;+}++async function frontmatterLoop(engine: BrainEngine, ctx: OperationContext): Promise<void> {+ await put(ctx, ORIGIN, 'person', 'founded: ["acme-toolkit"]\n');+ expect(await wanted(engine)).toEqual([{ producer: 'frontmatter', target_ref: 'acme-toolkit' }]);+ await extractStaleFromDB(engine, { dryRun: false, jsonMode: true, quiet: true, catchUp: true });+ await put(ctx, TARGET, 'company');+}++beforeAll(() => setCliOptions({ quiet: true, progressJson: false, progressInterval: 1000, explain: false, timeoutMs: null, brain: null }));++for (const kind of testBackends()) {+ describe(`stale extract continuations, unmanaged repo-less brain (${kind}) (#9295)`, () => {+ let engine: BrainEngine;+ let close: () => Promise<void>;+ beforeEach(async () => {+ if (kind === 'postgres') ({ engine, close } = await isolatedPersistencePostgres(process.env.DATABASE_URL!));+ else { const lite = new PGLiteEngine(); await lite.connect({}); await lite.initSchema(); engine = lite; close = () => lite.disconnect(); }+ }, 120_000);+ afterEach(async () => { await close?.(); });++ test('a frontmatter reference to a page written later does not chain the sweep without end', async () => {+ await frontmatterLoop(engine, ctxOf(engine));+ expect(await runChain(engine)).toBeLessThanOrEqual(1);+ expect((await readExtractionLag(engine)).actionable).toBe(0);+ expect(await wanted(engine)).toEqual([{ producer: 'frontmatter', target_ref: 'acme-toolkit' }]);+ }, 300_000);++ test('the unchecked frontmatter row is still found once the frontmatter sweep is turned on', async () => {+ await frontmatterLoop(engine, ctxOf(engine));+ await runChain(engine);+ await engine.setConfig('autopilot.incremental_extract_include_frontmatter', 'true');+ expect((await readExtractionLag(engine)).actionable).toBe(1);+ await extractStaleFromDB(engine, { dryRun: false, jsonMode: true, quiet: true, catchUp: true });+ expect((await readExtractionLag(engine)).actionable).toBe(0);+ expect(await wanted(engine)).toEqual([]);+ }, 300_000);++ test('a continuation whose sweep leaves as many pages stale as before chains nothing', async () => {+ await engine.setConfig('pages.segmented_storage', 'on');+ const body = Array.from({ length: Math.ceil((BLOCK_MAX_BYTES + 2 * 1024 * 1024) / 32) }, (_, i) => 'code line ' + String(i).padStart(10, '0') + ' x = 1;\n').join('');+ expect((await importFromContent(engine, 'notes/archive', '---\ntitle: Archive\ntype: note\n---\n\n\`\`\`ts\n' + body + '\`\`\`\n',+ { sourceId: 'default', noEmbed: true })).status).toBe('imported');+ await put(ctxOf(engine), 'notes/first', 'note');+ let written = 0;+ const chained = await runChain(engine, async () => { await put(ctxOf(engine), `notes/later-${++written}`, 'note'); });+ expect(chained).toBeLessThanOrEqual(1);+ }, 600_000);+ });+}++describe('stale extract continuations, managed brain (#9295)', () => {+ test('the managed sweep re-resolves the frontmatter row and the chain ends', async () => {+ await managedBrain(async ({ engine, ctx }) => {+ await frontmatterLoop(engine, ctx);+ expect(await runChain(engine)).toBeLessThanOrEqual(1);+ expect((await readExtractionLag(engine)).actionable).toBe(0);+ });+ }, 300_000);+});diff --git a/test/fixtures/goldens/exports/types.json b/test/fixtures/goldens/exports/types.jsonindex 81d57f5faa..a9794abfb6 100644--- a/test/fixtures/goldens/exports/types.json+++ b/test/fixtures/goldens/exports/types.json@@ -71,7 +71,7 @@"countPagesBy: (query: PageCountQuery, groupTypes?: TypeGrouper) => Promise<PageCounts>","countStaleChunks: (opts?: { sourceId?: string; signature?: string; includeNullSignature?: boolean; }) => Promise<number>","countStaleFrontmatterKeyPages: (opts: { sourceIds: string[]; }) => Promise<Array<{ source_id: string; n: number; }>>",- "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; }) => Promise<number>",+ "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; wantedProducers?: readonly string[]; }) => Promise<number>","countStaleTakes: (opts?: StaleTakeOpts) => Promise<number>","countUnconsolidatedFacts: (source_id: string) => Promise<number>","createSegmentedPageShell: (slug: string, meta: SegmentedPageMeta, opts: { sourceId: string; }) => Promise<{ id: number; }>",@@ -165,7 +165,7 @@"listPrefixSampledPages: (opts: DomainBankSampleOpts) => Promise<DomainBankRow[]>","listStaleChunks: (opts?: { batchSize?: number; afterPageId?: number; afterChunkIndex?: number; sourceId?: string; orderBy?: \"page_id\" | \"updated_desc\"; afterUpdatedAt?: string | null; }) => Promise<StaleChunkRow[]>","listStaleFrontmatterKeyPages: (opts: { sourceId: string; afterId: number; limit: number; }) => Promise<Array<{ id: number; slug: string; keys: string[]; }>>",- "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; }) => Promise<StalePageRow[]>",+ "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; wantedProducers?: readonly string[]; }) => Promise<StalePageRow[]>","listStaleTakes: (opts?: StaleTakeOpts) => Promise<StaleTakeRow[]>","listSupersessions: (source_id: string, opts?: { since?: Date; limit?: number; visibility?: FactVisibility[]; eligibility?: FactListOpts[\"eligibility\"]; excludePrivate?: boolean; }) => Promise<FactRow[]>","listTakes: (opts?: TakesListOpts) => Promise<Take[]>",@@ -1087,7 +1087,7 @@"countPagesBy: (query: PageCountQuery, groupTypes?: TypeGrouper) => Promise<PageCounts>","countStaleChunks: (opts?: { sourceId?: string; signature?: string; includeNullSignature?: boolean; }) => Promise<number>","countStaleFrontmatterKeyPages: (opts: { sourceIds: string[]; }) => Promise<{ source_id: string; n: number; }[]>",- "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; }) => Promise<number>",+ "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; wantedProducers?: readonly string[]; }) => Promise<number>","countStaleTakes: (opts?: StaleTakeOpts) => Promise<number>","countUnconsolidatedFacts: (source_id: string) => Promise<number>","createSegmentedPageShell: (slug: string, meta: SegmentedPageMeta, opts: { sourceId: string; }) => Promise<{ id: number; }>",@@ -1183,7 +1183,7 @@"listPrefixSampledPages: (opts: DomainBankSampleOpts) => Promise<DomainBankRow[]>","listStaleChunks: (opts?: { batchSize?: number; afterPageId?: number; afterChunkIndex?: number; sourceId?: string; orderBy?: \"page_id\" | \"updated_desc\"; afterUpdatedAt?: string | null; }) => Promise<StaleChunkRow[]>","listStaleFrontmatterKeyPages: (opts: { sourceId: string; afterId: number; limit: number; }) => Promise<{ id: number; slug: string; keys: string[]; }[]>",- "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; }) => Promise<StalePageRow[]>",+ "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; wantedProducers?: readonly string[]; }) => Promise<StalePageRow[]>","listStaleTakes: (opts?: StaleTakeOpts) => Promise<StaleTakeRow[]>","listSupersessions: (source_id: string, opts?: { since?: Date; limit?: number; visibility?: (\"private\" | \"world\")[]; eligibility?: FactListOpts[\"eligibility\"]; excludePrivate?: boolean; }) => Promise<FactRow[]>","listTakes: (opts?: TakesListOpts) => Promise<Take[]>",@@ -3641,7 +3641,7 @@"countPagesBy: (query: PageCountQuery, groupTypes?: TypeGrouper) => Promise<PageCounts>","countStaleChunks: (opts?: { sourceId?: string; signature?: string; includeNullSignature?: boolean; }) => Promise<number>","countStaleFrontmatterKeyPages: (opts: { sourceIds: string[]; }) => Promise<Array<{ source_id: string; n: number; }>>",- "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; }) => Promise<number>",+ "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; wantedProducers?: readonly string[]; }) => Promise<number>","countStaleTakes: (opts?: StaleTakeOpts) => Promise<number>","countUnconsolidatedFacts: (source_id: string) => Promise<number>","createSegmentedPageShell: (slug: string, meta: SegmentedPageMeta, opts: { sourceId: string; }) => Promise<{ id: number; }>",@@ -3735,7 +3735,7 @@"listPrefixSampledPages: (opts: DomainBankSampleOpts) => Promise<DomainBankRow[]>","listStaleChunks: (opts?: { batchSize?: number; afterPageId?: number; afterChunkIndex?: number; sourceId?: string; orderBy?: \"page_id\" | \"updated_desc\"; afterUpdatedAt?: string | null; }) => Promise<StaleChunkRow[]>","listStaleFrontmatterKeyPages: (opts: { sourceId: string; afterId: number; limit: number; }) => Promise<Array<{ id: number; slug: string; keys: string[]; }>>",- "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; }) => Promise<StalePageRow[]>",+ "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; wantedProducers?: readonly string[]; }) => Promise<StalePageRow[]>","listStaleTakes: (opts?: StaleTakeOpts) => Promise<StaleTakeRow[]>","listSupersessions: (source_id: string, opts?: { since?: Date; limit?: number; visibility?: FactVisibility[]; eligibility?: FactListOpts[\"eligibility\"]; excludePrivate?: boolean; }) => Promise<FactRow[]>","listTakes: (opts?: TakesListOpts) => Promise<Take[]>",@@ -7104,7 +7104,7 @@"countPagesBy: (query: PageCountQuery, groupTypes?: TypeGrouper) => Promise<PageCounts>","countStaleChunks: (opts?: { sourceId?: string; signature?: string; includeNullSignature?: boolean; }) => Promise<number>","countStaleFrontmatterKeyPages: (opts: { sourceIds: string[]; }) => Promise<{ source_id: string; n: number; }[]>",- "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; }) => Promise<number>",+ "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; wantedProducers?: readonly string[]; }) => Promise<number>","countStaleTakes: (opts?: StaleTakeOpts) => Promise<number>","countUnconsolidatedFacts: (source_id: string) => Promise<number>","createSegmentedPageShell: (slug: string, meta: SegmentedPageMeta, opts: { sourceId: string; }) => Promise<{ id: number; }>",@@ -7198,7 +7198,7 @@"listPrefixSampledPages: (opts: DomainBankSampleOpts) => Promise<DomainBankRow[]>","listStaleChunks: (opts?: { batchSize?: number; afterPageId?: number; afterChunkIndex?: number; sourceId?: string; orderBy?: \"page_id\" | \"updated_desc\"; afterUpdatedAt?: string | null; }) => Promise<StaleChunkRow[]>","listStaleFrontmatterKeyPages: (opts: { sourceId: string; afterId: number; limit: number; }) => Promise<{ id: number; slug: string; keys: string[]; }[]>",- "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; }) => Promise<StalePageRow[]>",+ "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; wantedProducers?: readonly string[]; }) => Promise<StalePageRow[]>","listStaleTakes: (opts?: StaleTakeOpts) => Promise<StaleTakeRow[]>","listSupersessions: (source_id: string, opts?: { since?: Date; limit?: number; visibility?: (\"private\" | \"world\")[]; eligibility?: FactListOpts[\"eligibility\"]; excludePrivate?: boolean; }) => Promise<FactRow[]>","listTakes: (opts?: TakesListOpts) => Promise<Take[]>",diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txtindex 85a4307a4d..0c5cba6d1a 100644--- a/test/postgres-unit-arms.txt+++ b/test/postgres-unit-arms.txt@@ -111,6 +111,7 @@ test/extract-atoms-world-fence-rows.test.tstest/extract-endpoint-revision.serial.test.tstest/extract-entities-managed.test.tstest/extract-facts-fence-failure-health-8758.test.ts+test/extract-stale-frontmatter-loop-9295.test.tstest/extract-timeline-from-meetings-managed.test.tstest/extraction-watermark-atoms.test.tstest/fact-embedding-oversized-7211.test.ts