Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: stop the stale extract sweep chaining forever on frontmatter references

garrytan6331e9f60309
diff --git a/src/commands/extract.ts b/src/commands/extract.ts
index 155d8372cc..4994d0d5f3 100644
--- a/src/commands/extract.ts
+++ b/src/commands/extract.ts
@@ -2152,7 +2152,10 @@ export async function extractStaleFromDB(
const versionTs = linkExtractorWatermarkFor(grammar.generation);
// Pre-flight count — cheap indexed COUNT. dry-run reports and returns.
- const totalStale = await engine.countStalePagesForExtraction({ sourceId: sourceIdFilter, versionTs });
+ // #9295: a body-only sweep counts only body wanted rows; it cannot settle a frontmatter row. The managed sweep re-resolves both.
+ const managed = await managedPersistenceEnabled(engine);
+ const wantedProducers = managed || includeFrontmatter ? undefined : ['body'];
+ const totalStale = await engine.countStalePagesForExtraction({ sourceId: sourceIdFilter, versionTs, wantedProducers });
if (dryRun) {
const m = await previewMentionPass(engine, sourceIdFilter), counts = opts.quiet ? null : await staleExtractionCounts(engine, sourceIdFilter);
if (jsonMode && !opts.quiet) {
@@ -2165,7 +2168,7 @@ export async function extractStaleFromDB(
// Managed brains: the writer guard owns canonical rows, so links, missing
// canonical timeline rows and the watermark publish on the one managed path
// (the mention pass runs inside it).
- if (await managedPersistenceEnabled(engine)) {
+ if (managed) {
const { runManagedStaleExtraction, formatManagedStaleExtraction } = await import('../core/persistence/links-maintenance.ts');
const r = await runManagedStaleExtraction(engine, { sourceId: sourceIdFilter, catchUp, timeBudgetMs, counts: !opts.quiet, retryHeld: opts.retryHeld });
if (!jsonMode) log(formatManagedStaleExtraction(r, false, false));
@@ -2240,7 +2243,7 @@ export async function extractStaleFromDB(
for (;;) {
await plannerTick(pagesProcessed);
const rows = await engine.listStalePagesForExtraction({
- batchSize: STALE_BATCH_SIZE, afterPageId, sourceId: sourceIdFilter, versionTs,
+ batchSize: STALE_BATCH_SIZE, afterPageId, sourceId: sourceIdFilter, versionTs, wantedProducers,
});
if (rows.length === 0) break;
await loadSourceLinkPacks(engine, rows.map(page => page.source_id), packs);
@@ -2369,7 +2372,7 @@ export async function extractStaleFromDB(
progress.finish();
if (packUnavailable) throw new Error('Cannot extract links: active schema pack is unavailable.');
const mentions = await runMentionPass(engine, { sourceId: sourceIdFilter, ...(catchUp ? {} : { deadline: startMs + timeBudgetMs }) });
- const staleRemaining = await engine.countStalePagesForExtraction({ sourceId: sourceIdFilter, versionTs }) + mentions.remaining;
+ const staleRemaining = await engine.countStalePagesForExtraction({ sourceId: sourceIdFilter, versionTs, wantedProducers }) + mentions.remaining;
const counts = opts.quiet ? null : await staleExtractionCounts(engine, sourceIdFilter);
if (!jsonMode) {
diff --git a/src/core/engine-sql/pages.ts b/src/core/engine-sql/pages.ts
index 5f5a4b7092..f415c54e9b 100644
--- a/src/core/engine-sql/pages.ts
+++ b/src/core/engine-sql/pages.ts
@@ -772,12 +772,20 @@ export async function resolveSlugs(
* origin last resolved it. Their next extraction creates the edge
* (src/core/wanted-links.ts).
*/
-const WANTED_ORIGIN_IS_STALE = sqlFragment`id IN (SELECT w.origin_page_id FROM wanted_links w
+/**
+ * #9295: `producers` limits the wanted rows to the producers the caller's sweep re-resolves. A body-only sweep cannot
+ * settle a frontmatter row, so counting one made the page stale after every sweep; the row stays unchecked and a
+ * frontmatter sweep still finds it.
+ */
+function wantedOriginIsStale(producers?: readonly string[]) {
+ const producer = producers ? sqlFragment` AND w.producer = ANY(${[...producers]}::text[])` : sqlFragment``;
+ return sqlFragment`id IN (SELECT w.origin_page_id FROM wanted_links w
JOIN pages t ON t.source_id = w.target_source_id AND t.slug = w.target_ref AND t.deleted_at IS NULL
- WHERE t.updated_at > w.checked_at
+ WHERE t.updated_at > w.checked_at${producer}
UNION SELECT w.origin_page_id FROM wanted_links w
JOIN pages t ON t.source_id = w.target_source_id AND regexp_replace(t.slug, '^.*/', '') = w.target_ref AND t.deleted_at IS NULL
- WHERE w.ref_kind = 'name' AND t.updated_at > w.checked_at)`;
+ WHERE w.ref_kind = 'name' AND t.updated_at > w.checked_at${producer})`;
+}
/**
* GBRA-130 A-1: a page whose link extraction is held: its failure record is `held` at the page's current knowledge
@@ -788,6 +796,8 @@ function linkExtractionHeld(versionTs?: string) {
return sqlFragment`COALESCE(links_extract_failure->>'state' = 'held' AND links_extract_failure->>'revision' = knowledge_revision::text${generation}, false)`;
}
+type StaleOpts = { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only'; wantedProducers?: readonly string[] };
+
/**
* Shared stale-for-extraction predicate. `attendance` narrows it by the #5761
* marker: a page is attendance-blocked while its marker equals its current
@@ -795,10 +805,11 @@ function linkExtractionHeld(versionTs?: string) {
* reconsidering blocked pages. `held` (GBRA-130 A-1) excludes or selects pages
* whose link extraction is held (linkExtractionHeld); managed extraction excludes them.
*/
-function stalePagesWhere(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only' }) {
+function stalePagesWhere(opts?: StaleOpts) {
+ const wanted = wantedOriginIsStale(opts?.wantedProducers);
const version = opts?.versionTs
- ? sqlFragment`(links_extracted_at IS NULL OR links_extracted_at < ${opts.versionTs}::timestamptz OR updated_at > links_extracted_at OR ${WANTED_ORIGIN_IS_STALE})`
- : sqlFragment`(links_extracted_at IS NULL OR updated_at > links_extracted_at OR ${WANTED_ORIGIN_IS_STALE})`;
+ ? sqlFragment`(links_extracted_at IS NULL OR links_extracted_at < ${opts.versionTs}::timestamptz OR updated_at > links_extracted_at OR ${wanted})`
+ : sqlFragment`(links_extracted_at IS NULL OR updated_at > links_extracted_at OR ${wanted})`;
const source = opts?.sourceId ? sqlFragment` AND source_id = ${opts.sourceId}` : sqlFragment``;
const attendance = opts?.attendance === 'exclude'
? sqlFragment` AND links_attendance_blocked_revision IS DISTINCT FROM knowledge_revision`
@@ -820,11 +831,12 @@ export function staleForExtractionSql(versionTs: string): SqlFragment {
* `attendance: false` is the pre-v180 shape (no marker column: nothing is attendance-blocked). `held` (GBRA-130 A-1) also
* excludes held pages from the stale count and counts them (`held: false` is the pre-v258 shape: nothing is held).
*/
-export function extractionLagStatement(opts: { sourceId?: string; versionTs: string; attendance: boolean; held?: boolean }): { text: string; params: unknown[] } {
+export function extractionLagStatement(opts: { sourceId?: string; versionTs: string; attendance: boolean; held?: boolean; wantedProducers?: readonly string[] }): { text: string; params: unknown[] } {
const source = opts.sourceId ? sqlFragment` AND source_id = ${opts.sourceId}` : sqlFragment``;
- const stale = stalePagesWhere({ versionTs: opts.versionTs, ...(opts.attendance ? { attendance: 'exclude' as const } : {}), ...(opts.held ? { held: 'exclude' as const } : {}) });
- const blocked = opts.attendance ? sqlFragment`count(*) FILTER (WHERE ${stalePagesWhere({ versionTs: opts.versionTs, attendance: 'blocked' })})::int` : sqlFragment`0`;
- const held = opts.held ? sqlFragment`count(*) FILTER (WHERE ${stalePagesWhere({ versionTs: opts.versionTs, held: 'only' })})::int` : sqlFragment`0`;
+ const base = { versionTs: opts.versionTs, wantedProducers: opts.wantedProducers };
+ const stale = stalePagesWhere({ ...base, ...(opts.attendance ? { attendance: 'exclude' as const } : {}), ...(opts.held ? { held: 'exclude' as const } : {}) });
+ const blocked = opts.attendance ? sqlFragment`count(*) FILTER (WHERE ${stalePagesWhere({ ...base, attendance: 'blocked' })})::int` : sqlFragment`0`;
+ const held = opts.held ? sqlFragment`count(*) FILTER (WHERE ${stalePagesWhere({ ...base, held: 'only' })})::int` : sqlFragment`0`;
return renderFragment(sqlFragment`SELECT count(*)::int AS total, count(*) FILTER (WHERE ${stale})::int AS stale, ${blocked} AS attendance_blocked, ${held} AS held
FROM pages WHERE deleted_at IS NULL AND ${trustedSql(quarantineFilterFragment('pages'))}${source}`);
}
@@ -835,7 +847,7 @@ export function heldLinkExtractionStatement(opts: { sourceId?: string; versionTs
WHERE ${stalePagesWhere({ sourceId: opts.sourceId, versionTs: opts.versionTs, held: 'only' })} ORDER BY id LIMIT ${opts.limit}`);
}
-export async function countStalePagesForExtraction(exec: ScopedRead, opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only' }): Promise<number> {
+export async function countStalePagesForExtraction(exec: ScopedRead, opts?: StaleOpts): Promise<number> {
const { text, params } = renderFragment(sqlFragment`SELECT count(*)::int AS count FROM pages WHERE ${stalePagesWhere(opts)}`);
const { rows } = await exec.unsafe<{ count?: number }>(text, params);
return Number(rows[0]?.count ?? 0);
@@ -847,6 +859,7 @@ export async function listStalePagesForExtraction(exec: ScopedRead, opts: {
sourceId?: string;
versionTs?: string;
held?: 'exclude';
+ wantedProducers?: readonly string[];
}): Promise<StalePageRow[]> {
const afterClause = opts.afterPageId != null ? sqlFragment` AND id > ${opts.afterPageId}` : sqlFragment``;
// #1768: project a deterministic full-µs UTC string alongside updated_at.
diff --git a/src/core/engine.ts b/src/core/engine.ts
index 1c7c846989..e66b6d4c21 100644
--- a/src/core/engine.ts
+++ b/src/core/engine.ts
@@ -1435,7 +1435,7 @@ export interface BrainEngine {
* `attendance` (#5761) excludes or selects pages whose attendance marker
* equals their current knowledge revision.
*/
- countStalePagesForExtraction(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only' }): Promise<number>;
+ countStalePagesForExtraction(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only'; wantedProducers?: readonly string[] }): Promise<number>;
/**
* List a keyset page (ordered by `id`, `id > afterPageId`) of stale pages
* WITH their content so the caller extracts without an N+1 `getPage`. Same
@@ -1449,6 +1449,7 @@ export interface BrainEngine {
sourceId?: string;
versionTs?: string;
held?: 'exclude';
+ wantedProducers?: readonly string[];
}): Promise<StalePageRow[]>;
/**
* Stamp `links_extracted_at` for a batch of pages keyed on the unique
diff --git a/src/core/extraction-lag.ts b/src/core/extraction-lag.ts
index 337d165485..7245d3d162 100644
--- a/src/core/extraction-lag.ts
+++ b/src/core/extraction-lag.ts
@@ -16,6 +16,8 @@ import { previewMentionPass } from './mentions/stale.ts';
import { readMentionPolicy } from './mentions/policy.ts';
import { resolveEnvNumber, warnOnceForEnv } from './env-number.ts';
import { isMissingSchemaObject } from './utils.ts';
+import { managedPersistenceEnabled } from './persistence/ownership.ts';
+import { resolveIncludeFrontmatter } from './extract-frontmatter.ts';
import { LINK_HOLD_THRESHOLD_RUNS, linkExtractionRetryFix } from './persistence/links-extraction-holds.ts';
/** Warn threshold (percent of live pages actionable-stale); `GBRAIN_EXTRACTION_LAG_WARN_PCT` overrides it. */
@@ -58,8 +60,10 @@ export async function readExtractionLag(engine: BrainEngine, opts: { sourceId?:
async function readLagRow(engine: BrainEngine, opts: { sourceId?: string; timeoutMs?: number }) {
const versionTs = await effectiveLinkExtractorWatermark(engine);
+ // #9295: count the wanted rows the configured sweep re-resolves (a managed or frontmatter sweep: every producer).
+ const wantedProducers = await managedPersistenceEnabled(engine) || await resolveIncludeFrontmatter(engine) ? undefined : ['body'];
const read = async (attendance: boolean, held: boolean) => {
- const { text, params } = extractionLagStatement({ sourceId: opts.sourceId, versionTs, attendance, held });
+ const { text, params } = extractionLagStatement({ sourceId: opts.sourceId, versionTs, attendance, held, wantedProducers });
return (await engine.executeRaw<{ total: number; stale: number; attendance_blocked: number; held: number }>(text, params, { timeoutMs: opts.timeoutMs ?? 10_000 }))[0];
};
let row;
diff --git a/src/core/minions/handlers/extract.ts b/src/core/minions/handlers/extract.ts
index 330e57ab67..7ccd571ae7 100644
--- a/src/core/minions/handlers/extract.ts
+++ b/src/core/minions/handlers/extract.ts
@@ -29,8 +29,10 @@ export function makeExtractHandler(engine: BrainEngine): MinionHandler {
// waiting for the next sync. Forward-progress guard (pagesProcessed >
// 0) prevents an infinite chain if the sweep can't advance. #8672: the raw
// stale count includes attendance-blocked and held pages, which no rerun
- // clears, so a sweep that left only those chains nothing.
- if (!job.data.dryRun && r.staleRemaining > 0 && (r.pagesProcessed > 0 || (r.mentions?.pages ?? 0) > 0)
+ // clears, so a sweep that left only those chains nothing. #9295: a continuation chains again only when it
+ // left fewer pages stale than the sweep before it, so a page no sweep settles cannot keep the chain alive.
+ const previous = typeof job.data.stale_remaining === 'number' ? job.data.stale_remaining : Infinity;
+ if (!job.data.dryRun && r.staleRemaining > 0 && r.staleRemaining < previous && (r.pagesProcessed > 0 || (r.mentions?.pages ?? 0) > 0)
&& !(await staleExtractionCounts(engine, sourceIdFilter)).complete) {
try {
const queue = new MinionQueue(engine);
@@ -41,7 +43,7 @@ export function makeExtractHandler(engine: BrainEngine): MinionHandler {
// so there is no pile-up to guard against.
await queue.add(
'extract',
- { ...job.data, continuation_of: job.id },
+ { ...job.data, continuation_of: job.id, stale_remaining: r.staleRemaining },
{ timeout_ms: STALE_TIME_BUDGET_MS + 5 * 60 * 1000 },
);
} catch { /* best-effort: next sync/manual sweep picks up the rest */ }
diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts
index 1e424a3b99..a5294fbb3d 100644
--- a/src/core/pglite-engine.ts
+++ b/src/core/pglite-engine.ts
@@ -1900,7 +1900,7 @@ export class PGLiteEngine implements BrainEngine {
return chunksImpl.deleteChunks(this.engineSql, slug, opts);
}
- async countStalePagesForExtraction(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only' }): Promise<number> {
+ async countStalePagesForExtraction(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only'; wantedProducers?: readonly string[] }): Promise<number> {
return pagesImpl.countStalePagesForExtraction(scopedRead(this.engineSql), opts);
}
@@ -1910,6 +1910,7 @@ export class PGLiteEngine implements BrainEngine {
sourceId?: string;
versionTs?: string;
held?: 'exclude';
+ wantedProducers?: readonly string[];
}): Promise<StalePageRow[]> {
return pagesImpl.listStalePagesForExtraction(scopedRead(this.engineSql), opts);
}
diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts
index d9d95ac7d8..e5baee50ab 100644
--- a/src/core/postgres-engine.ts
+++ b/src/core/postgres-engine.ts
@@ -1725,7 +1725,7 @@ export class PostgresEngine implements BrainEngine {
return chunksImpl.deleteChunks(this.engineSql, slug, opts);
}
- async countStalePagesForExtraction(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only' }): Promise<number> {
+ async countStalePagesForExtraction(opts?: { sourceId?: string; versionTs?: string; attendance?: 'exclude' | 'blocked'; held?: 'exclude' | 'only'; wantedProducers?: readonly string[] }): Promise<number> {
return this.withScopedReadTransaction(undefined, opts?.sourceId, tx => pagesImpl.countStalePagesForExtraction(scopedRead(this.engineSqlOn(tx)), opts));
}
@@ -1735,6 +1735,7 @@ export class PostgresEngine implements BrainEngine {
sourceId?: string;
versionTs?: string;
held?: 'exclude';
+ wantedProducers?: readonly string[];
}): Promise<StalePageRow[]> {
return this.withScopedReadTransaction(undefined, opts.sourceId, tx => pagesImpl.listStalePagesForExtraction(scopedRead(this.engineSqlOn(tx)), opts));
}
diff --git a/test/extract-stale-frontmatter-loop-9295.test.ts b/test/extract-stale-frontmatter-loop-9295.test.ts
new file mode 100644
index 0000000000..8aed47d338
--- /dev/null
+++ b/test/extract-stale-frontmatter-loop-9295.test.ts
@@ -0,0 +1,109 @@
+/**
+ * #9295: on the default config (`autopilot.incremental_extract_include_frontmatter` off) the stale sweep rewrites only
+ * an origin's body wanted-link rows, but the staleness check also counted its frontmatter rows, so a page whose
+ * frontmatter names a page written later stayed stale after every sweep and the `extract` job chained continuations
+ * without end. Now a sweep counts only the producers it re-resolves (the frontmatter row stays unchecked, so turning
+ * the frontmatter sweep on still finds it), and a continuation chains only while the remaining count falls.
+ * PGLite always; PostgreSQL when DATABASE_URL is set (test/postgres-unit-arms.txt). A managed brain re-resolves both
+ * producers, so its sweep settles the page.
+ */
+import { afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
+import { randomUUID } from 'node:crypto';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { makeExtractHandler } from '../src/core/minions/handlers/extract.ts';
+import { extractStaleFromDB } from '../src/commands/extract.ts';
+import { readExtractionLag } from '../src/core/extraction-lag.ts';
+import { submitPageMutation } from '../src/core/persistence/page-mutations.ts';
+import { importFromContent } from '../src/core/import-file.ts';
+import { BLOCK_MAX_BYTES } from '../src/core/page-body/segment-blocks.ts';
+import { setCliOptions } from '../src/core/cli-options.ts';
+import type { OperationContext } from '../src/core/ops/contract.ts';
+import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
+import { managedBrain } from './helpers/managed-brain.ts';
+import { testBackends } from './helpers/test-backends.ts';
+
+const ORIGIN = 'people/alice-example';
+const TARGET = 'companies/acme-toolkit';
+const MAX_CHAIN = 6;
+
+const ctxOf = (engine: BrainEngine): OperationContext => ({ engine, config: { engine: engine.kind, embedding_disabled: true }, sourceId: 'default',
+ remote: false, dryRun: false, logger: { info() {}, warn() {}, error() {} } }) as OperationContext;
+const put = (ctx: OperationContext, slug: string, type: string, frontmatter = '') => submitPageMutation(ctx, { operation: 'put_page',
+ params: { slug, request_id: randomUUID(), content: `---\ntype: ${type}\ntitle: ${slug}\n${frontmatter}---\n\n${slug} notes.\n` } });
+const wanted = (engine: BrainEngine) => engine.executeRaw<{ producer: string; target_ref: string }>('SELECT producer, target_ref FROM wanted_links ORDER BY producer');
+const queued = async (engine: BrainEngine) => engine.executeRaw<{ id: number; data: Record<string, unknown> }>(
+ "SELECT id, data FROM minion_jobs WHERE name = 'extract' AND status = 'waiting' ORDER BY id");
+
+/** Runs the stale job, then each continuation it queues, as the worker would; returns how many continuations ran. */
+async function runChain(engine: BrainEngine, between?: () => Promise<void>): Promise<number> {
+ const handler = makeExtractHandler(engine);
+ await handler({ id: 1, data: { stale: true } } as never);
+ for (let n = 1; n <= MAX_CHAIN; n++) {
+ const [next] = await queued(engine);
+ if (!next) return n - 1;
+ await engine.executeRaw("UPDATE minion_jobs SET status = 'completed' WHERE id = $1", [next.id]);
+ await between?.();
+ await handler({ id: Number(next.id), data: next.data } as never);
+ }
+ return MAX_CHAIN + 1;
+}
+
+async function frontmatterLoop(engine: BrainEngine, ctx: OperationContext): Promise<void> {
+ await put(ctx, ORIGIN, 'person', 'founded: ["acme-toolkit"]\n');
+ expect(await wanted(engine)).toEqual([{ producer: 'frontmatter', target_ref: 'acme-toolkit' }]);
+ await extractStaleFromDB(engine, { dryRun: false, jsonMode: true, quiet: true, catchUp: true });
+ await put(ctx, TARGET, 'company');
+}
+
+beforeAll(() => setCliOptions({ quiet: true, progressJson: false, progressInterval: 1000, explain: false, timeoutMs: null, brain: null }));
+
+for (const kind of testBackends()) {
+ describe(`stale extract continuations, unmanaged repo-less brain (${kind}) (#9295)`, () => {
+ let engine: BrainEngine;
+ let close: () => Promise<void>;
+ beforeEach(async () => {
+ if (kind === 'postgres') ({ engine, close } = await isolatedPersistencePostgres(process.env.DATABASE_URL!));
+ else { const lite = new PGLiteEngine(); await lite.connect({}); await lite.initSchema(); engine = lite; close = () => lite.disconnect(); }
+ }, 120_000);
+ afterEach(async () => { await close?.(); });
+
+ test('a frontmatter reference to a page written later does not chain the sweep without end', async () => {
+ await frontmatterLoop(engine, ctxOf(engine));
+ expect(await runChain(engine)).toBeLessThanOrEqual(1);
+ expect((await readExtractionLag(engine)).actionable).toBe(0);
+ expect(await wanted(engine)).toEqual([{ producer: 'frontmatter', target_ref: 'acme-toolkit' }]);
+ }, 300_000);
+
+ test('the unchecked frontmatter row is still found once the frontmatter sweep is turned on', async () => {
+ await frontmatterLoop(engine, ctxOf(engine));
+ await runChain(engine);
+ await engine.setConfig('autopilot.incremental_extract_include_frontmatter', 'true');
+ expect((await readExtractionLag(engine)).actionable).toBe(1);
+ await extractStaleFromDB(engine, { dryRun: false, jsonMode: true, quiet: true, catchUp: true });
+ expect((await readExtractionLag(engine)).actionable).toBe(0);
+ expect(await wanted(engine)).toEqual([]);
+ }, 300_000);
+
+ test('a continuation whose sweep leaves as many pages stale as before chains nothing', async () => {
+ await engine.setConfig('pages.segmented_storage', 'on');
+ const body = Array.from({ length: Math.ceil((BLOCK_MAX_BYTES + 2 * 1024 * 1024) / 32) }, (_, i) => 'code line ' + String(i).padStart(10, '0') + ' x = 1;\n').join('');
+ expect((await importFromContent(engine, 'notes/archive', '---\ntitle: Archive\ntype: note\n---\n\n\`\`\`ts\n' + body + '\`\`\`\n',
+ { sourceId: 'default', noEmbed: true })).status).toBe('imported');
+ await put(ctxOf(engine), 'notes/first', 'note');
+ let written = 0;
+ const chained = await runChain(engine, async () => { await put(ctxOf(engine), `notes/later-${++written}`, 'note'); });
+ expect(chained).toBeLessThanOrEqual(1);
+ }, 600_000);
+ });
+}
+
+describe('stale extract continuations, managed brain (#9295)', () => {
+ test('the managed sweep re-resolves the frontmatter row and the chain ends', async () => {
+ await managedBrain(async ({ engine, ctx }) => {
+ await frontmatterLoop(engine, ctx);
+ expect(await runChain(engine)).toBeLessThanOrEqual(1);
+ expect((await readExtractionLag(engine)).actionable).toBe(0);
+ });
+ }, 300_000);
+});
diff --git a/test/fixtures/goldens/exports/types.json b/test/fixtures/goldens/exports/types.json
index 81d57f5faa..a9794abfb6 100644
--- a/test/fixtures/goldens/exports/types.json
+++ b/test/fixtures/goldens/exports/types.json
@@ -71,7 +71,7 @@
"countPagesBy: (query: PageCountQuery, groupTypes?: TypeGrouper) => Promise<PageCounts>",
"countStaleChunks: (opts?: { sourceId?: string; signature?: string; includeNullSignature?: boolean; }) => Promise<number>",
"countStaleFrontmatterKeyPages: (opts: { sourceIds: string[]; }) => Promise<Array<{ source_id: string; n: number; }>>",
- "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; }) => Promise<number>",
+ "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; wantedProducers?: readonly string[]; }) => Promise<number>",
"countStaleTakes: (opts?: StaleTakeOpts) => Promise<number>",
"countUnconsolidatedFacts: (source_id: string) => Promise<number>",
"createSegmentedPageShell: (slug: string, meta: SegmentedPageMeta, opts: { sourceId: string; }) => Promise<{ id: number; }>",
@@ -165,7 +165,7 @@
"listPrefixSampledPages: (opts: DomainBankSampleOpts) => Promise<DomainBankRow[]>",
"listStaleChunks: (opts?: { batchSize?: number; afterPageId?: number; afterChunkIndex?: number; sourceId?: string; orderBy?: \"page_id\" | \"updated_desc\"; afterUpdatedAt?: string | null; }) => Promise<StaleChunkRow[]>",
"listStaleFrontmatterKeyPages: (opts: { sourceId: string; afterId: number; limit: number; }) => Promise<Array<{ id: number; slug: string; keys: string[]; }>>",
- "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; }) => Promise<StalePageRow[]>",
+ "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; wantedProducers?: readonly string[]; }) => Promise<StalePageRow[]>",
"listStaleTakes: (opts?: StaleTakeOpts) => Promise<StaleTakeRow[]>",
"listSupersessions: (source_id: string, opts?: { since?: Date; limit?: number; visibility?: FactVisibility[]; eligibility?: FactListOpts[\"eligibility\"]; excludePrivate?: boolean; }) => Promise<FactRow[]>",
"listTakes: (opts?: TakesListOpts) => Promise<Take[]>",
@@ -1087,7 +1087,7 @@
"countPagesBy: (query: PageCountQuery, groupTypes?: TypeGrouper) => Promise<PageCounts>",
"countStaleChunks: (opts?: { sourceId?: string; signature?: string; includeNullSignature?: boolean; }) => Promise<number>",
"countStaleFrontmatterKeyPages: (opts: { sourceIds: string[]; }) => Promise<{ source_id: string; n: number; }[]>",
- "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; }) => Promise<number>",
+ "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; wantedProducers?: readonly string[]; }) => Promise<number>",
"countStaleTakes: (opts?: StaleTakeOpts) => Promise<number>",
"countUnconsolidatedFacts: (source_id: string) => Promise<number>",
"createSegmentedPageShell: (slug: string, meta: SegmentedPageMeta, opts: { sourceId: string; }) => Promise<{ id: number; }>",
@@ -1183,7 +1183,7 @@
"listPrefixSampledPages: (opts: DomainBankSampleOpts) => Promise<DomainBankRow[]>",
"listStaleChunks: (opts?: { batchSize?: number; afterPageId?: number; afterChunkIndex?: number; sourceId?: string; orderBy?: \"page_id\" | \"updated_desc\"; afterUpdatedAt?: string | null; }) => Promise<StaleChunkRow[]>",
"listStaleFrontmatterKeyPages: (opts: { sourceId: string; afterId: number; limit: number; }) => Promise<{ id: number; slug: string; keys: string[]; }[]>",
- "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; }) => Promise<StalePageRow[]>",
+ "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; wantedProducers?: readonly string[]; }) => Promise<StalePageRow[]>",
"listStaleTakes: (opts?: StaleTakeOpts) => Promise<StaleTakeRow[]>",
"listSupersessions: (source_id: string, opts?: { since?: Date; limit?: number; visibility?: (\"private\" | \"world\")[]; eligibility?: FactListOpts[\"eligibility\"]; excludePrivate?: boolean; }) => Promise<FactRow[]>",
"listTakes: (opts?: TakesListOpts) => Promise<Take[]>",
@@ -3641,7 +3641,7 @@
"countPagesBy: (query: PageCountQuery, groupTypes?: TypeGrouper) => Promise<PageCounts>",
"countStaleChunks: (opts?: { sourceId?: string; signature?: string; includeNullSignature?: boolean; }) => Promise<number>",
"countStaleFrontmatterKeyPages: (opts: { sourceIds: string[]; }) => Promise<Array<{ source_id: string; n: number; }>>",
- "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; }) => Promise<number>",
+ "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; wantedProducers?: readonly string[]; }) => Promise<number>",
"countStaleTakes: (opts?: StaleTakeOpts) => Promise<number>",
"countUnconsolidatedFacts: (source_id: string) => Promise<number>",
"createSegmentedPageShell: (slug: string, meta: SegmentedPageMeta, opts: { sourceId: string; }) => Promise<{ id: number; }>",
@@ -3735,7 +3735,7 @@
"listPrefixSampledPages: (opts: DomainBankSampleOpts) => Promise<DomainBankRow[]>",
"listStaleChunks: (opts?: { batchSize?: number; afterPageId?: number; afterChunkIndex?: number; sourceId?: string; orderBy?: \"page_id\" | \"updated_desc\"; afterUpdatedAt?: string | null; }) => Promise<StaleChunkRow[]>",
"listStaleFrontmatterKeyPages: (opts: { sourceId: string; afterId: number; limit: number; }) => Promise<Array<{ id: number; slug: string; keys: string[]; }>>",
- "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; }) => Promise<StalePageRow[]>",
+ "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; wantedProducers?: readonly string[]; }) => Promise<StalePageRow[]>",
"listStaleTakes: (opts?: StaleTakeOpts) => Promise<StaleTakeRow[]>",
"listSupersessions: (source_id: string, opts?: { since?: Date; limit?: number; visibility?: FactVisibility[]; eligibility?: FactListOpts[\"eligibility\"]; excludePrivate?: boolean; }) => Promise<FactRow[]>",
"listTakes: (opts?: TakesListOpts) => Promise<Take[]>",
@@ -7104,7 +7104,7 @@
"countPagesBy: (query: PageCountQuery, groupTypes?: TypeGrouper) => Promise<PageCounts>",
"countStaleChunks: (opts?: { sourceId?: string; signature?: string; includeNullSignature?: boolean; }) => Promise<number>",
"countStaleFrontmatterKeyPages: (opts: { sourceIds: string[]; }) => Promise<{ source_id: string; n: number; }[]>",
- "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; }) => Promise<number>",
+ "countStalePagesForExtraction: (opts?: { sourceId?: string; versionTs?: string; attendance?: \"exclude\" | \"blocked\"; held?: \"exclude\" | \"only\"; wantedProducers?: readonly string[]; }) => Promise<number>",
"countStaleTakes: (opts?: StaleTakeOpts) => Promise<number>",
"countUnconsolidatedFacts: (source_id: string) => Promise<number>",
"createSegmentedPageShell: (slug: string, meta: SegmentedPageMeta, opts: { sourceId: string; }) => Promise<{ id: number; }>",
@@ -7198,7 +7198,7 @@
"listPrefixSampledPages: (opts: DomainBankSampleOpts) => Promise<DomainBankRow[]>",
"listStaleChunks: (opts?: { batchSize?: number; afterPageId?: number; afterChunkIndex?: number; sourceId?: string; orderBy?: \"page_id\" | \"updated_desc\"; afterUpdatedAt?: string | null; }) => Promise<StaleChunkRow[]>",
"listStaleFrontmatterKeyPages: (opts: { sourceId: string; afterId: number; limit: number; }) => Promise<{ id: number; slug: string; keys: string[]; }[]>",
- "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; }) => Promise<StalePageRow[]>",
+ "listStalePagesForExtraction: (opts: { batchSize: number; afterPageId?: number; sourceId?: string; versionTs?: string; held?: \"exclude\"; wantedProducers?: readonly string[]; }) => Promise<StalePageRow[]>",
"listStaleTakes: (opts?: StaleTakeOpts) => Promise<StaleTakeRow[]>",
"listSupersessions: (source_id: string, opts?: { since?: Date; limit?: number; visibility?: (\"private\" | \"world\")[]; eligibility?: FactListOpts[\"eligibility\"]; excludePrivate?: boolean; }) => Promise<FactRow[]>",
"listTakes: (opts?: TakesListOpts) => Promise<Take[]>",
diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txt
index 85a4307a4d..0c5cba6d1a 100644
--- a/test/postgres-unit-arms.txt
+++ b/test/postgres-unit-arms.txt
@@ -111,6 +111,7 @@ test/extract-atoms-world-fence-rows.test.ts
test/extract-endpoint-revision.serial.test.ts
test/extract-entities-managed.test.ts
test/extract-facts-fence-failure-health-8758.test.ts
+test/extract-stale-frontmatter-loop-9295.test.ts
test/extract-timeline-from-meetings-managed.test.ts
test/extraction-watermark-atoms.test.ts
test/fact-embedding-oversized-7211.test.ts