Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(extract): report attendance-blocked skips apart from edited pages in managed extract --stale (#6820)
garrytan39875b963654
diff --git a/src/commands/extract.ts b/src/commands/extract.tsindex f238a05b0e..873da4cf41 100644--- a/src/commands/extract.ts+++ b/src/commands/extract.ts@@ -2171,7 +2171,7 @@ export async function extractStaleFromDB(if (!jsonMode) log(formatManagedStaleExtraction(r, false, false));else if (!opts.quiet) process.stdout.write(formatManagedStaleExtraction(r, false, true) + '\n');return { linksCreated: r.created, timelineCreated: r.timeline, pagesProcessed: r.pages, staleRemaining: r.remaining + (r.mentions?.remaining ?? 0),- ...(r.skipped ? { skippedChanged: r.skipped } : {}), ...(r.mentions ? { mentions: r.mentions } : {}),+ ...(r.skipped > (r.attendance_skipped ?? 0) ? { skippedChanged: r.skipped - (r.attendance_skipped ?? 0) } : {}), ...(r.attendance_skipped ? { skippedAttendanceIncomplete: r.attendance_skipped } : {}), ...(r.mentions ? { mentions: r.mentions } : {}),...(r.failed ? { linkFailed: r.failed, linkFailures: r.failures } : {}), ...(r.systemic?.length ? { linkSystemic: r.systemic } : {}) };}diff --git a/src/core/persistence/links-maintenance.ts b/src/core/persistence/links-maintenance.tsindex bcf180b569..3f77c9adf6 100644--- a/src/core/persistence/links-maintenance.ts+++ b/src/core/persistence/links-maintenance.ts@@ -17,6 +17,8 @@ import { clearLinkExtractionHolds, linkExtractionRetryFix, recordLinkExtractionFexport type { LinkExtractionFailure };export interface ManagedLinkExtraction { pages: number; created: number; removed: number; timeline: number; skipped: number; remaining: number;+ /** #6820: of `skipped`, the pages skipped for unresolved attendance (the rest changed while the run read them). */+ attendance_skipped?: number;/** The mention pass (absent when the caller ran links only, as sync does). */mentions?: MentionPassResult; mention_due?: number; mention_last_pass_at?: string | null;/** #6803 (D3): the terminal counts (`gbrain extract --stale` only; sync's tail does not read them). */@@ -56,7 +58,8 @@ export function formatManagedStaleExtraction(result: ManagedLinkExtraction, dryRmention_last_pass_at: result.mention_last_pass_at ?? null } : {action: 'extract_stale_done', links_created: result.created, links_removed: result.removed, timeline_created: result.timeline,pages_processed: result.pages, stale_remaining: result.remaining + (result.mentions?.remaining ?? 0),- ...(result.skipped ? { skipped_changed: result.skipped } : {}), ...(result.mentions ? mentionJsonFields(result.mentions) : {}),+ ...(changedSkips(result) ? { skipped_changed: changedSkips(result) } : {}), ...(result.attendance_skipped ? { skipped_attendance_incomplete: result.attendance_skipped } : {}),+ ...(result.mentions ? mentionJsonFields(result.mentions) : {}),...(result.failed ? { failed: result.failed, failures: result.failures } : {}), ...(result.systemic?.length ? { systemic: result.systemic } : {}),}), ...(counts ?? {}) });}@@ -67,10 +70,14 @@ export function formatManagedStaleExtraction(result: ManagedLinkExtraction, dryR}const mentionLine = result.mentions ? formatMentionSummary(result.mentions) : null;return `Extract --stale: ${result.created} link(s) created, ${result.removed} removed, ${result.timeline} timeline entr(ies) from ${result.pages} page(s).` +- (result.skipped ? ` Skipped ${result.skipped} page(s) edited during extraction or with unresolved attendance; they stay stale.` : '') ++ (changedSkips(result) ? ` Skipped ${changedSkips(result)} page(s) edited during extraction; they stay stale.` : '') ++ (result.attendance_skipped ? ` Skipped ${result.attendance_skipped} page(s) with unresolved attendance; they wait for the attendee's person page.` : '') +(result.remaining ? ` ${result.remaining} page(s) remain stale (raw total).` : '') + formatLinkFailures(result) + (mentionLine ? `\n${mentionLine}` : '') + countsLine;}+/** Pages a run skipped because they changed while it read them (the rest of `skipped` waits on attendance). */+const changedSkips = (result: ManagedLinkExtraction) => result.skipped - (result.attendance_skipped ?? 0);+/** The failure lines of a run: each listed page with its code and state, then each source the breaker stopped, each with its fix. */function formatLinkFailures(result: ManagedLinkExtraction): string {const lines = (result.failures ?? []).map(f => `\n ${f.source_id}:${f.slug}: ${f.code} (${f.held ? 'held; fix the page, then run' : f.class === 'content' ? `failed ${f.attempts} run(s); retried by` : 'transient; retried by'} ${f.fix.argv!.join(' ')}): ${f.message}`);@@ -107,7 +114,7 @@ export const MANAGED_LINK_EXTRACTION_WIDTH = 4;*/export async function extractManagedStaleLinks(engine: BrainEngine,opts: { sourceId?: string; slugs?: readonly string[]; maxPages?: number; timeBudgetMs?: number; signal?: AbortSignal; mentions?: boolean } = {}): Promise<ManagedLinkExtraction> {- const result: ManagedLinkExtraction = { pages: 0, created: 0, removed: 0, timeline: 0, skipped: 0, remaining: 0 };+ const result: ManagedLinkExtraction = { pages: 0, created: 0, removed: 0, timeline: 0, skipped: 0, attendance_skipped: 0, remaining: 0 };const startMs = Date.now();const withMentions = opts.mentions !== false;const deadline = withMentions ? await linkPhaseDeadline(engine, opts.sourceId, startMs, opts.timeBudgetMs)@@ -134,7 +141,7 @@ export async function extractManagedStaleLinks(engine: BrainEngine,if (current?.revision !== snapshot.revision) return null;if (!prepared.attendanceComplete) {await tx.markPagesAttendanceBlocked([{ slug, source_id: sourceId, revision: current.revision }]);- return null;+ return 'attendance' as const;}const written = await prepared.apply(tx);if (written.errors) return null;@@ -145,7 +152,7 @@ export async function extractManagedStaleLinks(engine: BrainEngine,await tx.markPagesExtractedBatch([{ slug, source_id: sourceId, extractedAt: at }], at);return { ...written, timeline: added };}, attribution));- if (!outcome) { result.skipped++; return; }+ if (!outcome || outcome === 'attendance') { result.skipped++; if (outcome) result.attendance_skipped!++; return; }result.pages++;result.created += outcome.created;result.removed += outcome.removed;diff --git a/test/links-attendance-blocked.test.ts b/test/links-attendance-blocked.test.tsindex 4520f85c06..d0de80fd00 100644--- a/test/links-attendance-blocked.test.ts+++ b/test/links-attendance-blocked.test.ts@@ -18,7 +18,7 @@ import { PGLiteEngine } from '../src/core/pglite-engine.ts';import { claimWorktree } from '../src/core/persistence/ownership.ts';import { submitPageMutation } from '../src/core/persistence/page-mutations.ts';import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';-import { extractManagedStaleLinks } from '../src/core/persistence/links-maintenance.ts';+import { extractManagedStaleLinks, formatManagedStaleExtraction } from '../src/core/persistence/links-maintenance.ts';import { checkLinksExtractionLag } from '../src/commands/doctor.ts';import { extractStaleFromDB } from '../src/commands/extract.ts';import { configureGateway, resetGateway } from '../src/core/ai/gateway.ts';@@ -143,6 +143,25 @@ test('#5761: creating the missing attendee lets the next extraction publish and});});+test('#6820: managed extract --stale reports an attendance-blocked page as an attendance skip, not as an edited page', async () => {+ await managedFixture(async (engine, sourceId) => {+ const opts = { includeFrontmatter: true, dryRun: false, jsonMode: false, quiet: true, sourceIdFilter: sourceId, catchUp: true };+ for (const run of [1, 2]) {+ const result = await extractManagedStaleLinks(engine, { sourceId });+ const json = JSON.parse(formatManagedStaleExtraction(result, false, true));+ expect(json.skipped_changed).toBeUndefined();+ expect(json.skipped_attendance_incomplete).toBe(1);+ expect(result).toMatchObject({ pages: run === 1 ? 1 : 0, skipped: 1, attendance_skipped: 1 });+ const text = formatManagedStaleExtraction(result, false, false);+ expect(text).toContain('Skipped 1 page(s) with unresolved attendance');+ expect(text).not.toContain('edited during extraction');+ }+ const swept = await extractStaleFromDB(engine, opts);+ expect(swept.skippedAttendanceIncomplete).toBe(1);+ expect(swept.skippedChanged).toBeUndefined();+ });+});+test('#5761: the unmanaged extract --stale sweep marks and clears the same way', async () => {for (const engine of engines) {const sourceId = `att-${randomUUID().slice(0, 8)}`;