Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
27.3 KB
Raw Blame
/**
 * #6865: files of a managed Git source that are committed, eligible for sync,
 * and neither indexed nor held. An older gbrain refused some files before
 * refusals became holds (a stray frontmatter `slug:` line, unreadable YAML);
 * incremental discovery never visits an unchanged file with no page and no
 * hold, so those files stay absent from search and nothing reports them.
 *
 * `findUnindexedFiles` streams the tree of the source's last synced commit
 * (`git ls-tree -r`, the committed set discovery walks), filters it through
 * discovery's own eligibility (`loadSyncEligibility`) and compares in
 * source-path space: a file counts as indexed when a live page records its
 * origin, held when it has a sync hold, and `scheduled` when a re-screen is
 * already requested for it. A file whose path slug belongs to a live page with
 * another or no origin is not indexed either; it is counted as `slug_taken`,
 * and one whose page was deleted while the file stayed committed as
 * `deleted_page`. Neither is ever scheduled: a re-screen cannot import them
 * (a collision, or a page sync keeps deleted), so a person renames, deletes or
 * restores, and doctor names them read-only.
 * Lookups run in batches on the existing indexes, never over every page at
 * once. A file sync skips on purpose as folder scaffolding (`index.md`,
 * `README.md` and the other `isSyncMetafile` names) that would otherwise be eligible
 * and whose frontmatter has a `type:` or `title:` is counted separately as
 * `metafile_skipped`: read-only, never scheduled, because sync skips it again.
 * That happens only under `sync.metafile_pages off` (GBRA-150): under the default
 * `typed` such a nested file is eligible, so it is found and scheduled like any file.
 * Each found file is screened read-only (the import screen's codes, or
 * `clean`) up to a cap. A time or count budget stops the scan with
 * `complete: false` and a cursor (the last Git path examined, in tree order),
 * so the next pass resumes there and every path is eventually covered; an
 * incomplete scan is never reported clean.
 *
 * `refreshUnindexedFiles` runs one bounded pass, schedules the found paths
 * for a re-screen on the next sync (`requestGitHoldRetry`, at most
 * `UNINDEXED_SCHEDULE_CAP` per pass; a scheduling write, never a content
 * write) and stores the result per source in `op_checkpoints` (op
 * `unindexed-files`), where doctor `unindexed_files` reads it through
 * `judgeUnindexedFiles`. An incomplete stored result is continued and
 * accumulated by the next pass while its incarnation and eligibility
 * fingerprint are unchanged, also when the source's commit moved meanwhile
 * (#7149: a busy source commits between every two maintenance runs, and a scan
 * that started over each time never finished): the continuation first
 * re-examines the paths before its cursor that changed between the two
 * commits, then resumes after the cursor. When those changes cannot be listed
 * (the old commit is gone) or number more than `UNINDEXED_REVISIT_CAP`, the
 * pass starts over. Each source's time budget grows with its size
 * (`unindexedScanBudgetMs`). Blobs are read
 * and screened in slices bounded by bytes, with the deadline checked between
 * slices (a file left unread counts as `unscreened`).
 */
import type { BrainEngine } from '../engine.ts';
import type { SyncOpts } from '../../commands/sync.ts';
import { spawn } from '../spawn.ts';
import { gitChildEnv, literalPathspecs } from '../git-env.ts';
import { HARDENED_GIT_ARGS, NO_LAZY_FETCH_ENV } from '../hardened-git.ts';
import { isSyncMetafile, matchesAnyGlob, metafileIsPage, resolveSlugForPath } from '../sync.ts';
import { screenImportContent } from '../import-screen.ts';
import { isReservedSkillBundlePath } from '../skill-reserved-paths.ts';
import { loadSyncEligibility, resolveManagedSyncContext, syncExcludePatterns, syncGit, type ManagedSyncContext } from './sync-discovery.ts';
import { heldGitPaths, readGitHoldRetryPaths, requestGitHoldRetry } from './sync-holds.ts';
import { readBlobContents, type TreeBlob } from './sync-blobs.ts';
import { isWindowsColonTarget } from './native-file-target.ts';
import { danglingLink } from './sync-entry-fault.ts';
import { createHash } from 'node:crypto';

export const UNINDEXED_FILES_OP = 'unindexed-files';
/** Paths one pass schedules for a re-screen, and the most scheduled paths a stored result names. */
export const UNINDEXED_SCHEDULE_CAP = 5_000;
/** Found files one pass screens; the rest count as `unscreened`. */
export const UNINDEXED_SCREEN_LIMIT = 2_000;
export const UNINDEXED_SAMPLE_LIMIT = 20;
/** The time one live pass (the maintenance phase, `sources retry-held --unindexed`) spends per `UNINDEXED_BUDGET_PAGES` live pages of its source. */
export const UNINDEXED_SCAN_BUDGET_MS = 60_000;
export const UNINDEXED_BUDGET_PAGES = 100_000;
/** The most one pass of one source spends, and the most the maintenance phase spends over all sources. */
export const UNINDEXED_SCAN_MAX_BUDGET_MS = 300_000;
/** Changed paths before a stored cursor a continuation re-examines; with more, the pass starts over. */
export const UNINDEXED_REVISIT_CAP = 5_000;
/** A stored result older than this is stale. */
export const UNINDEXED_STALE_MS = 7 * 24 * 3_600_000;
const BATCH = 1_000;
const SYMLINK_MODE = '120000';
/** Blob bytes one screening slice holds in memory at most (a single larger blob is read alone); each slice is released before the next. */
const SCREEN_SLICE_BYTES = 32 * 1024 ** 2;

export interface UnindexedFile { path: string; source_path: string; code: string; slug_taken: boolean; deleted_page: boolean }
export interface UnindexedSample { path: string; code: string; slug_taken?: true; deleted_page?: true }

export interface UnindexedScan {
  source_id: string;
  incarnation: string;
  /** The commit whose tree was scanned (the source's last synced commit); null when the source never synced. */
  target: string | null;
  /** Fingerprint of what decides eligibility (scope, slug mode, sync excludes); a continuation needs the same one. */
  eligibility: string;
  complete: boolean;
  /** The last Git path examined when the budget stopped the scan; null when complete. */
  cursor: string | null;
  /** Eligible files examined. */
  scanned: number;
  /** Files with no page, no hold and no scheduled re-screen. */
  unindexed: number;
  /** Of `unindexed`: the path slug belongs to a live page with another or no recorded origin; never scheduled (a re-screen cannot import it). */
  slug_taken: number;
  /** Of `unindexed`: the file's page was deleted while the file stayed committed; never scheduled (sync keeps the page deleted). */
  deleted_page: number;
  /** Files with no page and no hold whose re-screen is already scheduled. */
  scheduled: number;
  by_code: Record<string, number>;
  samples: UnindexedSample[];
  /** Folder scaffolding names sync skips on purpose (`isSyncMetafile`) that are otherwise eligible and carry a frontmatter `type:` or `title:`; never scheduled. */
  metafile_skipped: number;
  metafile_samples: string[];
  /** The first unindexed files (at most `keep`), root-relative. */
  files: UnindexedFile[];
  /** The first already scheduled paths seen (at most `keep`), root-relative. */
  scheduled_paths: string[];
}

export interface UnindexedScanOpts {
  context?: ManagedSyncContext;
  /** Resume after this Git path (a previous incomplete scan's cursor). */
  cursor?: string | null;
  /** Git paths at or before `cursor` examined again first (paths that changed since the scan passed them). */
  revisit?: ReadonlySet<string>;
  /** Absolute epoch ms; checked after each batch, so every pass examines at least one batch. */
  deadlineAt?: number;
  /** Stop after examining this many eligible files. */
  maxScanned?: number;
  screenLimit?: number;
  keep?: number;
  /** Tree entries per lookup batch (default 1,000). */
  batch?: number;
  now?: () => number;
}

/** The stored result of the latest pass(es) for one source (`op_checkpoints` op `unindexed-files`, fingerprint the source id). */
export interface UnindexedRecord extends Omit<UnindexedScan, 'files' | 'scheduled_paths'> {
  version: 1;
  started_at: string;
  computed_at: string;
  /** Root-relative paths whose re-screen the stored passes scheduled or found scheduled, at most `UNINDEXED_SCHEDULE_CAP`. */
  pending: string[];
  /** Some scheduled paths did not fit in `pending`, so the result cannot confirm them. */
  pending_truncated: boolean;
  /** Of `unindexed`: how many the stored passes scheduled; the rest wait for a pass with room. */
  scheduled_now: number;
  /** Prepended to a root-relative path to get its page origin (`<scope>/` for a subdirectory source in git-root mode). */
  source_prefix: string;
}

interface TreeEntry { path: string; oid: string; size: number; mode: string }

/** Reads `entries` in slices of at most `SCREEN_SLICE_BYTES` and visits each with its text (undefined when too large to read); stops before a slice once `stop()` is true. */
function visitBlobSlices<T extends TreeBlob>(gitRoot: string, entries: T[], visit: (entry: T, content: string | undefined) => void, stop: () => boolean): void {
  for (let i = 0; i < entries.length && !stop();) {
    const slice: T[] = [];
    for (let bytes = 0; i < entries.length && (!slice.length || bytes + entries[i]!.size <= SCREEN_SLICE_BYTES); i++) { bytes += entries[i]!.size; slice.push(entries[i]!); }
    const contents = readBlobContents(gitRoot, slice);
    for (const entry of slice) visit(entry, contents.get(entry.oid));
  }
}

/** The blobs of `target` under `scope` in tree order, after `after`, in batches; the child stops when the consumer does. */
async function* treeBatches(gitRoot: string, target: string, scope: string, after: string | null, size: number, revisit?: ReadonlySet<string>): AsyncGenerator<TreeEntry[]> {
  const child = spawn('git', ['-c', 'core.quotepath=false', ...HARDENED_GIT_ARGS, '-C', gitRoot, 'ls-tree', '-r', '-l', '-z', target, ...(scope ? ['--', ...literalPathspecs([scope])] : [])],
    { env: gitChildEnv(NO_LAZY_FETCH_ENV), stdio: ['ignore', 'pipe', 'pipe'] });
  let stderr = '';
  child.stderr!.on('data', (chunk: Buffer) => { stderr = (stderr + chunk.toString('utf8')).slice(0, 2000); });
  const exit = new Promise<number | null>((resolve, reject) => { child.on('error', reject); child.on('close', resolve); });
  const floor = after === null ? null : Buffer.from(after, 'utf8');
  let pending: Buffer = Buffer.alloc(0), batch: TreeEntry[] = [];
  try {
    for await (const chunk of child.stdout! as AsyncIterable<Buffer>) {
      pending = pending.length ? Buffer.concat([pending, chunk]) : chunk;
      for (let nul = pending.indexOf(0); nul >= 0; nul = pending.indexOf(0)) {
        const row = pending.subarray(0, nul).toString('utf8');
        pending = pending.subarray(nul + 1);
        const tab = row.indexOf('\t');
        const [mode, kind, oid, bytes] = row.slice(0, tab).split(/\s+/);
        const path = row.slice(tab + 1);
        if (kind !== 'blob' || floor && Buffer.compare(Buffer.from(path, 'utf8'), floor) <= 0 && !revisit?.has(path)) continue;
        batch.push({ path, oid: oid!, size: Number(bytes), mode: mode! });
        if (batch.length >= size) { yield batch; batch = []; }
      }
    }
    if (batch.length) yield batch;
    const code = await exit;
    if (code !== 0) throw new Error(`git ls-tree failed in ${gitRoot} (exit ${code}): ${stderr.trim().slice(0, 300)}`);
  } finally {
    if (child.exitCode === null) child.kill();
  }
}

const eligibilityKey = (scope: string, slugMode: string, exclude: string[]) => createHash('sha256').update(JSON.stringify([scope, slugMode, exclude])).digest('hex').slice(0, 16);

/** Files of one managed Git source with no live page, no hold and no scheduled re-screen; bounded by `opts`. */
export async function findUnindexedFiles(engine: BrainEngine, sourceId: string, opts: UnindexedScanOpts = {}): Promise<UnindexedScan> {
  const context = opts.context ?? await resolveManagedSyncContext(engine, { sourceId, noPull: true } as SyncOpts);
  const { gitRoot, incarnation } = context;
  const now = opts.now ?? Date.now;
  const keep = opts.keep ?? UNINDEXED_SCHEDULE_CAP;
  const screenLimit = opts.screenLimit ?? UNINDEXED_SCREEN_LIMIT;
  const { scope, slugMode, eligible, sourcePath, metafileHeads } = await loadSyncEligibility(engine, context);
  const exclude = await syncExcludePatterns(engine);
  const scan: UnindexedScan = { source_id: sourceId, incarnation, target: context.source.last_commit, eligibility: eligibilityKey(scope, slugMode, exclude), complete: true, cursor: null, scanned: 0, unindexed: 0,
    slug_taken: 0, deleted_page: 0, scheduled: 0, by_code: {}, samples: [], metafile_skipped: 0, metafile_samples: [], files: [], scheduled_paths: [] };
  if (!scan.target) return scan;
  const scaffolding = (path: string) => isSyncMetafile(path) && !matchesAnyGlob(scope ? path.slice(scope.length + 1) : path, exclude)
    && !isReservedSkillBundlePath(sourcePath(path)) && eligible(`${path.slice(0, path.lastIndexOf('/') + 1)}gbrain-metafile-probe${path.slice(path.lastIndexOf('.'))}`);
  const retry = new Set(await readGitHoldRetryPaths(engine, sourceId, incarnation));
  const pastDeadline = () => opts.deadlineAt !== undefined && now() >= opts.deadlineAt;
  let screened = 0;
  const floor = opts.cursor ? Buffer.from(opts.cursor, 'utf8') : null;
  for await (const batch of treeBatches(gitRoot, scan.target, scope, opts.cursor ?? null, opts.batch ?? BATCH, opts.revisit)) {
    metafileHeads.prime(batch);
    const rows = batch.filter(entry => eligible(entry.path) && !isWindowsColonTarget(entry.path)).map(entry => {
      const root = scope ? entry.path.slice(scope.length + 1) : entry.path;
      return { ...entry, root, origin: sourcePath(entry.path), legacy: slugMode === 'source-root' && scope ? entry.path : null };
    });
    scan.scanned += rows.length;
    const metafiles = batch.filter(entry => !eligible(entry.path) && scaffolding(entry.path));
    if (metafiles.length) {
      const indexed = new Set((await engine.executeRaw<{ source_path: string }>('SELECT source_path FROM pages WHERE source_id=$1 AND deleted_at IS NULL AND source_path = ANY($2::text[])',
        [sourceId, metafiles.map(entry => sourcePath(entry.path))])).map(page => page.source_path));
      const unread = metafiles.filter(entry => !indexed.has(sourcePath(entry.path))).slice(0, Math.max(0, screenLimit - screened));
      screened += unread.length;
      visitBlobSlices(gitRoot, unread, (entry, content) => {
        if (!metafileIsPage(scope ? entry.path.slice(scope.length + 1) : entry.path, content, 'typed')) return;
        scan.metafile_skipped++;
        if (scan.metafile_samples.length < UNINDEXED_SAMPLE_LIMIT) scan.metafile_samples.push(scope ? entry.path.slice(scope.length + 1) : entry.path);
      }, () => false);
    }
    if (rows.length) {
      const origins = rows.flatMap(row => row.legacy ? [row.origin, row.legacy] : [row.origin]);
      const pages = await engine.executeRaw<{ source_path: string; slug: string }>(
        'SELECT source_path, slug FROM pages WHERE source_id=$1 AND deleted_at IS NULL AND source_path = ANY($2::text[])', [sourceId, origins]);
      const byOrigin = new Map(pages.map(page => [page.source_path, page.slug]));
      const held = await heldGitPaths(engine, sourceId, incarnation, rows.map(row => row.root));
      // GBRA-542: a committed symlink whose target does not exist is skipped by sync on every pass, so it is not unindexed either.
      const missing = rows.filter(row => !byOrigin.has(row.origin) && !(row.legacy && byOrigin.get(row.legacy) === resolveSlugForPath(row.root)) && !held.has(row.root)
        && !(row.mode === SYMLINK_MODE && danglingLink(gitRoot, row.path)));
      const fresh = missing.filter(row => {
        if (!retry.has(row.root)) return true;
        scan.scheduled++;
        if (scan.scheduled_paths.length < keep) scan.scheduled_paths.push(row.root);
        return false;
      });
      const slugs = fresh.map(row => resolveSlugForPath(row.origin)).filter(Boolean);
      const taken = new Set(slugs.length ? (await engine.executeRaw<{ slug: string }>(
        'SELECT slug FROM pages WHERE source_id=$1 AND deleted_at IS NULL AND slug = ANY($2::text[])', [sourceId, slugs])).map(page => page.slug) : []);
      const deleted = new Set(fresh.length ? (await engine.executeRaw<{ source_path: string }>(
        'SELECT source_path FROM pages WHERE source_id=$1 AND deleted_at IS NOT NULL AND source_path = ANY($2::text[])', [sourceId, fresh.map(row => row.origin)])).map(page => page.source_path) : []);
      // #7149: a committed symlink's blob is its target path, not page content; it is named as one instead of screened.
      const codes = new Map<string, string>(fresh.filter(row => row.mode === SYMLINK_MODE).map(row => [row.root, 'symlink']));
      const toScreen = fresh.filter(row => row.mode !== SYMLINK_MODE).slice(0, Math.max(0, screenLimit - screened));
      screened += toScreen.length;
      visitBlobSlices(gitRoot, toScreen, (row, content) => {
        if (content === undefined) { codes.set(row.root, 'file_too_large'); return; }
        const result = screenImportContent({ content, path: row.origin, byteLength: row.size, expectedSlug: resolveSlugForPath(row.origin) || null, holdNoncanonicalPrivate: true });
        codes.set(row.root, result.status === 'refused' ? result.refusal.code : 'clean');
      }, pastDeadline);
      for (const row of fresh) {
        const code = codes.get(row.root) ?? 'unscreened', slugTaken = taken.has(resolveSlugForPath(row.origin)), deletedPage = deleted.has(row.origin);
        scan.unindexed++;
        if (slugTaken) scan.slug_taken++;
        if (deletedPage) scan.deleted_page++;
        scan.by_code[code] = (scan.by_code[code] ?? 0) + 1;
        if (scan.samples.length < UNINDEXED_SAMPLE_LIMIT) scan.samples.push({ path: row.root, code, ...(slugTaken ? { slug_taken: true as const } : {}), ...(deletedPage ? { deleted_page: true as const } : {}) });
        if (scan.files.length < keep) scan.files.push({ path: row.root, source_path: row.origin, code, slug_taken: slugTaken, deleted_page: deletedPage });
      }
    }
    if (pastDeadline() || (opts.maxScanned !== undefined && scan.scanned >= opts.maxScanned)) {
      scan.complete = false;
      const last = batch[batch.length - 1]!.path;
      scan.cursor = floor && Buffer.compare(Buffer.from(last, 'utf8'), floor) <= 0 ? opts.cursor! : last;
      break;
    }
  }
  return scan;
}

/** One pass's time budget for a source: `UNINDEXED_SCAN_BUDGET_MS` per started `UNINDEXED_BUDGET_PAGES` live pages, at most `UNINDEXED_SCAN_MAX_BUDGET_MS`. */
export async function unindexedScanBudgetMs(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string): Promise<number> {
  const [row] = await engine.executeRaw<{ n: number | string }>('SELECT count(*) AS n FROM pages WHERE source_id=$1 AND deleted_at IS NULL', [sourceId]);
  return unindexedBudgetForPages(Number(row?.n ?? 0));
}

export function unindexedBudgetForPages(pages: number): number {
  return Math.min(UNINDEXED_SCAN_MAX_BUDGET_MS, UNINDEXED_SCAN_BUDGET_MS * Math.max(1, Math.ceil(pages / UNINDEXED_BUDGET_PAGES)));
}

/** Git paths at or before `cursor` that differ between `from` and `to`; null when Git cannot list them or they number more than `UNINDEXED_REVISIT_CAP`. */
function changedBefore(gitRoot: string, from: string, to: string, scope: string, cursor: string): Set<string> | null {
  let listed: string;
  try { listed = syncGit(gitRoot, ['diff-tree', '-r', '-z', '--no-renames', '--name-only', from, to, ...(scope ? ['--', ...literalPathspecs([scope])] : [])]); } catch { return null; }
  const floor = Buffer.from(cursor, 'utf8');
  const paths = new Set(listed.split('\0').filter(path => path && Buffer.compare(Buffer.from(path, 'utf8'), floor) <= 0));
  return paths.size > UNINDEXED_REVISIT_CAP ? null : paths;
}

/** The stored results of these sources (sources with none are absent). */
export async function readUnindexedFiles(engine: Pick<BrainEngine, 'executeRaw'>, sourceIds: string[]): Promise<Map<string, UnindexedRecord>> {
  if (!sourceIds.length) return new Map();
  const rows = await engine.executeRaw<{ record: UnindexedRecord }>('SELECT completed_keys->0 AS record FROM op_checkpoints WHERE op=$1 AND fingerprint=ANY($2::text[])',
    [UNINDEXED_FILES_OP, sourceIds]);
  return new Map(rows.map(row => [row.record.source_id, row.record]));
}

const addCounts = (a: Record<string, number>, b: Record<string, number>) => {
  const out = { ...a };
  for (const [code, n] of Object.entries(b)) out[code] = (out[code] ?? 0) + n;
  return out;
};

export interface UnindexedRefresh { scan: UnindexedScan; record: UnindexedRecord; scheduled: string[] }

/**
 * One bounded pass for one source: continues an incomplete stored result of the same incarnation (accumulating its counts),
 * schedules the found paths for a re-screen unless `schedule` is false, and stores the merged result.
 */
export async function refreshUnindexedFiles(engine: BrainEngine, sourceId: string, opts: Omit<UnindexedScanOpts, 'cursor'> & { schedule?: boolean } = {}): Promise<UnindexedRefresh> {
  const context = opts.context ?? await resolveManagedSyncContext(engine, { sourceId, noPull: true } as SyncOpts);
  const now = opts.now ?? Date.now;
  const stored = (await readUnindexedFiles(engine, [sourceId])).get(sourceId);
  const eligibility = await loadSyncEligibility(engine, context);
  const key = eligibilityKey(eligibility.scope, eligibility.slugMode, await syncExcludePatterns(engine));
  const resumable = stored && !stored.complete && stored.incarnation === context.incarnation && stored.cursor && stored.target && context.source.last_commit
    && stored.eligibility === key ? stored : null;
  const revisit = resumable && resumable.target !== context.source.last_commit
    ? changedBefore(context.gitRoot, resumable.target!, context.source.last_commit!, eligibility.scope, resumable.cursor!) : new Set<string>();
  const previous = revisit ? resumable : null;
  const scan = await findUnindexedFiles(engine, sourceId, { ...opts, context, cursor: previous?.cursor ?? null, ...(previous && revisit?.size ? { revisit } : {}) });
  const scheduled = opts.schedule === false ? [] : scan.files.filter(file => !file.slug_taken && !file.deleted_page).map(file => file.path);
  if (scheduled.length) await requestGitHoldRetry(engine, sourceId, context.incarnation, scheduled);
  // Paths the continuation re-examined drop their earlier verdict: one still missing is found again, one deleted or imported is not.
  const revisited = new Set([...(previous && revisit ? revisit : [])].map(path => eligibility.scope ? path.slice(eligibility.scope.length + 1) : path));
  const pending = [...new Set([...(previous?.pending ?? []).filter(path => !revisited.has(path)), ...scan.scheduled_paths, ...scheduled])];
  const { scope, slugMode } = eligibility;
  const at = new Date(now()).toISOString();
  const record: UnindexedRecord = {
    version: 1, source_id: sourceId, incarnation: context.incarnation, target: scan.target, eligibility: scan.eligibility, complete: scan.complete, cursor: scan.cursor,
    started_at: previous?.started_at ?? at, computed_at: at,
    scanned: (previous?.scanned ?? 0) + scan.scanned, unindexed: (previous?.unindexed ?? 0) + scan.unindexed, slug_taken: (previous?.slug_taken ?? 0) + scan.slug_taken,
    deleted_page: (previous?.deleted_page ?? 0) + scan.deleted_page,
    scheduled: (previous?.scheduled ?? 0) + scan.scheduled, by_code: addCounts(previous?.by_code ?? {}, scan.by_code),
    samples: [...(previous?.samples ?? []), ...scan.samples].slice(0, UNINDEXED_SAMPLE_LIMIT),
    metafile_skipped: (previous?.metafile_skipped ?? 0) + scan.metafile_skipped,
    metafile_samples: [...(previous?.metafile_samples ?? []), ...scan.metafile_samples].slice(0, UNINDEXED_SAMPLE_LIMIT),
    pending: pending.slice(0, UNINDEXED_SCHEDULE_CAP),
    pending_truncated: (previous?.pending_truncated ?? false) || pending.length > UNINDEXED_SCHEDULE_CAP || scan.scheduled_paths.length < scan.scheduled,
    scheduled_now: (previous?.scheduled_now ?? 0) + scheduled.length,
    source_prefix: slugMode === 'git-root' && scope ? `${scope}/` : '',
  };
  await engine.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys) VALUES($1,$2,jsonb_build_array($3::text::jsonb))
    ON CONFLICT(op,fingerprint) DO UPDATE SET completed_keys=EXCLUDED.completed_keys,updated_at=now()`, [UNINDEXED_FILES_OP, sourceId, JSON.stringify(record)]);
  return { scan, record, scheduled };
}

/** A stored result as doctor judges it now. */
export interface UnindexedState {
  source_id: string;
  record: UnindexedRecord | null;
  /** `missing`: no result for the current incarnation; `partial`: nothing found so far but the scan is incomplete. */
  state: 'ok' | 'unindexed' | 'scheduled' | 'partial' | 'stale' | 'missing';
  /** Found files no pass scheduled that a re-screen could import (not `slug_taken`, not `deleted_page`). */
  unscheduled: number;
  /** Found files a re-screen cannot import (`slug_taken` + `deleted_page`): a person renames, deletes or restores; never scheduled. */
  blocked: number;
  /** Scheduled paths the next sync has not re-screened yet and that still have no page and no hold. */
  waiting: number;
  /** Scheduled paths a sync re-screened that still have no page and no hold. */
  still_unindexed: number;
  age_ms: number | null;
}

/**
 * Judges each source's stored result without scanning: one page read plus one hold read over the scheduled paths says
 * whether each was imported or held, and of the rest its retry row says which still wait for a sync. A scheduled path
 * with a page or a hold is settled even while the retry row still names it (#7149: a sync pass that stops before its
 * stamp leaves the row behind after importing the path).
 */
export async function judgeUnindexedFiles(engine: Pick<BrainEngine, 'executeRaw'>, sources: Array<{ id: string; incarnation: string }>, now = Date.now()): Promise<UnindexedState[]> {
  const records = await readUnindexedFiles(engine, sources.map(source => source.id));
  const out: UnindexedState[] = [];
  for (const source of sources) {
    const record = records.get(source.id) ?? null;
    const base = { source_id: source.id, record, unscheduled: 0, blocked: 0, waiting: 0, still_unindexed: 0, age_ms: record ? Math.max(0, now - Date.parse(record.computed_at)) : null };
    if (!record || record.incarnation !== source.incarnation) { out.push({ ...base, state: 'missing' }); continue; }
    if (base.age_ms! > UNINDEXED_STALE_MS) { out.push({ ...base, state: 'stale' }); continue; }
    const blocked = (record.slug_taken ?? 0) + (record.deleted_page ?? 0);
    const unscheduled = Math.max(0, record.unindexed - blocked - record.scheduled_now);
    const retry = new Set(await readGitHoldRetryPaths(engine, source.id, source.incarnation));
    let open = record.pending;
    if (open.length) {
      const indexed = new Set((await engine.executeRaw<{ source_path: string }>(
        'SELECT source_path FROM pages WHERE source_id=$1 AND deleted_at IS NULL AND source_path = ANY($2::text[])',
        [source.id, open.map(path => `${record.source_prefix}${path}`)])).map(page => page.source_path));
      const held = await heldGitPaths(engine, source.id, source.incarnation, open);
      open = open.filter(path => !indexed.has(`${record.source_prefix}${path}`) && !held.has(path));
    }
    const waiting = open.filter(path => retry.has(path)).length;
    const stillUnindexed = open.length - waiting;
    const state = unscheduled || stillUnindexed ? 'unindexed' : waiting ? 'scheduled' : record.complete && !record.pending_truncated ? 'ok' : 'partial';
    out.push({ ...base, state, unscheduled, blocked, waiting, still_unindexed: stillUnindexed });
  }
  return out;
}