Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.7 KB
Raw Blame
/**
 * `unindexed_files` cycle phase (#6865): one bounded pass of the unindexed-file
 * detector (src/core/persistence/unindexed-files.ts) over every managed Git
 * source this host owns, in the global maintenance lane right after
 * `content_repair`. It finds committed files with no page, no hold and no
 * scheduled re-screen (typically files an older gbrain refused before
 * refusals became holds), stores the result where doctor `unindexed_files`
 * reads it, and schedules the found paths for a re-screen on the next sync
 * (at most `UNINDEXED_SCHEDULE_CAP` per run), so each is imported or recorded
 * as a hold. Scheduling is the only write: no file or page changes here, and
 * the per-sync path never runs it.
 *
 * Deadline: min(5 min, a third of the maintenance job's remaining time),
 * shared by the sources, each of which also stops at its own budget (60 s per
 * started 100k live pages, `unindexedScanBudgetMs`). #7149: the sources run
 * least recently scanned first (no stored result first, then the oldest), so
 * one large source cannot keep the others from ever being scanned; a source
 * the budget stops is continued by the next run from its stored cursor. A dry run scans and reports without
 * scheduling or storing. The report is counts only, never a path. It never
 * fails the cycle: an error on a source reports `warn` and the others still run.
 */
import type { BrainEngine } from '../engine.ts';
import type { PhaseResult } from '../cycle.ts';
import type { SyncOpts } from '../../commands/sync.ts';
import { managedPersistenceEnabled } from '../persistence/ownership.ts';
import { resolveManagedSyncContext } from '../persistence/sync-discovery.ts';
import { companyBrainProfile } from '../company-brain/policy.ts';
import { findUnindexedFiles, readUnindexedFiles, refreshUnindexedFiles, unindexedScanBudgetMs, UNINDEXED_SCAN_MAX_BUDGET_MS, UNINDEXED_SCHEDULE_CAP, type UnindexedScan } from '../persistence/unindexed-files.ts';
import { isUncontainedPhaseError } from './phase-containment.ts';

export interface UnindexedFilesPhaseOpts {
  dryRun: boolean;
  signal?: AbortSignal;
  /** The enclosing maintenance job's absolute deadline (epoch ms); null or unset for a direct `gbrain dream`. */
  deadlineAtMs?: number | null;
  now?: () => number;
}

const errorCode = (error: unknown) => { const code = (error as { code?: unknown } | null)?.code; return typeof code === 'string' ? code : 'internal'; };

export async function runUnindexedFilesPhase(engine: BrainEngine | null, opts: UnindexedFilesPhaseOpts): Promise<PhaseResult> {
  const base = { phase: 'unindexed_files' as const, duration_ms: 0 };
  if (!engine) return { ...base, status: 'skipped', summary: 'no database connected', details: { reason: 'no_database' } };
  if (!(await managedPersistenceEnabled(engine))) {
    return { ...base, status: 'skipped', summary: 'managed persistence is off, so there is no managed Git source to check for unindexed files', details: { reason: 'not_managed' } };
  }
  const now = opts.now ?? Date.now;
  const start = now();
  const timeBudgetMs = opts.deadlineAtMs == null ? UNINDEXED_SCAN_MAX_BUDGET_MS : Math.min(UNINDEXED_SCAN_MAX_BUDGET_MS, Math.floor((opts.deadlineAtMs - start) / 3));
  if (timeBudgetMs <= 0) return { ...base, status: 'skipped', summary: 'no maintenance-job time left to look for unindexed files; the next run continues', details: { reason: 'deadline' } };
  const deadlineAt = start + timeBudgetMs;
  const rows = await engine.executeRaw<{ id: string; config: Record<string, unknown> | null; incarnation: string }>(
    "SELECT id, config, incarnation::text AS incarnation FROM sources WHERE archived IS NOT TRUE AND local_path IS NOT NULL AND (config->>'kind') IS NULL ORDER BY id");
  const stored = await readUnindexedFiles(engine, rows.map(row => row.id));
  const scannedAt = (row: { id: string; incarnation: string }) => { const record = stored.get(row.id); return record?.incarnation === row.incarnation ? record.computed_at : ''; };
  rows.sort((a, b) => scannedAt(a).localeCompare(scannedAt(b)));
  const sources: Array<{ source_id: string; budget_ms: number; complete: boolean; scanned: number; unindexed: number; slug_taken: number; scheduled_now: number; already_scheduled: number;
    metafile_skipped: number; by_code: Record<string, number> }> = [];
  const skipped: Array<{ source_id: string; reason: string }> = [];
  const errors: Array<{ source_id: string; code: string }> = [];
  let room = UNINDEXED_SCHEDULE_CAP;
  for (const row of rows) {
    if (now() >= deadlineAt) { skipped.push({ source_id: row.id, reason: 'deadline' }); continue; }
    try {
      if (companyBrainProfile(row.config ?? {})) { skipped.push({ source_id: row.id, reason: 'company_source' }); continue; }
      const context = await resolveManagedSyncContext(engine, { sourceId: row.id, noPull: true } as SyncOpts).catch((error: unknown) => {
        if (['owner_unavailable', 'source_changed', 'writer_coordinator_required'].includes(errorCode(error))) return errorCode(error);
        throw error;
      });
      if (typeof context === 'string') { skipped.push({ source_id: row.id, reason: context }); continue; }
      let scan: UnindexedScan, scheduledNow = 0;
      const budgetMs = await unindexedScanBudgetMs(engine, row.id);
      const sourceDeadlineAt = Math.min(deadlineAt, now() + budgetMs);
      if (opts.dryRun) scan = await findUnindexedFiles(engine, row.id, { context, deadlineAt: sourceDeadlineAt, now });
      else {
        const refresh = await refreshUnindexedFiles(engine, row.id, { context, deadlineAt: sourceDeadlineAt, now, keep: room });
        scan = refresh.scan; scheduledNow = refresh.scheduled.length; room -= scheduledNow;
      }
      sources.push({ source_id: row.id, budget_ms: budgetMs, complete: scan.complete, scanned: scan.scanned, unindexed: scan.unindexed, slug_taken: scan.slug_taken,
        scheduled_now: scheduledNow, already_scheduled: scan.scheduled, metafile_skipped: scan.metafile_skipped, by_code: scan.by_code });
    } catch (error) {
      if (isUncontainedPhaseError(error, opts.signal)) throw error;
      errors.push({ source_id: row.id, code: errorCode(error) });
    }
  }
  const total = (key: 'scanned' | 'unindexed' | 'slug_taken' | 'scheduled_now' | 'already_scheduled' | 'metafile_skipped') => sources.reduce((n, source) => n + source[key], 0);
  const byCode: Record<string, number> = {};
  for (const source of sources) for (const [code, n] of Object.entries(source.by_code)) byCode[code] = (byCode[code] ?? 0) + n;
  const incomplete = sources.filter(source => !source.complete).length;
  const codes = Object.entries(byCode).map(([code, n]) => `${code} ${n}`).join(', ');
  const parts = [
    `${opts.dryRun ? 'dry run: ' : ''}checked ${total('scanned')} committed file(s) in ${sources.length} managed Git source(s)`,
    total('unindexed') ? `${total('unindexed')} had no page and no hold (${codes}${total('slug_taken') ? `; ${total('slug_taken')} share a slug with another page` : ''})` : 'none was missing from the index',
    opts.dryRun ? (total('unindexed') ? 'nothing was scheduled' : '') : total('scheduled_now') ? `${total('scheduled_now')} scheduled for a re-screen on the next sync` : '',
    total('already_scheduled') ? `${total('already_scheduled')} already wait for a re-screen` : '',
    total('metafile_skipped') ? `${total('metafile_skipped')} titled file(s) are named like folder scaffolding (index.md and the like), which sync skips on purpose; renaming one is the user's call` : '',
  ].filter(Boolean);
  const notes = [
    incomplete ? `${incomplete} source(s) stopped at their time budget; the next maintenance run continues where they stopped.` : '',
    skipped.length ? `${skipped.length} source(s) skipped (${[...new Set(skipped.map(s => s.reason))].join(', ')}).` : '',
    errors.length ? `${errors.length} source(s) could not be checked (${[...new Set(errors.map(e => e.code))].join(', ')}); gbrain doctor --only unindexed_files --json shows each source's state.` : '',
  ].filter(Boolean);
  return {
    ...base,
    status: errors.length ? 'warn' : 'ok',
    summary: [`${parts.join('; ')}.`, ...notes].join(' '),
    details: {
      mode: opts.dryRun ? 'dry_run' : 'apply', sources: sources.map(({ by_code: _codes, ...source }) => source), scanned: total('scanned'), unindexed: total('unindexed'),
      slug_taken: total('slug_taken'), scheduled_now: total('scheduled_now'), already_scheduled: total('already_scheduled'), metafile_skipped: total('metafile_skipped'), by_code: byCode,
      complete: incomplete === 0 && errors.length === 0, time_budget_ms: timeBudgetMs, skipped, errors,
      ...(errors.length ? { fix: { argv: ['gbrain', 'doctor', '--only', 'unindexed_files', '--json'], consent: [], actor: 'agent', requires_exclusive: false,
        why: 'Shows each managed Git source\'s unindexed-file state and the next step; read-only.' } } : {}),
    },
  };
}