Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
7.1 KB
Raw Blame
/**
 * `gbrain pages unsearchable --source <id> [--state <state>] [--limit N] [--json]`
 * (#6696 B3): read-only listing of what search cannot fully see in one source,
 * one row per (page, embedding state) with its reason, chunk count, bytes and
 * the state's protocol fix. States and their exits: src/core/embedding-state.ts.
 * Without `--state` every state but `embedded` is listed. The listing is
 * bounded (`--limit`, default 50); `complete: false` means more rows exist.
 *
 * #7193: `unsearchable` keeps its doctor meaning; `semantic_unsearchable` adds
 * the `keyword_only` units vector search cannot reach, and the headline leads
 * with it. `awaiting_sync` counts files no state can see yet: the source's
 * held files (new files with no page, pages whose newer file is held) and its
 * sync freshness, when a resident reports it.
 */
import type { BrainEngine } from '../core/engine.ts';
import { cliRenderContext, renderAction } from '../core/agent-output.ts';
import { opError } from '../core/ops/contract.ts';
import { resolveSourceId } from '../core/source-resolver.ts';
import { embeddingsDisabled } from '../core/embedding-disabled.ts';
import { readHeldCoverage } from '../core/persistence/held-reads.ts';
import { readSourceFreshness } from '../core/notifier/status.ts';
import {
  DELETED_PAGE_PURGE_WINDOW_HOURS, EMBEDDING_STATE_DOCS, EMBEDDING_STATES, embeddingStateFix, readEmbeddingState,
  type EmbeddingState, type EmbeddingStateRow,
} from '../core/embedding-state.ts';

const DEFAULT_LIMIT = 50;
const MAX_LIMIT = 1000;

export const UNSEARCHABLE_USAGE = 'Usage: gbrain pages unsearchable --source <id> [--state <state>] [--limit N] [--json]';

export interface UnsearchableArgs { source: string | null; state: EmbeddingState | null; limit: number; json: boolean }

function flagValue(args: readonly string[], name: string): string | undefined {
  const i = args.findIndex(a => a === name || a.startsWith(`${name}=`));
  if (i < 0) return undefined;
  return args[i]!.startsWith(`${name}=`) ? args[i]!.slice(name.length + 1) : args[i + 1];
}

/** `--state` must name a state and `--limit` a whole number from 1 to 1000; anything else is `invalid_params`. */
export function parseUnsearchableArgs(args: readonly string[]): UnsearchableArgs {
  const state = flagValue(args, '--state');
  if (state !== undefined && !(EMBEDDING_STATES as readonly string[]).includes(state)) {
    throw opError('invalid_params', `Unknown --state "${state}". Valid states: ${EMBEDDING_STATES.join(', ')}.`,
      `Pass one of: ${EMBEDDING_STATES.join(', ')}.`, { docs: EMBEDDING_STATE_DOCS });
  }
  const limitRaw = flagValue(args, '--limit');
  const limit = limitRaw === undefined ? DEFAULT_LIMIT : Number(limitRaw);
  if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) {
    throw opError('invalid_params', `--limit must be a whole number from 1 to ${MAX_LIMIT} (got "${limitRaw}").`, UNSEARCHABLE_USAGE);
  }
  const source = flagValue(args, '--source');
  if (source !== undefined && (source === '' || source.startsWith('--'))) throw opError('invalid_params', '--source needs a source id.', UNSEARCHABLE_USAGE);
  return { source: source ?? null, state: (state as EmbeddingState | undefined) ?? null, limit, json: args.includes('--json') };
}

function reasonFor(row: EmbeddingStateRow, embeddingsEnabled: boolean): string {
  switch (row.state) {
    case 'embedded': return 'vector_present';
    case 'pending': return embeddingsEnabled ? 'no_vector' : 'embeddings_disabled';
    case 'keyword_only': return row.embed_skip_reason ?? 'embed_skip';
    case 'quarantined': return row.quarantine_reason ?? 'quarantine';
    case 'empty': return 'empty_body';
    case 'deleted_page_chunks': return row.overdue ? 'past_purge_window' : 'within_purge_window';
    case 'chunkless_healable': return 'no_chunk_rows';
  }
}

export async function runPagesUnsearchable(engine: BrainEngine, args: string[]): Promise<void> {
  const parsed = parseUnsearchableArgs(args);
  const sourceId = await resolveSourceId(engine, parsed.source);
  const states = parsed.state ? [parsed.state] : EMBEDDING_STATES.filter(state => state !== 'embedded');
  const embeddingsEnabled = !(await embeddingsDisabled(engine));
  const found = await engine.listEmbeddingStateRows({ sourceIds: [sourceId], states, limit: parsed.limit + 1, purgeWindowHours: DELETED_PAGE_PURGE_WINDOW_HOURS });
  const render = cliRenderContext();
  const summary = await readEmbeddingState(engine, { mode: 'full', sourceIds: [sourceId] });
  const blockBytes = (summary.keyword_only as { threshold_bytes?: number } | undefined)?.threshold_bytes;
  const rows = found.slice(0, parsed.limit).map(row => {
    const fix = embeddingStateFix(row.state, { sourceId, slug: row.slug, markerBytes: row.marker_bytes, blockBytes, embeddingsEnabled });
    return {
      slug: row.slug, source_id: row.source_id, state: row.state, reason: reasonFor(row, embeddingsEnabled), chunks: row.chunks, bytes: row.bytes,
      ...(row.marker_bytes !== null ? { marker_bytes: row.marker_bytes } : {}),
      ...(row.deleted_at ? { deleted_at: row.deleted_at.toISOString() } : {}),
      fix: fix ? renderAction(fix, render) : null,
    };
  });
  const complete = found.length <= parsed.limit;
  const semanticUnsearchable = summary.unsearchable === null ? null : summary.unsearchable + (summary.counts.keyword_only ?? 0);
  const [held] = await readHeldCoverage(engine, { sourceId });
  const [freshness] = await readSourceFreshness(engine, { sourceId }, held ? [held] : []);
  const awaitingSync = { held_files: (held?.missing ?? 0) + (held?.stale ?? 0), new_files: held?.missing ?? 0, stale_pages: held?.stale ?? 0,
    sync_state: freshness?.state ?? 'not_reported' };
  if (parsed.json) {
    console.log(JSON.stringify({ source_id: sourceId, state: parsed.state, limit: parsed.limit, complete, active_column: summary.active_column,
      counts: summary.counts, unsearchable: summary.unsearchable, semantic_unsearchable: semanticUnsearchable, awaiting_sync: awaitingSync, rows }, null, 2));
    return;
  }
  console.log(`Source ${sourceId} (active column '${summary.active_column}'): ${semanticUnsearchable} not reachable by vector search (${summary.unsearchable} unsearchable + `
    + `${summary.counts.keyword_only ?? 0} keyword_only); ${EMBEDDING_STATES.map(s => `${s} ${summary.counts[s] ?? 0}`).join(', ')}.`);
  if (awaitingSync.held_files > 0 || awaitingSync.sync_state === 'behind') {
    console.log(`Not yet in any state: ${awaitingSync.held_files} file(s) held by sync (${awaitingSync.new_files} new, ${awaitingSync.stale_pages} newer than their page); sync ${awaitingSync.sync_state}. `
      + `gbrain sync status --source ${sourceId} --json shows why.`);
  }
  if (rows.length === 0) { console.log(parsed.state ? `No pages in state ${parsed.state}.` : 'Nothing unsearchable.'); return; }
  for (const row of rows) {
    console.log(`  ${row.state.padEnd(20)} ${row.slug}  chunks=${row.chunks} bytes=${row.bytes}  ${row.reason}`);
    if (row.fix?.command) console.log(`      fix (${row.fix.next.replace(/_/g, ' ')}): ${row.fix.command}`);
  }
  if (!complete) console.log(`(first ${parsed.limit} rows; raise --limit or filter with --state to see more)`);
}