Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
16.8 KB
Raw Blame
/**
 * `gbrain sources status` view of each managed connector source's state row:
 * upgrade recovery, account pin continuity, pending receipts and the last
 * run's counts. Never includes the pinned account or installation id.
 *
 * #5988: also each Git source's held files (`readGitHoldStatuses`, limited
 * in SQL to `GBRAIN_SYNC_HOLD_CAP`) and the blocked sync requests recent runs
 * converted in place, shown beside connector holds. Only `sources status` and `sources retry-held` merge
 * the two; `readAllSourceHolds`, `gbrain waiting` and doctor
 * `connector_held_items` stay connector-only.
 */
import type { BrainEngine } from '../engine.ts';
import { CONNECTOR_KINDS_SQL } from './connector-kinds.ts';
import { opError } from '../ops/contract.ts';
import { readFix } from '../ops/op-fix.ts';
import { CONNECTOR_STOPPED_RUNS, readManagedConnectorState, type ConnectorPurgedItem, type ConnectorRunCounts, type ConnectorRunFailure, type UpgradeRecovery } from './connector-state.ts';
import { classifySyncFault, type SyncFaultVerdict } from './sync-fault-class.ts';
import { cliRenderContext, renderAction, type Action } from '../agent-output.ts';
import { managedBrain, readAllSourceHolds } from '../connectors/item-holds-store.ts';
import type { ItemHoldRecord } from '../connectors/item-holds.ts';
import { fenceAutoRepairFor, gitHoldItem, readGitHoldCodeCounts, readGitHoldListing, readSyncConversions, readSyncHoldPolicy, type GitHoldItem, type SyncConversion } from './sync-holds.ts';
import { holdLine } from '../../commands/sync-diagnostics.ts';
import { pageSortedRows, type ListingRequest } from '../bounded-listing.ts';

/** Fix wave 4: `sources status` lists this many held items per source, then "+N more". */
export const HELD_STATUS_LIMIT = 10;

export interface ConnectorSourceStatus {
  upgrade_recovery: UpgradeRecovery;
  resumed_from: string | null;
  account_pinned: boolean;
  continuity_unverified: boolean;
  pending: number;
  last_run: ConnectorRunCounts | null;
  /** Fix wave 4: every held item (the human view shows HELD_STATUS_LIMIT). */
  held: ItemHoldRecord[];
  /** Set when this source's hold state could not be read: coverage is unknown, not hold-free. */
  held_error?: string;
  /** #6831: recent items skipped because their content matches a page purge tombstone (newest first). */
  purged_items: ConnectorPurgedItem[];
  /** #6831: the error the source's connector runs keep ending on, or null when the last run finished. */
  last_failure: ConnectorRunFailure | null;
  /** #6831: `last_failure.consecutive` reached CONNECTOR_STOPPED_RUNS: nothing from this source is being imported. */
  stopped: boolean;
}

/** #6831: `sync status`'s view of a connector source's recorded run failure (`connector_last_error`). */
export interface ConnectorLastError extends ConnectorRunFailure, SyncFaultVerdict {
  stopped: boolean;
  fix: Action;
}

/** #6831: the read-only next step for a recorded connector run failure: its receipt when it names one, else the source's status. */
function connectorFailureFix(sourceId: string, failure: ConnectorRunFailure): Action {
  return failure.request_id
    ? { argv: ['gbrain', 'write-request', '--', failure.request_id], consent: [], actor: 'agent', requires_exclusive: false,
      why: `Read-only: the receipt of ${failure.slug ?? 'the item'} the last connector run of ${sourceId} ended on, with its error and fix.` }
    : { argv: ['gbrain', 'sources', 'status', sourceId, '--json'], consent: [], actor: 'agent', requires_exclusive: false,
      why: `Read-only: the connector state of ${sourceId}, including the error its last run ended on.` };
}

/** #6831: the connector run failure recorded for a source (null for a non-connector source or a last run that finished). */
export async function readConnectorLastError(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string): Promise<ConnectorLastError | null> {
  const [source] = await engine.executeRaw<{ incarnation: string }>(
    `SELECT incarnation::text FROM sources WHERE id=$1 AND NOT archived AND config->>'kind' IN (${CONNECTOR_KINDS_SQL})`, [sourceId]);
  const failure = source ? (await readManagedConnectorState(engine, sourceId, source.incarnation)).last_failure : null;
  if (!failure) return null;
  return { ...failure, stopped: failure.consecutive >= CONNECTOR_STOPPED_RUNS, ...classifySyncFault({ code: failure.code, message: failure.message }),
    fix: connectorFailureFix(sourceId, failure) };
}

export async function readConnectorSourceStatuses(engine: Pick<BrainEngine, 'executeRaw'>, sourceIds?: string[]): Promise<Map<string, ConnectorSourceStatus>> {
  const sources = await engine.executeRaw<{ id: string; incarnation: string }>(
    `SELECT id,incarnation::text FROM sources WHERE NOT archived AND config->>'kind' IN (${CONNECTOR_KINDS_SQL}) AND ($1::text[] IS NULL OR id=ANY($1::text[]))`,
    [sourceIds ?? null]);
  const statuses = new Map<string, ConnectorSourceStatus>();
  for (const source of sources) {
    const state = await readManagedConnectorState(engine, source.id, source.incarnation);
    let held: ItemHoldRecord[] = [];
    let heldError: string | undefined;
    try {
      held = (await readAllSourceHolds(engine, { sourceIds: [source.id] }))[0]?.held ?? [];
    } catch (error) {
      heldError = (error instanceof Error ? error.message : String(error)).split('\n')[0].slice(0, 160) || 'unknown error';
    }
    statuses.set(source.id, { upgrade_recovery: state.upgrade_recovery, resumed_from: state.resumed_from, account_pinned: state.account !== null,
      continuity_unverified: state.continuity_unverified, pending: state.pending.length, last_run: state.last_run, held,
      ...(heldError ? { held_error: heldError } : {}), purged_items: state.purged_items ?? [], last_failure: state.last_failure ?? null,
      stopped: (state.last_failure?.consecutive ?? 0) >= CONNECTOR_STOPPED_RUNS });
  }
  return statuses;
}

export function connectorStatusLines(sourceId: string, status: ConnectorSourceStatus): string[] {
  const lines: string[] = [];
  const run = status.last_run;
  lines.push(`  ${sourceId}: connector ${run ? `last run ${run.finished_at.slice(0, 19).replace('T', ' ')}: ${run.page_admissions} page admission(s), `
    + `${run.skipped_unchanged} skipped unchanged, ${run.pending} pending, ${run.checkpoint_admissions} checkpoint admission(s)`
    + `${run.stopped_on_wait_budget ? ', stopped on wait budget' : ''}${run.dropped_upstream ? `, ${run.dropped_upstream} dropped (deleted upstream)` : ''}`
    + `${run.purged_skipped ? `, ${run.purged_skipped} skipped as purged content` : ''}` : 'has not run since the upgrade'}`);
  const failure = status.last_failure;
  if (failure) {
    const where = failure.slug ? ` on ${failure.slug}${failure.request_id ? ` (request ${failure.request_id})` : ''}` : '';
    lines.push(`    ${status.stopped ? `STOPPED: the last ${failure.consecutive} connector syncs` : `the last ${failure.consecutive === 1 ? 'connector sync' : `${failure.consecutive} connector syncs`}`} ended on ${failure.code}${where} `
      + `at ${failure.at.slice(0, 19).replace('T', ' ')}: ${failure.message}`);
    lines.push(`    nothing after that point of the run was imported; next: ${connectorFailureFix(sourceId, failure).argv!.join(' ')}`);
  }
  if (status.purged_items.length) {
    lines.push(`    ${status.purged_items.length} item(s) recently skipped as purged content (the purge tombstone working; they do not stop the sync): `
      + status.purged_items.slice(0, HELD_STATUS_LIMIT).map(item => `${item.slug} (${day(item.at)})`).join(', ')
      + `${status.purged_items.length > HELD_STATUS_LIMIT ? `, +${status.purged_items.length - HELD_STATUS_LIMIT} more` : ''}`);
    lines.push('    to let one back in, the owner clears its tombstone on the brain host: gbrain pages unpurge <slug>');
  }
  if (status.upgrade_recovery === 'resumed') {
    lines.push(`    resumed from pre-upgrade checkpoint of ${status.resumed_from}; content selection since then is unverified; to re-walk: gbrain sync --source ${sourceId} --reset-checkpoint`);
  } else if (status.upgrade_recovery === 'rewalking_once') {
    lines.push('    re-walking its window once after the upgrade (an expected admission spike, not #5470 churn)');
  }
  if (status.continuity_unverified) lines.push('    account pinned on its first post-upgrade run; continuity before the upgrade is unverified');
  if (status.held_error) lines.push(`    hold state unreadable: ${status.held_error}; coverage is unknown until it can be read`);
  else lines.push(...heldItemLines(sourceId, status.held));
  return lines;
}

const day = (iso: string | null) => (iso ? iso.slice(0, 16).replace('T', ' ') : 'unknown');

/** Held items: up to HELD_STATUS_LIMIT per source, then "+N more", and the retry command once. */
export function heldItemLines(sourceId: string, held: ItemHoldRecord[]): string[] {
  if (!held.length) return [];
  const lines = [`    ${held.length} held item(s) (they do not block freshness; \`gbrain waiting\` reports partial coverage):`];
  for (const record of held.slice(0, HELD_STATUS_LIMIT)) {
    const label = [record.meta.sender, record.meta.subject ?? record.meta.title].filter(Boolean).join(' · ') || 'metadata unknown';
    lines.push(`      ${record.key}  ${label}  ${record.code} (${record.class})  first ${day(record.first_failed_at)}, last ${day(record.last_failed_at)}, `
      + `${record.attempts} attempt(s)${record.class === 'transient' ? `, next ${record.next_attempt_at ? day(record.next_attempt_at) : 'only on retry'}` : ''}`
      + `${record.legacy ? ' [carried from the pre-upgrade poison ledger]' : ''}`);
  }
  if (held.length > HELD_STATUS_LIMIT) lines.push(`      +${held.length - HELD_STATUS_LIMIT} more; --json lists all`);
  lines.push(`    re-attempt them on the next sync: gbrain sources retry-held ${sourceId}`);
  return lines;
}

/** `sources status <id>` naming no registered, unarchived source. */
export function statusSourceNotFound(sourceId: string) {
  return opError('not_found', `Source "${sourceId}" was not found (or is archived).`, 'List the registered sources, then run sources status with one of their ids.',
    { fix: readFix('Lists the registered sources and their ids, read-only.', { argv: ['gbrain', 'sources', 'list', '--json'] }) });
}

/** A Git source's held files: the outstanding count and up to `GBRAIN_SYNC_HOLD_CAP` of them in detail. */
export interface GitHoldStatus {
  count: number;
  items: GitHoldItem[];
  truncated?: true;
  /** GST-2843: holds per code over every hold of a truncated listing; absent when the listing is complete (its items count) or the count read failed. */
  code_counts?: Record<string, number>;
  /** Blocked sync requests a recent run converted in place (held or re-frozen) instead of staying blocked. */
  recent_conversions?: SyncConversion[];
  /** The command that runs the next sync of this source (managed brains sync with --no-pull). */
  sync_argv: string[];
}

/** How many converted requests `sources status` shows per source. */
export const CONVERSION_STATUS_LIMIT = 5;

/** Held files and recent conversions of each listed source that has any (sources with neither are absent). */
export async function readGitHoldStatuses(engine: Pick<BrainEngine, 'executeRaw' | 'getConfig'>, sourceIds: string[]): Promise<Map<string, GitHoldStatus>> {
  const out = new Map<string, GitHoldStatus>();
  if (!sourceIds.length) return out;
  const { cap } = await readSyncHoldPolicy(engine);
  const [sources, conversions] = await Promise.all([readGitHoldListing(engine, sourceIds, cap), readSyncConversions(engine, sourceIds, CONVERSION_STATUS_LIMIT)]);
  if (!sources.length && !conversions.size) return out;
  const managed = await managedBrain(engine);
  const auto = await fenceAutoRepairFor(engine, sources.flatMap(source => source.holds));
  const counts = new Map(sources.map(source => [source.sourceId, source]));
  const truncated = sources.filter(source => source.count > source.holds.length).map(source => source.sourceId);
  const codeCounts = truncated.length ? await readGitHoldCodeCounts(engine, truncated).catch(() => null) : null;
  for (const sourceId of sourceIds) {
    const source = counts.get(sourceId), converted = conversions.get(sourceId);
    if (!source && !converted) continue;
    const items = (source?.holds ?? []).map(hold => gitHoldItem(hold, auto));
    const perCode = codeCounts?.get(sourceId);
    out.set(sourceId, { count: source?.count ?? 0, items, ...((source?.count ?? 0) > items.length ? { truncated: true as const, ...(perCode ? { code_counts: perCode } : {}) } : {}),
      ...(converted ? { recent_conversions: converted } : {}),
      sync_argv: ['gbrain', 'sync', '--source', sourceId, ...(managed ? ['--no-pull'] : [])] });
  }
  return out;
}

/**
 * #7326: what `sources status --json` lists of a source's held files. `summary` (the default): the count, the
 * counts per code with one example each, and the first HELD_STATUS_LIMIT items; `all` (`--holds`): every item read;
 * `page` (`--limit` / `--cursor`): one keyset page of items ordered by path.
 */
export type GitHoldListing = { mode: 'summary' } | { mode: 'all' } | { mode: 'page'; request: ListingRequest };

/**
 * Held items per code, largest first, each with its first item as the example (its fix and docs apply to the rest). GST-2843:
 * `counts` (every hold per code) replaces the listed items' counts, so a truncated listing still counts each code in full; a code
 * none of the listed items has carries no example.
 */
function holdsByCode<T extends { code: string }>(items: readonly T[], counts?: Record<string, number>): Array<{ code: string; count: number; example?: T }> {
  const groups = new Map<string, { code: string; count: number; example?: T }>();
  for (const item of items) {
    const group = groups.get(item.code);
    if (group) group.count++;
    else groups.set(item.code, { code: item.code, count: 1, example: item });
  }
  if (counts) for (const [code, count] of Object.entries(counts)) groups.set(code, { ...groups.get(code), code, count });
  return [...groups.values()].sort((a, b) => b.count - a.count || (a.code < b.code ? -1 : 1));
}

/** A source's `git_holds` for `sources status --json`, summary first and items last, bounded by `listing`; each fix rendered with the CLI routing pin (#9319). */
export function gitHoldStatusJson(sourceId: string, status: GitHoldStatus, listing: GitHoldListing) {
  const ctx = cliRenderContext();
  const { items: held, code_counts: codeCounts, ...rest } = status;
  const items = held.map(item => ({ ...item, fix: renderAction(item.fix, ctx) }));
  const head = { ...rest, by_code: holdsByCode(items, codeCounts), ...(rest.truncated && !codeCounts ? { by_code_complete: false } : {}) };
  if (listing.mode === 'all') return { ...head, items };
  if (listing.mode === 'page') {
    const page = pageSortedRows(items, item => [item.path], listing.request, `sources_status_git_holds:${sourceId}`);
    const next = page.next_cursor === null ? null
      : ['gbrain', 'sources', 'status', sourceId, '--json', '--limit', String(page.limit), '--cursor', page.next_cursor];
    return { ...head, returned: page.returned, limit: page.limit, complete: page.complete, next_cursor: page.next_cursor, next_command: next, items: page.rows };
  }
  if (items.length <= HELD_STATUS_LIMIT) return { ...head, items };
  return { ...head, items_complete: false, items_command: ['gbrain', 'sources', 'status', sourceId, '--json', '--holds'], items: items.slice(0, HELD_STATUS_LIMIT) };
}

/** Held files: up to HELD_STATUS_LIMIT per source with each one's next step, then the re-screen and retry commands; then recent conversions. */
export function gitHoldStatusLines(sourceId: string, status: GitHoldStatus): string[] {
  const lines: string[] = [];
  if (status.count) {
    lines.push(`  ${sourceId}: ${status.count} held file(s): not imported, and they do not block sync. A page whose newer file is held keeps its last good revision and is read-only for put_page until the file is repaired.`);
    for (const item of status.items.slice(0, HELD_STATUS_LIMIT)) lines.push(`    ${holdLine(item, 'Held').trim()} Held since ${day(item.held_since)}.`);
    if (status.count > HELD_STATUS_LIMIT) lines.push(`      +${status.count - HELD_STATUS_LIMIT} more; --json lists ${status.truncated ? `the first ${status.items.length}` : 'all'}`);
    lines.push(`    Most holds re-screen on the next sync by themselves (the file changed or was deleted, or a newer gbrain can read it): ${status.sync_argv.join(' ')}`);
    lines.push(`    to re-screen the held files on that sync even if nothing changed: gbrain sources retry-held ${sourceId}`);
  }
  if (status.recent_conversions?.length) {
    lines.push(`  ${sourceId}: ${status.recent_conversions.length} blocked sync request(s) recently converted in place (the file was held or re-read instead of blocking the source):`);
    for (const c of status.recent_conversions) lines.push(`    ${c.request_id}${c.path ? ` (${c.path})` : ''}: ${c.outcome === 'held' ? 'held' : 're-read and resubmitted'}, ${day(c.converted_at)}, run ${c.run_id}`);
  }
  return lines;
}