Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
18.9 KB
Raw Blame
/**
 * `gbrain sources writer movement [<source>] [--wait <dur>] [--warn-only] [--json]`
 * (#6317, I2): a deploy ends with data movement, not process liveness.
 *
 * It snapshots every active managed source (`readSourceMovement`: committed
 * managed receipt watermark, head step, cursor, holds, who is live), waits
 * the window (default `max(300 s, persistence.sync_preparation_ms + 60 s)`,
 * never below one sync preparation budget, so a healthy multi-wave group under
 * a raised budget reads `within_allowance`, never `not_moving`; the output
 * prints the budget it judged against), snapshots again and judges each
 * source: `moved` (a new committed publication), `within_allowance` (no commit
 * yet, but the head's step or the cursor advanced), `held` (advanced only by
 * holds: #6298's containment, routed by hold kind), `parked` (pending work and
 * nothing live to move it), `not_moving`, `nothing_pending`. The window ends
 * early once every pending source has moved.
 *
 * Exit: `not_moving` → 1 with code `managed_sync_not_moving` (`reason:
 * movement_check`) and the writer summary; `parked` → 1 with the same code
 * and `state: parked` (after a restart nothing picked the catch-up up);
 * `held`/`within_allowance`/`moved`/`nothing_pending` → 0; `unknown` (#6781:
 * the backlog read failed) → 1 with its own code `movement_unknown` (`reason:
 * backlog_read_failed`); `--warn-only` prints the same envelopes and exits 0.
 * A read failure keeps its own code (`fix.next: report`), never
 * `managed_sync_not_moving`. `gbrain upgrade`
 * never runs this (it owns no relaunch); it prints the bare command for the
 * supervisor's step.
 */
import type { BrainEngine } from '../core/engine.ts';
import { getCliOptions } from '../core/cli-options.ts';
import { finishCliTeardown, setCliExitVerdict, writeStdoutFinal } from '../core/cli-force-exit.ts';
import { loadConfig, toEngineConfig } from '../core/config.ts';
import { resolveBrainId } from '../core/brain-resolver.ts';
import { loadMounts } from '../core/brain-registry.ts';
import { opError } from '../core/ops/contract.ts';
import { readFix } from '../core/ops/op-fix.ts';
import { cliRenderContext, renderAction, toAgentError, type Action, type AgentEnvelope, type RenderedAction } from '../core/agent-output.ts';
import { parseDurationSeconds } from '../core/sync-concurrency.ts';
import { persistenceConfigForBrain } from '../core/persistence/local-client.ts';
import { readPreparationPolicy } from '../core/persistence/switches.ts';
import { judgeMovement, readSourceMovement, writerStatusCommand, type MovementVerdict, type SourceMovement } from '../core/persistence/sync-movement.ts';
import { stalledHoldSteps } from '../core/persistence/sync-holds.ts';
import { adminHostConfig } from './persistence-admin.ts';
import { reportPersistenceCliError } from './persistence-delegate.ts';

export const MOVEMENT_HELP = `Usage:
  gbrain sources writer movement [<source>] [--source <id>] [--wait <dur>] [--warn-only] [--json] [--brain <id>]

Proves that managed sync data moves: snapshots every active managed source,
waits the window, and judges each one from committed pages and the head's step
(a renewed lease, a live lock or a healthy /health are not movement).
  --wait <dur>   The window (30s, 5m, 1h; seconds when bare). Default: the larger of 5m and
                 persistence.sync_preparation_ms + 60s, printed as the budget judged against.
  --warn-only    Print the same verdicts and envelopes but exit 0 (pipelines that cannot fail).
States: moved, within_allowance (a step or cursor advanced; a multi-wave group preparing
normally), held (advanced only by holds; the hold kind's repair route is printed), parked
(pending work and nothing live to move it), not_moving, nothing_pending, unknown (the
sync backlog could not be read, so pending work is not known; never nothing_pending).
Exit 1 with code managed_sync_not_moving for not_moving and parked, 1 with code
movement_unknown for unknown; 0 otherwise.
Run it after restarting serve and the workers: gbrain sources writer movement`;

export { MOVEMENT_SUPERVISOR_STEP } from './movement-step.ts';

export const MOVEMENT_DEFAULT_MIN_MS = 300_000;
export const MOVEMENT_BUDGET_GRACE_MS = 60_000;
const POLL_MS = 10_000;

export interface MovementArgs { sourceId?: string; waitMs?: number; warnOnly: boolean; json: boolean; brain?: string }
export function parseMovementArgs(args: string[]): MovementArgs {
  const out: MovementArgs = { warnOnly: false, json: false };
  const usage = (message: string, suggestion: string) => opError('invalid_params', message, suggestion,
    { fix: readFix('Prints the movement command\'s flags and states.', { argv: ['gbrain', 'sources', 'writer', 'movement', '--help'] }) });
  for (let i = 0; i < args.length; i++) {
    const token = args[i]!;
    const equal = token.indexOf('=');
    const flag = token.startsWith('-') && equal > 0 ? token.slice(0, equal) : token;
    const value = () => { const v = equal > 0 && token.startsWith('-') ? token.slice(equal + 1) : args[++i]; if (v === undefined || v.startsWith('--')) throw usage(`${flag} requires a value.`, `Give ${flag} its value right after it, as ${flag} <value>.`); return v; };
    if (!token.startsWith('-')) { if (out.sourceId !== undefined) throw usage('Specify the source once.', `Name the source either as the argument or with --source, not both (here ${out.sourceId} and ${token}).`); out.sourceId = token; }
    else if ((flag === '--json' || flag === '--warn-only') && flag !== token && token !== '--json=true') throw usage(`${flag} is a switch and takes no value.`, `Write ${flag} on its own.`);
    else if (flag === '--json') out.json = true;
    else if (flag === '--warn-only') out.warnOnly = true;
    else if (flag === '--source') { if (out.sourceId !== undefined) throw usage('Specify the source once.', 'Name the source either as the argument or with --source, not both.'); out.sourceId = value(); }
    else if (flag === '--brain') out.brain = value();
    else if (flag === '--wait') {
      const seconds = parseDurationSeconds(value(), '--wait');
      if (seconds === undefined || seconds === null) throw usage('--wait requires a duration.', 'Give --wait a duration such as 30s, 5m or 1h (bare numbers are seconds).');
      out.waitMs = seconds * 1000;
    }
    else throw usage(`Unknown option: ${flag}.`, 'The accepted options are --source, --wait, --warn-only, --json and --brain.');
  }
  return out;
}

/** The window the command judges against: never below one sync preparation budget, by default the larger of 5 minutes and the budget plus a minute. */
export function movementWindowMs(policy: { syncMs: number }, requested?: number): { window_ms: number; budget_ms: number; source: 'default' | '--wait' } {
  if (requested !== undefined) return { window_ms: Math.max(requested, policy.syncMs), budget_ms: policy.syncMs, source: '--wait' };
  return { window_ms: Math.max(MOVEMENT_DEFAULT_MIN_MS, policy.syncMs + MOVEMENT_BUDGET_GRACE_MS), budget_ms: policy.syncMs, source: 'default' };
}

export interface MovementReport {
  source_id: string;
  state: MovementVerdict;
  exit: 0 | 1;
  code: 'managed_sync_not_moving' | 'movement_unknown' | null;
  reason?: 'movement_check' | 'backlog_read_failed';
  why: string;
  fix: RenderedAction | null;
  retry_after_ms?: number;
  before: SourceMovement | undefined;
  after: SourceMovement;
}

/** The verdict, exit and agent-operator envelope for one source over the window. */
export function movementReport(before: SourceMovement | undefined, after: SourceMovement, windowMs: number): MovementReport {
  const state = judgeMovement(before, after);
  const ctx = cliRenderContext();
  const id = after.source_id;
  const status: Action = readFix('Read-only: the request at the head, its step and wait cause, the owner process and the next action on the running claim.',
    { argv: ['gbrain', 'sources', 'writer', 'status', '--source', id, '--json'] });
  const verify = { argv: ['gbrain', 'sources', 'writer', 'movement', id, '--json'] };
  const head = after.head?.step ? ` The head is at step ${after.head.step}${after.head.waiting_on && after.head.waiting_on !== 'unknown' ? ` (waiting on ${after.head.waiting_on})` : ''}${after.head.owner ? `, held by ${after.head.owner.kind} pid ${after.head.owner.pid}` : ''}.` : '';
  switch (state) {
    case 'unknown':
      return { source_id: id, state, exit: 1, code: 'movement_unknown', reason: 'backlog_read_failed',
        why: `${id}: the managed sync backlog could not be read (${after.unknown?.error ?? 'no detail'}), so whether sync work is pending is not known; this is not a nothing-pending verdict.`,
        fix: renderAction({ argv: ['gbrain', 'sources', 'writer', 'movement', id, '--json'], consent: [], actor: 'agent', requires_exclusive: false,
          why: 'Read-only: reads the backlog again. If it keeps failing, gbrain doctor --json checks the database connection.', verify }, ctx), before, after };
    case 'nothing_pending':
      return { source_id: id, state, exit: 0, code: null, why: `${id}: no managed sync is pending${after.holds.count ? ` (${after.holds.count} held file(s) await their repair)` : ''}.`, fix: null, before, after };
    case 'moved':
      return { source_id: id, state, exit: 0, code: null, why: `${id}: a page committed at ${after.last_commit_at} inside the ${Math.round(windowMs / 1000)}s window; ${after.cursor ? `${after.cursor.remaining} remaining` : after.pass ? `${after.pass.remaining} path(s) unsettled in the pass` : 'the cursor is done'}.`, fix: null, before, after };
    case 'within_allowance':
      return { source_id: id, state, exit: 0, code: null, retry_after_ms: windowMs,
        why: `${id}: no page committed yet, but the head's step or the cursor advanced inside the window (a multi-wave group commits nothing until its last wave).${head} Check again after one more window.`,
        fix: renderAction({ argv: ['gbrain', 'sources', 'writer', 'movement', id, '--json'], consent: [], actor: 'agent', requires_exclusive: false,
          why: 'The owner is preparing normally; rerun after retry_after_ms and the next window shows the commit.', verify }, ctx), before, after };
    case 'held': {
      const kinds = after.holds;
      const route = kinds.stalled > 0 ? stalledHoldSteps(id).status : kinds.fences > 0 ? ['gbrain', 'repair', 'fences', '--source', id] : ['gbrain', 'repair', 'frontmatter', '--source', id];
      const then = kinds.stalled > 0 ? ` then ${stalledHoldSteps(id).retry.join(' ')}` : '';
      return { source_id: id, state, exit: 0, code: null,
        why: `${id}: pending work advanced only by holds inside the window (${kinds.count} held: ${kinds.stalled} preparation_stalled, ${kinds.fences} fence, ${kinds.concurrent} concurrent-write). The containment is working, not a stall; the held files need their repair route.`,
        fix: renderAction({ argv: route, consent: [], actor: 'agent', requires_exclusive: false, docs: 'docs/guides/repair.md#held-files',
          why: kinds.stalled > 0 ? `Read-only: shows the owner the stalled preparations name;${then} once the cause is fixed.` : 'Read-only preview of the repair; it prints the apply command for the user to approve.', verify }, ctx), before, after };
    }
    case 'parked': {
      const work = after.cursor ?? after.pass;
      return { source_id: id, state, exit: 1, code: 'managed_sync_not_moving', reason: 'movement_check',
        why: `${id}: the managed sync ${after.cursor ? 'cursor' : 'pass'} is pending at ${work?.index ?? '?'}/${work?.total ?? '?'} and nothing live is moving it (no sync run holds the lock; ${after.admitted} admitted request(s), no live consumer on the owner host). After a restart this means the catch-up was not started again.`,
        fix: renderAction({ argv: ['gbrain', 'sync', ...(work?.resume_args ?? ['--source', id, '--no-pull'])], consent: [], actor: 'agent', requires_exclusive: false,
          why: `Resumes the managed catch-up ${after.cursor ? 'where its cursor stopped' : 'with a plain sync that reconciles the checkout'}; the same command is safe to rerun in a loop.`, verify }, ctx), before, after };
    }
    case 'not_moving':
      if (after.pass?.stuck) return { source_id: id, state, exit: 1, code: 'managed_sync_not_moving', reason: 'movement_check',
        why: `${id}: every managed sync pass since ${after.pass.stuck_since} stopped ${after.pass.stuck}${after.pass.stop_reason ? ` (${after.pass.stop_reason})` : ''} with ${after.pass.remaining} path(s) unsettled, so the source does not converge even when pages commit.`,
        fix: renderAction({ ...readFix('Read-only: the convergence of the source, the reason its passes stop and the next step.', { argv: ['gbrain', 'sync', 'status', '--source', id, '--json'] }), verify }, ctx), before, after };
      return { source_id: id, state, exit: 1, code: 'managed_sync_not_moving', reason: 'movement_check',
        why: `${id}: ${after.cursor ? `${after.cursor.remaining} entries pending, ` : ''}no page committed and no step advance inside the ${Math.round(windowMs / 1000)}s window while ${after.live.drain ? after.live.drain_job_id !== null && after.live.drain_pid === null ? `sync job #${after.live.drain_job_id} is draining it` : `a sync (pid ${after.live.drain_pid}) holds the lock` : 'a consumer is live'}; last progress ${after.last_progress_at ?? 'unknown'}.${head}`,
        fix: renderAction({ ...status, verify }, ctx), before, after };
  }
}

export function formatMovementReport(r: MovementReport): string[] {
  const lines = [`${r.exit !== 0 ? '✗' : '✓'} ${r.source_id}: ${r.state}${r.code ? ` [${r.code}${r.reason ? `/${r.reason}` : ''}]` : ''}`, `  ${r.why}`];
  if (r.fix?.command) lines.push(`  Next: ${r.fix.command}${r.retry_after_ms ? ` (after ${Math.round(r.retry_after_ms / 1000)}s)` : ''}${r.fix.next === 'tell_user_to_run' ? ' (the host operator runs it)' : ''}`);
  return lines;
}

export async function runWriterMovementCli(args: string[], connected?: BrainEngine): Promise<void> {
  if (args.some(arg => arg === '--help' || arg === '-h')) { console.log(MOVEMENT_HELP); return; }
  let owned: BrainEngine | undefined;
  let json = false;
  try {
    const parsed = parseMovementArgs(args);
    json = parsed.json;
    const brainId = resolveBrainId(parsed.brain ?? getCliOptions().brain);
    const config = adminHostConfig(persistenceConfigForBrain(loadConfig(), brainId, brainId === 'host' ? [] : loadMounts()), brainId, 'The movement check', 'sources writer movement');
    if (!connected) {
      const { createEngine } = await import('../core/engine-factory.ts');
      owned = await createEngine(toEngineConfig(config));
      await owned.connect(toEngineConfig(config));
    }
    const engine = connected ?? owned!;
    const result = await runMovementCheck(engine, parsed, line => { if (!json) console.error(line); });
    const exitCode = parsed.warnOnly ? 0 : result.exit_code;
    if (json) await writeStdoutFinal(JSON.stringify({ ...result, exit_code: exitCode, warn_only: parsed.warnOnly, selected_brain: brainId }, null, 2) + '\n');
    else for (const r of result.sources) for (const line of formatMovementReport(r)) console.log(line);
    if (!json && !result.sources.length) console.log('No active managed source: nothing to judge (exit 0).');
    if (!json && parsed.warnOnly && result.exit_code !== 0) console.log('--warn-only: exiting 0 despite the verdict above.');
    setCliExitVerdict(exitCode);
  } catch (error) {
    if (!await reportPersistenceCliError(error, json)) {
      console.error(error instanceof Error ? error.message : String(error));
      setCliExitVerdict(1);
    }
  } finally { if (owned) await finishCliTeardown({ engine: owned, drainTimeoutMs: 1000 }); }
}

export interface MovementCheckResult {
  schema_version: 1;
  window_ms: number;
  budget_ms: number;
  window_source: 'default' | '--wait';
  waited_ms: number;
  sources: MovementReport[];
  exit_code: 0 | 1;
  /** The envelope of the first failing source (what a caller reads first), when any. */
  error?: AgentEnvelope;
}

/** Snapshot, wait (ending early once every pending source moved), snapshot, judge. `note` receives the human progress lines. */
export async function runMovementCheck(engine: BrainEngine, opts: { sourceId?: string; waitMs?: number; pollMs?: number; sleep?: (ms: number) => Promise<void> }, note: (line: string) => void = () => {}): Promise<MovementCheckResult> {
  const policy = await readPreparationPolicy(engine);
  const window = movementWindowMs(policy, opts.waitMs);
  const scope = opts.sourceId ? { sourceIds: [opts.sourceId] } : {};
  const before = await readSourceMovement(engine, scope);
  const pendingBefore = before.filter(m => m.movement_state !== 'nothing_pending');
  note(`[movement] window ${Math.round(window.window_ms / 1000)}s (${window.source === 'default' ? `default: max(300s, sync preparation budget ${Math.round(window.budget_ms / 1000)}s + 60s)` : `--wait, floored at the ${Math.round(window.budget_ms / 1000)}s sync preparation budget`}); `
    + `${before.length} managed source(s), ${pendingBefore.length} with pending work.`);
  const byId = new Map(before.map(m => [m.source_id, m]));
  let after = before, waited = 0;
  if (pendingBefore.length) {
    const sleep = opts.sleep ?? (ms => new Promise<void>(resolve => setTimeout(resolve, ms)));
    const poll = Math.max(1, Math.min(opts.pollMs ?? POLL_MS, window.window_ms));
    const started = Date.now();
    for (;;) {
      await sleep(Math.min(poll, window.window_ms - waited));
      waited = Date.now() - started;
      after = await readSourceMovement(engine, scope);
      const settled = after.every(m => { const v = judgeMovement(byId.get(m.source_id), m); return v === 'moved' || v === 'nothing_pending'; });
      if (settled || waited >= window.window_ms) break;
      note(`[movement] ${Math.round(waited / 1000)}s: ${after.map(m => `${m.source_id} ${judgeMovement(byId.get(m.source_id), m)}`).join(', ')}`);
    }
  }
  const sources = after.map(m => movementReport(byId.get(m.source_id), m, window.window_ms));
  const failing = sources.find(r => r.exit !== 0);
  const error = !failing ? undefined : toAgentError(failing.code === 'movement_unknown'
    ? opError('movement_unknown', failing.why, `Rerun gbrain sources writer movement ${failing.source_id}; if the backlog read keeps failing, run gbrain doctor --json.`,
      { reason: failing.reason, fix: readFix('Read-only: reads the managed sync backlog again.', { argv: ['gbrain', 'sources', 'writer', 'movement', failing.source_id, '--json'] }) })
    : opError('managed_sync_not_moving', failing.why, `Run ${writerStatusCommand(failing.source_id)} and follow the running claim's next step; verify with gbrain sources writer movement ${failing.source_id}.`,
      { reason: failing.reason, fix: readFix('Read-only: the request at the head, its step, wait cause, owner and next action.', { argv: ['gbrain', 'sources', 'writer', 'status', '--source', failing.source_id, '--json'] }) }),
  { transport: 'cli', command: 'sources writer movement', render: cliRenderContext() });
  return { schema_version: 1, window_ms: window.window_ms, budget_ms: window.budget_ms, window_source: window.source, waited_ms: waited, sources, exit_code: failing ? 1 : 0, ...(error ? { error } : {}) };
}