Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
38.6 KB
Raw Blame
/**
 * #6317 (B4, I2, T4): the movement reader and `gbrain sources writer movement`
 * judge managed sync by committed pages and the head's step, never by a
 * renewed lease, a live lock or a green /health. Fixtures on in-memory PGLite:
 * one managed source (claimed worktree) with a pending `managed-sync` cursor,
 * committed `managed_sync_import` receipts stamped at chosen times, a running
 * head with a claim stamp, the per-source sync lock, and a hold summary row.
 *
 * Reader (B4): nothing pending → `nothing_pending`; a pending cursor with no
 * drain and no live consumer → `parked` (data_moving null, never counted); a
 * live drain whose last progress is older than the ceiling → `not_moving`
 * (data_moving false, `not_moving_since` is the watermark); a commit or a head
 * step inside the ceiling → `moving`; a hold newer than the last commit →
 * `held`. Doctor warns only for `not_moving`.
 *
 * Command (T4): a pending cursor nothing moves exits 1 with
 * `managed_sync_not_moving`; a moving one exits 0; nothing pending skips the
 * wait and exits 0; a healthy source beside a stalled one exits 1 naming the
 * stalled one; holds-only advance is `held`, exit 0, with the hold kind's
 * route; a group whose head step advances inside the window is
 * `within_allowance`, exit 0 with `retry_after_ms`; `--warn-only` exits 0 on
 * the not-moving fixture; the default window never drops below one sync
 * preparation budget; every state's envelope follows the v1 contract.
 */
import { afterAll, beforeAll, beforeEach, describe, expect, spyOn, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { claimWorktree } from '../src/core/persistence/ownership.ts';
import { syncLockId, tryAcquireDbLock, type DbLockHandle } from '../src/core/db-lock.ts';
import { withEnv } from './helpers/with-env.ts';
import { judgeMovement, readSourceMovement, startMovementWatch, formatSourceMovement, MOVEMENT_WATERMARK_SQL } from '../src/core/persistence/sync-movement.ts';
import { movementReport, movementWindowMs, parseMovementArgs, runMovementCheck, runWriterMovementCli, MOVEMENT_HELP, MOVEMENT_SUPERVISOR_STEP } from '../src/commands/sources-writer-movement-cli.ts';
import { managedSyncMovementEntry } from '../src/commands/doctor/checks/managed-sync-movement.ts';
import { deriveNext, cliRenderContext } from '../src/core/agent-output.ts';
import { CODES } from '../src/core/error-registry.ts';
import { _resetCliExitVerdictForTests, currentExitCode } from '../src/core/cli-force-exit.ts';
import { runSources } from '../src/commands/sources.ts';
import { runPersistenceAdministration } from '../src/core/persistence/administration.ts';

const home = mkdtempSync(join(tmpdir(), 'gbrain-movement-'));
let engine: BrainEngine;
const ago = (ms: number) => new Date(Date.now() - ms).toISOString();

interface Fixture { id: string; worktreeId: string; incarnation: string }
async function managedSource(): Promise<Fixture> {
  const id = `mv-${randomUUID().slice(0, 8)}`, root = join(home, id);
  mkdirSync(root, { recursive: true });
  await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
  await engine.executeRaw("INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,'{}')", [id, root]);
  await claimWorktree(engine, id, root);
  await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
  const [row] = await engine.executeRaw<{ worktree_id: string; incarnation: string }>(
    'SELECT b.worktree_id::text AS worktree_id, b.source_incarnation::text AS incarnation FROM persistence_source_bindings b WHERE b.source_id=$1', [id]);
  return { id, worktreeId: row!.worktree_id, incarnation: row!.incarnation };
}
async function cursor(f: Fixture, index: number, total: number, extra: Record<string, unknown> = {}): Promise<void> {
  const header = { sourceId: f.id, index, total, runId: randomUUID(), counts: { added: 0, modified: 0, deleted: 0, chunks: 0, ...(extra.counts as object ?? {}) }, ...extra };
  await engine.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys) VALUES('managed-sync',$1,$2::text::jsonb)
    ON CONFLICT(op,fingerprint) DO UPDATE SET completed_keys=EXCLUDED.completed_keys`, [`cursor:${f.id}`, JSON.stringify([header])]);
}
async function request(f: Fixture, state: 'committed' | 'running' | 'queued', opts: { completedAt?: string; stamp?: Record<string, unknown>; lapsed?: boolean; kind?: string } = {}): Promise<string> {
  const token = randomUUID(), requestId = randomUUID();
  const stamp = opts.stamp ? JSON.stringify({ phase: 'preparing', claimed_at: ago(90_000), since: ago(60_000), token, waiting_on: 'db', owner: { kind: 'serve', pid: process.pid + 1, version: 'test', nonce: 'other' }, ...opts.stamp }) : null;
  await engine.executeRaw(`INSERT INTO persistence_requests(principal_kind,principal_id,request_id,operation,source_id,source_incarnation,slug,worktree_id,digest,authority,intent,intent_bytes,terminal_reservation,
      state,execution_token,claim_expires_at,claim_phase,completed_at)
    VALUES('local_cli','cli:example',$1::uuid,'submit_job',$2,$3::uuid,$4,$5::uuid,'d','{}'::jsonb,$6::text::jsonb,1,16384,$7,
      ${state === 'running' ? '$8::uuid' : 'NULLIF($8::text,$8::text)::uuid'},${state === 'running' ? (opts.lapsed ? "now()-interval '1 second'" : "now()+interval '30 seconds'") : 'NULL'},$9::text::jsonb,$10::timestamptz)`,
  [requestId, f.id, f.incarnation, `notes/${requestId.slice(0, 6)}`, f.worktreeId, JSON.stringify({ kind: opts.kind ?? 'managed_sync_import', path: 'p.md' }), state, token, stamp, opts.completedAt ?? null]);
  return requestId;
}
async function holdSummary(f: Fixture, counts: { count: number; stalled?: number; fences?: number }, at: string): Promise<void> {
  await engine.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys,updated_at) VALUES('sync-hold-summary',$1,$2::text::jsonb,$3::timestamptz)
    ON CONFLICT(op,fingerprint) DO UPDATE SET completed_keys=EXCLUDED.completed_keys, updated_at=EXCLUDED.updated_at`,
  [`hold-summary:${f.id}`, JSON.stringify([{ source_id: f.id, incarnation: f.incarnation, count: counts.count, stale: 0, stalled: counts.stalled ?? 0, fences: counts.fences ?? 0, concurrent: 0 }]), at]);
}
/** Ends a fixture's pending work, so a later brain-wide check does not read it as parked. */
const finish = (f: Fixture) => engine.executeRaw("DELETE FROM op_checkpoints WHERE op='managed-sync' AND fingerprint=$1", [`cursor:${f.id}`]);
const one = async (f: Fixture, now?: number) => (await readSourceMovement(engine, { sourceIds: [f.id], ...(now ? { now } : {}) }))[0]!;

beforeAll(async () => {
  const lite = new PGLiteEngine(); await lite.connect({}); await lite.initSchema(); engine = lite;
  // A 60 s ceiling (its floor is the larger budget plus 30 s) and the smallest budgets, so the command's window floor is 1 s.
  for (const [key, value] of [['persistence.sync_preparation_ms', '1000'], ['persistence.maintenance_preparation_ms', '1000'], ['persistence.preparation_ceiling_ms', '60000']]) {
    await engine.executeRaw('INSERT INTO config(key,value) VALUES($1,$2) ON CONFLICT(key) DO UPDATE SET value=EXCLUDED.value', [key, value]);
  }
}, 120_000);
afterAll(async () => { await engine.disconnect(); rmSync(home, { recursive: true, force: true }); });

describe('readSourceMovement (B4)', () => {
  test('nothing pending, parked, not moving, moving by commit, moving by step, held', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    expect(await one(f)).toMatchObject({ source_id: f.id, movement_state: 'nothing_pending', data_moving: null, cursor: null, writer_status_command: `gbrain sources writer status --source ${f.id} --json` });

    await cursor(f, 3, 10, { progress: { startedAt: Date.now() - 300_000, startIndex: 0, lastAt: Date.now() - 200_000, lastIndex: 3 } });
    await request(f, 'committed', { completedAt: ago(200_000) });
    // Pending, but no drain holds the lock and no managed request is admitted: parked, informational.
    expect(await one(f)).toMatchObject({ movement_state: 'parked', data_moving: null, not_moving_since: null, live: { drain: false, consumer: null } });
    expect(formatSourceMovement(await one(f))).toContain('parked at 3/10');

    // A live drain and a watermark 200 s old against a 60 s ceiling: not moving since the watermark.
    const lock = await tryAcquireDbLock(engine, syncLockId(f.id));
    try {
      const stuck = await one(f);
      expect(stuck).toMatchObject({ movement_state: 'not_moving', data_moving: false, live: { drain: true, drain_pid: process.pid }, ceiling_ms: 60_000, cursor: { index: 3, total: 10, remaining: 7, held: 0 } });
      expect(stuck.not_moving_since).toBe(stuck.last_commit_at);
      expect(Date.now() - Date.parse(stuck.not_moving_since!)).toBeGreaterThanOrEqual(199_000);
      expect(formatSourceMovement(stuck)).toContain('NOT MOVING since');
      // A commit inside the ceiling: moving.
      await request(f, 'committed', { completedAt: ago(5_000) });
      expect(await one(f)).toMatchObject({ movement_state: 'moving', data_moving: true, not_moving_since: null });
    } finally { await lock!.release(); }

    // No drain, but an admitted head whose owner renews a stamped claim: the claim proves a consumer; its step advance is progress.
    const g = await managedSource();
    await cursor(g, 0, 16);
    await request(g, 'committed', { completedAt: ago(500_000) });
    await request(g, 'running', { stamp: { step: 'import_screen', step_since: ago(10_000), last_sql: { label: 'SELECT pages', at: ago(4_000) } } });
    const stepping = await one(g);
    expect(stepping).toMatchObject({ movement_state: 'moving', data_moving: true, admitted: 1, live: { drain: false, consumer: 'claim' },
      head: { state: 'running', phase: 'preparing', step: 'import_screen', waiting_on: 'db', owner: { kind: 'serve', pid: process.pid + 1, nonce: 'other' }, last_sql: { label: 'SELECT pages' } } });
    expect(stepping.last_progress_at).toBe(stepping.head!.step_since);
    // The same head parked on its step for longer than the ceiling: not moving, even though the lease renews.
    await engine.executeRaw(`UPDATE persistence_requests SET claim_phase = claim_phase || $2::text::jsonb WHERE worktree_id=$1::uuid AND state='running'`, [g.worktreeId, JSON.stringify({ step_since: ago(120_000) })]);
    expect(await one(g)).toMatchObject({ movement_state: 'not_moving', data_moving: false });

    // Holds newer than the last commit: held (the containment, not a stall).
    await holdSummary(g, { count: 2, stalled: 2 }, ago(1_000));
    expect(await one(g)).toMatchObject({ movement_state: 'held', data_moving: true, holds: { count: 2, stalled: 2 } });
    await finish(f); await finish(g);
  }), 60_000);
});

describe('movement watermark index', () => {
  test('the watermark read returns the newest committed sync receipt of the incarnation, with and without persistence_requests_sync_watermark', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const indexes = await engine.executeRaw<{ name: string }>(`SELECT indexname AS name FROM pg_indexes WHERE indexname IN ('persistence_requests_sync_watermark','persistence_requests_committed_watermark')`);
    expect(indexes.map(r => r.name)).toEqual(['persistence_requests_sync_watermark']);
    const f = await managedSource();
    const read = async () => {
      const [row] = await engine.executeRaw<{ last_commit_at: unknown; admitted: number | string }>(MOVEMENT_WATERMARK_SQL, [f.worktreeId, f.incarnation]);
      return { last_commit_at: row!.last_commit_at === null ? null : new Date(row!.last_commit_at as string).toISOString(), admitted: Number(row!.admitted) };
    };
    const unindexed = () => engine.transaction(async tx => {
      await tx.executeRaw('SET LOCAL enable_indexscan = off');
      await tx.executeRaw('SET LOCAL enable_indexonlyscan = off');
      await tx.executeRaw('SET LOCAL enable_bitmapscan = off');
      const [row] = await tx.executeRaw<{ last_commit_at: unknown; admitted: number | string }>(MOVEMENT_WATERMARK_SQL, [f.worktreeId, f.incarnation]);
      return { last_commit_at: row!.last_commit_at === null ? null : new Date(row!.last_commit_at as string).toISOString(), admitted: Number(row!.admitted) };
    });
    expect(await read()).toEqual({ last_commit_at: null, admitted: 0 });
    for (let i = 0; i < 30; i++) await request(f, 'committed', { completedAt: ago(1_000 + i * 1_000), kind: 'managed_file_import' });
    expect(await read()).toEqual(await unindexed());
    expect((await read()).last_commit_at).toBeNull();
    await request(f, 'committed', { completedAt: ago(90_000) });
    const newest = await request(f, 'committed', { completedAt: ago(60_000), kind: 'managed_sync_delete' });
    await request(f, 'committed', { completedAt: ago(30_000), kind: 'managed_sync_import' });
    await request(f, 'queued', { kind: 'managed_sync_import' });
    await request(f, 'queued', { kind: 'managed_file_import' });
    await engine.executeRaw(`UPDATE persistence_requests SET source_incarnation=gen_random_uuid() WHERE worktree_id=$1::uuid AND completed_at > now()-interval '45 seconds' AND intent->>'kind'='managed_sync_import'`, [f.worktreeId]);
    const expected = await unindexed();
    expect(await read()).toEqual(expected);
    expect(expected.admitted).toBe(1);
    const [stamp] = await engine.executeRaw<{ at: string }>('SELECT completed_at AS at FROM persistence_requests WHERE request_id=$1::uuid', [newest]);
    expect(expected.last_commit_at).toBe(new Date(stamp!.at).toISOString());
    await engine.executeRaw(`UPDATE persistence_requests SET intent=NULL, compacted=true WHERE request_id=$1::uuid`, [newest]);
    expect(await read()).toEqual(await unindexed());
    expect(Date.parse((await read()).last_commit_at!)).toBeLessThan(Date.parse(stamp!.at));
  }), 60_000);
});

describe('gbrain sources writer movement (I2, T4)', () => {
  const fast = { pollMs: 100 };

  test('the default window never drops below one sync preparation budget and the flags parse', () => {
    expect(movementWindowMs({ syncMs: 120_000 })).toEqual({ window_ms: 300_000, budget_ms: 120_000, source: 'default' });
    expect(movementWindowMs({ syncMs: 900_000 })).toEqual({ window_ms: 960_000, budget_ms: 900_000, source: 'default' });
    expect(movementWindowMs({ syncMs: 120_000 }, 30_000)).toEqual({ window_ms: 120_000, budget_ms: 120_000, source: '--wait' });
    expect(parseMovementArgs(['default', '--wait', '5m', '--warn-only', '--json'])).toEqual({ sourceId: 'default', waitMs: 300_000, warnOnly: true, json: true });
    expect(parseMovementArgs(['--source=notes', '--wait=45'])).toMatchObject({ sourceId: 'notes', waitMs: 45_000 });
    expect(() => parseMovementArgs(['a', 'b'])).toThrow(/Specify the source once/);
    expect(() => parseMovementArgs(['--bogus'])).toThrow(/Unknown option/);
    expect(MOVEMENT_SUPERVISOR_STEP).toBe('after restarting serve and the workers, run: gbrain sources writer movement');
  });

  test('a pending cursor nothing moves exits 1 with managed_sync_not_moving; moving exits 0; nothing pending skips the wait', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    // Nothing pending: no wait, exit 0.
    const idle = await runMovementCheck(engine, { sourceId: f.id, waitMs: 60_000, ...fast });
    expect(idle).toMatchObject({ exit_code: 0, waited_ms: 0, window_ms: 60_000, budget_ms: 1_000, sources: [{ state: 'nothing_pending', exit: 0, code: null }] });
    expect(idle.error).toBeUndefined();

    await cursor(f, 2, 10);
    await request(f, 'committed', { completedAt: ago(300_000) });
    // Parked: pending and nothing live to move it (after a restart, the catch-up was not started again) → exit 1, resume command as the fix.
    const parked = await runMovementCheck(engine, { sourceId: f.id, waitMs: 1_000, ...fast });
    expect(parked).toMatchObject({ exit_code: 1, sources: [{ state: 'parked', exit: 1, code: 'managed_sync_not_moving', reason: 'movement_check' }] });
    expect(parked.sources[0]!.fix).toMatchObject({ next: 'run', command: `gbrain sync --source ${f.id} --no-pull` });
    expect(parked.error).toMatchObject({ code: 'managed_sync_not_moving', reason: 'movement_check', class: 'server', retryable: false, contract_version: 1 });
    expect(parked.error!.fix).toMatchObject({ next: 'run', argv: ['gbrain', 'sources', 'writer', 'status', '--source', f.id, '--json'] });
    expect(parked.error!.docs).toContain('troubleshooting.md#managed-sync-not-moving');

    const lock = await tryAcquireDbLock(engine, syncLockId(f.id));
    try {
      // A live drain and no commit, step or cursor advance inside the window: not_moving, exit 1 with the writer summary.
      const stuck = await runMovementCheck(engine, { sourceId: f.id, waitMs: 1_000, ...fast });
      expect(stuck).toMatchObject({ exit_code: 1, sources: [{ state: 'not_moving', exit: 1, code: 'managed_sync_not_moving', reason: 'movement_check' }] });
      expect(stuck.sources[0]!.why).toContain('no page committed and no step advance');
      expect(stuck.sources[0]!.fix).toMatchObject({ next: 'run', command: `gbrain sources writer status --source ${f.id} --json` });
      expect(stuck.sources[0]!.fix!.verify?.argv).toEqual(['gbrain', 'sources', 'writer', 'movement', f.id, '--json']);
      // The consumer commits during the window: moved, exit 0, and the window ends early.
      const started = Date.now();
      const moving = runMovementCheck(engine, { sourceId: f.id, waitMs: 120_000, pollMs: 100 });
      await new Promise(r => setTimeout(r, 50));
      await request(f, 'committed', { completedAt: ago(0) });
      const result = await moving;
      expect(result).toMatchObject({ exit_code: 0, sources: [{ state: 'moved', exit: 0, code: null }] });
      expect(Date.now() - started).toBeLessThan(10_000);
    } finally { await lock!.release(); await finish(f); }
  }), 60_000);

  test('a healthy source beside a stalled one exits 1 naming the stalled one; --warn-only is the caller\'s exit 0', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const healthy = await managedSource(), stalled = await managedSource();
    await cursor(stalled, 1, 5); await request(stalled, 'committed', { completedAt: ago(400_000) });
    const locks = [await tryAcquireDbLock(engine, syncLockId(stalled.id)), await tryAcquireDbLock(engine, syncLockId(healthy.id))];
    try {
      await cursor(healthy, 1, 5);
      const check = runMovementCheck(engine, { waitMs: 2_000, pollMs: 100 });
      await new Promise(r => setTimeout(r, 50));
      await request(healthy, 'committed', { completedAt: ago(0) });
      // The healthy one moved; the stalled one keeps the window pending, so the check runs its full (short) window.
      const result = await check;
      expect(result.waited_ms).toBeGreaterThanOrEqual(2_000);
      expect(result.exit_code).toBe(1);
      expect(result.sources.find(r => r.source_id === healthy.id)).toMatchObject({ state: 'moved', exit: 0 });
      expect(result.sources.find(r => r.source_id === stalled.id)).toMatchObject({ state: 'not_moving', exit: 1 });
      expect(result.error!.message).toContain(stalled.id);
    } finally { for (const lock of locks) await lock!.release(); await finish(healthy); await finish(stalled); }
  }), 90_000);

  test('holds-only advance is held (exit 0, the hold kind\'s route); a head step advance inside the window is within_allowance (exit 0, retry_after_ms)', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    await cursor(f, 4, 20, { counts: { held: 1 } });
    await request(f, 'committed', { completedAt: ago(500_000) });
    const lock = await tryAcquireDbLock(engine, syncLockId(f.id));
    try {
      const check = runMovementCheck(engine, { sourceId: f.id, waitMs: 1_000, pollMs: 100 });
      await new Promise(r => setTimeout(r, 50));
      await holdSummary(f, { count: 1, stalled: 1 }, ago(0));
      const held = await check;
      expect(held).toMatchObject({ exit_code: 0, sources: [{ state: 'held', exit: 0, code: null }] });
      expect(held.sources[0]!.fix).toMatchObject({ next: 'run', argv: ['gbrain', 'sources', 'writer', 'status', '--source', f.id, '--json'] });
      expect(held.sources[0]!.why).toContain('1 preparation_stalled');
      // A fence hold routes to the fence repair preview instead.
      await holdSummary(f, { count: 2, stalled: 0, fences: 2 }, ago(0));
      const fenceCheck = runMovementCheck(engine, { sourceId: f.id, waitMs: 1_000, pollMs: 100 });
      await new Promise(r => setTimeout(r, 50));
      await holdSummary(f, { count: 3, stalled: 0, fences: 3 }, ago(0));
      expect((await fenceCheck).sources[0]).toMatchObject({ state: 'held', fix: { argv: ['gbrain', 'repair', 'fences', '--source', f.id] } });

      // A 16-member group preparing in waves: the head's step advances, nothing commits: within_allowance.
      const g = await managedSource();
      await cursor(g, 0, 16);
      await request(g, 'running', { stamp: { step: 'raw_hash', step_since: ago(2_000) } });
      const lockG = await tryAcquireDbLock(engine, syncLockId(g.id));
      try {
        const waves = runMovementCheck(engine, { sourceId: g.id, waitMs: 1_000, pollMs: 100 });
        await new Promise(r => setTimeout(r, 50));
        await engine.executeRaw(`UPDATE persistence_requests SET claim_phase = claim_phase || $2::text::jsonb WHERE worktree_id=$1::uuid AND state='running'`, [g.worktreeId, JSON.stringify({ step: 'import_screen', step_since: ago(0) })]);
        const result = await waves;
        expect(result).toMatchObject({ exit_code: 0, sources: [{ state: 'within_allowance', exit: 0, code: null, retry_after_ms: 1_000 }] });
        expect(result.sources[0]!.fix).toMatchObject({ next: 'run', argv: ['gbrain', 'sources', 'writer', 'movement', g.id, '--json'] });
        expect(result.sources[0]!.why).toContain('multi-wave group');
      } finally { await lockG!.release(); await finish(g); }
    } finally { await lock!.release(); await finish(f); }
  }), 60_000);

  test('every state\'s envelope follows the v1 contract: a fix with argv, consent, actor, a derived next and a read-only verify', () => {
    // One sample clock for every `base`: `before` and `after` are built in one expression, and two `ago(1_000)` calls that
    // straddle a millisecond boundary made the not_moving pair read as `moved` (#6340 gate flake, root cause in the test).
    const sampledAt = ago(0), committedAt = ago(1_000);
    const base = (over: Record<string, unknown>) => ({ source_id: 's', sampled_at: sampledAt, movement_state: 'moving', data_moving: true, not_moving_since: null, last_commit_at: committedAt, last_progress_at: committedAt,
      cursor: { index: 1, total: 4, remaining: 3, held: 0, last_progress_at: null, resume_command: 'gbrain sync --source s --no-pull' }, head: null, holds: { count: 0, stalled: 0, fences: 0, concurrent: 0, last_hold_at: null },
      admitted: 1, live: { drain: true, drain_pid: 1, drain_job_id: null, consumer: null }, ceiling_ms: 600_000, owner_is_this_host: true, writer_status_command: 'gbrain sources writer status --source s --json', ...over }) as Parameters<typeof movementReport>[1];
    const states: Array<[ReturnType<typeof judgeMovement>, Parameters<typeof movementReport>[0], Parameters<typeof movementReport>[1]]> = [
      ['nothing_pending', undefined, base({ movement_state: 'nothing_pending', cursor: null, data_moving: null })],
      ['moved', base({ last_commit_at: ago(100_000) }), base({})],
      ['within_allowance', base({ head: { request_id: 'r', state: 'running', phase: 'preparing', step: 'a', step_since: ago(50_000), step_age_ms: 50_000, waiting_on: 'db', owner: null, last_sql: null, lapsed: false } }),
        base({ last_commit_at: null, head: { request_id: 'r', state: 'running', phase: 'preparing', step: 'b', step_since: ago(1_000), step_age_ms: 1_000, waiting_on: 'db', owner: null, last_sql: null, lapsed: false } })],
      ['held', base({ holds: { count: 0, stalled: 0, fences: 0, concurrent: 0, last_hold_at: null } }), base({ holds: { count: 1, stalled: 0, fences: 1, concurrent: 0, last_hold_at: ago(0) } })],
      ['parked', base({ movement_state: 'parked', live: { drain: false, drain_pid: null, drain_job_id: null, consumer: null } }), base({ movement_state: 'parked', live: { drain: false, drain_pid: null, drain_job_id: null, consumer: null } })],
      ['not_moving', base({}), base({})],
    ];
    for (const [expected, before, after] of states) {
      const report = movementReport(before, after, 300_000);
      expect(report.state).toBe(expected);
      expect(report.exit).toBe(expected === 'not_moving' || expected === 'parked' ? 1 : 0);
      expect(report.code).toBe(report.exit ? 'managed_sync_not_moving' : null);
      expect(report.why.length).toBeGreaterThan(20);
      if (report.fix) {
        expect(report.fix.argv?.[0]).toBe('gbrain');
        expect(report.fix.consent).toEqual([]);
        expect(['run', 'tell_user_to_run']).toContain(report.fix.next);
        expect(report.fix.verify?.argv).toEqual(['gbrain', 'sources', 'writer', 'movement', 's', '--json']);
      } else expect(['nothing_pending', 'moved']).toContain(expected);
    }
    // The registry entry the command's envelope rides on.
    const entry = CODES.managed_sync_not_moving;
    expect(entry.reasons).toContain('movement_check');
    expect(deriveNext(entry.fix!, cliRenderContext())).toBe('run');
  });
});

describe('doctor managed_sync_not_moving and the serve notice', () => {
  test('warns only for not_moving (structured fix), lists parked as info, ok when nothing is pending; the watch prints one notice per flip', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const ctx = { engine } as unknown as Parameters<typeof managedSyncMovementEntry.run>[0];
    const f = await managedSource();
    await cursor(f, 1, 9); await request(f, 'committed', { completedAt: ago(400_000) });
    const [parked] = await managedSyncMovementEntry.run(ctx) as unknown as Array<Record<string, unknown>>;
    expect(parked).toMatchObject({ name: 'managed_sync_not_moving', status: 'ok', severity: 'info' });
    expect(String(parked!.message)).toContain(`${f.id} is parked at 1/9`);
    const lines: string[] = [];
    const watch = startMovementWatch(engine, { log: line => lines.push(line), everyMs: 3_600_000 });
    const lock = await tryAcquireDbLock(engine, syncLockId(f.id));
    try {
      const [warn] = await managedSyncMovementEntry.run(ctx) as unknown as Array<Record<string, unknown>>;
      expect(warn).toMatchObject({ name: 'managed_sync_not_moving', status: 'warn', readiness_state: 'degraded', details: { count: 1 } });
      expect(warn!.fix).toMatchObject({ argv: ['gbrain', 'sources', 'writer', 'status', '--source', f.id, '--json'], actor: 'agent', consent: [] });
      await watch.tick(); await watch.tick();
      expect(lines.filter(line => line.startsWith('[gbrain notice managed_sync_not_moving kind=degraded]'))).toHaveLength(1);
      expect(lines[0]).toContain(`gbrain sources writer status --source ${f.id} --json`);
      await request(f, 'committed', { completedAt: ago(0) });
      await watch.tick();
      expect(lines[1]).toContain('moving again');
      const [ok] = await managedSyncMovementEntry.run(ctx) as unknown as Array<Record<string, unknown>>;
      expect(ok).toMatchObject({ status: 'ok' });
    } finally { watch.stop(); await lock!.release(); }
  }), 60_000);

  test('#6552 §2: parked with admitted requests, no consumer and no progress past the ceiling fails with the resume sync as its fix; recent progress or an empty queue stays informational', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const ctx = { engine } as unknown as Parameters<typeof managedSyncMovementEntry.run>[0];
    const f = await managedSource();
    try {
      await cursor(f, 2, 415, { progress: { startedAt: Date.now() - 900_000, startIndex: 0, lastAt: Date.now() - 600_000, lastIndex: 2 } });
      await request(f, 'committed', { completedAt: ago(600_000) });
      const [empty] = await managedSyncMovementEntry.run(ctx) as unknown as Array<Record<string, unknown>>;
      expect(empty).toMatchObject({ name: 'managed_sync_not_moving', status: 'ok', details: { stranded: 0 } });
      await request(f, 'queued');
      const before = (await readSourceMovement(engine, { sourceIds: [f.id] }))[0]!;
      expect(before).toMatchObject({ movement_state: 'parked', admitted: 1, live: { consumer: null } });
      const [fail] = await managedSyncMovementEntry.run(ctx) as unknown as Array<Record<string, unknown>>;
      expect(fail).toMatchObject({ name: 'managed_sync_not_moving', status: 'fail', readiness_state: 'degraded', details: { stranded: 1 } });
      expect(String(fail!.message)).toContain('non-empty write queue and no consumer');
      expect(fail!.fix).toMatchObject({ argv: ['gbrain', 'sync', '--source', f.id, '--no-pull'], actor: 'agent', consent: [] });
      // The same queue with progress inside the ceiling is a cursor between runs: informational.
      await cursor(f, 3, 415, { progress: { startedAt: Date.now() - 60_000, startIndex: 0, lastAt: Date.now() - 1_000, lastIndex: 3 } });
      const [recent] = await managedSyncMovementEntry.run(ctx) as unknown as Array<Record<string, unknown>>;
      expect(recent).toMatchObject({ status: 'ok', details: { stranded: 0 } });
    } finally {
      await finish(f);
      await engine.executeRaw("UPDATE persistence_requests SET state='cancelled', completed_at=now() WHERE source_id=$1 AND state IN ('queued','running')", [f.id]);
    }
  }), 60_000);

  test('#6552 §2: a queue stranded before its first commit (fresh cursor, no progress stamp) ages from its oldest admitted request', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const ctx = { engine } as unknown as Parameters<typeof managedSyncMovementEntry.run>[0];
    const f = await managedSource();
    try {
      await cursor(f, 0, 415);
      const queued = await request(f, 'queued');
      const [fresh] = await managedSyncMovementEntry.run(ctx) as unknown as Array<Record<string, unknown>>;
      expect(fresh).toMatchObject({ status: 'ok', details: { stranded: 0 } });
      await engine.executeRaw("UPDATE persistence_requests SET created_at=now()-interval '2 days' WHERE request_id=$1::uuid", [queued]);
      const movement = (await readSourceMovement(engine, { sourceIds: [f.id] }))[0]!;
      expect(movement).toMatchObject({ movement_state: 'parked', admitted: 1, last_progress_at: null });
      expect(Date.now() - Date.parse(movement.admitted_since!)).toBeGreaterThan(86_400_000);
      const [fail] = await managedSyncMovementEntry.run(ctx) as unknown as Array<Record<string, unknown>>;
      expect(fail).toMatchObject({ status: 'fail', details: { stranded: 1 } });
    } finally {
      await finish(f);
      await engine.executeRaw("UPDATE persistence_requests SET state='cancelled', completed_at=now() WHERE source_id=$1 AND state IN ('queued','running')", [f.id]);
    }
  }), 60_000);
});

describe('#6781: a failed backlog read is unknown, never nothing_pending', () => {
  /** The reporter's two arms: the same engine, except that the backlog SELECT alone rejects in the second. */
  const BACKLOG_SQL = /FROM op_checkpoints WHERE op='managed-sync' AND COALESCE\(completed_keys->0->>'done','false'\)<>'true'/;
  const failingBacklog = (target: BrainEngine): BrainEngine => new Proxy(target, {
    get(t, prop) {
      if (prop === 'executeRaw') {
        return (sql: string, params?: unknown[]) => BACKLOG_SQL.test(sql) ? Promise.reject(new Error('OFFLINE_SENTINEL backlog read failed')) : t.executeRaw(sql, params);
      }
      const value = Reflect.get(t, prop, t);
      return typeof value === 'function' ? value.bind(t) : value;
    },
  });
  beforeEach(() => _resetCliExitVerdictForTests());

  test('reader, window verdict and human line: control arm not_moving, injected arm unknown', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    await cursor(f, 2, 7, { progress: { startedAt: Date.now() - 400_000, startIndex: 0, lastAt: Date.now() - 300_000, lastIndex: 2 } });
    await request(f, 'committed', { completedAt: ago(300_000) });
    const lock = await tryAcquireDbLock(engine, syncLockId(f.id));
    try {
      const control = (await readSourceMovement(engine, { sourceIds: [f.id] }))[0]!;
      expect(control).toMatchObject({ movement_state: 'not_moving', data_moving: false, cursor: { remaining: 5 } });
      const injected = (await readSourceMovement(failingBacklog(engine), { sourceIds: [f.id] }))[0]!;
      expect(injected).toMatchObject({ source_id: f.id, movement_state: 'unknown', data_moving: null, not_moving_since: null, cursor: null,
        unknown: { reason: 'backlog_read_failed' }, live: { drain: true } });
      expect(injected.unknown!.error).toContain('OFFLINE_SENTINEL');
      expect(judgeMovement(undefined, injected)).toBe('unknown');
      expect(judgeMovement(control, injected)).toBe('unknown');
      const line = formatSourceMovement(injected);
      expect(line).toContain('UNKNOWN');
      expect(line).toContain('backlog_read_failed');
    } finally { await lock!.release(); await finish(f); }
  }), 60_000);

  test('a failed read between two not_moving ticks: the window verdict is unknown and the serve watch holds its flip (no false recovery, no repeat notice)', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    await cursor(f, 2, 7, { progress: { startedAt: Date.now() - 400_000, startIndex: 0, lastAt: Date.now() - 300_000, lastIndex: 2 } });
    await request(f, 'committed', { completedAt: ago(300_000) });
    const flaky = { fail: false };
    const broken = failingBacklog(engine);
    const toggling = new Proxy(engine, { get: (t, prop) => { const src = flaky.fail ? broken : t; const value = Reflect.get(src, prop, src); return typeof value === 'function' ? value.bind(src) : value; } }) as typeof engine;
    const lines: string[] = [];
    const watch = startMovementWatch(toggling, { log: line => lines.push(line), everyMs: 3_600_000 });
    const lock = await tryAcquireDbLock(engine, syncLockId(f.id));
    try {
      const control = (await readSourceMovement(engine, { sourceIds: [f.id] }))[0]!;
      expect(control.movement_state).toBe('not_moving');
      const injected = (await readSourceMovement(broken, { sourceIds: [f.id] }))[0]!;
      expect(judgeMovement(injected, control)).toBe('unknown');
      const mine = () => lines.filter(line => line.includes(f.id));
      await watch.tick();
      expect(mine().filter(line => line.includes('kind=degraded'))).toHaveLength(1);
      flaky.fail = true;
      await watch.tick();
      flaky.fail = false;
      await watch.tick();
      expect(mine().filter(line => line.includes('moving again'))).toEqual([]);
      expect(mine().filter(line => line.includes('kind=degraded'))).toHaveLength(1);
    } finally { watch.stop(); await lock!.release(); await finish(f); }
  }), 60_000);

  test('writer movement exits 1 with movement_unknown (never managed_sync_not_moving); --warn-only exits 0', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    await cursor(f, 2, 7);
    try {
      const result = await runMovementCheck(failingBacklog(engine), { sourceId: f.id, waitMs: 1_000, pollMs: 1, sleep: async () => {} });
      expect(result.exit_code).toBe(1);
      expect(result.sources[0]).toMatchObject({ source_id: f.id, state: 'unknown', exit: 1, code: 'movement_unknown', reason: 'backlog_read_failed' });
      expect(result.error).toMatchObject({ code: 'movement_unknown', reason: 'backlog_read_failed' });
      expect(JSON.stringify(result)).not.toContain('managed_sync_not_moving');
      expect(result.sources[0]!.fix).toMatchObject({ argv: ['gbrain', 'sources', 'writer', 'movement', f.id, '--json'], actor: 'agent', consent: [] });

      // The command resolves the brain host config before it uses the connected engine.
      mkdirSync(join(home, '.gbrain'), { recursive: true });
      writeFileSync(join(home, '.gbrain', 'config.json'), JSON.stringify({ engine: 'pglite', database_path: join(home, '.gbrain', 'brain.pglite') }));
      const out: string[] = [];
      const logSpy = spyOn(console, 'log').mockImplementation((...args: unknown[]) => { out.push(args.map(String).join(' ')); });
      const errSpy = spyOn(console, 'error').mockImplementation(() => {});
      try {
        await runWriterMovementCli([f.id, '--wait', '1s'], failingBacklog(engine));
        expect(currentExitCode()).toBe(1);
        expect(out.join('\n')).toContain(`✗ ${f.id}: unknown [movement_unknown/backlog_read_failed]`);
        _resetCliExitVerdictForTests();
        out.length = 0;
        await runWriterMovementCli([f.id, '--wait', '1s', '--warn-only'], failingBacklog(engine));
        expect(currentExitCode()).toBe(0);
        expect(out.join('\n')).toContain('--warn-only: exiting 0 despite the verdict above.');
      } finally { logSpy.mockRestore(); errSpy.mockRestore(); }
      expect(MOVEMENT_HELP).toContain('unknown');
      expect(MOVEMENT_HELP).toContain('movement_unknown');
    } finally { await finish(f); }
  }), 60_000);

  test('doctor warns with health unknown; sources status and the writer_status admin op show unknown instead of blanking', () => withEnv({ GBRAIN_HOME: home }, async () => {
    const f = await managedSource();
    await cursor(f, 2, 7);
    try {
      const broken = failingBacklog(engine);
      const [check] = await managedSyncMovementEntry.run({ engine: broken } as unknown as Parameters<typeof managedSyncMovementEntry.run>[0]) as unknown as Array<Record<string, unknown>>;
      expect(check).toMatchObject({ name: 'managed_sync_not_moving', status: 'warn', details: { health: 'unknown' } });
      const details = check!.details as { unknown: number; sources: Array<{ source_id: string; movement_state: string }> };
      expect(details.unknown).toBeGreaterThanOrEqual(1);
      expect(details.sources).toContainEqual(expect.objectContaining({ source_id: f.id, movement_state: 'unknown' }));
      expect(String(check!.message)).toContain(f.id);
      expect(String(check!.message)).toContain('could not be read');

      const logs: string[] = [];
      const spy = spyOn(console, 'log').mockImplementation((...args: unknown[]) => { logs.push(args.map(String).join(' ')); });
      try { await runSources(broken, ['status', f.id, '--json']); } finally { spy.mockRestore(); }
      const status = JSON.parse(logs.join('\n')) as { sources: Array<{ source_id: string; movement_state?: string; data_moving?: unknown }> };
      expect(status.sources.find(s => s.source_id === f.id)).toMatchObject({ movement_state: 'unknown', data_moving: null });

      const admin = await runPersistenceAdministration(broken, 'writer_status', { source_id: f.id }) as { movement: Array<{ source_id: string; movement_state: string }> };
      expect(admin.movement).toEqual([expect.objectContaining({ source_id: f.id, movement_state: 'unknown' })]);
    } finally { await finish(f); }
  }), 60_000);
});