Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
38.4 KB
Raw Blame
/**
 * #6317 (B2, T2): a live preparation owned by another process on this host no
 * longer stops the drain at its allowance (that stop turned #6298's
 * run-continuing containment into a `blocked` exit for every caller); the
 * drain prints the stall line naming the owner from the allowance on and
 * stops `drain_stalled` with `cause: owner_wedged_here` only past the ceiling
 * or when the owner's heartbeat row says it is wedged, with `retry_after_ms`
 * to the ceiling. A lapsed head a consumer here can reclaim gets exactly one
 * more window. The same-process check compares the nonce. A foreign-host
 * owner and an overdue publication keep the allowance stop.
 *
 * #6278 (B4): the drain's no-progress detector is claim-aware. Its fingerprint
 * keys on state, blocked reason, head id/state and the head claim's
 * phase/step/since, never on the lease columns every renewal bumps (before
 * this change a renewed-forever claim could never read as a stall, which is
 * how the reporter got the watchdog instead of `blocked` with `next`). A live
 * preparation is allowed its budget plus grace; a lapsed claim keeps the plain
 * window; a preparation this process's own consumer holds past its allowance
 * ends the drain `resumable` / `preparation_abandoned`; a tripped breaker ends
 * it `blocked` / `preparation_systemic`. The stall object and the timer line
 * name the step and wait cause. B6: the timer line also prints during a pass
 * (a grouped publish parked in preparation never returns to the drain loop),
 * from a bounded read of the source's head claim, and never while commits flow.
 */
import { afterAll, beforeAll, describe, expect, spyOn, test } from 'bun:test';
import { execFileSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { claimWorktree } from '../src/core/persistence/ownership.ts';
import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
import { performManagedSync } from '../src/core/persistence/sync-run.ts';
import { resolveBulkSettings } from '../src/core/persistence/sync-group.ts';
import { preparationConfigView } from '../src/core/persistence/config-snapshot.ts';
import { installFaultHook } from '../src/core/persistence/fault-points.ts';
import { withEnv } from './helpers/with-env.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { testBackends } from './helpers/test-backends.ts';
import { drainClaimOf, drainJsonFields, drainNext, engineStallProbe, formatDrainSummary, runDrain, STALL_GRACE_MS, syncOutcome, type DrainClaim, type StallProbe } from '../src/core/persistence/sync-drain.ts';
import { ERROR_CATALOGUE } from '../src/core/error-catalogue.ts';
import type { SyncResult } from '../src/commands/sync.ts';

const base: SyncResult = { status: 'synced', fromCommit: 'a', toCommit: 'b', added: 0, modified: 0, deleted: 0, renamed: 0, chunksCreated: 0, embedded: 0, pagesAffected: [] };
const pending = (index: number, requestId = '00000000-0000-0000-0000-000000000001', total = 10): SyncResult => ({ ...base, status: 'partial', reason: 'writer_pending', managedCursor: { index, total },
  managedWrite: { source_id: 's', slug: 'p', path: 'p.md', write_error: 'write_pending', reason: 'write_pending', message: 'm', suggestion: 's',
    write_request: { request_id: requestId, state: 'queued', retry_after_ms: 0 } as never } });
const done = (total = 10): SyncResult => ({ ...base, managedCursor: { index: total, total } });
const RESUME = 'gbrain sync --source s --no-pull';
const stall = { request_id: 'r', state: 'queued', blocked_reason: null, head_request_id: 'h', head_state: 'running', claimable_here: false, owner_is_this_host: true };
const claim = (over: Partial<DrainClaim> = {}): DrainClaim => ({ phase: 'preparing', step: 'origin_check', waiting_on: 'db', step_age_ms: 45_000, claim_age_ms: 50_000, lapsed: false,
  owner_pid: null, owner_kind: null, owner_nonce: null, last_sql: null, allowance_ms: 150_000, ceiling_ms: 600_000, ...over });
const probeWith = (c: DrainClaim | null, key = 'same', extra: Partial<StallProbe> = {}): StallProbe => ({ blockedHead: async () => null, fingerprint: async () => ({ key, stall, claim: c }), ...extra });
const foreign = { ...stall, owner_is_this_host: false };
const foreignProbe = (c: DrainClaim | null): StallProbe => ({ blockedHead: async () => null, fingerprint: async () => ({ key: 'same', stall: foreign, claim: c }) });

describe('claim-aware no-progress window', () => {
  test('a healthy preparation longer than the 30 s window and shorter than its budget is not cut off', async () => {
    let passes = 0;
    const started = Date.now();
    const result = await runDrain({ pass: async () => (++passes < 12 ? pending(2) : done()), probe: probeWith(claim()), pauseMs: 5, stallMs: 20 });
    // Twelve passes over more than the 20 ms window: the plain rule would have stopped it as drain_stalled at the fourth pass.
    expect(Date.now() - started).toBeGreaterThanOrEqual(40);
    expect(result.drain).toMatchObject({ outcome: 'synced', passes: 12 });
  });

  test('a live preparation past its allowance, owned on another host, stops blocked as drain_stalled with cause, step and wait cause', async () => {
    const result = await runDrain({ pass: async () => pending(2), probe: foreignProbe(claim({ step_age_ms: 160_000, owner_pid: process.pid + 1, owner_kind: 'serve' })), pauseMs: 1, stallMs: 20 });
    expect(result.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'drain_stalled',
      stall: { cause: 'preparation_overdue', step: 'origin_check', waiting_on: 'db', phase: 'preparing', stalled_seconds: 160, owner_pid: process.pid + 1, owner_kind: 'serve' } });
    const next = drainNext(result, RESUME, 's')!;
    expect(next).toMatchObject({ command: 'gbrain sources writer status s', safe_to_loop: false, docs: 'docs/guides/write-refusals.md#drain-stalled', code: 'drain_stalled', cause: 'preparation_overdue' });
    expect(next.why).toContain('stuck at step origin_check, waiting on db');
    expect(next.why).toContain('renewing the claim past the preparation budget');
    expect(formatDrainSummary(result, RESUME, 's').join('\n')).toContain('step=origin_check, waiting_on=db, cause=preparation_overdue');
  });

  describe('#6317 (B2, T2): a same-host owner that renews a preparing claim forever', () => {
    const other = { owner_pid: process.pid + 1, owner_kind: 'serve', owner_nonce: 'other-nonce', last_sql: { label: 'SELECT pages', age_ms: 155_000 } };

    test('past the allowance but inside the ceiling the drain keeps running (the containment, not a blocked exit) and the stall line names the owner and the triple', async () => {
      const err = captureStderr();
      try {
        let passes = 0;
        // Eight passes past the 150 s allowance: before #6317 the fourth pass stopped the drain as drain_stalled / preparation_overdue.
        const result = await runDrain({ announce: true, progressMs: 30, pass: async () => { await new Promise(r => setTimeout(r, 12)); return ++passes < 8 ? pending(2) : done(); },
          probe: probeWith(claim({ step_age_ms: 160_000, claim_age_ms: 165_000, ...other })), pauseMs: 1, stallMs: 20 });
        expect(result.drain).toMatchObject({ outcome: 'synced', passes: 8 });
        expect(err.stalled().length).toBeGreaterThanOrEqual(1);
        expect(err.stalled()[0]).toContain('stalled 160s on origin_check (waiting on db) · held by serve pid ' + (process.pid + 1) + ' · last_sql SELECT pages 155s ago · past the 2m30s allowance; the root is freed at the 10m00s ceiling');
      } finally { err.restore(); }
    });

    test('past the ceiling it stops drain_stalled / owner_wedged_here naming the owner pid, within 2x the window, with the restart as the fix', async () => {
      const started = Date.now();
      const result = await runDrain({ pass: async () => pending(2), probe: probeWith(claim({ step_age_ms: 600_000, claim_age_ms: 610_000, ...other })), pauseMs: 1, stallMs: 20 });
      expect(Date.now() - started).toBeLessThan(2 * 20 + 500);
      expect(result.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'drain_stalled', retry_after_ms: 0,
        stall: { cause: 'owner_wedged_here', owner_pid: process.pid + 1, owner_kind: 'serve', owner_nonce: 'other-nonce', step: 'origin_check', waiting_on: 'db', past_ceiling: true, ceiling_ms: 600_000, last_sql: { label: 'SELECT pages' } } });
      const next = drainNext(result, RESUME, 's')!;
      expect(next).toMatchObject({ code: 'drain_stalled', cause: 'owner_wedged_here', safe_to_loop: false, retry_after_ms: 0, command: 'gbrain sources writer status --source s --json' });
      expect(next.fix).toMatchObject({ next: 'tell_user_to_run', actor: 'host_admin' });
      expect(next.fix!.user_message).toContain(`pid ${process.pid + 1}`);
      expect(next.why).toContain(`held by the serve pid ${process.pid + 1} on this host, past the 10m00s ceiling`);
      expect(formatDrainSummary(result, RESUME, 's').join('\n')).toContain(`last_sql=SELECT pages (155s ago), cause=owner_wedged_here, owner=serve pid ${process.pid + 1}`);
      expect(drainJsonFields(result, RESUME, 's')).toMatchObject({ outcome: 'blocked', next: { fix: { next: 'tell_user_to_run' } } });
    });

    test('an owner whose heartbeat row is wedged stops the drain before the ceiling with retry_after_ms to it and a rerunnable fix (safe to loop)', async () => {
      const owner = async () => ({ kind: 'serve', pid: process.pid + 1, nonce: 'other-nonce', mode: 'full', live: true, restart_required: true, root_barrier_age_ms: null });
      const result = await runDrain({ pass: async () => pending(2), probe: probeWith(claim({ step_age_ms: 160_000, claim_age_ms: 200_000, ...other }), 'same', { owner }), pauseMs: 1, stallMs: 20 });
      expect(result.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'drain_stalled', stall: { cause: 'owner_wedged_here', past_ceiling: false, owner_row: { restart_required: true } } });
      expect(result.drain!.retry_after_ms).toBeGreaterThan(390_000);
      expect(result.drain!.retry_after_ms).toBeLessThanOrEqual(400_000);
      const next = drainNext(result, RESUME, 's')!;
      expect(next).toMatchObject({ safe_to_loop: true, command: RESUME, cause: 'owner_wedged_here' });
      // `wait` derives only from a provider actor (agent-output.ts deriveNext); a wedged owner is not one, so the agent reruns after retry_after_ms.
      expect(next.fix).toMatchObject({ next: 'run', actor: 'agent', command: RESUME });
      expect(next.fix!.verify?.argv).toEqual(['gbrain', 'sources', 'writer', 'status', '--source', 's', '--json']);
      expect(next.why).toContain('reports restart_required');
      // A live, healthy row keeps the drain running.
      let passes = 0;
      const healthy = async () => ({ ...await owner(), restart_required: false, root_barrier_age_ms: 10_000 });
      const kept = await runDrain({ pass: async () => (++passes < 8 ? pending(2) : done()), probe: probeWith(claim({ step_age_ms: 160_000, claim_age_ms: 200_000, ...other }), 'same', { owner: healthy }), pauseMs: 1, stallMs: 20 });
      expect(kept.drain).toMatchObject({ outcome: 'synced', passes: 8 });
    });

    test('a stamp with this pid but another nonce is another process (pid reuse), never preparation_abandoned', async () => {
      const result = await runDrain({ pass: async () => pending(2), probe: probeWith(claim({ step_age_ms: 700_000, claim_age_ms: 700_000, owner_pid: process.pid, owner_nonce: 'not-ours', owner_kind: 'sync' })), pauseMs: 1, stallMs: 20 });
      expect(result.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'drain_stalled', stall: { cause: 'owner_wedged_here', owner_pid: process.pid } });
    });

    test('a lapsed head a consumer here can reclaim gets exactly one extra window, then drain_stalled / owner_missing; without one it stops at once', async () => {
      let reclaimable = true;
      const probe = probeWith(claim({ lapsed: true, step_age_ms: 5_000 }), 'same', { reclaimableHere: async () => reclaimable });
      const startedAt = Date.now();
      const extended = await runDrain({ pass: async () => pending(2), probe, pauseMs: 1, stallMs: 40 });
      const endedAt = Date.now();
      expect(extended.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'drain_stalled', stall: { cause: 'owner_missing' } });
      // Two windows of 40 ms (three passes each at least), not one. Each window runs from the probe after a pass, so the bound is on the drain's wall time, not on pass timestamps.
      expect(endedAt - startedAt).toBeGreaterThanOrEqual(80);
      expect(extended.drain!.passes).toBeGreaterThanOrEqual(6);
      reclaimable = false;
      const at = Date.now();
      const immediate = await runDrain({ pass: async () => pending(2), probe, pauseMs: 1, stallMs: 40 });
      expect(immediate.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'drain_stalled', stall: { cause: 'owner_missing' } });
      expect(Date.now() - at).toBeLessThan(80 + 300);
      expect(drainNext(immediate, RESUME, 's')!.why).toContain('no consumer here reclaimed it');
    });
  });

  test('a preparation this process owns past its allowance ends the drain resumable as preparation_abandoned (exit 0, safe to loop)', async () => {
    const result = await runDrain({ pass: async () => pending(2), probe: probeWith(claim({ step_age_ms: 200_000, owner_pid: process.pid })), pauseMs: 1, stallMs: 20 });
    expect(result.drain).toMatchObject({ outcome: 'resumable', stop_reason: 'preparation_abandoned', stall: { cause: 'preparation_overdue', step: 'origin_check' } });
    expect(syncOutcome(result)).toBe('resumable');
    const next = drainNext(result, RESUME, 's')!;
    expect(next).toMatchObject({ command: RESUME, safe_to_loop: true, docs: 'docs/guides/write-refusals.md#drain-preparation-abandoned' });
    expect(next.why).toContain('outran its budget in this process');
    expect(drainJsonFields(result, RESUME, 's')).toMatchObject({ outcome: 'resumable', next: { safe_to_loop: true } });
  });

  test('a lapsed claim (owner missing) keeps the plain window and names the cause; an unstamped head keeps today\'s rule', async () => {
    const lapsed = await runDrain({ pass: async () => pending(2), probe: probeWith(claim({ lapsed: true, step_age_ms: 5_000 })), pauseMs: 1, stallMs: 20 });
    expect(lapsed.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'drain_stalled', stall: { cause: 'owner_missing', step: 'origin_check' } });
    expect(drainNext(lapsed, RESUME, 's')!.why).toContain('its claim lapsed');
    const unstamped = await runDrain({ pass: async () => pending(2), probe: probeWith(null), pauseMs: 1, stallMs: 20 });
    expect(unstamped.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'drain_stalled', stall: { cause: 'no_progress', step: null } });
    expect(unstamped.drain!.passes).toBeGreaterThanOrEqual(4);
  });

  test('an overdue publication is detected separately from an overdue preparation', async () => {
    const result = await runDrain({ pass: async () => pending(2), probe: probeWith(claim({ phase: 'publishing', step: 'apply', step_age_ms: 400_000, owner_pid: process.pid })), pauseMs: 1, stallMs: 20 });
    expect(result.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'drain_stalled', stall: { cause: 'publication_overdue', phase: 'publishing', step: 'apply' } });
  });

  test('a tripped breaker ends the drain blocked as preparation_systemic with writer status as the next command', async () => {
    const tripped: SyncResult = { ...base, status: 'blocked_by_failures', failedFiles: 1, failureCodes: [{ code: 'preparation_stalled', count: 5 }],
      breaker: { code: 'preparation_systemic', rule: 'consecutive', stalled: 5, consecutive: 5, step: 'knowledge_publication', request_id: 'r5',
        fix: { argv: ['gbrain', 'sources', 'writer', 'status', '--source', 's', '--json'], consent: [], actor: 'agent', requires_exclusive: false, why: 'w' } } };
    const result = await runDrain({ pass: async () => tripped, pauseMs: 1 });
    expect(result.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'preparation_systemic' });
    expect(syncOutcome(result)).toBe('blocked');
    const next = drainNext(result, RESUME, 's')!;
    expect(next).toMatchObject({ command: 'gbrain sources writer status --source s --json', safe_to_loop: false, docs: 'docs/guides/write-refusals.md#preparation_systemic' });
    expect(next.why).toContain('5 writes of this sync could not finish preparing at step knowledge_publication (5 in a row');
    expect(ERROR_CATALOGUE.sync_drain_preparation_systemic.code).toBe('preparation_systemic');
    expect(ERROR_CATALOGUE.sync_drain_preparation_abandoned.code).toBe('preparation_abandoned');
  });
});

describe('drainClaimOf', () => {
  const budgets = { syncMs: 120_000, maintenanceMs: 120_000, ceilingMs: 600_000 };
  const now = Date.parse('2026-10-07T12:00:00Z');
  const stamp = (over: Record<string, unknown> = {}) => ({ phase: 'preparing', claimed_at: '2026-10-07T11:58:00Z', since: '2026-10-07T11:59:00Z', token: 't', ...over });

  test('reads this claim\'s stamp: phase, step, wait cause, ages, pid and the allowance from the head\'s budget plus grace', () => {
    // The stamp claim-phase.ts writes: the pid lives under `owner`, and the step has its own `step_since`.
    const c = drainClaimOf({ head_state: 'running', head_claim_phase: stamp({ step: 'origin_check', step_since: '2026-10-07T11:59:30Z', waiting_on: 'pool', owner: { kind: 'sync', pid: 4242, version: '0.60.109.0' } }),
      head_token: 't', head_lapsed: false, head_kind: 'managed_sync_import' }, budgets, now);
    expect(c).toEqual({ phase: 'preparing', step: 'origin_check', waiting_on: 'pool', step_age_ms: 30_000, claim_age_ms: 120_000, lapsed: false, owner_pid: 4242, owner_kind: 'sync', owner_nonce: null, last_sql: null,
      allowance_ms: 120_000 + STALL_GRACE_MS, ceiling_ms: 600_000 });
    // #6317: the nonce and the last statement ride the stamp when the owner recorded them.
    expect(drainClaimOf({ head_state: 'running', head_claim_phase: stamp({ owner: { kind: 'serve', pid: 7, version: 'v', nonce: 'n1' }, last_sql: { label: 'SELECT pages', at: '2026-10-07T11:59:50Z' } }),
      head_token: 't', head_lapsed: false, head_kind: 'managed_sync_import' }, budgets, now)).toMatchObject({ owner_pid: 7, owner_kind: 'serve', owner_nonce: 'n1', last_sql: { label: 'SELECT pages', age_ms: 10_000 } });
    // A stamp without `step_since` falls back to the phase's `since`; a pre-release top-level `pid` is still read.
    expect(drainClaimOf({ head_state: 'running', head_claim_phase: stamp({ step: 'origin_check', pid: 4243 }), head_token: 't', head_lapsed: false, head_kind: 'managed_sync_import' }, budgets, now))
      .toMatchObject({ step_age_ms: 60_000, owner_pid: 4243 });
    expect(drainClaimOf({ head_state: 'running', head_claim_phase: JSON.stringify(stamp({ phase: 'publishing' })), head_token: 't', head_lapsed: false, head_kind: 'put_page' }, { ...budgets, maintenanceMs: 300_000 }, now))
      .toMatchObject({ phase: 'publishing', step: null, waiting_on: 'unknown', allowance_ms: 300_000 + STALL_GRACE_MS });
    // The ceiling caps the allowance.
    expect(drainClaimOf({ head_state: 'running', head_claim_phase: stamp(), head_token: 't', head_lapsed: false, head_kind: 'managed_sync_import' }, { ...budgets, ceilingMs: 60_000 }, now)!.allowance_ms).toBe(60_000 + STALL_GRACE_MS);
  });
  test('#6650: an undispatched group follower\'s stamp (owner, no phase) is unstamped for the stall window and still names its owner', () => {
    expect(drainClaimOf({ head_state: 'running', head_claim_phase: { token: 't', owner: { kind: 'sync', pid: 9, version: 'v', nonce: 'n9' } }, head_token: 't', head_lapsed: false, head_kind: 'managed_sync_import' }, budgets, now))
      .toMatchObject({ phase: null, step: null, owner_pid: 9, owner_nonce: 'n9' });
  });

  test('an older owner\'s stamp, a stamp of another claim, or no stamp read as null/unknown, never inferred; a lapsed lease is reported', () => {
    expect(drainClaimOf({ head_state: 'running', head_claim_phase: stamp({ token: 'other' }), head_token: 't', head_lapsed: false, head_kind: null }, budgets, now))
      .toMatchObject({ phase: null, step: null, waiting_on: null, step_age_ms: null, owner_pid: null, lapsed: false });
    expect(drainClaimOf({ head_state: 'running', head_claim_phase: null, head_token: 't', head_lapsed: true, head_kind: null }, budgets, now)).toMatchObject({ phase: null, lapsed: true });
    expect(drainClaimOf({ head_state: 'queued', head_claim_phase: stamp(), head_token: 't', head_lapsed: false, head_kind: null }, budgets, now)).toBeNull();
    expect(drainClaimOf({ head_state: 'running', head_claim_phase: stamp({ waiting_on: 'lock' }), head_token: 't', head_lapsed: false, head_kind: null }, budgets, now)!.waiting_on).toBe('unknown');
  });
});

describe('engineStallProbe fingerprint', () => {
  let engine: BrainEngine;
  beforeAll(async () => { const lite = new PGLiteEngine(); await lite.connect({}); await lite.initSchema(); engine = lite; }, 60_000);
  afterAll(async () => { await engine.disconnect(); });

  test('forced probe: lease renewals (updated_at, claim_expires_at) do not change the key; a step change does', async () => {
    const id = `probe-${randomUUID().slice(0, 8)}`;
    await engine.executeRaw("INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,'{}')", [id, `/tmp/${id}`]);
    const [wt] = await engine.executeRaw<{ id: string }>('INSERT INTO persistence_worktrees(owner_host_id) VALUES(gen_random_uuid()) RETURNING id::text AS id');
    const [src] = await engine.executeRaw<{ incarnation: string }>('SELECT incarnation::text AS incarnation FROM sources WHERE id=$1', [id]);
    const token = randomUUID(), requestId = randomUUID();
    const stampOf = (step: string) => JSON.stringify({ phase: 'preparing', claimed_at: new Date(Date.now() - 90_000).toISOString(), since: new Date(Date.now() - 60_000).toISOString(), token, step, step_since: new Date(Date.now() - 60_000).toISOString(), waiting_on: 'db', owner: { kind: 'sync', pid: process.pid, version: 'test' } });
    const [row] = await engine.executeRaw<{ id: string }>(`INSERT INTO persistence_requests(principal_kind,principal_id,request_id,operation,source_id,source_incarnation,slug,worktree_id,digest,authority,intent,intent_bytes,terminal_reservation,
        state,execution_token,claim_expires_at,claim_phase)
      VALUES('local_cli','cli:example',$1::uuid,'submit_job',$2,$3::uuid,'notes/p',$4::uuid,'d','{}'::jsonb,'{"kind":"managed_sync_import","path":"notes/p.md"}'::jsonb,1,16384,'running',$5::uuid,now()+interval '30 seconds',$6::text::jsonb)
      RETURNING id::text AS id`, [requestId, id, src!.incarnation, wt!.id, token, stampOf('origin_check')]);
    const probe = engineStallProbe(engine);
    const result = pending(1, row!.id);
    const first = (await probe.fingerprint(result))!;
    expect(first.claim).toMatchObject({ phase: 'preparing', step: 'origin_check', waiting_on: 'db', lapsed: false, owner_pid: process.pid, allowance_ms: 120_000 + STALL_GRACE_MS });
    expect(first.claim!.step_age_ms).toBeGreaterThanOrEqual(59_000);
    // A renewal: the lease columns move, the key does not.
    await engine.executeRaw("UPDATE persistence_requests SET updated_at=now()+interval '1 second',claim_expires_at=now()+interval '40 seconds' WHERE id=$1::uuid", [row!.id]);
    expect((await probe.fingerprint(result))!.key).toBe(first.key);
    // The owner reaches the next step: the key changes.
    await engine.executeRaw('UPDATE persistence_requests SET claim_phase=$2::text::jsonb WHERE id=$1::uuid', [row!.id, stampOf('knowledge_publication')]);
    const second = (await probe.fingerprint(result))!;
    expect(second.key).not.toBe(first.key);
    expect(second.claim!.step).toBe('knowledge_publication');
    // The lease lapses: the claim reads lapsed and the key changes again.
    await engine.executeRaw("UPDATE persistence_requests SET claim_expires_at=now()-interval '1 second' WHERE id=$1::uuid", [row!.id]);
    const lapsed = (await probe.fingerprint(result))!;
    expect(lapsed.claim!.lapsed).toBe(true);
    expect(lapsed.key).not.toBe(second.key);
  });
});

/** Captures the drain's stderr lines (`serr` writes through console.error without a source prefix). */
function captureStderr(): { lines: string[]; stalled: () => string[]; stalledAt: () => Array<{ line: string; at: number }>; restore(): void } {
  const lines: string[] = [], at: number[] = [];
  const spy = spyOn(console, 'error').mockImplementation((...args: unknown[]) => { lines.push(args.map(String).join(' ')); at.push(Date.now()); });
  const stalledAt = () => lines.flatMap((line, i) => line.includes('· stalled ') ? [{ line, at: at[i]! }] : []);
  return { lines, stalled: () => stalledAt().map(entry => entry.line), stalledAt, restore: () => spy.mockRestore() };
}
/** One head read of the stall probe: when it started and when it settled (the line it feeds prints right after it settles). */
interface HeadRead { startedAt: number; settledAt: number }
/** Wraps a probe's `head` so every read's start and settle times are kept, in settle order. */
function headReadsOf(probe: StallProbe): { probe: StallProbe; list: HeadRead[] } {
  const list: HeadRead[] = [];
  const head = probe.head;
  if (!head) return { probe, list };
  return { list, probe: { ...probe, head: async () => {
    const startedAt = Date.now();
    try { return await head(); } finally { list.push({ startedAt, settledAt: Date.now() }); }
  } } };
}
/**
 * The stall lines that belong to the park: each line is attributed to the latest head read settled at or before it was
 * printed, and counts only when that read started after `parkedAt`. A read in flight across the stamp shows the state
 * before it (a true "preparing" line), so counting lines by position misattributes it to the park.
 */
function parkedStallLines(lines: ReadonlyArray<{ line: string; at: number }>, reads: ReadonlyArray<HeadRead>, parkedAt: number): string[] {
  if (!parkedAt) return [];
  return lines.flatMap(({ line, at }) => {
    const read = [...reads].reverse().find(entry => entry.settledAt <= at);
    return read && read.startedAt >= parkedAt ? [line] : [];
  });
}
const until = async (check: () => boolean, ms: number) => { const end = Date.now() + ms; while (!check() && Date.now() < end) await new Promise(r => setTimeout(r, 10)); return check(); };
const committed = { phase: 'managed_sync.page_committed' } as Parameters<NonNullable<Parameters<typeof runDrain>[0]['onProgress']>>[0];

describe('live stall line during a pass (B6)', () => {
  const PROGRESS = 200;
  const headProbe = (head: StallProbe['head']): StallProbe => ({ blockedHead: async () => null, fingerprint: async () => null, head });

  test('while commits flow nothing prints; once they stop, the head claim is read and named; an empty head prints nothing', async () => {
    const err = captureStderr();
    try {
      let reads = 0;
      const head: StallProbe['head'] = async () => { reads++; return { head_state: 'running', claim: claim({ step_age_ms: 42_000 }) }; };
      // A wider interval here, so a loaded runner's timer gap between 75 ms commits never reads as a stall.
      const every = 500;
      await runDrain({ announce: true, progressMs: every, probe: headProbe(head), pass: async (_signal, onProgress) => {
        // Commits every 75 ms for three intervals: the ticker never reads the head and never prints a stall.
        for (let i = 0; i < 20; i++) { onProgress({ ...committed, total: 30, bankedFiles: i + 1 }); await new Promise(r => setTimeout(r, 75)); }
        expect(err.stalled()).toEqual([]);
        expect(reads).toBe(0);
        // Then nothing commits: within about two intervals the line names the head's step, wait cause, stall age and allowance.
        expect(await until(() => err.stalled().length > 0, 4 * every)).toBe(true);
        return done(30);
      } });
      expect(err.stalled()[0]).toBe('[sync] 20/30 processed · stalled 42s on origin_check (waiting on db) · allowed 2m30s');
      // Rate-limited to one line per interval.
      expect(err.stalled()).toHaveLength(1);
      err.lines.length = 0;
      await runDrain({ announce: true, progressMs: PROGRESS, probe: headProbe(async () => null), pass: async () => { await new Promise(r => setTimeout(r, 4 * PROGRESS)); return done(); } });
      expect(err.stalled()).toEqual([]);
    } finally { err.restore(); }
  });

  // The convergence nightly's owner-kill stall: after a serve died, the drain's one entry queued behind the dead serve's
  // 40-odd in-flight writes, which this process's consumer committed one group after another. The head moved every few
  // seconds, but the line counted from the drain's own last commit and printed "stalled 140s on queued".
  test('a queued head that keeps moving is not a stall: the age counts from when this head became the head', async () => {
    const err = captureStderr();
    try {
      let now = 0, reads = 0;
      const t0 = Date.now();
      const head: StallProbe['head'] = async () => { reads++; now += 10_000; return { head_state: 'queued', claim: null, request_id: `r${Math.min(reads, 4)}` }; };
      await runDrain({ announce: true, progressMs: PROGRESS, governMs: 60_000, clock: () => now + (Date.now() - t0) / 10, probe: headProbe(head), pass: async () => {
        expect(await until(() => err.stalled().length >= 6, 20 * PROGRESS)).toBe(true);
        return done();
      } });
      const ages = err.stalled().slice(0, 6).map(line => Number(/stalled (\d+)s on queued/.exec(line)?.[1]));
      expect(ages).toEqual([10, 10, 10, 10, 20, 30]);
    } finally { err.restore(); }
  });

  // The next night's owner-kill stall: between two queued heads the head was claimed (the consumer publishing the dead
  // serve's writes) for two minutes. Claimed reads print the claim's own age and never moved the queued-head clock, so the
  // next queued head counted from the last queued read and printed "stalled 150s on queued".
  test('a claimed head between two queued heads is progress: the next queued head counts from the read before it', async () => {
    const err = captureStderr();
    try {
      let now = 0, reads = 0;
      const t0 = Date.now();
      const publishing: DrainClaim = { phase: 'publishing', step: null, waiting_on: null, step_age_ms: 2_000, claim_age_ms: 2_000, lapsed: false,
        owner_pid: null, owner_kind: null, owner_nonce: null, last_sql: null, allowance_ms: 150_000, ceiling_ms: 600_000 };
      const head: StallProbe['head'] = async () => {
        reads++; now += 10_000;
        if (reads === 1) return { head_state: 'queued', claim: null, request_id: 'r1' };
        if (reads <= 7) return { head_state: 'claimed', claim: publishing, request_id: `c${reads}` };
        return { head_state: 'queued', claim: null, request_id: 'r2' };
      };
      await runDrain({ announce: true, progressMs: PROGRESS, governMs: 60_000, clock: () => now + (Date.now() - t0) / 10, probe: headProbe(head), pass: async () => {
        expect(await until(() => err.stalled().length >= 9, 30 * PROGRESS)).toBe(true);
        return done();
      } });
      const ages = err.stalled().slice(0, 9).map(line => Number(/stalled (\d+)s on (?:queued|publishing)/.exec(line)?.[1]));
      expect(ages).toEqual([10, 2, 2, 2, 2, 2, 2, 10, 20]);
    } finally { err.restore(); }
  });

  test('a head read that rejects or hangs never escapes the ticker or stacks; announce off never reads', async () => {
    const err = captureStderr();
    try {
      let reads = 0;
      const rejecting = await runDrain({ announce: true, progressMs: PROGRESS, probe: headProbe(async () => { reads++; throw new Error('lock timeout'); }),
        pass: async () => { await new Promise(r => setTimeout(r, 4 * PROGRESS)); return done(); } });
      expect(rejecting.drain?.outcome).toBe('synced');
      expect(reads).toBeGreaterThanOrEqual(1);
      reads = 0;
      await runDrain({ announce: true, progressMs: PROGRESS, probe: headProbe(() => { reads++; return new Promise(() => {}); }),
        pass: async () => { await new Promise(r => setTimeout(r, 5 * PROGRESS)); return done(); } });
      expect(reads).toBe(1);
      reads = 0;
      await runDrain({ announce: false, progressMs: PROGRESS, probe: headProbe(async () => { reads++; return null; }),
        pass: async () => { await new Promise(r => setTimeout(r, 3 * PROGRESS)); return done(); } });
      expect(reads).toBe(0);
      expect(err.stalled()).toEqual([]);
    } finally { err.restore(); }
  });

  describe('forced probe: a real managed sync parked in preparation', () => {
    // PGLite parks the single-write path; Postgres (DATABASE_URL set) parks a member of the grouped publish, the reporter's case.
    const engines: BrainEngine[] = [];
    let closePostgres: (() => Promise<void>) | undefined;
    const home = mkdtempSync(join(tmpdir(), 'gbrain-drain-live-'));
    beforeAll(async () => {
      if (testBackends().includes('pglite')) { const lite = new PGLiteEngine(); await lite.connect({}); await lite.initSchema(); engines.push(lite); }
      if (testBackends().includes('postgres')) { const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!); engines.push(pg.engine); closePostgres = pg.close; }
    }, 120_000);
    afterAll(async () => {
      installFaultHook(undefined);
      for (const engine of engines) { await disposePersistenceConsumer(engine); await engine.disconnect(); }
      await closePostgres?.(); rmSync(home, { recursive: true, force: true });
    });

    test('forced probe: a head read in flight across the park stamp prints a true "preparing" line that is not the park\'s first line', () => {
      // The master flake (#6271's unit-lane arm): read 1 starts before the stamp, settles after it, and its line prints after
      // the park snapshot. Counting lines by position took that line as the parked member's; attributing by read start does not.
      const parkedAt = 1_000;
      const reads: HeadRead[] = [{ startedAt: 990, settledAt: 1_005 }, { startedAt: 1_010, settledAt: 1_020 }];
      const lines = [{ line: '[sync] 1/3 processed · stalled 0s on preparing', at: 1_006 }, { line: '[sync] 1/3 processed · stalled 1s on origin_check (waiting on db) · allowed 2m', at: 1_021 }];
      const byPosition = lines.map(entry => entry.line);
      expect(byPosition[0]).toMatch(/on preparing$/);
      expect(parkedStallLines(lines, reads, parkedAt)).toEqual([lines[1]!.line]);
      // A line before the park, or with no settled read behind it, is never the park's.
      expect(parkedStallLines([{ line: lines[0]!.line, at: 900 }], reads, parkedAt)).toEqual([]);
      expect(parkedStallLines(lines, [], parkedAt)).toEqual([]);
      expect(parkedStallLines(lines, reads, 0)).toEqual([]);
    });

    test('the line names the parked member\'s step within about two intervals, then the sync finishes once it is released', () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_FAILURES_DIR: home }, async () => {
      for (const engine of engines) {
        const id = `live-${randomUUID().replace(/-/g, '').slice(0, 20)}`, root = join(home, id);
        const git = (...args: string[]) => execFileSync('git', ['-C', root, ...args], { stdio: ['ignore', 'pipe', 'pipe'] });
        mkdirSync(join(root, 'notes'), { recursive: true });
        git('init', '-q');
        for (const name of ['a', 'b', 'c']) writeFileSync(join(root, 'notes', `${name}.md`), `---\ntitle: ${name}\n---\nA synthetic observation about ${name}.\n`);
        git('add', '-A'); git('-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qm', 'fixture');
        await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
        await engine.executeRaw("INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,'{}')", [id, root]);
        await claimWorktree(engine, id, root);
        await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
        let release!: () => void;
        const released = new Promise<void>(resolve => { release = resolve; });
        let parkedAt = 0;
        const err = captureStderr();
        // A stall line comes from a head read; only a read that started after the park can show the parked member's stamp.
        // A read started before the stamp committed and settled after it prints a true-at-read-time "preparing" line, so the
        // lines are attributed to their read's start, never counted by position (#6355 fold-in: the master flake on #6271).
        const probe = engineStallProbe(engine, id);
        const reads = headReadsOf(probe);
        const parkedLines = () => parkedStallLines(err.stalledAt(), reads.list, parkedAt);
        // The first member parks in preparation, honouring its signal; the stamp gains the step a preparer would enter (the keys a renewal writes).
        installFaultHook(async (at, detail) => {
          if (at !== 'consumer:preparing' || detail.sourceId !== id || parkedAt) return;
          await engine.executeRaw(`UPDATE persistence_requests SET claim_phase = claim_phase || '{"step":"origin_check","waiting_on":"db"}'::jsonb WHERE request_id=$1::uuid`, [detail.requestId]);
          parkedAt = Date.now();
          await new Promise<void>(resolve => { void released.then(resolve); detail.signal?.addEventListener('abort', () => resolve(), { once: true }); });
        });
        try {
          const bulk = await resolveBulkSettings(await preparationConfigView(engine), false);
          expect(bulk.enabled).toBe(engine.kind === 'postgres');
          const drain = runDrain({ announce: true, progressMs: PROGRESS, probe: reads.probe, bulk: { enabled: bulk.enabled, reason: bulk.reason },
            pass: (signal, onProgress) => performManagedSync(engine, { sourceId: id, noPull: true, noEmbed: true, noExtract: true, signal, onProgress, ...(bulk.enabled ? { bulk } : {}) }) });
          expect(await until(() => parkedAt > 0, 30_000)).toBe(true);
          const seen = await until(() => parkedLines().length > 0, 10 * PROGRESS);
          const seenAfterMs = Date.now() - parkedAt;
          release();
          const result = await drain;
          expect(seen).toBe(true);
          expect(seenAfterMs).toBeLessThanOrEqual(2 * PROGRESS + 400);
          expect(parkedLines()[0]).toMatch(/^\[sync\] \d+\/3 processed · stalled \d+s on origin_check \(waiting on db\) · allowed \d/);
          // One pass: the stall never returned to the drain loop, so only the live read could have named it.
          expect(result.drain).toMatchObject({ outcome: 'synced', written: 3, passes: 1 });
          if (bulk.enabled) expect(result.drain!.bulk!.grouped_pages).toBeGreaterThan(0);
          expect(await engine.getPage('notes/a', { sourceId: id })).not.toBeNull();
        } finally { err.restore(); release(); installFaultHook(undefined); await disposePersistenceConsumer(engine); }
      }
    }), 120_000);
  });
});