Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
9.4 KB
Raw Blame
/**
 * #6706: a managed drain bounded by a caller deadline (the `sync` job's `timeout_at`).
 *
 * Protects: the drain stops `resumable / deadline` before the caller's deadline and never starts a pass after it; a past
 * deadline makes zero passes; the earlier of the CLI run deadline and the caller deadline wins; a far deadline does not
 * fire early (setTimeout overflow); a deadline landing in a retry backoff is a resumable report, not a throw; a pass that
 * ignores the abort is abandoned after the grace; and `jobDisposition` classifies every drained stop.
 */
import { afterEach, describe, expect, test } from 'bun:test';
import { OperationError } from '../src/core/ops/contract.ts';
import { registerRunDeadline } from '../src/core/forward-progress.ts';
import { deadlinePassed, jobDisposition, runDrain, type DrainReport } from '../src/core/persistence/sync-drain.ts';
import { stampDrainStep } from '../src/core/persistence/drain-step.ts';
import type { SyncResult } from '../src/commands/sync.ts';

const base: SyncResult = { status: 'synced', fromCommit: 'a', toCommit: 'b', added: 0, modified: 0, deleted: 0, renamed: 0,
  chunksCreated: 0, embedded: 0, pagesAffected: [] };
const pending = (index: number, total = 10): SyncResult => ({ ...base, status: 'partial', reason: 'writer_pending', managedCursor: { index, total } });

afterEach(() => registerRunDeadline(null));

describe('runDrain with a caller deadline', () => {
  test('stops resumable/deadline before the deadline and starts no pass after it', async () => {
    const deadlineAtMs = Date.now() + 400;
    const starts: number[] = [];
    const result = await runDrain({ deadlineAtMs, pauseMs: 20, pass: async () => { starts.push(Date.now()); return pending(starts.length); } });
    expect(result.drain).toMatchObject({ outcome: 'resumable', stop_reason: 'deadline' });
    expect(result.reason).toBe('timeout');
    expect(Date.now()).toBeLessThan(deadlineAtMs + 100);
    expect(starts.every(at => at < deadlineAtMs)).toBe(true);
    expect(starts.length).toBeGreaterThan(1);
  });

  test('a deadline already past makes exactly zero passes', async () => {
    let calls = 0;
    const result = await runDrain({ deadlineAtMs: Date.now() - 1, pass: async () => { calls++; return pending(1); } });
    expect(calls).toBe(0);
    expect(result.drain).toMatchObject({ outcome: 'resumable', stop_reason: 'deadline', passes: 0 });
  });

  test('the earlier of the strict run deadline and the caller deadline wins', async () => {
    registerRunDeadline({ atMs: Date.now() + 60_000, strict: true });
    const started = Date.now();
    const early = await runDrain({ deadlineAtMs: Date.now() + 200, pauseMs: 10, pass: async () => pending(1) });
    expect(early.drain?.stop_reason).toBe('deadline');
    expect(Date.now() - started).toBeLessThan(2_000);
    registerRunDeadline({ atMs: Date.now() + 200, strict: true });
    const runFirst = Date.now();
    const late = await runDrain({ deadlineAtMs: Date.now() + 60_000, pauseMs: 10, pass: async () => pending(1) });
    expect(late.drain?.stop_reason).toBe('deadline');
    expect(Date.now() - runFirst).toBeLessThan(2_000);
  });

  test('a deadline 30 days out does not fire early', async () => {
    let calls = 0;
    const result = await runDrain({ deadlineAtMs: Date.now() + 30 * 24 * 3_600_000, pauseMs: 1,
      pass: async () => (++calls < 3 ? pending(calls) : { ...base, managedCursor: { index: 10, total: 10 } }) });
    expect(calls).toBe(3);
    expect(result.drain?.outcome).toBe('synced');
  });

  test('a deadline that lands in a transient retry backoff returns resumable instead of throwing', async () => {
    let calls = 0;
    const result = await runDrain({ deadlineAtMs: Date.now() + 150, backoffMs: 400,
      pass: async () => { calls++; if (calls === 1) return pending(2); throw new OperationError('database_contention', 'busy'); } });
    expect(result.drain).toMatchObject({ outcome: 'resumable', stop_reason: 'deadline', remaining: 8 });
  });

  test('a fatal error after the deadline still throws', async () => {
    await expect(runDrain({ deadlineAtMs: Date.now() + 100, pauseMs: 1, pass: async () => {
      await new Promise(resolve => setTimeout(resolve, 150));
      throw new OperationError('permission_denied', 'no');
    } })).rejects.toThrow('no');
  });

  test('a pass that ignores the abort is abandoned after the grace and its late result is dropped', async () => {
    let lateSignal: AbortSignal | undefined;
    const started = Date.now();
    const result = await runDrain({ deadlineAtMs: Date.now() + 100, deadlineGraceMs: 50, pass: signal => { lateSignal = signal; return new Promise<SyncResult>(() => undefined); } });
    expect(result.drain).toMatchObject({ outcome: 'resumable', stop_reason: 'deadline' });
    expect(Date.now() - started).toBeLessThan(1_000);
    expect(lateSignal?.aborted).toBe(true);
  });

  test('an abandoned pass is fenced at its next step even though the deadline aborted its signal first', async () => {
    let passSignal: AbortSignal | undefined;
    await runDrain({ deadlineAtMs: Date.now() + 100, deadlineGraceMs: 30, pass: signal => { passSignal = signal; return new Promise<SyncResult>(() => undefined); } });
    expect(passSignal?.aborted).toBe(true);
    expect((passSignal!.reason as Error).message).toBe('run deadline');
    expect(() => stampDrainStep('admission', passSignal)).toThrow('pass abandoned');
  });

  test('a reconnect that hangs past the deadline is not awaited; the stop is resumable', async () => {
    const drop = Object.assign(new Error('Connection terminated unexpectedly'), { code: 'ECONNRESET' });
    let calls = 0;
    const started = Date.now();
    const result = await runDrain({ deadlineAtMs: Date.now() + 200, backoffMs: 1, reconnect: () => new Promise<void>(() => undefined),
      pass: async () => { calls++; if (calls === 1) return pending(3); throw drop; } });
    expect(result.drain).toMatchObject({ outcome: 'resumable', stop_reason: 'deadline' });
    expect(Date.now() - started).toBeLessThan(2_000);
  });

  test('deadlinePassed is true only inside the cooperative margin', () => {
    const now = 1_000_000;
    expect(deadlinePassed(undefined, now)).toBe(false);
    expect(deadlinePassed(now + 60_000, now)).toBe(false);
    expect(deadlinePassed(now + 10_000, now)).toBe(true);
    expect(deadlinePassed(now - 1, now)).toBe(true);
  });
});

describe('jobDisposition', () => {
  const drained = (drain: Partial<DrainReport>, over: Partial<SyncResult> = {}): SyncResult => ({ ...base, status: 'partial', reason: 'timeout', ...over,
    drain: { outcome: 'resumable', passes: 1, processed: 1, written: 1, waived: 0, remaining: 3, rate_pages_per_min: null, eta_seconds: null, ...drain } });

  test('a result with no drain keeps the master result (null)', () => {
    expect(jobDisposition(base, 's')).toBeNull();
    expect(jobDisposition({ ...base, status: 'partial', reason: 'timeout' }, 's')).toBeNull();
  });

  test.each([
    ['synced', drained({ outcome: 'synced', remaining: 0 }, { status: 'synced', reason: undefined }), 'synced'],
    ['deadline', drained({ stop_reason: 'deadline' }), 'continue'],
    ['preparation_abandoned', drained({ stop_reason: 'preparation_abandoned' }), 'continue'],
    ['connection_lost', drained({ outcome: 'blocked', stop_reason: 'connection_lost', connection: { drops: 3, last_error: 'x' } }), 'continue'],
    ['write_capacity', drained({ outcome: 'blocked', stop_reason: 'write_capacity' }), 'needs_operator'],
    ['database_contention', drained({ outcome: 'blocked', stop_reason: 'database_contention' }), 'needs_operator'],
    ['preparation_systemic', drained({ outcome: 'blocked', stop_reason: 'preparation_systemic' }), 'needs_operator'],
    ['blocked_by_failures', drained({ outcome: 'blocked', stop_reason: 'blocked_by_failures' }, { status: 'blocked_by_failures' }), 'needs_operator'],
    ['recovery_required', drained({ outcome: 'blocked', stop_reason: 'recovery_required' }), 'needs_operator'],
    ['owner_unavailable', drained({ outcome: 'blocked', stop_reason: 'owner_unavailable' }), 'needs_operator'],
    ['unexpected_file_bytes', drained({ outcome: 'blocked', stop_reason: 'unexpected_file_bytes' }), 'needs_operator'],
    ['unexpected_staging_bytes', drained({ outcome: 'blocked', stop_reason: 'unexpected_staging_bytes' }), 'needs_operator'],
    ['drain_stalled (no progress)', drained({ outcome: 'blocked', stop_reason: 'drain_stalled', stall: { request_id: 'r', state: 'running', blocked_reason: null, head_request_id: null, head_state: null, claimable_here: null, owner_is_this_host: null, stalled_seconds: 30, cause: 'no_progress' } }), 'needs_operator'],
    ['drain_stalled (owner wedged, ceiling ahead)', drained({ outcome: 'blocked', stop_reason: 'drain_stalled', retry_after_ms: 60_000, stall: { request_id: 'r', state: 'running', blocked_reason: null, head_request_id: null, head_state: null, claimable_here: null, owner_is_this_host: true, stalled_seconds: 300, cause: 'owner_wedged_here', past_ceiling: false, owner_pid: 1, owner_kind: 'serve' } }), 'continue'],
    ['drain_stalled (owner wedged, past ceiling)', drained({ outcome: 'blocked', stop_reason: 'drain_stalled', retry_after_ms: 0, stall: { request_id: 'r', state: 'running', blocked_reason: null, head_request_id: null, head_state: null, claimable_here: null, owner_is_this_host: true, stalled_seconds: 700, cause: 'owner_wedged_here', past_ceiling: true, owner_pid: 1, owner_kind: 'serve' } }), 'needs_operator'],
  ] as const)('%s', (_name, result, expected) => {
    expect(jobDisposition(result as SyncResult, 's')).toBe(expected);
  });
});