Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
12.8 KB
Raw Blame
import { afterEach, describe, expect, test } from 'bun:test';
import { OperationError } from '../src/core/ops/contract.ts';
import { lastForwardProgressAt, registerRunDeadline } from '../src/core/forward-progress.ts';
import { drainEstimate, drainJsonFields, drainNext, formatDuration, runDrain, syncOutcome, type StallProbe } from '../src/core/persistence/sync-drain.ts';
import type { SyncResult } from '../src/commands/sync.ts';

const base: SyncResult = { status: 'synced', fromCommit: 'a', toCommit: 'b', added: 0, modified: 0, deleted: 0, renamed: 0,
  chunksCreated: 0, embedded: 0, pagesAffected: [] };
const pending = (index: number, total = 10): SyncResult => ({ ...base, status: 'partial', reason: 'writer_pending', managedCursor: { index, total },
  managedWrite: { source_id: 's', slug: 'p', path: 'p.md', write_error: 'write_pending', reason: 'write_pending', message: 'm', suggestion: 's',
    write_request: { request_id: '00000000-0000-0000-0000-000000000001', state: 'queued', retry_after_ms: 0 } as never } });
const yielded = (index: number, total = 10): SyncResult => ({ ...base, status: 'partial', reason: 'writer_yield', managedCursor: { index, total } });
const done = (total = 10): SyncResult => ({ ...base, managedCursor: { index: total, total } });
const RESUME = 'gbrain sync --source s --no-pull';

function scripted(results: Array<SyncResult | Error>) {
  let i = 0;
  return { calls: () => i, pass: async () => { const next = results[Math.min(i++, results.length - 1)]; if (next instanceof Error) throw next; return next; } };
}

afterEach(() => registerRunDeadline(null));

describe('runDrain', () => {
  test('re-enters writer_pending and writer_yield passes until the cursor is done', async () => {
    const s = scripted([pending(1), yielded(3), pending(4), done()]);
    const result = await runDrain({ pass: s.pass, pauseMs: 1 });
    expect(s.calls()).toBe(4);
    expect(result.drain).toMatchObject({ outcome: 'synced', passes: 4, remaining: 0 });
    expect(syncOutcome(result)).toBe('synced');
  });

  test('a stopped signal ends the drain as resumable with the cursor intact', async () => {
    const stop = new AbortController();
    const s = scripted([pending(2)]);
    const result = await runDrain({ pass: async () => { stop.abort(); return s.pass(); }, signal: stop.signal, pauseMs: 1 });
    expect(result.drain).toMatchObject({ outcome: 'resumable', stop_reason: 'deadline', passes: 1, remaining: 8 });
    expect(result.reason).toBe('timeout');
    expect(drainNext(result, RESUME, 's')).toMatchObject({ command: RESUME, safe_to_loop: true });
  });

  test('a strict registered run deadline stops the drain before the watchdog', async () => {
    registerRunDeadline({ atMs: Date.now() + 60, strict: true });
    const started = Date.now();
    const result = await runDrain({ pass: async () => pending(1), pauseMs: 5 });
    expect(result.drain?.outcome).toBe('resumable');
    expect(Date.now() - started).toBeLessThan(2000);
  });

  test('a progress-aware run deadline does not stop a progressing drain', async () => {
    registerRunDeadline({ atMs: Date.now() - 1, strict: false });
    const s = scripted([pending(1), done()]);
    expect((await runDrain({ pass: s.pass, pauseMs: 1 })).drain?.outcome).toBe('synced');
  });

  test('blocked results end the drain as blocked with a retry command', async () => {
    const failed = await runDrain({ pass: async () => ({ ...base, status: 'blocked_by_failures', failedFiles: 1 }) });
    expect(failed.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'blocked_by_failures' });
    expect(drainNext(failed, RESUME, 's')).toMatchObject({ command: RESUME, safe_to_loop: false });
    const terminal = pending(3);
    terminal.managedWrite!.write_error = 'revision_conflict';
    terminal.status = 'blocked_by_failures';
    expect((await runDrain({ pass: async () => terminal })).drain?.outcome).toBe('blocked');
  });

  test('named transient failures retry with bounded attempts; others stop at once', async () => {
    const contention = () => new OperationError('database_contention', 'busy');
    const s = scripted([contention(), contention(), done()]);
    expect((await runDrain({ pass: s.pass, backoffMs: 1 })).drain?.outcome).toBe('synced');
    expect(s.calls()).toBe(3);
    const exhausted = scripted([contention(), contention(), contention(), contention(), done()]);
    await expect(runDrain({ pass: exhausted.pass, backoffMs: 1 })).rejects.toThrow('busy');
    const fatal = scripted([new OperationError('permission_denied', 'no'), done()]);
    await expect(runDrain({ pass: fatal.pass })).rejects.toThrow('no');
    expect(fatal.calls()).toBe(1);
  });

  test('admission contention keeps retrying the frozen request instead of ending the drain', async () => {
    const contention = new OperationError('storage_error', 'Write admission is temporarily blocked by database contention.');
    contention.detail = 'database_contention';
    const s = scripted([contention, contention, contention, contention, contention, done()]);
    const started = Date.now();
    expect((await runDrain({ pass: s.pass })).drain?.outcome).toBe('synced');
    expect(s.calls()).toBe(6);
    expect(Date.now() - started).toBeGreaterThanOrEqual(4500);
  }, 15_000);

  test('worktree_refreshing waits its retry_after_ms hint', async () => {
    const refreshing = new OperationError('worktree_refreshing', 'refreshing');
    refreshing.detail = 'retry_after_ms=5';
    const s = scripted([refreshing, refreshing, refreshing, refreshing, done()]);
    expect((await runDrain({ pass: s.pass })).drain?.outcome).toBe('synced');
    expect(s.calls()).toBe(5);
  });

  test('a recovery-blocked head stops the drain as blocked with the writer command', async () => {
    const stall = { request_id: 'r', state: 'queued', blocked_reason: 'recovery_required', head_request_id: 'h', head_state: 'recovering',
      claimable_here: false, owner_is_this_host: true, stalled_seconds: 0 };
    const probe: StallProbe = { blockedHead: async () => ({ reason: 'recovery_required', stall }), fingerprint: async () => null };
    const result = await runDrain({ pass: async () => pending(2), probe, pauseMs: 1 });
    expect(result.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'recovery_required', stall: { head_request_id: 'h' } });
    expect(drainNext(result, RESUME, 's')).toMatchObject({ command: 'gbrain sources writer status s', safe_to_loop: false });
  });

  test('no change in the awaited request across the stall window stops the drain as drain_stalled', async () => {
    const stall = { request_id: 'r', state: 'queued', blocked_reason: null, head_request_id: 'r', head_state: 'queued', claimable_here: false, owner_is_this_host: false };
    const probe: StallProbe = { blockedHead: async () => null, fingerprint: async () => ({ key: 'same', stall }) };
    const result = await runDrain({ pass: async () => pending(2), probe, pauseMs: 1, stallMs: 20 });
    expect(result.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'drain_stalled', stall: { request_id: 'r', claimable_here: false } });
    expect(result.drain!.passes).toBeGreaterThanOrEqual(4);
  });

  test('a changing fingerprint never counts as a stall', async () => {
    let n = 0;
    const stall = { request_id: 'r', state: 'running', blocked_reason: null, head_request_id: 'r', head_state: 'running', claimable_here: false, owner_is_this_host: true };
    const probe: StallProbe = { blockedHead: async () => null, fingerprint: async () => ({ key: String(n), stall }) };
    const result = await runDrain({ pass: async () => (++n < 8 ? pending(2) : done()), probe, pauseMs: 1, stallMs: 1 });
    expect(result.drain?.outcome).toBe('synced');
  });

  test('progress events count written and waived pages and note forward progress', async () => {
    const before = lastForwardProgressAt();
    const seen: string[] = [];
    const result = await runDrain({ onProgress: e => seen.push(e.phase), pass: async (_signal, onProgress) => {
      onProgress({ phase: 'managed_sync.start', bankedFiles: 0, total: 4 });
      onProgress({ phase: 'managed_sync.page_committed', bankedFiles: 1, total: 4 });
      onProgress({ phase: 'managed_sync.page_committed', bankedFiles: 2, total: 4, waived: true });
      onProgress({ phase: 'managed_sync.page_committed', bankedFiles: 3, total: 4, waived: true });
      return done(4);
    } });
    expect(result.drain).toMatchObject({ processed: 3, written: 1, waived: 2, remaining: 0 });
    expect(seen).toContain('managed_sync.start');
    expect(lastForwardProgressAt()).toBeGreaterThanOrEqual(before);
  });
});

describe('estimates and outcomes', () => {
  test('zero progress reports an unknown rate, never a zero or infinite ETA', () => {
    expect(drainEstimate(100, 0, 60_000)).toEqual({ rate_pages_per_min: null, eta_seconds: null });
    expect(drainEstimate(100, 10, 60_000)).toEqual({ rate_pages_per_min: 10, eta_seconds: 600 });
    expect(drainEstimate(null, 10, 60_000)).toEqual({ rate_pages_per_min: 10, eta_seconds: null });
  });

  test('formatDuration', () => {
    expect(formatDuration(42)).toBe('42s');
    expect(formatDuration(125)).toBe('2m05s');
    expect(formatDuration(3 * 3600 + 12 * 60)).toBe('3h12m');
  });

  test('a single-pass pending write keeps the historical blocked verdict; failed pulls stay blocked', () => {
    expect(syncOutcome(pending(1))).toBe('blocked');
    expect(syncOutcome({ ...base, status: 'partial', reason: 'pull_failed' })).toBe('blocked');
    expect(syncOutcome({ ...base, status: 'partial', reason: 'timeout' })).toBe('resumable');
    expect(syncOutcome({ ...base, status: 'up_to_date' })).toBe('synced');
  });

  test('JSON fields appear only for managed results', () => {
    expect(drainJsonFields(base, RESUME, 's')).toEqual({});
    expect(drainJsonFields(done(), RESUME, 's')).toEqual({ outcome: 'synced' });
  });
});

describe('managed sync backlog', () => {
  const engineWith = (headers: unknown[]) => ({ executeRaw: async () => headers.map(header => ({ header })) }) as never;

  test('estimates from the latest drain window and builds the resume command from stored options', async () => {
    const { readManagedSyncBacklog, formatManagedSyncBacklog } = await import('../src/core/persistence/sync-drain.ts');
    const now = Date.now();
    const [b] = await readManagedSyncBacklog(engineWith([{ sourceId: 'notes', index: 300, total: 1300,
      progress: { startedAt: now - 120_000, startIndex: 100, lastAt: now, lastIndex: 300 }, processingOptions: { noEmbed: true } }]));
    expect(b).toMatchObject({ source_id: 'notes', remaining: 1000, rate_pages_per_min: 100, eta_seconds: 600,
      resume_command: 'gbrain sync --source notes --no-pull --no-embed' });
    expect(formatManagedSyncBacklog(b!)).toContain('indexing ETA 10m00s');
  });

  test('a cursor with no drain window reports rate unknown, never an infinite ETA', async () => {
    const { readManagedSyncBacklog, formatManagedSyncBacklog } = await import('../src/core/persistence/sync-drain.ts');
    const [b] = await readManagedSyncBacklog(engineWith([{ sourceId: 'notes', index: 0, total: 50 }]));
    expect(b).toMatchObject({ remaining: 50, rate_pages_per_min: null, eta_seconds: null });
    expect(formatManagedSyncBacklog(b!)).toContain('rate unknown');
    expect(await readManagedSyncBacklog(engineWith([{ sourceId: 'other', index: 0, total: 5 }]), ['notes'])).toEqual([]);
  });

  test('a resumed drain window resets, so downtime never depresses the rate', async () => {
    const { readManagedSyncBacklog } = await import('../src/core/persistence/sync-drain.ts');
    const now = Date.now();
    const [b] = await readManagedSyncBacklog(engineWith([{ sourceId: 's', index: 500, total: 600,
      progress: { startedAt: now - 60_000, startIndex: 440, lastAt: now, lastIndex: 500 } }]));
    expect(b!.rate_pages_per_min).toBe(60);
  });
});

describe('classified write waits', () => {
  test('a blocked wait stops the drain with its cause; a repeated read failure stops it as database_contention', async () => {
    const blocked = { ...pending(2), writeWait: { status: 'blocked' as const, request_id: 'r', cause: 'owner_unavailable', command: 'gbrain sources writer status s --json' } };
    expect((await runDrain({ pass: async () => blocked, pauseMs: 1 })).drain).toMatchObject({ outcome: 'blocked', stop_reason: 'owner_unavailable' });
    const failing = { ...pending(2), writeWait: { status: 'read_failed' as const, request_id: 'r', reason: 'conn_dropped', transient: true, attempts: 1, message: 'm', why: 'retry' } };
    const s = scripted([failing, failing, failing, done()]);
    const result = await runDrain({ pass: s.pass, pauseMs: 1 });
    expect(result.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'database_contention' });
    expect(drainNext(result, RESUME, 's')).toMatchObject({ command: RESUME, safe_to_loop: false });
    const auth = { ...failing, writeWait: { ...failing.writeWait, reason: 'auth_failed', transient: false } };
    expect((await runDrain({ pass: async () => auth, pauseMs: 1 })).drain?.passes).toBe(1);
  });
});