Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
11.1 KB
Raw Blame
/**
 * v0.41.6.0 D4 — tryRunPendingMigrations + isDeadlockError.
 *
 * Hermetic unit tests using the `_hooks` test seam — no real DB needed.
 * Verifies the retry-and-poll contract:
 *
 *   1. first attempt succeeds → status: 'ok'
 *   2. 40P01 once → retry succeeds → status: 'ok'
 *   3. 40P01 twice + hasPending flips to false → status: 'race_resolved'
 *   4. 40P01 twice + hasPending stays true past deadline → status: 'persistent'
 *   5. non-40P01 error propagates as status: 'error'
 *   6. not_needed early-exit when hasPending returns false up front
 *   7. 250ms poll interval honored (custom interval respected)
 *   8. isDeadlockError matches SQLSTATE 40P01 in code field
 *   9. isDeadlockError matches "deadlock detected" in message text
 *  10. initSchema retry catches raw 57014 setup flakes
 *  11. initSchema retry does not wrap an exhausted per-migration retry
 */
import { describe, test, expect } from 'bun:test';
import {
  tryRunPendingMigrations,
  isDeadlockError,
  MigrationRetryExhausted,
  type TryRunPendingMigrationsResult,
} from '../src/core/migrate.ts';
import { runInitSchemaWithRetry } from '../src/core/init-schema-retry.ts';
import type { BrainEngine } from '../src/core/engine.ts';

// Fake engine for typing — never used because _hooks override everything.
const fakeEngine = {} as BrainEngine;

class FakeDeadlock extends Error {
  code = '40P01';
  constructor() { super('deadlock detected'); this.name = 'FakeDeadlock'; }
}

/**
 * Deterministic clock through the `_hooks.now`/`_hooks.sleep` seam: `sleep` advances `now` by exactly `ms`, so the
 * poll loop's deadline arithmetic is exact and a loaded runner's slow `setTimeout(1)` (7-10 ms, which admitted 3 polls
 * in 30 ms on CI) cannot change the count. Forced probe: a `sleep` that also stalls 10 ms of real time gives 4
 * `hasPending` calls on the real clock and 31 here.
 */
function fakeClock() {
  let t = 0;
  return {
    now: () => t,
    hooks: { now: () => t, sleep: async (ms: number) => { t += ms; } },
  };
}

class FakeStatementTimeout extends Error {
  code = '57014';
  constructor() {
    super('canceling statement due to statement timeout');
    this.name = 'FakeStatementTimeout';
  }
}

describe('isDeadlockError', () => {
  test('matches SQLSTATE 40P01 in `.code`', () => {
    expect(isDeadlockError({ code: '40P01', message: 'whatever' })).toBe(true);
  });

  test('matches SQLSTATE 40P01 in `.sqlState`', () => {
    expect(isDeadlockError({ sqlState: '40P01', message: 'whatever' })).toBe(true);
  });

  test('matches "deadlock detected" in message text (driver-shape independent)', () => {
    expect(isDeadlockError(new Error('ERROR: deadlock detected'))).toBe(true);
  });

  test('matches the literal "40P01" token in message', () => {
    expect(isDeadlockError(new Error('Postgres returned 40P01'))).toBe(true);
  });

  test('returns false for non-deadlock errors', () => {
    expect(isDeadlockError(new Error('connection refused'))).toBe(false);
    expect(isDeadlockError({ code: '23505', message: 'duplicate key' })).toBe(false);
  });

  test('returns false for null/undefined', () => {
    expect(isDeadlockError(null)).toBe(false);
    expect(isDeadlockError(undefined)).toBe(false);
  });
});

describe('runInitSchemaWithRetry', () => {
  test('retries raw statement_timeout failures from schema setup', async () => {
    let initCalls = 0;
    const sleeps: number[] = [];
    const logs: string[] = [];

    const result = await runInitSchemaWithRetry(fakeEngine, {
      maxAttempts: 5,
      backoffMs: 0,
      log: (line) => logs.push(line),
      _hooks: {
        sleep: async (ms) => { sleeps.push(ms); },
        initSchema: async () => {
          initCalls++;
          if (initCalls < 3) throw new FakeStatementTimeout();
        },
      },
    });

    expect(result.attempts).toBe(3);
    expect(initCalls).toBe(3);
    expect(sleeps).toEqual([0, 0]);
    expect(logs).toHaveLength(2);
    expect(logs[0]).toContain('statement_timeout');
  });

  test('#5227: a schema_lock_blocked refusal (or raw 55P03) retries with the short lock backoff, bounded', async () => {
    let initCalls = 0;
    const sleeps: number[] = [];
    const logs: string[] = [];
    const blocked = Object.assign(new Error('Schema replay waited 1s for a table lock another session holds'), { code: 'schema_lock_blocked' });
    // Blocker released after attempt 1: success on attempt 2.
    const result = await runInitSchemaWithRetry(fakeEngine, {
      maxAttempts: 5, backoffMs: 15_000, lockBackoffMs: 250, log: line => logs.push(line),
      _hooks: { sleep: async ms => { sleeps.push(ms); }, initSchema: async () => { initCalls++; if (initCalls < 2) throw blocked; } },
    });
    expect(result.attempts).toBe(2);
    expect(sleeps).toEqual([250]);
    expect(logs[0]).toContain('schema_lock_blocked');
    // Never released: a bounded failure with the typed error, not a hang.
    let calls = 0; const waits: number[] = [];
    await expect(runInitSchemaWithRetry(fakeEngine, {
      maxAttempts: 3, backoffMs: 15_000, lockBackoffMs: 250, log: () => {},
      _hooks: { sleep: async ms => { waits.push(ms); }, initSchema: async () => { calls++; throw { code: '55P03', message: 'canceling statement due to lock timeout' }; } },
    })).rejects.toMatchObject({ code: '55P03' });
    expect(calls).toBe(3);
    expect(waits).toEqual([250, 250]);
  });

  test('does not double-wrap an exhausted migration retry envelope', async () => {
    const exhausted = new MigrationRetryExhausted(
      16,
      'sample_migration',
      3,
      [],
      new FakeStatementTimeout(),
    );
    let initCalls = 0;
    const sleeps: number[] = [];

    await expect(runInitSchemaWithRetry(fakeEngine, {
      backoffMs: 0,
      _hooks: {
        sleep: async (ms) => { sleeps.push(ms); },
        initSchema: async () => {
          initCalls++;
          throw exhausted;
        },
      },
    })).rejects.toBe(exhausted);

    expect(initCalls).toBe(1);
    expect(sleeps).toEqual([]);
  });
});

describe('tryRunPendingMigrations', () => {
  test('returns not_needed when hasPending returns false up front', async () => {
    const result = await tryRunPendingMigrations(fakeEngine, {
      _hooks: {
        hasPending: async () => false,
        initSchema: async () => { throw new Error('should not be called'); },
      },
    });
    expect(result.status).toBe('not_needed');
  });

  test('returns ok when first initSchema succeeds', async () => {
    let initCalls = 0;
    const result = await tryRunPendingMigrations(fakeEngine, {
      _hooks: {
        hasPending: async () => true,
        initSchema: async () => { initCalls++; },
      },
    });
    expect(result.status).toBe('ok');
    if (result.status === 'ok') expect(result.attempts).toBe(1);
    expect(initCalls).toBe(1);
  });

  test('returns ok after one 40P01 + retry succeeds', async () => {
    let initCalls = 0;
    const result = await tryRunPendingMigrations(fakeEngine, {
      retryBackoffMs: 0, // skip the real backoff for test speed
      _hooks: {
        hasPending: async () => true,
        initSchema: async () => {
          initCalls++;
          if (initCalls === 1) throw new FakeDeadlock();
        },
      },
    });
    expect(result.status).toBe('ok');
    if (result.status === 'ok') expect(result.attempts).toBe(2);
    expect(initCalls).toBe(2);
  });

  test('returns race_resolved when both attempts deadlock but hasPending flips to false', async () => {
    let initCalls = 0;
    let pendingCallCount = 0;
    const clock = fakeClock();
    const result = await tryRunPendingMigrations(fakeEngine, {
      retryBackoffMs: 0,
      pollIntervalMs: 5,
      deadlineMs: 100,
      _hooks: {
        ...clock.hooks,
        hasPending: async () => {
          pendingCallCount++;
          // Entry-call always pending; after a couple of poll iterations,
          // flip to false (simulating another runner finishing the migration).
          if (pendingCallCount <= 3) return true;
          return false;
        },
        initSchema: async () => { initCalls++; throw new FakeDeadlock(); },
      },
    });
    expect(result.status).toBe('race_resolved');
    if (result.status === 'race_resolved') {
      expect(result.attempts).toBe(2); // exhausted retries
      expect(result.pollIterations).toBeGreaterThan(0);
      // Entry call + three pending polls, then the fourth poll sees the other runner finish: t = 3 × 5 ms.
      expect(result.pollIterations).toBe(3);
      expect(clock.now()).toBe(15);
    }
    expect(initCalls).toBe(2);
  });

  test('returns persistent when both attempts deadlock + hasPending stays true past deadline', async () => {
    const clock = fakeClock();
    const result = await tryRunPendingMigrations(fakeEngine, {
      retryBackoffMs: 0,
      pollIntervalMs: 10,
      deadlineMs: 50, // tight deadline for test speed
      _hooks: {
        ...clock.hooks,
        hasPending: async () => true, // never resolves
        initSchema: async () => { throw new FakeDeadlock(); },
      },
    });
    expect(result.status).toBe('persistent');
    if (result.status === 'persistent') {
      expect(result.attempts).toBe(2);
      expect(result.pollIterations).toBeGreaterThan(0);
      expect(result.pollIterations).toBe(5); // 50 ms deadline / 10 ms interval, exactly
      expect(isDeadlockError(result.error)).toBe(true);
    }
  });

  test('returns error when first initSchema throws a non-40P01 error', async () => {
    const realFailure = new Error('connection refused');
    const result = await tryRunPendingMigrations(fakeEngine, {
      _hooks: {
        hasPending: async () => true,
        initSchema: async () => { throw realFailure; },
      },
    });
    expect(result.status).toBe('error');
    if (result.status === 'error') expect(result.error).toBe(realFailure);
  });

  test('returns error when second attempt throws non-40P01 (real failure beats deadlock)', async () => {
    let initCalls = 0;
    const realFailure = new Error('disk full');
    const result = await tryRunPendingMigrations(fakeEngine, {
      retryBackoffMs: 0,
      _hooks: {
        hasPending: async () => true,
        initSchema: async () => {
          initCalls++;
          if (initCalls === 1) throw new FakeDeadlock();
          throw realFailure;
        },
      },
    });
    expect(result.status).toBe('error');
    if (result.status === 'error') expect(result.error).toBe(realFailure);
  });

  test('honors custom pollIntervalMs (test seam allows tight intervals)', async () => {
    let pollCalls = 0;
    const clock = fakeClock();
    const result = await tryRunPendingMigrations(fakeEngine, {
      retryBackoffMs: 0,
      pollIntervalMs: 1,
      deadlineMs: 30,
      _hooks: {
        ...clock.hooks,
        hasPending: async () => { pollCalls++; return true; },
        initSchema: async () => { throw new FakeDeadlock(); },
      },
    });
    expect(result.status).toBe('persistent');
    // pollCalls = 1 (initial check) + many polling iterations.
    expect(pollCalls).toBeGreaterThan(5);
    // On the fake clock the count is exact: 30 ms deadline / 1 ms interval, plus the entry call.
    if (result.status === 'persistent') expect(result.pollIterations).toBe(30);
    expect(pollCalls).toBe(31);
  });
});