Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
5.5 KB
Raw Blame
/**
 * #2308 — bounded wait for a dead CROSS-HOST lock holder.
 *
 * A holder that died on another host leaves a row whose TTL is still live;
 * tryAcquireDbLock correctly refuses to steal it (cross_host classification,
 * process.kill is meaningless remotely) — so one-shot callers exited
 * LOCK_HELD even though the lock frees itself within TTL + steal-grace.
 * waitForDbLockTakeover polls the normal acquire up to that bound, bailing
 * fast when the holder's heartbeat advances (genuinely alive).
 */

import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import {
  tryAcquireDbLock,
  waitForDbLockTakeover,
} from '../src/core/db-lock.ts';
import { resolveSupervisorLockWaitSeconds } from '../src/core/minions/supervisor.ts';
import { withEnv } from './helpers/with-env.ts';

let eng: PGLiteEngine;

beforeAll(async () => {
  eng = new PGLiteEngine();
  await eng.connect({});
  await eng.initSchema();
}, 60_000);

afterAll(async () => {
  await eng.disconnect();
});

beforeEach(async () => {
  await eng.executeRaw('DELETE FROM gbrain_cycle_locks', []);
});

const LOCK_ID = 'cross-host-wait-test';

/** Seed a cross-host holder row. ttlMs/refreshAgoMs are relative to NOW(). */
async function seedHolder(opts: { ttlMs: number; refreshAgoMs: number; pid?: number }) {
  await eng.executeRaw(
    `INSERT INTO gbrain_cycle_locks (id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at)
     VALUES ($1, $2, 'other-host-zzz', NOW() - INTERVAL '10 minutes',
             NOW() + ($3 || ' milliseconds')::interval,
             NOW() - ($4 || ' milliseconds')::interval)`,
    [LOCK_ID, opts.pid ?? 99999, String(opts.ttlMs), String(opts.refreshAgoMs)],
  );
}

describe('waitForDbLockTakeover (#2308)', () => {
  test('precondition: dead cross-host holder with live TTL blocks the one-shot acquire', async () => {
    await seedHolder({ ttlMs: 60_000, refreshAgoMs: 10 * 60_000 });
    expect(await tryAcquireDbLock(eng, LOCK_ID, 1)).toBeNull();
  });

  test('dead holder: acquired once the TTL lapses (within the bound)', async () => {
    // TTL lapses in ~1.2s; last refresh is 10min ago (past the 60s grace for
    // a 1-min TTL) — the normal upsert takes over on the first post-lapse poll.
    await seedHolder({ ttlMs: 1_200, refreshAgoMs: 10 * 60_000 });
    const handle = await waitForDbLockTakeover(eng, LOCK_ID, 1, {
      pollMs: 200,
      maxWaitMs: 15_000,
    });
    expect(handle).not.toBeNull();
    // the row now names US
    const rows = await eng.executeRaw<{ holder_pid: number }>(
      'SELECT holder_pid FROM gbrain_cycle_locks WHERE id = $1',
      [LOCK_ID],
    );
    expect(Number(rows[0].holder_pid)).toBe(process.pid);
    await handle!.release();
  }, 30_000);

  test('refreshing holder: heartbeat advance bails to null fast', async () => {
    await seedHolder({ ttlMs: 5 * 60_000, refreshAgoMs: 0 });
    const started = Date.now();
    let bumped = false;
    const handle = await waitForDbLockTakeover(eng, LOCK_ID, 1, {
      pollMs: 100,
      maxWaitMs: 30_000,
      sleep: async (ms) => {
        // Simulate the remote holder's heartbeat firing between our polls.
        if (!bumped) {
          bumped = true;
          await eng.executeRaw(
            `UPDATE gbrain_cycle_locks SET last_refreshed_at = NOW() + INTERVAL '1 second' WHERE id = $1`,
            [LOCK_ID],
          );
        }
        await new Promise((r) => setTimeout(r, ms));
      },
    });
    expect(handle).toBeNull();
    // bailed on the heartbeat, nowhere near the 30s bound
    expect(Date.now() - started).toBeLessThan(10_000);
  }, 30_000);

  test('bound honored: unreapable holder returns null at maxWaitMs', async () => {
    await seedHolder({ ttlMs: 10 * 60_000, refreshAgoMs: 0 });
    const started = Date.now();
    const handle = await waitForDbLockTakeover(eng, LOCK_ID, 1, {
      pollMs: 150,
      maxWaitMs: 900,
    });
    expect(handle).toBeNull();
    const elapsed = Date.now() - started;
    expect(elapsed).toBeGreaterThanOrEqual(850);
    expect(elapsed).toBeLessThan(10_000);
  }, 30_000);

  test('holder replaced by a different (pid,host) while waiting → null (they are alive)', async () => {
    await seedHolder({ ttlMs: 5 * 60_000, refreshAgoMs: 0, pid: 11111 });
    let swapped = false;
    const handle = await waitForDbLockTakeover(eng, LOCK_ID, 1, {
      pollMs: 100,
      maxWaitMs: 30_000,
      sleep: async (ms) => {
        if (!swapped) {
          swapped = true;
          await eng.executeRaw(
            `UPDATE gbrain_cycle_locks SET holder_pid = 22222, acquisition_token = gen_random_uuid() WHERE id = $1`,
            [LOCK_ID],
          );
        }
        await new Promise((r) => setTimeout(r, ms));
      },
    });
    expect(handle).toBeNull();
  }, 30_000);
});

describe('resolveSupervisorLockWaitSeconds — env hatch', () => {
  const KEY = 'GBRAIN_SUPERVISOR_LOCK_WAIT_SECONDS';

  test('unset → -1 (derived TTL+grace bound)', async () => {
    await withEnv({ [KEY]: undefined }, () => {
      expect(resolveSupervisorLockWaitSeconds()).toBe(-1);
    });
  });

  test('0 disables the wait', async () => {
    await withEnv({ [KEY]: '0' }, () => {
      expect(resolveSupervisorLockWaitSeconds()).toBe(0);
    });
  });

  test('positive integer is the hard bound; junk falls back to derived', async () => {
    await withEnv({ [KEY]: '90' }, () => {
      expect(resolveSupervisorLockWaitSeconds()).toBe(90);
    });
    await withEnv({ [KEY]: 'nonsense' }, () => {
      expect(resolveSupervisorLockWaitSeconds()).toBe(-1);
    });
  });
});