Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
5.1 KB
Raw Blame
/**
 * #6028 diagnostic: when the fenced refresh of a held migration lease matches
 * no row but the row still names this process (same pid and host), the
 * runner did not lose the lease to another runner, so it must not say
 * "another apply-migrations is running" naming its own pid. `assertHeld`
 * raises `migration_lease_lost` with which part of the fence no longer
 * matches (no token values), and the runner reports it (exit 1).
 * Run: DATABASE_URL=... bun test test/e2e/migration-lease-lost.test.ts
 */
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { hostname } from 'node:os';
import { rmSync } from 'node:fs';
import { hasDatabase, setupDB, teardownDB, getConn } from './helpers.ts';
import { collect, logHas, makeHome, spawnDriver, writeDriver } from '../helpers/apply-migrations-lock-driver.ts';
import {
  acquireMigrationOrchestrationLock,
  MIGRATION_ORCHESTRATION_LOCK_ID,
  MigrationsRunningError,
} from '../../src/core/migration-orchestration-lock.ts';

const describeE2E = hasDatabase() ? describe : describe.skip;
const config = () => ({ engine: 'postgres' as const, database_url: process.env.DATABASE_URL! });

async function heldLock() {
  const lock = await acquireMigrationOrchestrationLock(config());
  if (!lock) throw new Error('expected the Postgres migration lease');
  return lock;
}

type LeaseLost = Error & { code: string; details: Record<string, unknown> };

async function caught(promise: Promise<unknown>): Promise<unknown> {
  try { await promise; } catch (error) { return error; }
  throw new Error('expected assertHeld to throw');
}

describeE2E('migration lease lost under this runner (#6028, Postgres)', () => {
  beforeAll(async () => { await setupDB(); });
  afterAll(async () => { await teardownDB(); });
  beforeEach(async () => {
    await getConn().unsafe('DELETE FROM gbrain_cycle_locks WHERE id = $1', [MIGRATION_ORCHESTRATION_LOCK_ID]);
  });

  test('own row with a rewritten acquisition token: migration_lease_lost, token mismatch, fence match', async () => {
    const lock = await heldLock();
    try {
      await getConn().unsafe('UPDATE gbrain_cycle_locks SET acquisition_token = gen_random_uuid() WHERE id = $1', [MIGRATION_ORCHESTRATION_LOCK_ID]);
      const lost = await caught(lock.assertHeld()) as LeaseLost;
      expect(lost.code).toBe('migration_lease_lost');
      expect(lost.details).toMatchObject({ holder_pid: process.pid, holder_host: hostname(), token_match: false, fence_match: true });
      expect(lost.message).not.toContain('another apply-migrations is running');
      expect(JSON.stringify(lost.details)).not.toContain(lock.leaseToken!);
    } finally {
      await lock.release();
    }
  });

  // #6028 (wave 14 PR4): the lease predicate matches (id, holder_pid, acquisition_token); the acquisition
  // time is diagnostic only, so a clock or storage rewrite of acquired_at alone no longer loses the lease.
  test('own row with a moved acquisition time alone stays held; moved time plus a rotated token reports both', async () => {
    const lock = await heldLock();
    try {
      await getConn().unsafe(`UPDATE gbrain_cycle_locks SET acquired_at = acquired_at - interval '1 second' WHERE id = $1`, [MIGRATION_ORCHESTRATION_LOCK_ID]);
      await lock.assertHeld();
      await getConn().unsafe('UPDATE gbrain_cycle_locks SET acquisition_token = gen_random_uuid() WHERE id = $1', [MIGRATION_ORCHESTRATION_LOCK_ID]);
      const lost = await caught(lock.assertHeld()) as LeaseLost;
      expect(lost.code).toBe('migration_lease_lost');
      expect(lost.details).toMatchObject({ token_match: false, fence_match: false });
    } finally {
      await lock.release();
    }
  });

  test('a row that names another holder still refuses as migrations_running', async () => {
    const lock = await heldLock();
    try {
      await getConn().unsafe(`UPDATE gbrain_cycle_locks SET holder_pid = 4242, holder_host = 'host-b', acquisition_token = gen_random_uuid() WHERE id = $1`, [MIGRATION_ORCHESTRATION_LOCK_ID]);
      const error = await caught(lock.assertHeld());
      expect(error).toBeInstanceOf(MigrationsRunningError);
      expect((error as Error).message).toContain('host host-b, pid 4242');
    } finally {
      await lock.release();
      await getConn().unsafe('DELETE FROM gbrain_cycle_locks WHERE id = $1', [MIGRATION_ORCHESTRATION_LOCK_ID]);
    }
  });

  test('the runner reports migration_lease_lost (exit 1) and stops before the next migration', async () => {
    const home = makeHome({ engine: 'postgres', database_url: process.env.DATABASE_URL! });
    try {
      const run = await collect(spawnDriver(home, writeDriver(home, { holdMs: 0, openDatastore: false, rotateOwnToken: true })));
      expect(run.code, run.stderr + run.stdout).toBe(1);
      expect(run.stderr).toContain('Error [migration_lease_lost]');
      expect(run.stderr).not.toContain('another apply-migrations is running');
      expect(logHas(home, 'start')).toBe(false);
    } finally {
      await getConn().unsafe('DELETE FROM gbrain_cycle_locks WHERE id = $1', [MIGRATION_ORCHESTRATION_LOCK_ID]);
      rmSync(home, { recursive: true, force: true });
    }
  }, 120_000);
});