Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

7.3 KB
Raw Blame
import { beforeEach, expect, mock, test } from 'bun:test';
import { getRedisCache } from '@openpanel/redis';
import type { Logger } from '../../logger';
import {
  type CodeMigrationRunner,
  startCodeMigrations,
} from './code-migration.service';

const LOCK_KEY = 'lock:code-migrations';
const FAST = { lockTtlMs: 300, lockRenewIntervalMs: 50, retryIntervalMs: 50 };

const redis = getRedisCache();
const STILL_RUNNING = 'query still running';

/** The real client, except that every Lua script call goes through `script`. */
function withScripts(script: () => Promise<unknown>): typeof redis {
  return new Proxy(redis, {
    get(target, property) {
      if (property === 'evalsha' || property === 'eval') {
        return script;
      }
      const value = Reflect.get(target, property, target);
      return typeof value === 'function' ? value.bind(target) : value;
    },
  });
}

function silentLogger(): Logger {
  const log = () => undefined;
  const logger: Logger = {
    fatal: log,
    error: log,
    warn: log,
    info: log,
    debug: log,
    trace: log,
    child: () => logger,
  };
  return logger;
}

const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));

/** A runner over an in-memory list; `run` removes a migration once it ran. */
function fakeRunner(
  pending: string[],
  run: (file: string) => Promise<void> = async () => undefined
) {
  const ran: string[] = [];
  const runner = {
    getPending: mock(async () => [...pending]),
    run: mock(async (file: string) => {
      await run(file);
      ran.push(file);
      pending.splice(pending.indexOf(file), 1);
    }),
    abort: mock((): void => undefined),
    allow: mock((): void => undefined),
    isQueryStillRunning: (error: unknown) =>
      error instanceof Error && error.message === STILL_RUNNING,
  } satisfies CodeMigrationRunner;
  return { runner, ran };
}

beforeEach(async () => {
  await redis.del(LOCK_KEY);
});

test('does nothing and never locks when nothing is pending', async () => {
  const { runner } = fakeRunner([]);
  const set = mock(redis.set.bind(redis));
  const handle = startCodeMigrations(
    {
      redis: { ...redis, set } as unknown as typeof redis,
      logger: silentLogger(),
      migrations: runner,
    },
    FAST
  );
  await handle.done;
  expect(runner.getPending).toHaveBeenCalled();
  expect(set).not.toHaveBeenCalled();
  expect(runner.run).not.toHaveBeenCalled();
});

test('runs pending migrations in order and releases the lock', async () => {
  const { runner, ran } = fakeRunner(['27-a.ts', '28-b.ts']);
  const handle = startCodeMigrations(
    { redis, logger: silentLogger(), migrations: runner },
    FAST
  );
  await handle.done;
  expect(ran).toEqual(['27-a.ts', '28-b.ts']);
  expect(runner.allow).toHaveBeenCalled();
  expect(await redis.get(LOCK_KEY)).toBeNull();
});

test('waits while another process holds the lock, then takes over', async () => {
  await redis.set(LOCK_KEY, 'someone-else', 'PX', 10_000);
  const { runner, ran } = fakeRunner(['27-a.ts']);
  const handle = startCodeMigrations(
    { redis, logger: silentLogger(), migrations: runner },
    FAST
  );
  await sleep(200);
  expect(runner.run).not.toHaveBeenCalled();

  await redis.del(LOCK_KEY);
  await handle.done;
  expect(ran).toEqual(['27-a.ts']);
});

test('retries a failed migration and releases the lock in between', async () => {
  let attempts = 0;
  let lockDuringRetryWait: string | null = 'unset';
  const { runner, ran } = fakeRunner(['27-a.ts'], async () => {
    attempts += 1;
    if (attempts === 1) {
      setTimeout(async () => {
        lockDuringRetryWait = await redis.get(LOCK_KEY);
      }, 20);
      throw new Error('clickhouse went away');
    }
  });
  const handle = startCodeMigrations(
    { redis, logger: silentLogger(), migrations: runner },
    FAST
  );
  await handle.done;
  expect(attempts).toBe(2);
  expect(ran).toEqual(['27-a.ts']);
  expect(lockDuringRetryWait).toBeNull();
});

test('aborts when the lock is taken over mid-run', async () => {
  let release: () => void = () => undefined;
  const { runner } = fakeRunner(['27-a.ts'], async () => {
    // Another process steals the lock, then the abort ends this run.
    await redis.set(LOCK_KEY, 'thief', 'PX', 10_000);
    await new Promise<void>((resolve) => {
      release = resolve;
    });
    throw new Error('Migration commands aborted');
  });
  runner.abort.mockImplementation(() => release());
  const handle = startCodeMigrations(
    { redis, logger: silentLogger(), migrations: runner },
    FAST
  );
  await sleep(200);
  expect(runner.abort).toHaveBeenCalled();
  // The thief's lock is not ours to release.
  expect(await redis.get(LOCK_KEY)).toBe('thief');
  await handle.stop();
  await handle.done;
});

test('stop() aborts the running migration and stops retrying', async () => {
  let release: () => void = () => undefined;
  const { runner } = fakeRunner(['27-a.ts'], async () => {
    await new Promise<void>((resolve) => {
      release = resolve;
    });
    throw new Error('Migration commands aborted');
  });
  runner.abort.mockImplementation(() => release());
  const handle = startCodeMigrations(
    { redis, logger: silentLogger(), migrations: runner },
    FAST
  );
  await sleep(50);
  await handle.stop();
  expect(runner.abort).toHaveBeenCalled();
  // Released by the time stop() settles, not only after the TTL.
  expect(await redis.get(LOCK_KEY)).toBeNull();
  await sleep(150);
  expect(runner.run).toHaveBeenCalledTimes(1);
});

test('aborts before the lock can expire when renewals keep failing', async () => {
  let release: () => void = () => undefined;
  const { runner } = fakeRunner(['28-a.ts'], async () => {
    await new Promise<void>((resolve) => {
      release = resolve;
    });
    throw new Error('Migration commands aborted');
  });
  let abortedAfterMs = 0;
  const startedAt = performance.now();
  runner.abort.mockImplementation(() => {
    abortedAfterMs = performance.now() - startedAt;
    release();
  });
  const handle = startCodeMigrations(
    {
      redis: withScripts(() => Promise.reject(new Error('redis unreachable'))),
      logger: silentLogger(),
      migrations: runner,
    },
    FAST
  );
  await sleep(FAST.lockTtlMs + 100);
  expect(runner.abort).toHaveBeenCalled();
  expect(abortedAfterMs).toBeLessThan(FAST.lockTtlMs);
  await handle.stop();
});

test('keeps the lock when an aborted query may still be running', async () => {
  const { runner } = fakeRunner(['28-a.ts'], async () => {
    throw new Error(STILL_RUNNING);
  });
  const handle = startCodeMigrations(
    { redis, logger: silentLogger(), migrations: runner },
    { ...FAST, retryIntervalMs: 10_000 }
  );
  await sleep(50);
  expect(runner.run).toHaveBeenCalledTimes(1);
  expect(await redis.get(LOCK_KEY)).not.toBeNull();
  await handle.stop();
});

test('ignores a renewal answer that arrives after the run finished', async () => {
  const { runner } = fakeRunner(['28-a.ts'], async () => {
    await sleep(FAST.lockRenewIntervalMs + 10);
  });
  // The renewal sent during the run answers "not yours" only after it ended.
  const slowNotOwned = () => sleep(100).then(() => 0);
  const handle = startCodeMigrations(
    {
      redis: withScripts(slowNotOwned),
      logger: silentLogger(),
      migrations: runner,
    },
    FAST
  );
  await handle.done;
  await sleep(150);
  expect(runner.run).toHaveBeenCalledTimes(1);
  expect(runner.abort).not.toHaveBeenCalled();
});