Skip to content

armgit / kaneo

Public Imported from GitHub
0

⚡ Project management that doesn't become the project. Boards, Gantt charts, and GitHub integration. Self-host it or use Kaneo Cloud.

5.2 KB
Raw Blame
import { sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vite-plus/test";
import db from "../../apps/api/src/database";
import { withJobLease } from "../../apps/api/src/scheduler/leader-lock";
import { ensureTestDatabaseMigrated } from "./helpers/database";

const LEASE = "integration-test-lease";

async function clearLease() {
  await db.execute(sql`DELETE FROM job_lease WHERE "name" = ${LEASE};`);
}

async function readLease() {
  const rows = await db.execute(
    sql`SELECT "owner", "expires_at" FROM job_lease WHERE "name" = ${LEASE};`,
  );
  return rows.rows[0];
}

beforeAll(async () => {
  await ensureTestDatabaseMigrated();
});

describe("withJobLease", () => {
  it("lets a second caller skip while the first still holds the lease", async () => {
    await clearLease();

    let firstHasLease = false;
    let secondRan = false;
    let release!: () => void;
    const held = new Promise<void>((resolve) => {
      release = resolve;
    });

    const first = withJobLease(
      LEASE,
      async () => {
        firstHasLease = true;
        await held;
        return "ran";
      },
      () => "skipped",
    );

    try {
      await vi.waitFor(() => {
        expect(firstHasLease).toBe(true);
      });

      const second = await withJobLease(
        LEASE,
        async () => {
          secondRan = true;
          return "ran";
        },
        () => "skipped",
      );

      expect(second).toBe("skipped");
      expect(secondRan).toBe(false);
    } finally {
      release();
      await expect(first).resolves.toBe("ran");
    }
  });

  it("releases the lease so a later run can take it", async () => {
    await clearLease();

    const first = await withJobLease(
      LEASE,
      async () => "ran",
      () => "skipped",
    );
    const second = await withJobLease(
      LEASE,
      async () => "ran",
      () => "skipped",
    );

    expect(first).toBe("ran");
    expect(second).toBe("ran");
    expect(await readLease()).toBeUndefined();
  });

  it("releases the lease when the job throws", async () => {
    await clearLease();

    await expect(
      withJobLease(
        LEASE,
        async () => {
          throw new Error("job blew up");
        },
        () => "skipped",
      ),
    ).rejects.toThrow("job blew up");

    expect(await readLease()).toBeUndefined();

    const after = await withJobLease(
      LEASE,
      async () => "ran",
      () => "skipped",
    );
    expect(after).toBe("ran");
  });

  it("takes over a lease left behind by a crashed replica", async () => {
    await clearLease();

    await db.execute(sql`
      INSERT INTO job_lease ("name", "owner", "expires_at")
      VALUES (${LEASE}, 'dead-replica', timezone('UTC', now()) - interval '1 minute');
    `);

    const result = await withJobLease(
      LEASE,
      async () => "ran",
      () => "skipped",
    );

    expect(result).toBe("ran");
  });

  it("does not take over a lease that is still live", async () => {
    await clearLease();

    await db.execute(sql`
      INSERT INTO job_lease ("name", "owner", "expires_at")
      VALUES (${LEASE}, 'other-replica', timezone('UTC', now()) + interval '10 minutes');
    `);

    const result = await withJobLease(
      LEASE,
      async () => "ran",
      () => "skipped",
    );

    expect(result).toBe("skipped");
    expect((await readLease())?.owner).toBe("other-replica");

    await clearLease();
  });

  it("does not block a different lease name", async () => {
    await clearLease();

    let holderHasLease = false;
    let release!: () => void;
    const held = new Promise<void>((resolve) => {
      release = resolve;
    });

    const holder = withJobLease(
      LEASE,
      async () => {
        holderHasLease = true;
        await held;
        return "ran";
      },
      () => "skipped",
    );

    try {
      await vi.waitFor(() => {
        expect(holderHasLease).toBe(true);
      });

      const other = await withJobLease(
        `${LEASE}-other`,
        async () => "ran",
        () => "skipped",
      );

      expect(other).toBe("ran");
    } finally {
      release();
      await holder;
    }
  });
});

describe.each(["America/New_York", "Asia/Tokyo"])(
  "withJobLease when the process runs in %s",
  (timeZone) => {
    const originalTimeZone = process.env.TZ;

    beforeAll(() => {
      // Node applies TZ changes to Date immediately, so this stands in for a
      // server whose local time is not UTC.
      process.env.TZ = timeZone;
    });

    afterAll(() => {
      if (originalTimeZone === undefined) delete process.env.TZ;
      else process.env.TZ = originalTimeZone;
    });

    it("stores the expiry one lease window from now in UTC", async () => {
      await clearLease();

      const remaining = await withJobLease(
        LEASE,
        async () => {
          const rows = await db.execute(sql`
            SELECT extract(epoch FROM "expires_at" - timezone('UTC', now())) AS "seconds"
            FROM job_lease WHERE "name" = ${LEASE};
          `);
          return Number(rows.rows[0]?.seconds);
        },
        () => Number.NaN,
        60_000,
      );

      expect(remaining).toBeGreaterThan(50);
      expect(remaining).toBeLessThanOrEqual(60);
    });
  },
);