Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

19.2 KB
Raw Blame
import {
  afterAll,
  afterEach,
  beforeAll,
  describe,
  expect,
  it,
  vi,
} from "vitest";

import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
import {
  MonitorScheduler,
  type MonitorQueueEventInput,
} from "@langfuse/shared/monitors/server";
import {
  MonitorNoDataModeSchema,
  MonitorStatusSchema,
  MonitorThresholdOperatorSchema,
} from "@langfuse/shared/monitors";
import type { Prisma, PrismaClient } from "@prisma/client";
import { createIsolatedMonitorTable } from "./helpers/isolatedMonitorTable";

let prisma: PrismaClient;
let isolatedDb:
  | Awaited<ReturnType<typeof createIsolatedMonitorTable>>
  | undefined;

beforeAll(async () => {
  isolatedDb = await createIsolatedMonitorTable();
  prisma = isolatedDb.db;
});

afterAll(async () => {
  await isolatedDb?.cleanup();
});

type MonitorStatus = "ACTIVE" | "PAUSED" | "ERROR_BAD_QUERY";
type MonitorView =
  | "OBSERVATIONS"
  | "SCORES_NUMERIC"
  | "SCORES_BOOLEAN"
  | "SCORES_CATEGORICAL";

type SeedOverrides = Partial<{
  id: string;
  schedulerBatchId: bigint;
  cadenceMs: bigint;
  windowMs: bigint;
  nextRunAt: Date | null;
  lastPublishedAt: Date | null;
  lastCompletedAt: Date | null;
  status: MonitorStatus;
  view: MonitorView;
  metric: { measure: string; aggregation: string };
}>;

type MonitorSeed = { id: string } & SeedOverrides;

type ExpectedEvent = {
  schedulerBatchId: string;
  runAt: Date;
  monitorIds: string[];
};

type ExpectedRow = {
  id: string;
  nextRunAt: Date | null;
  lastPublishedAt: Date | null;
  lastCompletedAt: Date | null;
};

type SchedulerCase = {
  name: string;
  scheduler?: { id: number; total: number };
  tick: Date;
  monitors: MonitorSeed[];
  expect: { events: ExpectedEvent[]; rows: ExpectedRow[] };
};

const oneMinuteMs = 60n * 1000n;
const thirtyMinutesMs = 30n * 60n * 1000n;

// Scheduler times
const now = new Date("2026-05-27T12:00:30.000Z");
const twoMinutesAgo = new Date("2026-05-27T11:58:30.000Z");
const threeMinutesAgo = new Date("2026-05-27T11:57:30.000Z");
const fourMinutesAgo = new Date("2026-05-27T11:56:30.000Z");
const sixMinutesAgo = new Date("2026-05-27T11:54:30.000Z");
const tenMinutesAgo = new Date("2026-05-27T11:50:30.000Z");
const oneMinuteFromNow = new Date("2026-05-27T12:01:30.000Z");
const prevCadence30m = new Date("2026-05-27T12:00:00.000Z");
const nextCadence30m = new Date("2026-05-27T12:30:00.000Z");

// Cadence-aligned boundaries (cadence=1m), offset by (batchId % 60) seconds.
const prevCadence = new Date("2026-05-27T12:00:00.000Z");
const nextCadence = new Date("2026-05-27T12:01:00.000Z");
const prevCadenceBatch1 = new Date("2026-05-27T12:00:01.000Z");
const nextCadenceBatch1 = new Date("2026-05-27T12:01:01.000Z");
const prevCadenceBatch2 = new Date("2026-05-27T12:00:02.000Z");
const nextCadenceBatch2 = new Date("2026-05-27T12:01:02.000Z");
const prevCadenceBatch5 = new Date("2026-05-27T12:00:05.000Z");
const nextCadenceBatch5 = new Date("2026-05-27T12:01:05.000Z");
const prevCadenceBatch7 = new Date("2026-05-27T12:00:07.000Z");
const nextCadenceBatch7 = new Date("2026-05-27T12:01:07.000Z");

async function seedMonitor(projectId: string, seed: MonitorSeed) {
  return prisma.monitor.create({
    data: {
      id: seed.id,
      projectId,
      view: seed.view ?? "OBSERVATIONS",
      filters: [] as unknown as Prisma.InputJsonValue,
      metric: (seed.metric ?? {
        measure: "count",
        aggregation: "count",
      }) as unknown as Prisma.InputJsonValue,
      windowMs: seed.windowMs ?? 5n * oneMinuteMs,
      cadenceMs: seed.cadenceMs ?? oneMinuteMs,
      thresholdOperator: MonitorThresholdOperatorSchema.enum.GT,
      alertThreshold: 100,
      warningThreshold: null,
      noData: {
        mode: MonitorNoDataModeSchema.enum.SHOW_NO_DATA,
      } as unknown as Prisma.InputJsonValue,
      renotify: { mode: "OFF" } as unknown as Prisma.InputJsonValue,
      status: seed.status ?? MonitorStatusSchema.enum.ACTIVE,
      schedulerBatchId: seed.schedulerBatchId ?? 0n,
      nextRunAt: seed.nextRunAt === undefined ? null : seed.nextRunAt,
      lastPublishedAt: seed.lastPublishedAt ?? null,
      lastCompletedAt: seed.lastCompletedAt ?? null,
      name: `Test ${seed.id}`,
      tags: [],
    },
  });
}

function makeScheduler(
  publish: (event: MonitorQueueEventInput) => Promise<void>,
  shard: { id: number; total: number } = { id: 0, total: 1 },
) {
  return new MonitorScheduler({
    schedulerId: shard.id,
    totalSchedulers: shard.total,
    db: prisma,
    publish,
  });
}

const cases: SchedulerCase[] = [
  {
    name: "no monitors on the system: schedule returns 0, publish not called",
    tick: now,
    monitors: [],
    expect: { events: [], rows: [] },
  },
  {
    name: "all monitors ahead: every row filtered by WHERE, nothing published or advanced",
    tick: now,
    monitors: [
      { id: "m_ahead_a", nextRunAt: oneMinuteFromNow },
      { id: "m_ahead_b", nextRunAt: oneMinuteFromNow },
    ],
    expect: {
      events: [],
      rows: [
        {
          id: "m_ahead_a",
          nextRunAt: oneMinuteFromNow,
          lastPublishedAt: null,
          lastCompletedAt: null,
        },
        {
          id: "m_ahead_b",
          nextRunAt: oneMinuteFromNow,
          lastPublishedAt: null,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "new monitor: publishes at tick, advances to next boundary",
    tick: now,
    monitors: [{ id: "m_new", nextRunAt: null }],
    expect: {
      events: [{ schedulerBatchId: "0", runAt: now, monitorIds: ["m_new"] }],
      rows: [
        {
          id: "m_new",
          nextRunAt: nextCadence,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "mixed batch: new monitor pulls existing sibling forward to tick",
    tick: now,
    monitors: [
      { id: "m_existing", nextRunAt: prevCadence },
      { id: "m_new", nextRunAt: null },
    ],
    expect: {
      events: [
        {
          schedulerBatchId: "0",
          runAt: now,
          monitorIds: ["m_existing", "m_new"],
        },
      ],
      rows: [
        {
          id: "m_existing",
          nextRunAt: nextCadence,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
        {
          id: "m_new",
          nextRunAt: nextCadence,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "behind monitor: publishes at prior next_run_at, advances",
    tick: now,
    monitors: [{ id: "m_behind", nextRunAt: prevCadence }],
    expect: {
      events: [
        {
          schedulerBatchId: "0",
          runAt: prevCadence,
          monitorIds: ["m_behind"],
        },
      ],
      rows: [
        {
          id: "m_behind",
          nextRunAt: nextCadence,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "in-flight + last_completed=NULL within TTL: not published, advanced",
    tick: now,
    monitors: [
      {
        id: "m_pending",
        nextRunAt: prevCadence,
        lastPublishedAt: twoMinutesAgo,
        lastCompletedAt: null,
      },
    ],
    expect: {
      events: [],
      rows: [
        {
          id: "m_pending",
          nextRunAt: nextCadence,
          lastPublishedAt: twoMinutesAgo,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "in-flight + completed-before-published within TTL: not published, advanced",
    tick: now,
    monitors: [
      {
        id: "m_inflight_completed",
        nextRunAt: prevCadence,
        lastPublishedAt: twoMinutesAgo,
        lastCompletedAt: threeMinutesAgo,
      },
    ],
    expect: {
      events: [],
      rows: [
        {
          id: "m_inflight_completed",
          nextRunAt: nextCadence,
          lastPublishedAt: twoMinutesAgo,
          lastCompletedAt: threeMinutesAgo,
        },
      ],
    },
  },
  {
    name: "in-flight past TTL: republishes (rescue), advances",
    tick: now,
    monitors: [
      {
        id: "m_stuck",
        nextRunAt: prevCadence,
        lastPublishedAt: tenMinutesAgo,
        lastCompletedAt: null,
      },
    ],
    expect: {
      events: [
        {
          schedulerBatchId: "0",
          runAt: prevCadence,
          monitorIds: ["m_stuck"],
        },
      ],
      rows: [
        {
          id: "m_stuck",
          nextRunAt: nextCadence,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "long-cadence in-flight past TTL: rescue branch republishes",
    tick: now,
    monitors: [
      {
        id: "m_long_stuck",
        cadenceMs: thirtyMinutesMs,
        nextRunAt: nextCadence30m,
        lastPublishedAt: sixMinutesAgo,
        lastCompletedAt: null,
      },
    ],
    expect: {
      events: [
        {
          schedulerBatchId: "0",
          runAt: prevCadence30m,
          monitorIds: ["m_long_stuck"],
        },
      ],
      rows: [
        {
          id: "m_long_stuck",
          nextRunAt: nextCadence30m,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "long-cadence in-flight within TTL: rescue branch does not fire",
    tick: now,
    monitors: [
      {
        id: "m_long_pending",
        cadenceMs: thirtyMinutesMs,
        nextRunAt: nextCadence30m,
        lastPublishedAt: fourMinutesAgo,
        lastCompletedAt: null,
      },
    ],
    expect: {
      events: [],
      rows: [
        {
          id: "m_long_pending",
          nextRunAt: nextCadence30m,
          lastPublishedAt: fourMinutesAgo,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "very behind: snaps forward to most recent boundary, skips intermediates",
    tick: now,
    monitors: [{ id: "m_far_behind", nextRunAt: tenMinutesAgo }],
    expect: {
      events: [
        {
          schedulerBatchId: "0",
          runAt: prevCadence,
          monitorIds: ["m_far_behind"],
        },
      ],
      rows: [
        {
          id: "m_far_behind",
          nextRunAt: nextCadence,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "paused monitor: untouched",
    tick: now,
    monitors: [
      {
        id: "m_paused",
        status: MonitorStatusSchema.enum.PAUSED,
        nextRunAt: prevCadence,
      },
    ],
    expect: {
      events: [],
      rows: [
        {
          id: "m_paused",
          nextRunAt: prevCadence,
          lastPublishedAt: null,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "error monitor: untouched",
    tick: now,
    monitors: [
      {
        id: "m_error",
        status: MonitorStatusSchema.enum.ERROR_BAD_QUERY,
        nextRunAt: prevCadence,
      },
    ],
    expect: {
      events: [],
      rows: [
        {
          id: "m_error",
          nextRunAt: prevCadence,
          lastPublishedAt: null,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "multi-monitor batch: one event, both monitors, deduped metrics",
    tick: now,
    monitors: [
      {
        id: "m_a",
        schedulerBatchId: 7n,
        nextRunAt: prevCadence,
        metric: { measure: "count", aggregation: "count" },
      },
      {
        id: "m_b",
        schedulerBatchId: 7n,
        nextRunAt: prevCadence,
        metric: { measure: "latency", aggregation: "p95" },
      },
    ],
    expect: {
      events: [
        {
          schedulerBatchId: "7",
          runAt: prevCadenceBatch7,
          monitorIds: ["m_a", "m_b"],
        },
      ],
      rows: [
        {
          id: "m_a",
          nextRunAt: nextCadenceBatch7,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
        {
          id: "m_b",
          nextRunAt: nextCadenceBatch7,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "5 monitors batched into 2 events (batching + ordering)",
    tick: now,
    monitors: [
      { id: "m_a1", schedulerBatchId: 1n, nextRunAt: twoMinutesAgo },
      { id: "m_a2", schedulerBatchId: 1n, nextRunAt: twoMinutesAgo },
      { id: "m_a3", schedulerBatchId: 1n, nextRunAt: twoMinutesAgo },
      { id: "m_b1", schedulerBatchId: 2n, nextRunAt: tenMinutesAgo },
      { id: "m_b2", schedulerBatchId: 2n, nextRunAt: tenMinutesAgo },
    ],
    expect: {
      events: [
        {
          schedulerBatchId: "1",
          runAt: prevCadenceBatch1,
          monitorIds: ["m_a1", "m_a2", "m_a3"],
        },
        {
          schedulerBatchId: "2",
          runAt: prevCadenceBatch2,
          monitorIds: ["m_b1", "m_b2"],
        },
      ],
      rows: [
        {
          id: "m_a1",
          nextRunAt: nextCadenceBatch1,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
        {
          id: "m_a2",
          nextRunAt: nextCadenceBatch1,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
        {
          id: "m_a3",
          nextRunAt: nextCadenceBatch1,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
        {
          id: "m_b1",
          nextRunAt: nextCadenceBatch2,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
        {
          id: "m_b2",
          nextRunAt: nextCadenceBatch2,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "sharding: only this scheduler's slot is claimed",
    scheduler: { id: 1, total: 4 },
    tick: now,
    monitors: [
      { id: "m_in", schedulerBatchId: 5n, nextRunAt: prevCadence },
      { id: "m_out", schedulerBatchId: 4n, nextRunAt: prevCadence },
    ],
    expect: {
      events: [
        {
          schedulerBatchId: "5",
          runAt: prevCadenceBatch5,
          monitorIds: ["m_in"],
        },
      ],
      rows: [
        {
          id: "m_in",
          nextRunAt: nextCadenceBatch5,
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
        {
          id: "m_out",
          nextRunAt: prevCadence,
          lastPublishedAt: null,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "new monitor, 1m cadence: publishes at tick, advances to cadence-aligned + 17s slot",
    tick: now,
    monitors: [
      { id: "m_cadence_1m", schedulerBatchId: 17n, cadenceMs: oneMinuteMs },
    ],
    expect: {
      events: [
        { schedulerBatchId: "17", runAt: now, monitorIds: ["m_cadence_1m"] },
      ],
      rows: [
        {
          id: "m_cadence_1m",
          nextRunAt: new Date("2026-05-27T12:01:17.000Z"),
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "new monitor, 30m cadence: publishes at tick, advances to cadence-aligned + 17s slot",
    tick: now,
    monitors: [
      {
        id: "m_cadence_30m",
        schedulerBatchId: 17n,
        cadenceMs: thirtyMinutesMs,
      },
    ],
    expect: {
      events: [
        { schedulerBatchId: "17", runAt: now, monitorIds: ["m_cadence_30m"] },
      ],
      rows: [
        {
          id: "m_cadence_30m",
          nextRunAt: new Date("2026-05-27T12:30:17.000Z"),
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
      ],
    },
  },
  {
    name: "new monitor, 1d cadence: publishes at tick, advances to cadence-aligned + 17s slot",
    tick: now,
    monitors: [
      {
        id: "m_cadence_1d",
        schedulerBatchId: 17n,
        cadenceMs: 24n * 60n * 60n * 1000n,
      },
    ],
    expect: {
      events: [
        { schedulerBatchId: "17", runAt: now, monitorIds: ["m_cadence_1d"] },
      ],
      rows: [
        {
          id: "m_cadence_1d",
          nextRunAt: new Date("2026-05-28T00:00:17.000Z"),
          lastPublishedAt: now,
          lastCompletedAt: null,
        },
      ],
    },
  },
];

describe("MonitorScheduler (integration)", () => {
  let projectId: string;

  beforeAll(async () => {
    const org = await createOrgProjectAndApiKey();
    projectId = org.projectId;
  });

  afterEach(async () => {
    await prisma.monitor.deleteMany({ where: { projectId } });
  });

  it.each(cases)("$name", async (c) => {
    for (const m of c.monitors) {
      await seedMonitor(projectId, m);
    }

    const publish = vi.fn<(event: MonitorQueueEventInput) => Promise<void>>();
    await makeScheduler(publish, c.scheduler).schedule(c.tick);

    // Filter to this project's events — the scheduler is global, so concurrent
    // test files in the same shard can leak monitors into the publish call.
    const myEvents = publish.mock.calls
      .flatMap((call) => call[0])
      .filter((e) => e.projectId === projectId);

    expect(myEvents).toHaveLength(c.expect.events.length);
    myEvents.forEach((event, i) => {
      const exp = c.expect.events[i];
      expect(event.schedulerBatchId).toBe(exp.schedulerBatchId);
      expect(event.runAt).toEqual(exp.runAt);
      expect(event.monitors.map((m) => m.monitorId).sort()).toEqual(
        [...exp.monitorIds].sort(),
      );
    });

    for (const exp of c.expect.rows) {
      const row = await prisma.monitor.findUniqueOrThrow({
        where: { id: exp.id },
      });
      expect(row.nextRunAt?.toISOString() ?? null).toBe(
        exp.nextRunAt?.toISOString() ?? null,
      );
      expect(row.lastPublishedAt?.toISOString() ?? null).toBe(
        exp.lastPublishedAt?.toISOString() ?? null,
      );
      expect(row.lastCompletedAt?.toISOString() ?? null).toBe(
        exp.lastCompletedAt?.toISOString() ?? null,
      );
    }
  });

  it("computes the same next_run_at across two ticks with the same (cadence, schedulerBatchId)", async () => {
    await seedMonitor(projectId, {
      id: "m_det",
      schedulerBatchId: 17n,
      nextRunAt: prevCadence,
    });

    const publish1 = vi.fn<(event: MonitorQueueEventInput) => Promise<void>>();
    await makeScheduler(publish1).schedule(now);
    const firstNext = (
      await prisma.monitor.findUniqueOrThrow({ where: { id: "m_det" } })
    ).nextRunAt;

    await prisma.monitor.update({
      where: { id: "m_det" },
      data: { nextRunAt: prevCadence, lastPublishedAt: null },
    });

    const publish2 = vi.fn<(event: MonitorQueueEventInput) => Promise<void>>();
    await makeScheduler(publish2).schedule(now);
    const secondNext = (
      await prisma.monitor.findUniqueOrThrow({ where: { id: "m_det" } })
    ).nextRunAt;

    expect(firstNext?.toISOString()).toBe(secondNext?.toISOString());
  });
});

describe("MonitorScheduler (timeouts)", () => {
  const stubDb = (rows: unknown[]) => {
    const executeRawUnsafe = vi.fn().mockResolvedValue(0);
    const queryRaw = vi.fn().mockResolvedValue(rows);
    const transaction = vi.fn(async (ops: Promise<unknown>[]) =>
      Promise.all(ops),
    );
    return {
      db: {
        $executeRawUnsafe: executeRawUnsafe,
        $queryRaw: queryRaw,
        $transaction: transaction,
      } as unknown as PrismaClient,
      executeRawUnsafe,
      transaction,
    };
  };

  it("claim: bounds the query with a statement timeout", async () => {
    const { db, executeRawUnsafe, transaction } = stubDb([]);

    await new MonitorScheduler({
      schedulerId: 0,
      totalSchedulers: 1,
      db,
      publish: vi.fn(),
    }).schedule(now);

    expect(executeRawUnsafe).toHaveBeenCalledWith(
      "SET LOCAL statement_timeout = 20000",
    );
    expect(transaction).toHaveBeenCalledWith(expect.any(Array));
  });
});