Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

13.3 KB
Raw Blame
/** monitorWorkerIntegration.test.ts drives the full scheduler → processor → webhook dispatch → webhook received flow;
 * BullMq and ClickHouse are mocked. Posgtgress is injected.
 */
import {
  describe,
  it,
  expect,
  beforeAll,
  beforeEach,
  afterAll,
  afterEach,
  vi,
} from "vitest";
import { v4 } from "uuid";
import { setupServer } from "msw/node";
import { http, HttpResponse } from "msw";
import { Prisma, PrismaClient } from "@prisma/client";

import { JobConfigState } from "@langfuse/shared";
import {
  createOrgProjectAndApiKey,
  redis,
  type WebhookInput,
} from "@langfuse/shared/src/server";
import {
  MonitorProcessor,
  type MonitorQueueEvent,
  MonitorQueueEventSchema,
  MonitorScheduler,
} from "@langfuse/shared/monitors/server";
import {
  MonitorNoDataModeSchema,
  MonitorSeveritySchema,
  MonitorStatusSchema,
  MonitorThresholdOperatorSchema,
} from "@langfuse/shared/monitors";
import { prisma } from "@langfuse/shared/src/db";
import { encrypt, generateWebhookSecret } from "@langfuse/shared/encryption";

import { executeWebhook } from "../queues/webhooks";

describe("monitor-alert e2e (scheduler → processor → dispatcher → webhook URL)", () => {
  let projectId: string;

  beforeAll(() => {
    webhookServer.setup();
  });

  beforeEach(async () => {
    webhookServer.reset();
    ({ projectId } = await createOrgProjectAndApiKey());
  });

  afterEach(async () => {
    vi.restoreAllMocks();
    await prisma.monitor.deleteMany({ where: { projectId } });
    webhookServer.reset();
  });

  afterAll(() => {
    webhookServer.teardown();
  });

  it("posts a monitor-alert envelope to the webhook URL", async () => {
    const { triggerId } = await seedWebhookAutomation({
      projectId,
      url: "https://webhook.example.com/monitor-e2e",
      triggerFilter: [
        {
          column: "severity",
          operator: "any of",
          value: [MonitorSeveritySchema.enum.ALERT],
          type: "stringOptions",
        },
      ],
    });
    const monitorId = await seedMonitor({
      projectId,
      alertThreshold: 100,
      triggerIds: [triggerId],
    });
    // Capture now after seeding so the monitor's nextRunAt is strictly in the past.
    const now = new Date();

    // 1. Scheduler publishes monitor queue events for due Monitors
    let monitorQueueEvents: MonitorQueueEvent[] = [];
    const scheduler = new MonitorScheduler({
      schedulerId: 0,
      totalSchedulers: 1,
      db: prisma,
      publish: async (event) => {
        monitorQueueEvents.push(MonitorQueueEventSchema.parse(event));
      },
    });
    await scheduler.schedule(now);
    monitorQueueEvents = monitorQueueEvents.filter(
      (e) => e.projectId === projectId, // isolate the scheduler check to this test project
    );
    expect(monitorQueueEvents).toHaveLength(1);

    // 2. Processor executes the scheduled job (ClickHouse mocked above threshold).
    const webhookInputs: WebhookInput[] = [];
    const processor = new MonitorProcessor(
      prisma,
      async (event) => {
        webhookInputs.push(event);
      },
      async () => [{ count_count: 150 }],
    );
    await Promise.all(
      monitorQueueEvents.map((monitorQueueEvent) =>
        processor.process(monitorQueueEvent, now),
      ),
    );
    expect(webhookInputs).toHaveLength(1);

    // 3. Webhook processor executes the webhook inputs.
    for (const input of webhookInputs) {
      await executeWebhook(input, { skipValidation: true });
    }

    // 4. Webhook server receives one monitor-alert envelope per scheduled event.
    const requests = webhookServer.getReceivedRequests();
    expect(requests).toHaveLength(monitorQueueEvents.length);
    monitorQueueEvents.forEach((event, i) => {
      const body = JSON.parse(requests[i].body);
      // Window edges anchor to the scheduler's runAt (the cadence boundary), not now.
      const expectedTo = new Date(event.runAt.getTime() - 30_000).toISOString();
      const expectedFrom = new Date(
        event.runAt.getTime() - 30_000 - 5 * 60_000,
      ).toISOString();
      expect(body).toMatchObject({
        type: "monitor-alert",
        apiVersion: "v1",
        payload: {
          monitorId,
          severity: MonitorSeveritySchema.enum.ALERT,
          fromTimestamp: expectedFrom,
          toTimestamp: expectedTo,
        },
      });
    });
  });

  it("redis.del failure on a delivered monitor-alert webhook: does not increment the failure counter", async () => {
    const { triggerId } = await seedWebhookAutomation({
      projectId,
      url: "https://webhook.example.com/monitor-redis-del-fail",
      triggerFilter: [
        {
          column: "severity",
          operator: "any of",
          value: [MonitorSeveritySchema.enum.ALERT],
          type: "stringOptions",
        },
      ],
    });
    const monitorId = await seedMonitor({
      projectId,
      alertThreshold: 100,
      triggerIds: [triggerId],
    });
    const now = new Date();

    const delSpy = vi
      .spyOn(redis!, "del")
      .mockRejectedValueOnce(new Error("READONLY simulated failover"));
    const multiSpy = vi.spyOn(redis!, "multi");

    const monitorQueueEvents: MonitorQueueEvent[] = [];
    const scheduler = new MonitorScheduler({
      schedulerId: 0,
      totalSchedulers: 1,
      db: prisma,
      publish: async (event) => {
        monitorQueueEvents.push(MonitorQueueEventSchema.parse(event));
      },
    });
    await scheduler.schedule(now);
    const projectEvents = monitorQueueEvents.filter(
      (e) => e.projectId === projectId,
    );
    expect(projectEvents).toHaveLength(1);

    const webhookInputs: WebhookInput[] = [];
    const processor = new MonitorProcessor(
      prisma,
      async (event) => {
        webhookInputs.push(event);
      },
      async () => [{ count_count: 150 }],
    );
    await Promise.all(
      projectEvents.map((event) => processor.process(event, now)),
    );
    expect(webhookInputs).toHaveLength(1);

    for (const input of webhookInputs) {
      await expect(
        executeWebhook(input, { skipValidation: true }),
      ).resolves.not.toThrow();
    }

    expect(delSpy).toHaveBeenCalled();
    expect(multiSpy).not.toHaveBeenCalled();

    const trigger = await prisma.trigger.findUnique({
      where: { id: triggerId, projectId },
    });
    expect(trigger?.status).toBe(JobConfigState.ACTIVE);
    expect(monitorId).toBeTruthy();
  });

  it("does not write AutomationExecution rows for monitor-alert dispatches", async () => {
    const now = new Date();
    const { triggerId } = await seedWebhookAutomation({
      projectId,
      url: "https://webhook.example.com/monitor-no-exec",
      triggerFilter: [
        {
          column: "severity",
          operator: "any of",
          value: [MonitorSeveritySchema.enum.ALERT],
          type: "stringOptions",
        },
      ],
    });
    await seedMonitor({
      projectId,
      alertThreshold: 100,
      triggerIds: [triggerId],
    });

    const before = await prisma.automationExecution.count({
      where: { projectId },
    });

    const monitorQueueEvents: MonitorQueueEvent[] = [];
    const scheduler = new MonitorScheduler({
      schedulerId: 0,
      totalSchedulers: 1,
      db: prisma,
      publish: async (event) => {
        monitorQueueEvents.push(MonitorQueueEventSchema.parse(event));
      },
    });
    await scheduler.schedule(now);
    const projectEvents = monitorQueueEvents.filter(
      (e) => e.projectId === projectId,
    );
    expect(projectEvents).toHaveLength(1);

    const captured: WebhookInput[] = [];
    const processor = new MonitorProcessor(
      prisma,
      async (input) => {
        captured.push(input);
      },
      async () => [{ count_count: 150 }],
    );
    await Promise.all(
      projectEvents.map((event) => processor.process(event, now)),
    );
    expect(captured).toHaveLength(1);
    for (const input of captured) {
      await executeWebhook(input, { skipValidation: true });
    }

    const after = await prisma.automationExecution.count({
      where: { projectId },
    });
    expect(after).toBe(before);
  });
});

describe("monitor claim/complete under a non-UTC Postgres session TimeZone", () => {
  let projectId: string;
  let tzDb: PrismaClient;

  beforeAll(() => {
    const url = process.env.DATABASE_URL!;
    const sep = url.includes("?") ? "&" : "?";
    tzDb = new PrismaClient({
      datasources: {
        db: { url: `${url}${sep}options=-c%20timezone%3DAsia%2FShanghai` },
      },
    });
  });

  afterAll(async () => {
    await tzDb.$disconnect();
  });

  beforeEach(async () => {
    ({ projectId } = await createOrgProjectAndApiKey());
  });

  afterEach(async () => {
    await prisma.monitor.deleteMany({ where: { projectId } });
  });

  it("claims and completes a due monitor instead of staying PENDING/UNKNOWN", async () => {
    const session =
      await tzDb.$queryRawUnsafe<Array<Record<string, string>>>(
        "SHOW TIME ZONE",
      );
    expect(Object.values(session[0])[0]).toBe("Asia/Shanghai");

    const monitorId = await seedMonitor({ projectId, alertThreshold: 100 });
    const now = new Date();

    const monitorQueueEvents: MonitorQueueEvent[] = [];
    const scheduler = new MonitorScheduler({
      schedulerId: 0,
      totalSchedulers: 1,
      db: tzDb,
      publish: async (event) => {
        monitorQueueEvents.push(MonitorQueueEventSchema.parse(event));
      },
    });
    await scheduler.schedule(now);
    const projectEvents = monitorQueueEvents.filter(
      (e) => e.projectId === projectId,
    );
    expect(projectEvents).toHaveLength(1);

    const processor = new MonitorProcessor(
      tzDb,
      async () => {},
      async () => [{ count_count: 150 }],
    );
    await Promise.all(
      projectEvents.map((event) => processor.process(event, now)),
    );

    const row = await tzDb.monitor.findUniqueOrThrow({
      where: { id: monitorId },
    });
    expect(row.lastClaimedAt).not.toBeNull();
    expect(row.lastCompletedAt).not.toBeNull();
    expect(row.severity).not.toBe(MonitorSeveritySchema.enum.UNKNOWN);
  });
});

/** WebhookTestServer records webhook POSTs via MSW so tests can assert on them. */
class WebhookTestServer {
  private server;
  private receivedRequests: Array<{
    url: string;
    method: string;
    body: string;
  }> = [];

  constructor() {
    this.server = setupServer(
      http.post("https://webhook.example.com/*", async ({ request }) => {
        this.receivedRequests.push({
          url: request.url,
          method: request.method,
          body: JSON.stringify(await request.json()),
        });
        return HttpResponse.json({ success: true }, { status: 200 });
      }),
    );
  }

  setup() {
    this.server.listen();
  }
  reset() {
    this.receivedRequests = [];
    this.server.resetHandlers();
  }
  teardown() {
    this.server.close();
  }
  getReceivedRequests() {
    return this.receivedRequests;
  }
}

/** webhookServer is the suite-wide MSW capture server. */
const webhookServer = new WebhookTestServer();

/** seedWebhookAutomation creates a monitor-source trigger and webhook automation whose URL MSW captures, returning the created ids for assertions. */
async function seedWebhookAutomation(args: {
  projectId: string;
  url: string;
  triggerFilter: {
    column: string;
    operator: string;
    value: unknown;
    type: string;
  }[];
}) {
  const triggerId = v4();
  await prisma.trigger.create({
    data: {
      id: triggerId,
      projectId: args.projectId,
      eventSource: "monitor",
      eventActions: [],
      filter: args.triggerFilter as unknown as Prisma.InputJsonValue,
      status: JobConfigState.ACTIVE,
    },
  });

  const actionId = v4();
  const { secretKey, displaySecretKey } = generateWebhookSecret();
  await prisma.action.create({
    data: {
      id: actionId,
      projectId: args.projectId,
      type: "WEBHOOK",
      config: {
        type: "WEBHOOK",
        url: args.url,
        apiVersion: { monitor: "v1" },
        secretKey: encrypt(secretKey),
        displaySecretKey,
      },
    },
  });

  const automationId = v4();
  await prisma.automation.create({
    data: {
      id: automationId,
      projectId: args.projectId,
      triggerId,
      actionId,
      name: "Monitor e2e Automation",
    },
  });

  return { triggerId, actionId, automationId };
}

/** seedMonitor inserts a due monitor row; triggerIds gates which monitor-source triggers the processor routes its alerts to. */
async function seedMonitor(args: {
  projectId: string;
  alertThreshold: number;
  triggerIds?: string[];
}) {
  const id = `mon_${v4()}`;
  await prisma.monitor.create({
    data: {
      id,
      projectId: args.projectId,
      view: "OBSERVATIONS",
      filters: [],
      metric: { measure: "count", aggregation: "count" },
      windowMs: 5n * 60n * 1000n,
      cadenceMs: 60n * 1000n,
      thresholdOperator: MonitorThresholdOperatorSchema.enum.GT,
      alertThreshold: args.alertThreshold,
      warningThreshold: null,
      noData: { mode: MonitorNoDataModeSchema.enum.SHOW_NO_DATA },
      renotify: { mode: "OFF" },
      status: MonitorStatusSchema.enum.ACTIVE,
      schedulerBatchId: 0n,
      nextRunAt: new Date(Date.now() - 1000),
      lastPublishedAt: null,
      lastClaimedAt: null,
      lastCompletedAt: null,
      severity: MonitorSeveritySchema.enum.UNKNOWN,
      severityChangedAt: null,
      alertedAt: null,
      tags: [],
      triggerIds: args.triggerIds ?? [],
      name: "e2e monitor",
    },
  });
  return id;
}