Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

88.3 KB
Raw Blame
import {
  describe,
  it,
  expect,
  beforeEach,
  beforeAll,
  afterAll,
  afterEach,
  vi,
} from "vitest";
import { v4 } from "uuid";
import dns from "node:dns/promises";
import { backOff } from "exponential-backoff";
import {
  ActionExecutionStatus,
  JobConfigState,
  PromptDomainSchema,
  WebhookActionConfigWithSecrets,
} from "@langfuse/shared";
import {
  WebhookInput,
  createOrgProjectAndApiKey,
  getActionById,
  getActionByIdWithSecrets,
  logger,
  redis,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import {
  decrypt,
  encrypt,
  generateWebhookSignature,
} from "@langfuse/shared/encryption";
import { generateWebhookSecret } from "@langfuse/shared/encryption";
import { setupServer } from "msw/node";
import { http, HttpResponse } from "msw";
import { executeWebhook } from "../queues/webhooks";

const outboundWhitelistBuilders = vi.hoisted(() => ({
  webhookWhitelistFromEnv: vi.fn(),
  whitelistFromEnv: vi.fn(),
  realWebhookWhitelistFromEnv: (): {
    hosts: string[];
    ips: string[];
    ip_ranges: string[];
    allowedHosts?: string[];
  } => ({ hosts: [], ips: [], ip_ranges: [] }),
  realWhitelistFromEnv: (): {
    hosts: string[];
    ips: string[];
    ip_ranges: string[];
    allowedHosts?: string[];
  } => ({ hosts: [], ips: [], ip_ranges: [] }),
}));

vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
  const actual =
    await importOriginal<typeof import("@langfuse/shared/src/server")>();
  outboundWhitelistBuilders.realWebhookWhitelistFromEnv =
    actual.webhookWhitelistFromEnv;
  outboundWhitelistBuilders.realWhitelistFromEnv = actual.whitelistFromEnv;
  outboundWhitelistBuilders.webhookWhitelistFromEnv.mockImplementation(
    actual.webhookWhitelistFromEnv,
  );
  outboundWhitelistBuilders.whitelistFromEnv.mockImplementation(
    actual.whitelistFromEnv,
  );
  return {
    ...actual,
    webhookWhitelistFromEnv: outboundWhitelistBuilders.webhookWhitelistFromEnv,
    whitelistFromEnv: outboundWhitelistBuilders.whitelistFromEnv,
  };
});

vi.mock("exponential-backoff", async (importOriginal) => {
  const actual = await importOriginal<typeof import("exponential-backoff")>();
  return {
    ...actual,
    // State-transition tests exercise every HTTP attempt without waiting between them.
    // The endpoint-error test restores the real delays to cover the production policy.
    backOff: vi.fn((...args: Parameters<typeof actual.backOff>) =>
      actual.backOff(args[0], { ...args[1], startingDelay: 0 }),
    ),
  };
});

// Mock webhook server for testing HTTP requests
class WebhookTestServer {
  private server;
  private receivedRequests: Array<{
    url: string;
    method: string;
    headers: Record<string, string>;
    body: string;
  }> = [];
  pagerDutyStatus = 202;

  constructor() {
    const recordPagerDutyEvent = async ({ request }: { request: Request }) => {
      this.receivedRequests.push({
        url: request.url,
        method: request.method,
        headers: Object.fromEntries(request.headers.entries()),
        body: JSON.stringify(await request.json()),
      });
      return HttpResponse.json(
        { status: "success", message: "Event processed" },
        { status: this.pagerDutyStatus },
      );
    };

    this.server = setupServer(
      http.post("https://events.pagerduty.com/*", recordPagerDutyEvent),
      http.post("https://events.eu.pagerduty.com/*", recordPagerDutyEvent),

      // Default success response
      http.post("https://webhook.example.com/*", async ({ request }) => {
        this.receivedRequests.push({
          url: request.url,
          method: request.method,
          headers: Object.fromEntries(request.headers.entries()),
          body: JSON.stringify(await request.json()),
        });
        return HttpResponse.json({ success: true }, { status: 200 });
      }),

      // Error response endpoint
      http.post("https://webhook-error.example.com/*", async ({ request }) => {
        this.receivedRequests.push({
          url: request.url,
          method: request.method,
          headers: Object.fromEntries(request.headers.entries()),
          body: JSON.stringify(await request.json()),
        });
        return HttpResponse.json(
          { error: "Internal Server Error" },
          { status: 500 },
        );
      }),

      // 201 Created response endpoint (GitLab use case)
      http.post("https://webhook-201.example.com/*", async ({ request }) => {
        this.receivedRequests.push({
          url: request.url,
          method: request.method,
          headers: Object.fromEntries(request.headers.entries()),
          body: JSON.stringify(await request.json()),
        });
        return HttpResponse.json({ success: true }, { status: 201 });
      }),

      // Timeout endpoint
      http.post("https://webhook-timeout.example.com/*", () => {
        return HttpResponse.error();
      }),
    );
  }

  setup() {
    this.server.listen();
  }

  reset() {
    this.receivedRequests = [];
    this.pagerDutyStatus = 202;
    this.server.resetHandlers();
  }

  teardown() {
    this.server.close();
  }

  getReceivedRequests() {
    return this.receivedRequests;
  }

  getLastRequest() {
    return this.receivedRequests[this.receivedRequests.length - 1];
  }
}

const webhookServer = new WebhookTestServer();

/** createMonitor writes a minimal Monitor row so monitor-alert jobs pass the stale-alert guard. */
async function createMonitor(projectId: string): Promise<string> {
  const monitor = await prisma.monitor.create({
    data: {
      projectId,
      name: "Test monitor",
      view: "OBSERVATIONS",
      filters: [],
      metric: { measure: "count", aggregation: "count" },
      windowMs: 5n * 60_000n,
      cadenceMs: 60_000n,
      thresholdOperator: "GT",
      alertThreshold: 100,
      noData: { mode: "SHOW_NO_DATA" },
      renotify: { mode: "OFF" },
      schedulerBatchId: 0n,
      severity: "ALERT",
    },
  });
  return monitor.id;
}
const GITHUB_REPOSITORY_DISPATCH_TRUNCATION_MARKER =
  "[TRUNCATED: GitHub repository_dispatch payload exceeded size limit]";
const GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES = 64 * 1024;

describe("Webhook Integration Tests", () => {
  let projectId: string;
  let triggerId: string;
  let actionId: string;
  let automationId: string;
  let promptId: string;
  let executionId: string;
  let monitorId: string;

  beforeAll(() => {
    webhookServer.setup();
  });

  beforeEach(async () => {
    webhookServer.reset();

    // Create test project
    ({ projectId } = await createOrgProjectAndApiKey());

    // Create test prompt
    promptId = v4();
    await prisma.prompt.create({
      data: {
        id: promptId,
        projectId,
        name: "test-prompt",
        version: 1,
        type: "text",
        prompt: { text: "Hello {{name}}" },
        createdBy: "test-user",
      },
    });

    // Create test trigger
    triggerId = v4();
    await prisma.trigger.create({
      data: {
        id: triggerId,
        projectId,
        eventSource: "prompt-version",
        eventActions: ["created"],
        filter: [],
        status: JobConfigState.ACTIVE,
      },
    });

    // Create test action
    actionId = v4();
    const { secretKey, displaySecretKey } = generateWebhookSecret();
    await prisma.action.create({
      data: {
        id: actionId,
        projectId,
        type: "WEBHOOK",
        config: {
          type: "WEBHOOK",
          url: "https://webhook.example.com/test",
          headers: {
            "X-Custom-Header-2": "test-value-2",
            "X-Custom-Header": "test-value",
          },
          apiVersion: { prompt: "v1" },
          secretKey: encrypt(secretKey),
          displaySecretKey,
        },
      },
    });

    // Link trigger to action
    automationId = v4();
    await prisma.automation.create({
      data: {
        id: automationId,
        projectId,
        triggerId,
        actionId,
        name: "Test Automation",
      },
    });
    executionId = v4();
    monitorId = await createMonitor(projectId);
  });

  afterEach(() => {
    webhookServer.reset();
  });

  afterAll(() => {
    webhookServer.teardown();
  });

  describe("skill version delivery", () => {
    it.each([
      { github: false, oversized: false },
      { github: true, oversized: true },
    ])(
      "serializes skill metadata (github=$github, oversized=$oversized)",
      async ({ github, oversized }) => {
        if (github) {
          await prisma.action.update({
            where: { id: actionId, projectId },
            data: {
              type: "GITHUB_DISPATCH",
              config: {
                type: "GITHUB_DISPATCH",
                url: "https://webhook.example.com/dispatches",
                eventType: "skill-update",
                githubToken: encrypt("github-test-token"),
                displayGitHubToken: "github-...",
              },
            },
          });
        }
        const skillId = v4();
        const webhookInput: WebhookInput = {
          projectId,
          automationId,
          executionId,
          payload: {
            type: "skill-version",
            action: "created",
            skill: {
              id: skillId,
              projectId,
              name: "test-skill",
              description: "Test skill",
              version: 2,
              frontmatter: oversized ? { detail: "x".repeat(70_000) } : {},
              tags: [],
              labels: ["latest"],
              commitMessage: null,
              createdBy: "test-user",
              createdAt: new Date("2026-01-01T00:00:00Z"),
              updatedAt: new Date("2026-01-01T00:00:00Z"),
              files: [
                {
                  id: "file-id",
                  blobId: "blob-id",
                  path: "SKILL.md",
                  sha256Hash: "file-hash",
                  contentType: "text/markdown",
                  contentLength: 42,
                },
              ],
            },
            user: {
              id: "internal-user",
              name: "Test user",
              email: "[email protected]",
            },
          },
        };
        Object.assign(webhookInput.payload.skill.files[0], {
          content: "must not be exported",
        });
        await prisma.automationExecution.create({
          data: {
            id: executionId,
            projectId,
            triggerId,
            actionId,
            automationId,
            status: ActionExecutionStatus.PENDING,
            sourceId: skillId,
            input: {},
          },
        });
        await executeWebhook(webhookInput, { skipValidation: true });
        const requests = webhookServer.getReceivedRequests();
        expect(requests).toHaveLength(1);
        const request = requests[0];
        const parsed = JSON.parse(request.body);
        const payload = github ? parsed.client_payload : parsed;
        expect(payload).toMatchObject({
          id: executionId,
          type: "skill-version",
          action: "created",
          apiVersion: "v1",
          skill: { id: skillId, projectId, name: "test-skill", version: 2 },
        });
        expect(payload).not.toHaveProperty("prompt");
        expect(payload.user?.id).toBeUndefined();
        if (github) {
          expect(parsed.event_type).toBe("skill-update");
          expect(Buffer.byteLength(request.body, "utf8")).toBeLessThan(
            64 * 1024,
          );
        }
        if (oversized) {
          expect(payload.truncation.payloadTruncated).toBe(true);
          expect(payload.skill.files).toBeUndefined();
        } else {
          expect(payload.skill.createdAt).toBe("2026-01-01T00:00:00.000Z");
          expect(payload.skill.files).toMatchObject([
            { path: "SKILL.md", sha256Hash: "file-hash", contentLength: 42 },
          ]);
          expect(payload.skill.files[0].content).toBeUndefined();
        }
        expect(
          await prisma.automationExecution.findUnique({
            where: { id: executionId },
          }),
        ).toMatchObject({ status: ActionExecutionStatus.COMPLETED });
      },
    );
  });

  describe("executeWebhook function", () => {
    it("should execute webhook successfully with signature", async () => {
      // Get the full prompt for the payload
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      const webhookInput: WebhookInput = {
        projectId,
        automationId,
        executionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
          user: {
            id: "user-123",
            name: "Test User",
            email: "[email protected]",
          },
        },
      };

      await prisma.automationExecution.create({
        data: {
          id: executionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: webhookInput.executionId,
          input: webhookInput,
        },
      });

      await executeWebhook(webhookInput, { skipValidation: true });

      // Verify webhook request was made
      const requests = webhookServer.getReceivedRequests();
      expect(requests).toHaveLength(1);

      const request = requests[0];
      expect(request.url).toBe("https://webhook.example.com/test");
      expect(request.method).toBe("POST");
      expect(request.headers["x-custom-header-2"]).toBe("test-value-2");
      expect(request.headers["x-custom-header"]).toBe("test-value");

      expect(request.headers["x-langfuse-signature"]).toMatch(
        /^t=\d+,v1=[a-f0-9]+$/,
      );

      // check signature
      const signature = request.headers["x-langfuse-signature"];
      const payload = JSON.parse(request.body);

      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });

      const secretKey = (action?.config as WebhookActionConfigWithSecrets)
        ?.secretKey;

      if (!secretKey) {
        throw new Error("Action has no secret key");
      }

      const decryptedSecret = decrypt(secretKey);

      // Extract timestamp from the actual signature to avoid timing issues
      const timestampMatch = signature.match(/^t=(\d+),v1=/);
      if (!timestampMatch) {
        throw new Error("Invalid signature format");
      }
      const timestamp = parseInt(timestampMatch[1]);

      // Generate expected signature using the same timestamp
      const expectedSignatureHash = generateWebhookSignature(
        JSON.stringify(payload),
        timestamp,
        decryptedSecret,
      );
      const expectedSignature = `t=${timestamp},v1=${expectedSignatureHash}`;

      expect(signature).toBe(expectedSignature);

      expect(payload.id).toBe(webhookInput.executionId);
      expect(payload.type).toBe("prompt-version");
      expect(payload.action).toBe("created");
      expect(payload.prompt.name).toBe("test-prompt");
      expect(payload.prompt.version).toBe(1);
      expect(payload.prompt.createdBy).toBe("test-user");
      expect(payload.timestamp).toBeDefined();
      expect(payload.prompt.createdAt).toBeDefined();
      expect(payload.prompt.updatedAt).toBeDefined();

      // Verify prompt is the last field in the payload
      const payloadKeys = Object.keys(payload);
      expect(payloadKeys[payloadKeys.length - 1]).toBe("prompt");

      // Verify database execution record was updated
      const execution = await prisma.automationExecution.findUnique({
        where: { id: executionId },
      });
      expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
      expect(execution?.output).toBeNull();
      expect(execution?.startedAt).toBeDefined();
      expect(execution?.finishedAt).toBeDefined();
    });

    it("should fail webhook execution if secret key does not exist and retry the bull job", async () => {
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      const executionId = v4();

      await prisma.action.update({
        where: { id: actionId },
        data: {
          config: {
            type: "WEBHOOK",
            url: "https://webhook-error.example.com/test",
            headers: { "Content-Type": "application/json" },
            apiVersion: { prompt: "v1" },
            secretKey: null, // Explicitly set secret to null
          },
        },
      });

      await prisma.automationExecution.create({
        data: {
          id: executionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: v4(),
          input: {
            promptName: "test-prompt",
            promptVersion: 1,
            action: "created",
            type: "prompt-version",
          },
        },
      });

      const webhookInput: WebhookInput = {
        projectId,
        automationId,
        executionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
          user: {
            id: "user-123",
            name: "Test User",
            email: "[email protected]",
          },
        },
      };

      await expect(
        executeWebhook(webhookInput, { skipValidation: true }),
      ).rejects.toThrow("Action config is not a valid webhook configuration");

      const execution = await prisma.automationExecution.findUnique({
        where: { id: executionId },
      });

      expect(execution?.status).toBe(ActionExecutionStatus.PENDING);
    });

    it("should accept 201 status code as success (GitLab use case)", async () => {
      // Get the full prompt for the payload
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });

      if (!action) {
        throw new Error("Action not found");
      }

      // Update action to use 201 endpoint
      await prisma.action.update({
        where: { id: actionId },
        data: {
          projectId,
          type: "WEBHOOK",
          config: {
            ...(action.config as WebhookActionConfigWithSecrets),
            url: "https://webhook-201.example.com/test",
          },
        },
      });

      await prisma.automationExecution.create({
        data: {
          id: executionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: v4(),
          input: {
            promptName: "test-prompt",
            promptVersion: 1,
            action: "created",
            type: "prompt-version",
          },
        },
      });

      const webhookInput: WebhookInput = {
        projectId,
        automationId,
        executionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
          user: {
            id: "user-123",
            name: "Test User",
            email: "[email protected]",
          },
        },
      };

      await executeWebhook(webhookInput, { skipValidation: true });

      // Verify webhook request was made
      const requests = webhookServer.getReceivedRequests();
      expect(requests).toHaveLength(1);

      const request = requests[0];
      expect(request.url).toBe("https://webhook-201.example.com/test");
      expect(request.method).toBe("POST");

      // Verify execution was marked as completed (not error)
      const execution = await prisma.automationExecution.findUnique({
        where: { id: executionId },
      });
      expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
      expect(execution?.startedAt).toBeDefined();
      expect(execution?.finishedAt).toBeDefined();
    });

    it("should handle webhook endpoint returning error", async () => {
      const actual = await vi.importActual<
        typeof import("exponential-backoff")
      >("exponential-backoff");
      vi.mocked(backOff).mockImplementationOnce(actual.backOff);
      vi.mocked(backOff).mockClear();
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });

      if (!action) {
        throw new Error("Action not found");
      }

      await prisma.action.update({
        where: { id: actionId },
        data: {
          projectId,
          type: "WEBHOOK",
          config: {
            ...(action.config as WebhookActionConfigWithSecrets),
            url: "https://webhook-error.example.com/test",
          },
        },
      });

      await prisma.automationExecution.create({
        data: {
          id: executionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: v4(),
          input: {
            promptName: "test-prompt",
            promptVersion: 1,
            action: "created",
            type: "prompt-version",
          },
        },
      });
      // Create action pointing to error endpoint
      const webhookInput: WebhookInput = {
        projectId,
        automationId,
        executionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
          user: {
            id: "user-123",
            name: "Test User",
            email: "[email protected]",
          },
        },
      };

      await executeWebhook(webhookInput, { skipValidation: true });

      expect(backOff).toHaveBeenCalledWith(expect.any(Function), {
        numOfAttempts: 4,
      });
      expect(webhookServer.getReceivedRequests()).toHaveLength(4);

      // Verify execution was marked as error
      const execution = await prisma.automationExecution.findUnique({
        where: { id: executionId },
      });
      expect(execution?.status).toBe(ActionExecutionStatus.ERROR);
      expect(execution?.error).toContain(
        `Webhook does not return 2xx status: failed with status 500 for url https://webhook-error.example.com/test and project ${projectId}`,
      );
      expect(execution?.output).toMatchObject({
        httpStatus: 500,
        responseBody: '{"error":"Internal Server Error"}',
      });
    });

    it("sends a query-string secret and keeps it out of logs and the execution error", async () => {
      const signedUrl =
        "https://webhook-error.example.com/test?api-version=2016-06-01&sig=super-secret-sig";
      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });
      if (!action) {
        throw new Error("Action not found");
      }

      await prisma.action.update({
        where: { id: actionId },
        data: {
          projectId,
          type: "WEBHOOK",
          config: {
            ...(action.config as WebhookActionConfigWithSecrets),
            url: signedUrl,
          },
        },
      });

      await prisma.automationExecution.create({
        data: {
          id: executionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: v4(),
          input: {
            promptName: "test-prompt",
            promptVersion: 1,
            action: "created",
            type: "prompt-version",
          },
        },
      });

      const info = vi.spyOn(logger, "info");
      const warn = vi.spyOn(logger, "warn");
      const error = vi.spyOn(logger, "error");

      try {
        await executeWebhook(
          {
            projectId,
            automationId,
            executionId,
            payload: {
              prompt: PromptDomainSchema.parse(
                await prisma.prompt.findUnique({ where: { id: promptId } }),
              ),
              action: "created",
              type: "prompt-version",
              user: {
                id: "user-123",
                name: "Test User",
                email: "[email protected]",
              },
            },
          },
          { skipValidation: true },
        );

        const requests = webhookServer.getReceivedRequests();
        expect(requests.length).toBeGreaterThan(0);
        expect(requests.every((request) => request.url === signedUrl)).toBe(
          true,
        );

        const execution = await prisma.automationExecution.findUnique({
          where: { id: executionId },
        });
        expect(execution?.error).toContain(
          "https://webhook-error.example.com/test",
        );
        expect(execution?.error).not.toContain("super-secret-sig");

        const logged = [info, warn, error]
          .flatMap((spy) => spy.mock.calls)
          .map((call) =>
            call
              .map((arg) => {
                if (arg instanceof Error) {
                  return `${arg.name} ${arg.message} ${arg.stack ?? ""} ${JSON.stringify(arg)}`;
                }
                return typeof arg === "string" ? arg : JSON.stringify(arg);
              })
              .join(" "),
          )
          .join("\n");
        expect(logged).toContain("https://webhook-error.example.com/test");
        expect(logged).not.toContain("super-secret-sig");
      } finally {
        info.mockRestore();
        warn.mockRestore();
        error.mockRestore();
      }
    });

    it("should disable trigger after 5 consecutive failures and store lastFailingExecutionId", async () => {
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });

      if (!action) {
        throw new Error("Action not found");
      }

      const secretHeaderValue = "secret-api-key-value";
      const encryptedSecretHeaderValue = encrypt(secretHeaderValue);

      await prisma.action.update({
        where: { id: actionId },
        data: {
          projectId,
          type: "WEBHOOK",
          config: {
            ...(action.config as WebhookActionConfigWithSecrets),
            url: "https://webhook-error.example.com/test",
            requestHeaders: {
              "x-secret-api-key": {
                secret: true,
                value: encryptedSecretHeaderValue,
              },
            },
            displayHeaders: {
              "x-secret-api-key": { secret: true, value: "secr...alue" },
            },
          },
        },
      });

      // Execute webhook 5 times to trigger consecutive failures
      for (let i = 0; i < 5; i++) {
        const executionId = v4();

        await prisma.automationExecution.create({
          data: {
            id: executionId,
            projectId,
            triggerId,
            automationId,
            actionId,
            status: ActionExecutionStatus.PENDING,
            sourceId: v4(),
            input: {
              promptName: "test-prompt",
              promptVersion: 1,
              action: "created",
              type: "prompt-version",
            },
          },
        });

        const webhookInput: WebhookInput = {
          projectId,
          automationId,
          executionId,
          payload: {
            prompt: PromptDomainSchema.parse(fullPrompt),
            action: "created",
            type: "prompt-version",
            user: {
              id: "user-123",
              name: "Test User",
              email: "[email protected]",
            },
          },
        };

        await executeWebhook(webhookInput, { skipValidation: true });

        // Verify execution was marked as error
        const execution = await prisma.automationExecution.findUnique({
          where: { id: executionId },
        });

        expect(execution?.status).toBe(ActionExecutionStatus.ERROR);
        expect(execution?.error).toContain(
          `Webhook does not return 2xx status: failed with status 500 for url https://webhook-error.example.com/test and project ${projectId}`,
        );
      }

      // Verify trigger was disabled after 5 consecutive failures
      const trigger = await prisma.trigger.findUnique({
        where: { id: triggerId },
      });
      expect(trigger?.status).toBe(JobConfigState.INACTIVE);

      // Verify lastFailingExecutionId was stored in action config
      const updatedAction = await prisma.action.findUnique({
        where: { id: actionId },
      });
      const config = updatedAction?.config as any;
      expect(config.lastFailingExecutionId).toBeDefined();
      expect(typeof config.lastFailingExecutionId).toBe("string");

      // Encrypted request headers are preserved when lastFailingExecutionId is written.
      expect(config.requestHeaders["x-secret-api-key"].value).toBe(
        encryptedSecretHeaderValue,
      );
      expect(config.requestHeaders["x-secret-api-key"].value).not.toBe(
        secretHeaderValue,
      );
      expect(decrypt(config.requestHeaders["x-secret-api-key"].value)).toBe(
        secretHeaderValue,
      );
    });

    it("clears the monitor-alert failure counter when auto-disabling the trigger", async () => {
      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });
      if (!action) {
        throw new Error("Action not found");
      }

      await prisma.action.update({
        where: { id: actionId },
        data: {
          projectId,
          type: "WEBHOOK",
          config: {
            ...(action.config as WebhookActionConfigWithSecrets),
            url: "https://webhook-error.example.com/test",
          },
        },
      });

      const failureKey = `automation-failures:${projectId}:${automationId}`;
      await redis!.del(failureKey);

      for (let i = 0; i < 5; i++) {
        const webhookInput: WebhookInput = {
          projectId,
          automationId,
          executionId: v4(),
          payload: {
            id: v4(),
            timestamp: new Date(),
            type: "monitor-alert",
            apiVersion: "v1",
            payload: {
              monitorId,
              projectId,
              severity: "ALERT",
              permalink: `https://cloud.langfuse.com/project/${projectId}/monitors/mon`,
              timestamp: new Date(),
              fromTimestamp: new Date(Date.now() - 5 * 60_000),
              toTimestamp: new Date(),
              message: { title: "High error rate", body: "errors > 100" },
              view: "observations",
              filters: [],
              window: "5m",
            },
          },
        };

        await executeWebhook(webhookInput, { skipValidation: true });
      }

      const trigger = await prisma.trigger.findUnique({
        where: { id: triggerId },
      });
      expect(trigger?.status).toBe(JobConfigState.INACTIVE);

      expect(await redis!.get(failureKey)).toBeNull();
    });

    it("delivers a project notification envelope verbatim and completes the execution row", async () => {
      const event = {
        eventType: "blob-export-failed" as const,
        severity: "ALERT" as const,
        projectId,
        projectName: "Test Project",
        resourceId: projectId,
        resourceName: "test-export-bucket",
        message: "Blob storage export failed.",
        url: `https://cloud.langfuse.com/project/${projectId}/settings`,
      };
      const executionId = v4();
      const webhookInput: WebhookInput = {
        projectId,
        automationId,
        executionId,
        payload: {
          id: executionId,
          timestamp: new Date(),
          type: "project-notification",
          apiVersion: "v1",
          event,
        },
      };

      // dispatchProjectNotification creates a PENDING row per enqueued job.
      await prisma.automationExecution.create({
        data: {
          id: executionId,
          projectId,
          automationId,
          triggerId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: event.resourceId,
          input: event,
        },
      });

      await executeWebhook(webhookInput, { skipValidation: true });

      const requests = webhookServer.getReceivedRequests();
      expect(requests).toHaveLength(1);
      const body = JSON.parse(requests[0].body);
      expect(body.type).toBe("project-notification");
      expect(body.apiVersion).toBe("v1");
      expect(body.id).toBe(executionId);
      expect(body.event).toEqual(event);
      // like the prompt-version path, the execution row is resolved to COMPLETED
      const execution = await prisma.automationExecution.findUnique({
        where: { id: executionId },
      });
      expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
    });

    it("project-notification webhook: records ERROR rows on failure but never disables the trigger", async () => {
      // Point the webhook action at the always-500 endpoint.
      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });
      if (!action) throw new Error("Action not found");
      await prisma.action.update({
        where: { id: actionId },
        data: {
          config: {
            ...(action.config as WebhookActionConfigWithSecrets),
            url: "https://webhook-error.example.com/test",
          },
        },
      });

      // Fail more times than the (prompt/monitor) disable threshold of 5.
      for (let i = 0; i < 6; i++) {
        const executionId = v4();
        await prisma.automationExecution.create({
          data: {
            id: executionId,
            projectId,
            automationId,
            triggerId,
            actionId,
            status: ActionExecutionStatus.PENDING,
            sourceId: projectId,
            input: {},
          },
        });
        const webhookInput: WebhookInput = {
          projectId,
          automationId,
          executionId,
          payload: {
            id: executionId,
            timestamp: new Date(),
            type: "project-notification",
            apiVersion: "v1",
            event: {
              eventType: "blob-export-failed",
              severity: "ALERT",
              projectId,
              projectName: "Test Project",
              resourceId: projectId,
              resourceName: "test-export-bucket",
              message: "Blob storage export failed.",
            },
          },
        };
        await executeWebhook(webhookInput, { skipValidation: true });

        const execution = await prisma.automationExecution.findUnique({
          where: { id: executionId },
        });
        expect(execution?.status).toBe(ActionExecutionStatus.ERROR);
      }

      // Project-notification channels never auto-disable.
      const trigger = await prisma.trigger.findUnique({
        where: { id: triggerId },
      });
      expect(trigger?.status).toBe(JobConfigState.ACTIVE);
    });

    it("auto-disables despite a redis.del failure on the failure-side reset", async () => {
      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });
      if (!action) {
        throw new Error("Action not found");
      }

      await prisma.action.update({
        where: { id: actionId },
        data: {
          projectId,
          type: "WEBHOOK",
          config: {
            ...(action.config as WebhookActionConfigWithSecrets),
            url: "https://webhook-error.example.com/test",
          },
        },
      });

      const failureKey = `automation-failures:${projectId}:${automationId}`;
      await redis!.del(failureKey);

      const buildInput = (): WebhookInput => ({
        projectId,
        automationId,
        executionId: v4(),
        payload: {
          id: v4(),
          timestamp: new Date(),
          type: "monitor-alert",
          apiVersion: "v1",
          payload: {
            monitorId,
            projectId,
            severity: "ALERT",
            permalink: `https://cloud.langfuse.com/project/${projectId}/monitors/mon`,
            timestamp: new Date(),
            fromTimestamp: new Date(Date.now() - 5 * 60_000),
            toTimestamp: new Date(),
            message: { title: "High error rate", body: "errors > 100" },
            view: "observations",
            filters: [],
            window: "5m",
          },
        },
      });

      for (let i = 0; i < 4; i++) {
        await executeWebhook(buildInput(), { skipValidation: true });
      }

      const delSpy = vi
        .spyOn(redis!, "del")
        .mockRejectedValue(new Error("READONLY during failover"));
      try {
        await executeWebhook(buildInput(), { skipValidation: true });
      } finally {
        delSpy.mockRestore();
      }

      const trigger = await prisma.trigger.findUnique({
        where: { id: triggerId },
      });
      expect(trigger?.status).toBe(JobConfigState.INACTIVE);
    });

    it("should execute webhook with secret headers correctly", async () => {
      // Get the full prompt for the payload
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      // Update action config to include both public and secret headers
      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });

      if (!action) {
        throw new Error("Action not found");
      }

      const updatedConfig = {
        ...(action.config as WebhookActionConfigWithSecrets),
        requestHeaders: {
          "x-public-header": { secret: false, value: "public-value" },
          "x-secret-api-key": {
            secret: true,
            value: encrypt("secret-api-key-value"),
          },
          "x-secret-token": {
            secret: true,
            value: encrypt("bearer-token-value"),
          },
        },
        displayHeaders: {
          "x-secret-api-key": { secret: true, value: "secr...alue" },
          "x-secret-token": { secret: true, value: "bear...alue" },
        },
      };

      await prisma.action.update({
        where: { id: actionId },
        data: {
          config: updatedConfig,
        },
      });

      const webhookInput: WebhookInput = {
        projectId,
        automationId,
        executionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
          user: {
            id: "user-123",
            name: "Test User",
            email: "[email protected]",
          },
        },
      };

      await prisma.automationExecution.create({
        data: {
          id: executionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: webhookInput.executionId,
          input: webhookInput,
        },
      });

      await executeWebhook(webhookInput, { skipValidation: true });

      // Verify webhook request was made with decrypted secret headers
      const requests = webhookServer.getReceivedRequests();
      expect(requests).toHaveLength(1);

      const request = requests[0];
      expect(request.url).toBe("https://webhook.example.com/test");
      expect(request.method).toBe("POST");

      // Verify public header is present
      expect(request.headers["x-public-header"]).toBe("public-value");

      // Verify secret headers are present and decrypted
      expect(request.headers["x-secret-api-key"]).toBe("secret-api-key-value");
      expect(request.headers["x-secret-token"]).toBe("bearer-token-value");

      // Verify signature is present
      expect(request.headers["x-langfuse-signature"]).toMatch(
        /^t=\d+,v1=[a-f0-9]+$/,
      );

      // Verify database execution record was updated
      const execution = await prisma.automationExecution.findUnique({
        where: { id: executionId },
      });
      expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
    });

    it("should handle webhook with only secret headers", async () => {
      // Get the full prompt for the payload
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      // Update action config to include only secret headers
      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });

      if (!action) {
        throw new Error("Action not found");
      }

      const updatedConfig = {
        ...(action.config as WebhookActionConfigWithSecrets),
        requestHeaders: {
          authorization: {
            secret: true,
            value: encrypt("Bearer secret-token-12345"),
          },
          "x-api-key": { secret: true, value: encrypt("api-key-67890") },
        },
        displayHeaders: {
          authorization: { secret: true, value: "Bear***12345" },
          "x-api-key": { secret: true, value: "api-***67890" },
        },
      };

      await prisma.action.update({
        where: { id: actionId },
        data: {
          config: updatedConfig,
        },
      });

      const newExecutionId = v4();
      const webhookInput: WebhookInput = {
        projectId,
        automationId,
        executionId: newExecutionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
          user: {
            id: "user-123",
            name: "Test User",
            email: "[email protected]",
          },
        },
      };

      await prisma.automationExecution.create({
        data: {
          id: newExecutionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: webhookInput.executionId,
          input: webhookInput,
        },
      });

      await executeWebhook(webhookInput, { skipValidation: true });

      // Verify webhook request was made with decrypted secret headers
      const requests = webhookServer.getReceivedRequests();
      expect(requests).toHaveLength(1); // This test only

      const request = requests[0]; // Get the request
      expect(request.url).toBe("https://webhook.example.com/test");

      // Verify secret headers are present and decrypted
      expect(request.headers["authorization"]).toBe(
        "Bearer secret-token-12345",
      );
      expect(request.headers["x-api-key"]).toBe("api-key-67890");

      // Verify database execution record was updated
      const execution = await prisma.automationExecution.findUnique({
        where: { id: newExecutionId },
      });
      expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
    });

    it("should handle webhook with only legacy public headers", async () => {
      // Get the full prompt for the payload
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      // Update action config to include only public headers
      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });

      if (!action) {
        throw new Error("Action not found");
      }

      const updatedConfig = {
        ...(action.config as WebhookActionConfigWithSecrets),
        headers: {
          "x-public-header-1": "public-value-1",
          "x-public-header-2": "public-value-2",
        },
        secretHeaderKeys: [], // No secret headers
        displayHeaders: {},
      };

      await prisma.action.update({
        where: { id: actionId },
        data: {
          config: updatedConfig,
        },
      });

      const newExecutionId = v4();
      const webhookInput: WebhookInput = {
        projectId,
        automationId,
        executionId: newExecutionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
          user: {
            id: "user-123",
            name: "Test User",
            email: "[email protected]",
          },
        },
      };

      await prisma.automationExecution.create({
        data: {
          id: newExecutionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: webhookInput.executionId,
          input: webhookInput,
        },
      });

      await executeWebhook(webhookInput, { skipValidation: true });

      // Verify webhook request was made with public headers
      const requests = webhookServer.getReceivedRequests();
      expect(requests).toHaveLength(1); // This test only

      const request = requests[0]; // Get the request
      expect(request.url).toBe("https://webhook.example.com/test");

      // Verify public headers are present
      expect(request.headers["x-public-header-1"]).toBe("public-value-1");
      expect(request.headers["x-public-header-2"]).toBe("public-value-2");

      // Verify database execution record was updated
      const execution = await prisma.automationExecution.findUnique({
        where: { id: newExecutionId },
      });
      expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
    });

    it("should handle webhook with legacy public headers + new public and secret headers", async () => {
      // Get the full prompt for the payload
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      // Update action config to include only public headers
      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });

      if (!action) {
        throw new Error("Action not found");
      }

      const updatedConfig = {
        ...(action.config as WebhookActionConfigWithSecrets),
        headers: {
          "x-public-header-1": "public-value-1",
          "x-public-header-2": "public-value-2",
        },
        requestHeaders: {
          "x-public-header-3": { secret: false, value: "public-value-3" },
          "x-secret-header-4": {
            secret: true,
            value: encrypt("secret-value-4"),
          },
          "x-public-header-1": { secret: false, value: "public-value-5" },
        },
        displayHeaders: {},
      };

      await prisma.action.update({
        where: { id: actionId },
        data: {
          config: updatedConfig,
        },
      });

      const newExecutionId = v4();
      const webhookInput: WebhookInput = {
        projectId,
        automationId,
        executionId: newExecutionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
          user: {
            id: "user-123",
            name: "Test User",
            email: "[email protected]",
          },
        },
      };

      await prisma.automationExecution.create({
        data: {
          id: newExecutionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: webhookInput.executionId,
          input: webhookInput,
        },
      });

      await executeWebhook(webhookInput, { skipValidation: true });

      // Verify webhook request was made with public headers
      const requests = webhookServer.getReceivedRequests();
      expect(requests).toHaveLength(1); // This test only

      const request = requests[0]; // Get the request
      expect(request.url).toBe("https://webhook.example.com/test");

      // Verify public headers are present
      expect(request.headers["x-public-header-1"]).toBe("public-value-5");
      expect(request.headers["x-public-header-2"]).toBe("public-value-2");
      expect(request.headers["x-public-header-3"]).toBe("public-value-3");
      expect(request.headers["x-secret-header-4"]).toBe("secret-value-4");

      // Verify database execution record was updated
      const execution = await prisma.automationExecution.findUnique({
        where: { id: newExecutionId },
      });
      expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
    });

    it("should handle decryption failure gracefully", async () => {
      // Get the full prompt for the payload
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      // Update action config with invalid encrypted header
      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });

      if (!action) {
        throw new Error("Action not found");
      }

      const updatedConfig = {
        ...(action.config as WebhookActionConfigWithSecrets),
        requestHeaders: {
          "x-valid-header": { secret: false, value: "valid-public-value" },
          "x-invalid-secret": {
            secret: true,
            value: "invalid-encrypted-value", // This is not properly encrypted
          },
        },
        displayHeaders: {
          "x-invalid-secret": { secret: true, value: "inva...alue" },
        },
      };

      await prisma.action.update({
        where: { id: actionId },
        data: {
          config: updatedConfig,
        },
      });

      const newExecutionId = v4();
      const webhookInput: WebhookInput = {
        projectId,
        automationId,
        executionId: newExecutionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
          user: {
            id: "user-123",
            name: "Test User",
            email: "[email protected]",
          },
        },
      };

      await prisma.automationExecution.create({
        data: {
          id: newExecutionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: webhookInput.executionId,
          input: webhookInput,
        },
      });

      await executeWebhook(webhookInput, { skipValidation: true });

      // Verify webhook request was made with valid headers only
      const requests = webhookServer.getReceivedRequests();
      expect(requests).toHaveLength(1); // This test only

      const request = requests[0]; // Get the request
      expect(request.url).toBe("https://webhook.example.com/test");

      // Verify valid header is present
      expect(request.headers["x-valid-header"]).toBe("valid-public-value");

      // Verify invalid secret header is not present (skipped due to decryption failure)
      expect(request.headers["x-invalid-secret"]).toBeUndefined();

      // Verify database execution record was updated (should still succeed)
      const execution = await prisma.automationExecution.findUnique({
        where: { id: newExecutionId },
      });
      expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
    });

    it("should handle missing automation gracefully without throwing error", async () => {
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      const executionId = v4();
      const nonExistentAutomationId = v4(); // Use a random ID that doesn't exist

      const webhookInput: WebhookInput = {
        projectId,
        automationId: nonExistentAutomationId,
        executionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
          user: {
            id: "user-123",
            name: "Test User",
            email: "[email protected]",
          },
        },
      };

      // Should not throw an error, but return gracefully
      await expect(
        executeWebhook(webhookInput, { skipValidation: true }),
      ).resolves.toBeUndefined();

      // Verify that no execution record was created since automation doesn't exist
      const execution = await prisma.automationExecution.findUnique({
        where: { id: executionId },
      });
      expect(execution).toBeNull();
    });

    it("should reset failure count correctly with lastFailingExecutionId", async () => {
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      const action = await prisma.action.findUnique({
        where: { id: actionId },
      });

      if (!action) {
        throw new Error("Action not found");
      }

      // Set a lastFailingExecutionId in the action config
      const lastFailingExecutionId = v4();
      await prisma.action.update({
        where: { id: actionId },
        data: {
          config: {
            ...(action.config as WebhookActionConfigWithSecrets),
            lastFailingExecutionId,
          },
        },
      });

      // Create the execution that matches the lastFailingExecutionId
      await prisma.automationExecution.create({
        data: {
          id: lastFailingExecutionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.ERROR,
          sourceId: v4(),
          input: { test: "old failing execution" },
          error: "Old failure",
          createdAt: new Date(Date.now() - 60000), // 1 minute ago
        },
      });

      // Import the function to test it directly
      const { getConsecutiveAutomationFailures } =
        await import("@langfuse/shared/src/server");

      // Check that consecutive failures is 0 since there are no executions after the lastFailingExecutionId
      const failures = await getConsecutiveAutomationFailures({
        automationId,
        projectId,
      });

      expect(failures).toBe(0);
    });

    it(
      "should reset failure count when webhook is re-enabled after being disabled",
      { timeout: 20000 },
      async () => {
        const fullPrompt = await prisma.prompt.findUnique({
          where: { id: promptId },
        });

        const action = await getActionByIdWithSecrets({
          projectId,
          actionId,
        });

        if (!action) {
          throw new Error("Action not found");
        }

        // Update action to point to error endpoint
        await prisma.action.update({
          where: { id: actionId },
          data: {
            config: {
              ...(action.config as WebhookActionConfigWithSecrets),
              url: "https://webhook-error.example.com/test",
            },
          },
        });

        const executionIds: string[] = [];

        // Execute webhook 5 times to trigger disable
        for (let i = 0; i < 5; i++) {
          const executionId = v4();
          executionIds.push(executionId);

          await prisma.automationExecution.create({
            data: {
              id: executionId,
              projectId,
              triggerId,
              automationId,
              actionId,
              status: ActionExecutionStatus.PENDING,
              sourceId: v4(),
              input: {
                promptName: "test-prompt",
                promptVersion: 1,
                action: "created",
                type: "prompt-version",
              },
            },
          });

          const webhookInput: WebhookInput = {
            projectId,
            automationId,
            executionId,
            payload: {
              prompt: PromptDomainSchema.parse(fullPrompt),
              action: "created",
              type: "prompt-version",
            },
          };

          await executeWebhook(webhookInput, { skipValidation: true });
        }

        // Verify trigger was disabled and lastFailingExecutionId was stored
        const trigger = await prisma.trigger.findUnique({
          where: { id: triggerId },
        });
        expect(trigger?.status).toBe(JobConfigState.INACTIVE);

        const actionAfterDisable = await prisma.action.findUnique({
          where: { id: actionId },
        });
        const configAfterDisable = actionAfterDisable?.config as any;
        expect(configAfterDisable.lastFailingExecutionId).toBe(executionIds[4]);

        // Re-enable the trigger (simulates user action)
        await prisma.trigger.update({
          where: { id: triggerId },
          data: { status: JobConfigState.ACTIVE },
        });

        // Create 5 more failing executions to trigger disable again
        const newExecutionIds: string[] = [];
        for (let i = 0; i < 5; i++) {
          const newExecutionId = v4();
          newExecutionIds.push(newExecutionId);

          await prisma.automationExecution.create({
            data: {
              id: newExecutionId,
              projectId,
              triggerId,
              automationId,
              actionId,
              status: ActionExecutionStatus.PENDING,
              sourceId: v4(),
              input: {
                promptName: "test-prompt",
                promptVersion: 1,
                action: "created",
                type: "prompt-version",
              },
            },
          });

          const newWebhookInput: WebhookInput = {
            projectId,
            automationId,
            executionId: newExecutionId,
            payload: {
              prompt: PromptDomainSchema.parse(fullPrompt),
              action: "created",
              type: "prompt-version",
            },
          };

          await executeWebhook(newWebhookInput, { skipValidation: true });
        }

        // Verify trigger was disabled again and lastFailingExecutionId was updated
        const triggerAfterSecondDisable = await prisma.trigger.findUnique({
          where: { id: triggerId },
        });
        expect(triggerAfterSecondDisable?.status).toBe(JobConfigState.INACTIVE);

        const actionAfterSecondDisable = await prisma.action.findUnique({
          where: { id: actionId },
        });
        const configAfterSecondDisable =
          actionAfterSecondDisable?.config as any;
        expect(configAfterSecondDisable.lastFailingExecutionId).toBe(
          newExecutionIds[4],
        );
      },
    );

    it("should include user info in webhook payload when provided", async () => {
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      const testUser = {
        id: "user-123",
        name: "Test User",
        email: "[email protected]",
      };

      const webhookInput: WebhookInput = {
        projectId,
        automationId,
        executionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
          user: testUser,
        },
      };

      await prisma.automationExecution.create({
        data: {
          id: executionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: webhookInput.executionId,
          input: webhookInput,
        },
      });

      await executeWebhook(webhookInput, { skipValidation: true });

      const requests = webhookServer.getReceivedRequests();
      expect(requests).toHaveLength(1);

      const payload = JSON.parse(requests[0].body);
      expect(payload.user).toEqual({
        name: testUser.name,
        email: testUser.email,
      });
      expect(payload.user.id).toBeUndefined();
      // Verify prompt is still the last field
      const payloadKeys = Object.keys(payload);
      expect(payloadKeys[payloadKeys.length - 1]).toBe("prompt");
    });

    it("should omit user field from webhook payload when not provided", async () => {
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      const webhookInput: WebhookInput = {
        projectId,
        automationId,
        executionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
        },
      };

      await prisma.automationExecution.create({
        data: {
          id: executionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: webhookInput.executionId,
          input: webhookInput,
        },
      });

      await executeWebhook(webhookInput, { skipValidation: true });

      const requests = webhookServer.getReceivedRequests();
      expect(requests).toHaveLength(1);

      const payload = JSON.parse(requests[0].body);
      expect(payload.user).toBeUndefined();
    });

    it("should include user info in GitHub dispatch payload when provided", async () => {
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });

      const testUser = {
        id: "user-456",
        name: "GitHub User",
        email: "[email protected]",
      };

      // Create a GitHub dispatch action
      const ghActionId = v4();
      await prisma.action.create({
        data: {
          id: ghActionId,
          projectId,
          type: "GITHUB_DISPATCH",
          config: {
            type: "GITHUB_DISPATCH",
            url: "https://webhook.example.com/dispatches",
            eventType: "prompt-update",
            githubToken: encrypt("ghp_test_token"),
            displayGitHubToken: "ghp_...n",
          },
        },
      });

      // Create automation linking trigger and GitHub dispatch action
      const ghAutomationId = v4();
      await prisma.automation.create({
        data: {
          id: ghAutomationId,
          projectId,
          triggerId,
          actionId: ghActionId,
          name: "GitHub Dispatch Automation",
        },
      });

      const ghExecutionId = v4();
      await prisma.automationExecution.create({
        data: {
          id: ghExecutionId,
          projectId,
          triggerId,
          automationId: ghAutomationId,
          actionId: ghActionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: promptId,
          input: {},
        },
      });

      const webhookInput: WebhookInput = {
        projectId,
        automationId: ghAutomationId,
        executionId: ghExecutionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
          user: testUser,
        },
      };

      await executeWebhook(webhookInput, { skipValidation: true });

      const requests = webhookServer.getReceivedRequests();
      expect(requests).toHaveLength(1);

      const payload = JSON.parse(requests[0].body);
      expect(payload.event_type).toBe("prompt-update");
      expect(payload.client_payload.user).toEqual({
        name: testUser.name,
        email: testUser.email,
      });
      expect(payload.client_payload.user.id).toBeUndefined();
      // Verify prompt is still the last field in client_payload
      const clientPayloadKeys = Object.keys(payload.client_payload);
      expect(clientPayloadKeys[clientPayloadKeys.length - 1]).toBe("prompt");
    });

    it("applies the webhook host allowlist to webhook delivery and not GitHub dispatch", async () => {
      const resolve4 = vi
        .spyOn(dns, "resolve4")
        .mockResolvedValue(["93.184.216.34"]);
      const resolve6 = vi
        .spyOn(dns, "resolve6")
        .mockRejectedValue(
          Object.assign(new Error("ENODATA"), { code: "ENODATA" }),
        );
      const lookup = vi
        .spyOn(dns, "lookup")
        .mockResolvedValue([{ address: "93.184.216.34", family: 4 }] as never);
      outboundWhitelistBuilders.webhookWhitelistFromEnv.mockReturnValue({
        hosts: [],
        ips: [],
        ip_ranges: [],
        allowedHosts: ["hooks.example.com"],
      });
      outboundWhitelistBuilders.whitelistFromEnv.mockReturnValue({
        hosts: [],
        ips: [],
        ip_ranges: [],
      });

      try {
        const fullPrompt = await prisma.prompt.findUnique({
          where: { id: promptId },
        });
        const promptPayload = {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created" as const,
          type: "prompt-version" as const,
        };

        const webhookInput: WebhookInput = {
          projectId,
          automationId,
          executionId,
          payload: promptPayload,
        };
        await prisma.automationExecution.create({
          data: {
            id: executionId,
            projectId,
            triggerId,
            automationId,
            actionId,
            status: ActionExecutionStatus.PENDING,
            sourceId: promptId,
            input: webhookInput,
          },
        });

        await executeWebhook(webhookInput);

        const webhookExecution = await prisma.automationExecution.findUnique({
          where: { id: executionId },
        });
        expect(webhookExecution?.status).toBe(ActionExecutionStatus.ERROR);
        expect(webhookExecution?.error).toContain(
          "Host is not in the outbound URL allowlist",
        );
        expect(webhookServer.getReceivedRequests()).toHaveLength(0);

        const ghActionId = v4();
        await prisma.action.create({
          data: {
            id: ghActionId,
            projectId,
            type: "GITHUB_DISPATCH",
            config: {
              type: "GITHUB_DISPATCH",
              url: "https://webhook.example.com/dispatches",
              eventType: "prompt-update",
              githubToken: encrypt("ghp_test_token"),
              displayGitHubToken: "ghp_...n",
            },
          },
        });
        const ghAutomationId = v4();
        await prisma.automation.create({
          data: {
            id: ghAutomationId,
            projectId,
            triggerId,
            actionId: ghActionId,
            name: "GitHub Dispatch Automation",
          },
        });
        const ghExecutionId = v4();
        await prisma.automationExecution.create({
          data: {
            id: ghExecutionId,
            projectId,
            triggerId,
            automationId: ghAutomationId,
            actionId: ghActionId,
            status: ActionExecutionStatus.PENDING,
            sourceId: promptId,
            input: {},
          },
        });

        await executeWebhook({
          projectId,
          automationId: ghAutomationId,
          executionId: ghExecutionId,
          payload: promptPayload,
        });

        expect(webhookServer.getReceivedRequests()).toHaveLength(1);
        const dispatchExecution = await prisma.automationExecution.findUnique({
          where: { id: ghExecutionId },
        });
        expect(dispatchExecution?.status).toBe(ActionExecutionStatus.COMPLETED);
      } finally {
        outboundWhitelistBuilders.webhookWhitelistFromEnv.mockImplementation(
          outboundWhitelistBuilders.realWebhookWhitelistFromEnv,
        );
        outboundWhitelistBuilders.whitelistFromEnv.mockImplementation(
          outboundWhitelistBuilders.realWhitelistFromEnv,
        );
        resolve4.mockRestore();
        resolve6.mockRestore();
        lookup.mockRestore();
      }
    });

    it("should truncate oversized GitHub dispatch prompt fields", async () => {
      const fullPrompt = await prisma.prompt.findUnique({
        where: { id: promptId },
      });
      if (!fullPrompt) {
        throw new Error("Prompt not found");
      }

      const testUser = {
        id: "user-456",
        name: "GitHub User",
        email: "[email protected]",
      };

      const ghActionId = v4();
      await prisma.action.create({
        data: {
          id: ghActionId,
          projectId,
          type: "GITHUB_DISPATCH",
          config: {
            type: "GITHUB_DISPATCH",
            url: "https://webhook.example.com/dispatches",
            eventType: "prompt-update",
            githubToken: encrypt("ghp_test_token"),
            displayGitHubToken: "ghp_...n",
          },
        },
      });

      const ghAutomationId = v4();
      await prisma.automation.create({
        data: {
          id: ghAutomationId,
          projectId,
          triggerId,
          actionId: ghActionId,
          name: "GitHub Dispatch Automation",
        },
      });

      const ghExecutionId = v4();
      await prisma.automationExecution.create({
        data: {
          id: ghExecutionId,
          projectId,
          triggerId,
          automationId: ghAutomationId,
          actionId: ghActionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: promptId,
          input: {},
        },
      });

      const webhookInput: WebhookInput = {
        projectId,
        automationId: ghAutomationId,
        executionId: ghExecutionId,
        payload: {
          prompt: PromptDomainSchema.parse({
            ...fullPrompt,
            prompt: {
              content: "a".repeat(GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES),
            },
            config: {
              schema: {
                content: "b".repeat(
                  GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES,
                ),
              },
            },
          }),
          action: "created",
          type: "prompt-version",
          user: testUser,
        },
      };

      await executeWebhook(webhookInput, { skipValidation: true });

      const requests = webhookServer.getReceivedRequests();
      expect(requests).toHaveLength(1);
      expect(Buffer.byteLength(requests[0].body, "utf8")).toBeLessThan(
        GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES,
      );

      const payload = JSON.parse(requests[0].body);
      expect(payload.event_type).toBe("prompt-update");
      expect(Object.keys(payload.client_payload)).toHaveLength(8);
      expect(payload.client_payload.truncation).toEqual({
        payloadTruncated: true,
        truncatedFields: ["prompt.prompt", "prompt.config"],
      });
      expect(payload.client_payload.prompt.prompt).toBe(
        GITHUB_REPOSITORY_DISPATCH_TRUNCATION_MARKER,
      );
      expect(payload.client_payload.prompt.config).toEqual({});

      const clientPayloadKeys = Object.keys(payload.client_payload);
      expect(clientPayloadKeys[clientPayloadKeys.length - 1]).toBe("prompt");
    });

    it("should truncate oversized GitHub dispatch monitor-alert filters", async () => {
      const ghActionId = v4();
      await prisma.action.create({
        data: {
          id: ghActionId,
          projectId,
          type: "GITHUB_DISPATCH",
          config: {
            type: "GITHUB_DISPATCH",
            url: "https://webhook.example.com/dispatches",
            eventType: "monitor-alert",
            githubToken: encrypt("ghp_test_token"),
            displayGitHubToken: "ghp_...n",
          },
        },
      });

      const ghAutomationId = v4();
      await prisma.automation.create({
        data: {
          id: ghAutomationId,
          projectId,
          triggerId,
          actionId: ghActionId,
          name: "GitHub Dispatch Monitor Automation",
        },
      });

      const ghExecutionId = v4();
      await prisma.automationExecution.create({
        data: {
          id: ghExecutionId,
          projectId,
          triggerId,
          automationId: ghAutomationId,
          actionId: ghActionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: v4(),
          input: {},
        },
      });

      const webhookInput: WebhookInput = {
        projectId,
        automationId: ghAutomationId,
        executionId: ghExecutionId,
        payload: {
          id: v4(),
          timestamp: new Date(),
          type: "monitor-alert",
          apiVersion: "v1",
          payload: {
            monitorId,
            projectId,
            severity: "ALERT",
            permalink: `https://cloud.langfuse.com/project/${projectId}/monitors/mon`,
            timestamp: new Date(),
            fromTimestamp: new Date(Date.now() - 5 * 60_000),
            toTimestamp: new Date(),
            message: { title: "High error rate", body: "errors > 100" },
            view: "observations",
            filters: [
              {
                column: "metadata.user_id",
                operator: "any of",
                value: [
                  "x".repeat(
                    GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES + 1000,
                  ),
                ],
                type: "stringOptions",
              },
            ],
            window: "5m",
          },
        },
      };

      await executeWebhook(webhookInput, { skipValidation: true });

      const requests = webhookServer.getReceivedRequests();
      expect(requests).toHaveLength(1);
      expect(Buffer.byteLength(requests[0].body, "utf8")).toBeLessThan(
        GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES,
      );

      const payload = JSON.parse(requests[0].body);
      expect(payload.event_type).toBe("monitor-alert");
      expect(payload.client_payload.payload.filters).toEqual([]);
      expect(payload.client_payload.truncation).toEqual({
        payloadTruncated: true,
        truncatedFields: ["payload.filters", "payload.message.body"],
      });
    });
  });

  describe("stale monitor alerts", () => {
    const alertedAt = new Date("2026-05-27T14:00:00.000Z");

    const buildMonitorAlertInput = (alertAlertedAt?: Date) =>
      ({
        projectId,
        automationId,
        executionId: v4(),
        payload: {
          id: v4(),
          timestamp: new Date(),
          type: "monitor-alert",
          apiVersion: "v1",
          payload: {
            monitorId,
            projectId,
            severity: "ALERT",
            timestamp: new Date(),
            fromTimestamp: new Date(Date.now() - 5 * 60_000),
            toTimestamp: new Date(),
            message: { title: "High error rate", body: "errors > 100" },
            view: "observations",
            filters: [],
            window: "5m",
            alertedAt: alertAlertedAt,
          },
        },
      }) satisfies WebhookInput;

    const useErrorEndpoint = async () => {
      const action = await prisma.action.findUniqueOrThrow({
        where: { id: actionId },
      });
      await prisma.action.update({
        where: { id: actionId },
        data: {
          config: {
            ...(action.config as WebhookActionConfigWithSecrets),
            url: "https://webhook-error.example.com/test",
          },
        },
      });
    };

    it("drops the alert without counting a failure once the monitor emitted a newer alert", async () => {
      await prisma.monitor.update({
        where: { id: monitorId },
        data: { alertedAt: new Date(alertedAt.getTime() + 1) },
      });
      await useErrorEndpoint();
      const failureKey = `automation-failures:${projectId}:${automationId}`;
      await redis!.del(failureKey);

      await executeWebhook(buildMonitorAlertInput(alertedAt), {
        skipValidation: true,
      });

      expect(webhookServer.getReceivedRequests()).toHaveLength(0);
      expect(await redis!.get(failureKey)).toBeNull();
    });

    it("drops the alert when its monitor was deleted", async () => {
      await prisma.monitor.delete({ where: { id: monitorId } });

      await executeWebhook(buildMonitorAlertInput(alertedAt), {
        skipValidation: true,
      });

      expect(webhookServer.getReceivedRequests()).toHaveLength(0);
    });

    it("drops the alert for every action type, not only webhooks", async () => {
      await prisma.monitor.update({
        where: { id: monitorId },
        data: { alertedAt: new Date(alertedAt.getTime() + 1) },
      });
      await prisma.action.update({
        where: { id: actionId },
        data: {
          type: "GITHUB_DISPATCH",
          config: {
            type: "GITHUB_DISPATCH",
            url: "https://webhook.example.com/dispatches",
            eventType: "monitor-alert",
            githubToken: encrypt("ghp_test_token"),
            displayGitHubToken: "ghp_...n",
          },
        },
      });

      await executeWebhook(buildMonitorAlertInput(alertedAt), {
        skipValidation: true,
      });

      expect(webhookServer.getReceivedRequests()).toHaveLength(0);
    });

    it.each([
      // The processor publishes before it saves alertedAt.
      ["before the monitor saved it", new Date(0)],
      ["after the monitor saved it", alertedAt],
    ])("delivers the alert %s", async (_, monitorAlertedAt) => {
      await prisma.monitor.update({
        where: { id: monitorId },
        data: { alertedAt: monitorAlertedAt },
      });

      await executeWebhook(buildMonitorAlertInput(alertedAt), {
        skipValidation: true,
      });

      expect(webhookServer.getReceivedRequests()).toHaveLength(1);
    });

    it("delivers the alert when the monitor changed severity without emitting an alert", async () => {
      await prisma.monitor.update({
        where: { id: monitorId },
        data: {
          severityChangedAt: new Date(alertedAt.getTime() + 1),
          alertedAt,
        },
      });

      await executeWebhook(buildMonitorAlertInput(alertedAt), {
        skipValidation: true,
      });

      expect(webhookServer.getReceivedRequests()).toHaveLength(1);
    });

    it("delivers alerts queued without alertedAt", async () => {
      await prisma.monitor.update({
        where: { id: monitorId },
        data: { alertedAt: new Date() },
      });

      await executeWebhook(buildMonitorAlertInput(), {
        skipValidation: true,
      });

      expect(webhookServer.getReceivedRequests()).toHaveLength(1);
    });
  });

  describe("PagerDuty action", () => {
    const routingKey = "R0abcdefghijklmnopqrstuvwxyz9xYz";

    const setPagerDutyConfig = (region: "US" | "EU") =>
      prisma.action.update({
        where: { id: actionId },
        data: {
          type: "PAGERDUTY",
          config: {
            type: "PAGERDUTY",
            routingKey: encrypt(routingKey),
            displayRoutingKey: "R0ab...9xYz",
            region,
          },
        },
      });

    const buildMonitorInput = (
      severity: "ALERT" | "WARNING" | "OK" | "NO_DATA",
    ): WebhookInput => ({
      projectId,
      automationId,
      executionId: v4(),
      payload: {
        id: v4(),
        timestamp: new Date(),
        type: "monitor-alert",
        apiVersion: "v1",
        payload: {
          monitorId,
          projectId,
          severity,
          permalink: `https://cloud.langfuse.com/project/${projectId}/alerts/${monitorId}`,
          timestamp: new Date("2026-05-27T14:32:00.000Z"),
          fromTimestamp: new Date("2026-05-27T14:00:00.000Z"),
          toTimestamp: new Date("2026-05-27T14:30:00.000Z"),
          message: { title: "Hallucination rate breached", body: "0.42 > 0.3" },
          view: "observations",
          filters: [
            {
              column: "providedModelName",
              type: "stringOptions",
              operator: "any of",
              value: ["gpt-4o"],
            },
          ],
          window: "30m",
          monitorName: "Hallucination rate",
          metric: { measure: "latency", aggregation: "p95" },
          metricValue: 0.42,
          thresholdOperator: "GT",
          alertThreshold: 0.3,
          warningThreshold: 0.2,
        },
      },
    });

    const pagerDutyRequests = () =>
      webhookServer.getReceivedRequests().map((request) => ({
        url: request.url,
        body: JSON.parse(request.body),
      }));

    const lastPagerDutyRequest = () => pagerDutyRequests().at(-1)!;

    beforeEach(async () => {
      await setPagerDutyConfig("US");
      await redis!.del(`automation-failures:${projectId}:${automationId}`);
    });

    it("triggers a critical alert keyed by the monitor with LLMOps details", async () => {
      await executeWebhook(buildMonitorInput("ALERT"));

      const [{ url, body }] = pagerDutyRequests();
      const project = await prisma.project.findUniqueOrThrow({
        where: { id: projectId },
      });
      expect(url).toBe("https://events.pagerduty.com/v2/enqueue");
      expect(body).toMatchObject({
        routing_key: routingKey,
        event_action: "trigger",
        dedup_key: `langfuse-${projectId}-${monitorId}`,
        client: "Langfuse",
        links: [
          {
            href: `https://cloud.langfuse.com/project/${projectId}/alerts/${monitorId}`,
          },
        ],
        payload: {
          summary: "Hallucination rate breached",
          source: `langfuse/${project.name}`,
          severity: "critical",
          timestamp: "2026-05-27T14:32:00.000Z",
          class: "monitor-alert",
          component: "observations",
          custom_details: {
            llmops: {
              model_name: "gpt-4o",
              project_name: project.name,
              monitor_name: "Hallucination rate",
              monitor_id: monitorId,
              monitor_alert_start: "2026-05-27T14:00:00.000Z",
              monitor_alert_end: "2026-05-27T14:30:00.000Z",
              metric_value: 0.42,
              threshold_value: 0.3,
            },
            langfuse: { window: "30m" },
          },
        },
      });
    });

    it.each([
      ["WARNING", "warning"],
      ["NO_DATA", "error"],
    ] as const)(
      "maps %s to PagerDuty severity %s",
      async (severity, expected) => {
        await executeWebhook(buildMonitorInput(severity));

        expect(pagerDutyRequests()[0].body.payload.severity).toBe(expected);
      },
    );

    it("re-triggers the monitor's alert under the same key on a severity change", async () => {
      await executeWebhook(buildMonitorInput("WARNING"));
      await executeWebhook(buildMonitorInput("ALERT"));

      expect(
        pagerDutyRequests().map(({ body }) => [
          body.event_action,
          body.dedup_key,
          body.payload.severity,
        ]),
      ).toEqual([
        ["trigger", `langfuse-${projectId}-${monitorId}`, "warning"],
        ["trigger", `langfuse-${projectId}-${monitorId}`, "critical"],
      ]);
    });

    it("resolves the monitor's alert on recovery", async () => {
      await executeWebhook(buildMonitorInput("OK"));

      expect(pagerDutyRequests().map(({ body }) => body)).toEqual([
        {
          routing_key: routingKey,
          event_action: "resolve",
          dedup_key: `langfuse-${projectId}-${monitorId}`,
        },
      ]);
    });

    it("sends to the EU host for EU routing keys", async () => {
      await setPagerDutyConfig("EU");

      await executeWebhook(buildMonitorInput("ALERT"));

      expect(pagerDutyRequests()[0].url).toBe(
        "https://events.eu.pagerduty.com/v2/enqueue",
      );
    });

    it.each(["prompt", "skill"] as const)(
      "sends %s changes as change events and completes the execution",
      async (entity) => {
        const fullPrompt = await prisma.prompt.findUniqueOrThrow({
          where: { id: promptId },
        });
        const input: WebhookInput = {
          projectId,
          automationId,
          executionId,
          payload:
            entity === "prompt"
              ? {
                  prompt: PromptDomainSchema.parse(fullPrompt),
                  action: "created",
                  type: "prompt-version",
                }
              : {
                  type: "skill-version",
                  action: "created",
                  skill: {
                    id: v4(),
                    projectId,
                    name: "test-skill",
                    version: 1,
                    description: "Test skill",
                    frontmatter: {},
                    files: [],
                    tags: [],
                    labels: ["latest"],
                    commitMessage: null,
                    createdBy: "test-user",
                    createdAt: new Date(),
                    updatedAt: new Date(),
                  },
                },
        };
        await prisma.automationExecution.create({
          data: {
            id: executionId,
            projectId,
            triggerId,
            automationId,
            actionId,
            status: ActionExecutionStatus.PENDING,
            sourceId: executionId,
            input: {},
          },
        });

        await executeWebhook(input);

        const { url, body } = lastPagerDutyRequest();
        expect(url).toBe("https://events.pagerduty.com/v2/change/enqueue");
        expect(body).toMatchObject({
          routing_key: routingKey,
          payload: {
            summary: `Langfuse ${entity} "test-${entity}" version 1 created`,
            custom_details: {
              [`${entity}_name`]: `test-${entity}`,
              [`${entity}_version`]: 1,
            },
          },
        });
        expect(body).not.toHaveProperty("event_action");
        expect(body.links[0].href).toContain(
          `/project/${projectId}/${entity}s/`,
        );
        expect(body.links[0].href).toContain("version=1");
        const execution = await prisma.automationExecution.findUniqueOrThrow({
          where: { id: executionId },
        });
        expect(execution.status).toBe(ActionExecutionStatus.COMPLETED);
      },
    );

    it("always triggers project notifications keyed by event type and resource", async () => {
      await prisma.automationExecution.create({
        data: {
          id: executionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: executionId,
          input: {},
        },
      });

      await executeWebhook({
        projectId,
        automationId,
        executionId,
        payload: {
          id: executionId,
          timestamp: new Date(),
          type: "project-notification",
          apiVersion: "v1",
          event: {
            eventType: "blob-export-failed",
            severity: "ALERT",
            projectId,
            projectName: "Test Project",
            resourceId: "res_1",
            resourceName: "my-bucket",
            message: "Blob storage export failed.",
          },
        },
      });

      expect(lastPagerDutyRequest().body).toMatchObject({
        event_action: "trigger",
        dedup_key: `langfuse-${projectId}-blob-export-failed-res_1`,
        payload: { severity: "error", class: "blob-export-failed" },
      });
    });

    it.each([429, 500])(
      "retries a %i in-process, then counts a failure once retries run out",
      async (status) => {
        webhookServer.pagerDutyStatus = status;

        await executeWebhook(buildMonitorInput("ALERT"));

        expect(webhookServer.getReceivedRequests()).toHaveLength(4);
        expect(
          await redis!.get(`automation-failures:${projectId}:${automationId}`),
        ).toBe("1");
      },
    );

    it("counts a rejected event as a failure without retrying", async () => {
      webhookServer.pagerDutyStatus = 400;

      await executeWebhook(buildMonitorInput("ALERT"));

      expect(
        await redis!.get(`automation-failures:${projectId}:${automationId}`),
      ).toBe("1");
    });

    it("records PagerDuty's status on a rejected prompt change", async () => {
      webhookServer.pagerDutyStatus = 400;
      const fullPrompt = await prisma.prompt.findUniqueOrThrow({
        where: { id: promptId },
      });
      await prisma.automationExecution.create({
        data: {
          id: executionId,
          projectId,
          triggerId,
          automationId,
          actionId,
          status: ActionExecutionStatus.PENDING,
          sourceId: executionId,
          input: {},
        },
      });

      await executeWebhook({
        projectId,
        automationId,
        executionId,
        payload: {
          prompt: PromptDomainSchema.parse(fullPrompt),
          action: "created",
          type: "prompt-version",
        },
      });

      const execution = await prisma.automationExecution.findUniqueOrThrow({
        where: { id: executionId },
      });
      expect(execution).toMatchObject({
        status: ActionExecutionStatus.ERROR,
        output: { httpStatus: 400 },
      });
    });

    it("returns the routing key only on the execution read path", async () => {
      const withSecrets = await getActionByIdWithSecrets({
        projectId,
        actionId,
      });
      const safe = await getActionById({ projectId, actionId });

      const config = withSecrets?.config as { routingKey: string };
      expect(decrypt(config.routingKey)).toBe(routingKey);
      expect(safe?.config).toEqual({
        type: "PAGERDUTY",
        displayRoutingKey: "R0ab...9xYz",
        region: "US",
      });
      expect(JSON.stringify(safe)).not.toContain(routingKey);
    });
  });
});

describe("getActionByIdWithSecrets legacy headers", () => {
  it("sends legacy headers as secret and never falls back to a legacy value its requestHeaders entry overrides", async () => {
    const { projectId } = await createOrgProjectAndApiKey();
    const action = await prisma.action.create({
      data: {
        id: v4(),
        projectId,
        type: "WEBHOOK",
        config: {
          type: "WEBHOOK",
          url: "https://example.com/webhook",
          headers: {
            "x-api-key": "legacy-api-key",
            Authorization: "Bearer stale-token",
          },
          requestHeaders: {
            Authorization: { secret: true, value: "not-a-valid-ciphertext" },
          },
          apiVersion: { prompt: "v1" },
          secretKey: encrypt("whsec"),
          displaySecretKey: "lf-whsec_...",
        },
      },
    });

    const result = await getActionByIdWithSecrets({
      projectId,
      actionId: action.id,
    });

    expect(
      (result?.config as WebhookActionConfigWithSecrets).requestHeaders,
    ).toEqual({ "x-api-key": { secret: true, value: "legacy-api-key" } });
  });
});