Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

40.8 KB
Raw Blame
/* eslint-disable @typescript-eslint/no-use-before-define */
import {
  InternalServerError,
  PromptWebhookOutboundSchema,
  SkillWebhookOutboundSchema,
  WebhookDefaultHeaders,
  ActionExecutionStatus,
  LangfuseNotFoundError,
  JobConfigState,
  isSlackActionConfig,
  isWebhookAction,
  isGitHubDispatchAction,
  isPagerDutyActionConfig,
  isBaseError,
  type AutomationDomain,
  type ActionDomainWithSecrets,
  type PagerDutyActionConfigWithSecrets,
} from "@langfuse/shared";
import { decrypt, createSignatureHeader } from "@langfuse/shared/encryption";
import { Prisma, prisma } from "@langfuse/shared/src/db";
import {
  validateWebhookURL,
  webhookWhitelistFromEnv,
  whitelistFromEnv,
  fetchWithSecureRedirects,
  urlForLog,
  type WebhookValidationWhitelist,
  WEBHOOK_URL_VALIDATION_LOG_CONTEXT,
  buildWebhookRequestHeaders,
  recordIncrement,
} from "@langfuse/shared/src/server";
import {
  TQueueJobTypes,
  QueueName,
  WebhookInput,
  getAutomationById,
  getActionByIdWithSecrets,
  getActionById,
  getConsecutiveAutomationFailures,
  SlackService,
  logger,
  redis,
  ProjectNotificationWebhookQueueEventSchema,
  buildProjectNotificationSlackMessage,
} from "@langfuse/shared/src/server";
import { type MonitorAlert } from "@langfuse/shared/monitors";
import {
  MonitorWebhookQueueEventSchema,
  buildMonitorAlertSlackMessage,
} from "@langfuse/shared/monitors/server";
import {
  pagerDutyClient,
  type PagerDutyRegion,
} from "@langfuse/shared/pagerduty/server";
import { Processor, Job } from "bullmq";
import { backOff } from "exponential-backoff";
import { env } from "../env";
import { SlackMessageBuilder } from "../features/slack/slackMessageBuilder";
import {
  buildPagerDutyEvent,
  type PagerDutyEvent,
} from "../features/pagerduty/buildPagerDutyEvent";

/** WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS lets a retransmitted SYN finish before Node closes the socket. */
const WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS = 2_000;

// GitHub repository_dispatch client_payload: max 10 top-level properties and <64KB.
// https://docs.github.com/en/rest/repos/repos#create-a-repository-dispatch-event
const GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES = 64 * 1024;
const GITHUB_REPOSITORY_DISPATCH_TRUNCATION_MARKER =
  "[TRUNCATED: GitHub repository_dispatch payload exceeded size limit]";
const GITHUB_REPOSITORY_DISPATCH_TRUNCATED_FIELDS = [
  "prompt.prompt",
  "prompt.config",
];

/** automationFailureThreshold is the consecutive-failure count after which a monitor-alert trigger is auto-disabled. Mirrors prompt-version's getConsecutiveAutomationFailures threshold (>= 5 failures since the last success). */
const automationFailureThreshold = 5;
const automationFailureTtlSeconds = 24 * 60 * 60;
const automationFailureKey = (projectId: string, automationId: string) =>
  `automation-failures:${projectId}:${automationId}`;

/** incrementAutomationFailure bumps the consecutive-failure counter for a monitor-alert Automation and refreshes the 24h TTL; returns the new count. Inline because monitor-alert is the only consumer today. */
async function incrementAutomationFailure(args: {
  projectId: string;
  automationId: string;
}): Promise<number> {
  if (!redis) return 0;
  const key = automationFailureKey(args.projectId, args.automationId);
  const results = await redis
    .multi()
    .incr(key)
    .expire(key, automationFailureTtlSeconds)
    .exec();
  return Number(results?.[0]?.[1] ?? 0);
}

/** resetAutomationFailures clears the streak after a successful delivery. */
async function resetAutomationFailures(args: {
  projectId: string;
  automationId: string;
}): Promise<void> {
  if (!redis) return;
  await redis.del(automationFailureKey(args.projectId, args.automationId));
}

/** WebhookOutboundPayload is the validated HTTP body of a WebhookInput. */
type WebhookOutboundPayload = ReturnType<typeof buildWebhookOutboundPayload>;

/** buildWebhookOutboundPayload validates and returns the HTTP body for a WebhookInput, discriminating on payload.type. monitor-alert: the processor already built the unified envelope — pass through. prompt-version: wrap at dispatch time with id = executionId and timestamp = now. */
function buildWebhookOutboundPayload(input: WebhookInput) {
  if (input.payload.type === "monitor-alert") {
    const parsed = MonitorWebhookQueueEventSchema.safeParse(input.payload);
    if (!parsed.success) {
      throw new InternalServerError(
        `Invalid monitor-alert payload: ${parsed.error.message}`,
      );
    }
    return parsed.data;
  }
  if (input.payload.type === "project-notification") {
    const parsed = ProjectNotificationWebhookQueueEventSchema.safeParse(
      input.payload,
    );
    if (!parsed.success) {
      throw new InternalServerError(
        `Invalid project notification payload: ${parsed.error.message}`,
      );
    }
    return parsed.data;
  }
  if (input.payload.type === "skill-version") {
    return SkillWebhookOutboundSchema.parse({
      id: input.executionId,
      timestamp: new Date(),
      apiVersion: "v1",
      ...input.payload,
    });
  }
  const promptPayload = input.payload;
  const parsed = PromptWebhookOutboundSchema.safeParse({
    id: input.executionId,
    timestamp: new Date(),
    type: promptPayload.type,
    apiVersion: "v1",
    action: promptPayload.action,
    prompt: promptPayload.prompt,
    user: promptPayload.user
      ? { name: promptPayload.user.name, email: promptPayload.user.email }
      : undefined,
  });
  if (!parsed.success) {
    throw new InternalServerError(
      `Invalid webhook payload: ${parsed.error.message}`,
    );
  }
  return parsed.data;
}

// Handles both webhook and slack actions
export const webhookProcessor: Processor = async (
  job: Job<TQueueJobTypes[QueueName.WebhookQueue]>,
) => {
  try {
    return await executeWebhook(job.data.payload);
  } catch (error) {
    logger.error("Error executing WebhookJob", error);
    throw error;
  }
};

// TODO: Webhook outgoing API versioning
export const executeWebhook = async (
  input: WebhookInput,
  options?: { skipValidation?: boolean },
) => {
  const { projectId, automationId } = input;

  try {
    logger.debug(`Executing action for automation ${automationId}`);

    const automation = await getAutomationById({
      projectId,
      automationId,
    });

    if (!automation) {
      logger.warn(
        `Automation ${automationId} not found for project ${projectId}. We ack the job and will not retry.`,
      );
      return;
    }

    if (
      await isStaleMonitorAlert({
        projectId,
        automationId,
        actionType: automation.action.type,
        input,
      })
    ) {
      return;
    }

    // Route to appropriate handler based on action type
    if (automation.action.type === "WEBHOOK") {
      await executeWebhookAction({
        input,
        automation,
        skipValidation: options?.skipValidation,
      });
    } else if (automation.action.type === "SLACK") {
      await executeSlackAction({
        input,
        automation,
      });
    } else if (automation.action.type === "GITHUB_DISPATCH") {
      await executeGitHubDispatchAction({
        input,
        automation,
        skipValidation: options?.skipValidation,
      });
    } else if (automation.action.type === "PAGERDUTY") {
      await executePagerDutyAction({ input, automation });
    } else {
      throw new InternalServerError(
        `Unsupported action type: ${automation.action.type}`,
      );
    }

    logger.debug(
      `Action executed successfully for action ${automation.action.id}`,
    );
  } catch (error) {
    logger.error("Error executing action", error);
    throw error;
  }
};

/** isStaleMonitorAlert prevents older queued alerts from overwriting newer published alerts by reporting which jobs to drop. */
async function isStaleMonitorAlert({
  projectId,
  automationId,
  actionType,
  input,
}: {
  projectId: string;
  automationId: string;
  actionType: string;
  input: WebhookInput;
}): Promise<boolean> {
  if (input.payload.type !== "monitor-alert") return false;
  const envelope = buildWebhookOutboundPayload(input);
  if (envelope.type !== "monitor-alert") return false;
  const alert = envelope.payload;

  const reason = await getStaleMonitorAlertReason({ projectId, alert });
  if (!reason) return false;

  recordStaleMonitorAlertDrop({
    projectId,
    automationId,
    actionType,
    monitorId: alert.monitorId,
    reason,
  });
  return true;
}

/** getStaleMonitorAlertReason says why an alert is stale, or undefined when it is current. */
async function getStaleMonitorAlertReason({
  projectId,
  alert,
}: {
  projectId: string;
  alert: MonitorAlert;
}): Promise<StaleMonitorAlertReason | undefined> {
  const monitor = await prisma.monitor.findUnique({
    where: { id: alert.monitorId },
    select: { projectId: true, alertedAt: true },
  });
  if (!monitor || monitor.projectId !== projectId) return "monitor_deleted";
  // The processor publishes before it saves alertedAt, so only a later alertedAt is newer.
  if (
    alert.alertedAt &&
    monitor.alertedAt &&
    monitor.alertedAt > alert.alertedAt
  ) {
    return "newer_alert";
  }
  return undefined;
}

/** recordStaleMonitorAlertDrop logs a dropped stale alert and counts it by reason and action type. */
function recordStaleMonitorAlertDrop({
  projectId,
  automationId,
  actionType,
  monitorId,
  reason,
}: {
  projectId: string;
  automationId: string;
  actionType: string;
  monitorId: string;
  reason: StaleMonitorAlertReason;
}): void {
  logger.info("Dropping stale monitor alert", {
    projectId,
    automationId,
    monitorId,
    reason,
  });
  recordIncrement(
    "langfuse.queue.webhook_queue.stale_monitor_alerts_dropped",
    1,
    {
      reason,
      actionType,
    },
  );
}

/**
 * Shared HTTP execution logic for webhook and GitHub dispatch actions
 * Handles: fetch with timeout, retry logic, status validation, execution tracking
 */
async function executeHttpAction({
  url,
  payload,
  headers,
  projectId,
  skipValidation,
  automation,
  executionId,
  executionStart,
  actionConfig,
  additionalSensitiveHeaders,
  payloadType,
  whitelist,
  addressAttemptTimeoutMs,
}: {
  url: string;
  payload: string;
  headers: Record<string, string>;
  projectId: string;
  skipValidation?: boolean;
  automation: AutomationDomain;
  executionId: string;
  executionStart: Date;
  actionConfig: ActionDomainWithSecrets;
  additionalSensitiveHeaders?: string[];
  payloadType: WebhookInput["payload"]["type"];
  whitelist: WebhookValidationWhitelist;
  addressAttemptTimeoutMs?: number;
}): Promise<{ httpStatus: number; responseBody: string }> {
  // monitor-alert does not write AutomationExecution rows; auto-disable rides a Redis counter instead.
  const isMonitorAlert = payloadType === "monitor-alert";
  let httpStatus: number | undefined;
  let responseBody: string | undefined;

  try {
    // Execute HTTP request with retries
    await backOff(
      async () => {
        logger.info(
          `Sending HTTP request to ${urlForLog(url)} for action ${automation.action.id}`,
        );

        // Create AbortController for timeout
        const abortController = new AbortController();
        const timeoutId = setTimeout(() => {
          abortController.abort();
        }, env.LANGFUSE_WEBHOOK_TIMEOUT_MS);

        try {
          // Skip validation when flag is set (for tests with MSW mocking)
          if (!skipValidation) {
            await validateWebhookURL(url, whitelist);
          }

          const redirectOptions = skipValidation
            ? {
                maxRedirects: env.LANGFUSE_WEBHOOK_MAX_REDIRECTS,
                skipValidation: true as const,
                additionalSensitiveHeaders,
              }
            : {
                maxRedirects: env.LANGFUSE_WEBHOOK_MAX_REDIRECTS,
                redirectValidation: {
                  validateUrl: validateWebhookURL,
                  whitelist,
                  logContext: WEBHOOK_URL_VALIDATION_LOG_CONTEXT,
                  addressAttemptTimeoutMs,
                },
                additionalSensitiveHeaders,
              };

          const redirectResult = await fetchWithSecureRedirects(
            url,
            {
              method: "POST",
              body: payload,
              headers,
              signal: abortController.signal,
            },
            redirectOptions,
          );

          const res = redirectResult.response;

          // Log redirect chain if any redirects occurred (security audit trail)
          if (redirectResult.redirectChain.length > 0) {
            logger.info("Webhook followed redirects", {
              actionId: automation.action.id,
              initialUrl: urlForLog(url),
              finalUrl: urlForLog(redirectResult.finalUrl),
              redirectCount: redirectResult.redirectChain.length,
              redirectChain: redirectResult.redirectChain.map(urlForLog),
            });
          }

          httpStatus = res.status;
          responseBody = await res.text();

          if (!res.ok) {
            logger.warn(
              `Webhook does not return 2xx status: failed with status ${res.status} for url ${urlForLog(url)} and project ${projectId}. Body: ${responseBody}`,
            );
            throw new Error(
              `Webhook does not return 2xx status: failed with status ${res.status} for url ${urlForLog(url)} and project ${projectId}`,
            );
          }
        } catch (error) {
          if (error instanceof Error && error.name === "AbortError") {
            logger.warn(
              `Webhook timeout after ${env.LANGFUSE_WEBHOOK_TIMEOUT_MS}ms for url ${urlForLog(url)} and project ${projectId}`,
            );
            throw new Error(
              `Webhook timeout after ${env.LANGFUSE_WEBHOOK_TIMEOUT_MS}ms for url ${urlForLog(url)} and project ${projectId}`,
            );
          }
          throw error;
        } finally {
          clearTimeout(timeoutId);
        }
      },
      {
        numOfAttempts: 4,
      },
    );

    // Update execution status on success
    if (isMonitorAlert) {
      try {
        await resetAutomationFailures({
          projectId,
          automationId: automation.id,
        });
      } catch (resetError) {
        logger.warn(
          `Failed to reset automation failure counter after successful delivery for automation ${automation.id} in project ${projectId}`,
          resetError,
        );
      }
    } else {
      await prisma.automationExecution.update({
        where: {
          projectId,
          triggerId: automation.trigger.id,
          actionId: automation.action.id,
          id: executionId,
        },
        data: {
          status: ActionExecutionStatus.COMPLETED,
          startedAt: executionStart,
          finishedAt: new Date(),
        },
      });
    }

    return { httpStatus: httpStatus!, responseBody: responseBody! };
  } catch (error) {
    logger.error("Error executing HTTP action", error);

    // Handle action failure with retry logic and trigger disabling
    const shouldRetryJob =
      error instanceof LangfuseNotFoundError ||
      error instanceof InternalServerError;

    if (shouldRetryJob) {
      logger.warn(`Retrying BullMQ for action ${automation.action.id}`);
      throw error; // Trigger BullMQ retry
    }

    if (!actionConfig) {
      logger.error("Action config not found for failure handling");
      throw error;
    }

    if (isMonitorAlert) {
      // monitor-alert: Redis counter compensates for the missing
      // AutomationExecution streak. Disable on the 5th consecutive failure.
      const count = await incrementAutomationFailure({
        projectId,
        automationId: automation.id,
      });
      if (count >= automationFailureThreshold) {
        await prisma.trigger.update({
          where: { id: automation.trigger.id, projectId },
          data: { status: JobConfigState.INACTIVE },
        });
        try {
          await resetAutomationFailures({
            projectId,
            automationId: automation.id,
          });
        } catch (resetError) {
          logger.warn(
            `Failed to reset automation failure counter after auto-disable for automation ${automation.id} in project ${projectId}`,
            resetError,
          );
        }
        logger.warn(
          `Automation ${automation.trigger.id} disabled after ${count} consecutive failures in project ${projectId} (monitor-alert)`,
        );
      }
      return { httpStatus: httpStatus || 0, responseBody: responseBody || "" };
    }

    // project-notification records the ERROR row for visibility but never
    // auto-disables; prompt-version / github keep the consecutive-failure
    // disable behavior below.
    const disablesOnFailure = payloadType !== "project-notification";

    // Update execution status and check if we should disable trigger
    await prisma.$transaction(async (tx) => {
      // Update execution status
      await tx.automationExecution.update({
        where: {
          id: executionId,
          projectId,
          triggerId: automation.trigger.id,
          actionId: automation.action.id,
        },
        data: {
          status: ActionExecutionStatus.ERROR,
          startedAt: executionStart,
          finishedAt: new Date(),
          error: error instanceof Error ? error.message : "Unknown error",
          output: httpStatus
            ? {
                httpStatus,
                responseBody: responseBody?.substring(0, 1000),
              }
            : undefined,
        },
      });

      if (!disablesOnFailure) {
        return;
      }

      // Check consecutive failures from execution history
      const consecutiveFailures = await getConsecutiveAutomationFailures({
        automationId: automation.id,
        projectId,
      });

      logger.info(
        `Consecutive failures: ${consecutiveFailures} for trigger ${automation.trigger.id} in project ${projectId}`,
      );

      // Check if trigger should be disabled (this is the 5th failure, looking for 4 in the past.)
      if (consecutiveFailures >= 4) {
        // Update trigger to inactive status
        await tx.trigger.update({
          where: { id: automation.trigger.id, projectId },
          data: { status: JobConfigState.INACTIVE },
        });

        // Update action config to store the failing execution ID
        // Type guard to ensure we only update webhook or GitHub dispatch configs
        if (
          isWebhookAction(actionConfig) ||
          isGitHubDispatchAction(actionConfig)
        ) {
          await setActionLastFailingExecutionId({
            tx,
            actionId: automation.action.id,
            projectId,
            executionId,
          });
        }

        logger.warn(
          `Automation ${automation.trigger.id} disabled after ${consecutiveFailures} consecutive failures in project ${projectId}`,
        );
      }
    });

    logger.debug(
      `HTTP action failed for action ${automation.action.id} in project ${projectId}`,
    );

    // Error has been handled - don't rethrow
    // Return empty response to indicate failure was handled
    return { httpStatus: httpStatus || 0, responseBody: responseBody || "" };
  }
}

/**
 * Execute webhook action with HTTP request and signature validation
 */
async function executeWebhookAction({
  input,
  automation,
  skipValidation,
}: {
  input: WebhookInput;
  automation: Awaited<ReturnType<typeof getAutomationById>>;
  skipValidation?: boolean;
}) {
  if (!automation) return;

  const { projectId, executionId } = input;
  const executionStart = new Date();

  const actionConfig = await getActionByIdWithSecrets({
    projectId,
    actionId: automation.action.id,
  });

  if (!actionConfig) {
    throw new InternalServerError("Action config not found");
  }

  if (!isWebhookAction(actionConfig)) {
    throw new InternalServerError(
      "Action config is not a valid webhook configuration",
    );
  }

  const webhookConfig = actionConfig.config;

  const envelope = buildWebhookOutboundPayload(input);
  let webhookPayload: string;
  if (input.payload.type === "prompt-version") {
    // Prompt-version contract: prompt key always last.
    const { prompt, user, ...otherFields } = envelope as Extract<
      typeof envelope,
      { type: "prompt-version" }
    >;
    webhookPayload = JSON.stringify({
      ...otherFields,
      ...(user ? { user } : {}),
      prompt,
    });
  } else {
    // monitor-alert: post the envelope verbatim.
    webhookPayload = JSON.stringify(envelope);
  }

  let requestHeaders: Record<string, string>;
  let additionalSensitiveHeaders: string[];
  try {
    const builtRequestHeaders = buildWebhookRequestHeaders({
      customHeaders: webhookConfig.requestHeaders,
      body: webhookPayload,
      signingSecret: decrypt(webhookConfig.secretKey),
    });
    requestHeaders = builtRequestHeaders.headers;
    additionalSensitiveHeaders = builtRequestHeaders.sensitiveHeaderNames;
  } catch (error) {
    logger.error(
      "Failed to decrypt webhook secret or generate signature",
      error,
    );
    throw new InternalServerError("Failed to generate webhook signature");
  }

  // Execute HTTP request using shared function
  await executeHttpAction({
    url: webhookConfig.url,
    payload: webhookPayload,
    headers: requestHeaders,
    projectId,
    skipValidation,
    automation,
    executionId,
    executionStart,
    actionConfig,
    additionalSensitiveHeaders,
    payloadType: input.payload.type,
    whitelist: webhookWhitelistFromEnv(),
    addressAttemptTimeoutMs: WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS,
  });
}

function buildSkillDispatchPayload(
  validated: Extract<
    ReturnType<typeof buildWebhookOutboundPayload>,
    { type: "skill-version" }
  >,
  eventType: string,
): string {
  let githubPayload = JSON.stringify({
    event_type: eventType,
    client_payload: validated,
  });
  if (
    Buffer.byteLength(githubPayload, "utf8") >=
    GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES
  ) {
    githubPayload = JSON.stringify({
      event_type: eventType,
      client_payload: {
        id: validated.id,
        timestamp: validated.timestamp,
        type: validated.type,
        apiVersion: validated.apiVersion,
        action: validated.action,
        skill: {
          id: validated.skill.id,
          name: validated.skill.name,
          version: validated.skill.version,
          projectId: validated.skill.projectId,
        },
        truncation: {
          payloadTruncated: true,
          truncatedFields: [
            "skill.files",
            "skill.frontmatter",
            "skill.description",
            "skill.tags",
            "skill.labels",
            "skill.commitMessage",
            "skill.createdBy",
            "skill.createdAt",
            "skill.updatedAt",
            "user",
          ],
        },
      },
    });
  }
  if (
    Buffer.byteLength(githubPayload, "utf8") >=
    GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES
  ) {
    throw new InternalServerError(
      "Skill repository dispatch payload exceeds GitHub's size limit",
    );
  }
  return githubPayload;
}

/**
 * Execute GitHub Dispatch action with repository dispatch API
 */
async function executeGitHubDispatchAction({
  input,
  automation,
  skipValidation,
}: {
  input: WebhookInput;
  automation: Awaited<ReturnType<typeof getAutomationById>>;
  skipValidation?: boolean;
}) {
  if (!automation) return;

  const { projectId, executionId } = input;
  const executionStart = new Date();

  const actionConfig = await getActionByIdWithSecrets({
    projectId,
    actionId: automation.action.id,
  });

  if (!actionConfig) {
    throw new InternalServerError("Action config not found");
  }

  if (!isGitHubDispatchAction(actionConfig)) {
    throw new InternalServerError(
      "Action config is not a valid GitHub dispatch configuration",
    );
  }

  const githubConfig = actionConfig.config;
  const eventType = githubConfig.eventType;
  const envelope = buildWebhookOutboundPayload(input);

  let githubPayload: string;
  if (input.payload.type === "prompt-version") {
    const { prompt, user, ...otherFields } = envelope as Extract<
      typeof envelope,
      { type: "prompt-version" }
    >;
    const fullGithubPayload = JSON.stringify({
      event_type: eventType,
      client_payload: {
        ...otherFields,
        ...(user ? { user } : {}),
        prompt,
      },
    });
    githubPayload =
      Buffer.byteLength(fullGithubPayload, "utf8") <
      GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES
        ? fullGithubPayload
        : JSON.stringify({
            event_type: eventType,
            client_payload: {
              ...otherFields,
              ...(user ? { user } : {}),
              truncation: {
                payloadTruncated: true,
                truncatedFields: GITHUB_REPOSITORY_DISPATCH_TRUNCATED_FIELDS,
              },
              prompt: {
                ...prompt,
                prompt: GITHUB_REPOSITORY_DISPATCH_TRUNCATION_MARKER,
                config: {},
              },
            },
          });
  } else if (envelope.type === "skill-version") {
    githubPayload = buildSkillDispatchPayload(envelope, eventType);
  } else {
    // monitor-alert: post the envelope as client_payload.
    const monitorEnvelope = envelope as Extract<
      typeof envelope,
      { type: "monitor-alert" }
    >;
    const fullGithubPayload = JSON.stringify({
      event_type: eventType,
      client_payload: monitorEnvelope,
    });
    githubPayload =
      Buffer.byteLength(fullGithubPayload, "utf8") <
      GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES
        ? fullGithubPayload
        : JSON.stringify({
            event_type: eventType,
            client_payload: {
              ...monitorEnvelope,
              payload: {
                ...monitorEnvelope.payload,
                filters: [],
                message: {
                  ...monitorEnvelope.payload.message,
                  body: GITHUB_REPOSITORY_DISPATCH_TRUNCATION_MARKER,
                },
              },
              truncation: {
                payloadTruncated: true,
                truncatedFields: ["payload.filters", "payload.message.body"],
              },
            },
          });
  }

  // Prepare headers with GitHub token
  const requestHeaders: Record<string, string> = {};

  // Add default headers
  for (const [key, value] of Object.entries(WebhookDefaultHeaders)) {
    requestHeaders[key] = value;
  }

  try {
    const decryptedToken = decrypt(githubConfig.githubToken);

    // Add GitHub token as Bearer auth
    requestHeaders["Authorization"] = `Bearer ${decryptedToken}`;

    // Add signature for optional verification (using GitHub token as secret)
    const signature = createSignatureHeader(githubPayload, decryptedToken);
    requestHeaders["x-langfuse-signature"] = signature;
  } catch (error) {
    logger.error("Failed to decrypt GitHub token or generate signature", error);
    throw new InternalServerError("Failed to generate GitHub authentication");
  }

  // Execute HTTP request using shared function
  await executeHttpAction({
    url: githubConfig.url,
    payload: githubPayload,
    headers: requestHeaders,
    projectId,
    skipValidation,
    automation,
    executionId,
    executionStart,
    actionConfig,
    payloadType: input.payload.type,
    whitelist: whitelistFromEnv(),
  });
}

/**
 * Execute Slack action with message sending via SlackService
 */
async function executeSlackAction({
  input,
  automation,
}: {
  input: WebhookInput;
  automation: Awaited<ReturnType<typeof getAutomationById>>;
}) {
  if (!automation) return;

  const { projectId, executionId } = input;
  const executionStart = new Date();

  const actionConfig = await getActionById({
    projectId,
    actionId: automation.action.id,
  });

  if (!actionConfig) {
    throw new InternalServerError("Action config not found");
  }

  if (!isSlackActionConfig(actionConfig.config)) {
    throw new InternalServerError(
      "Action config is not a valid Slack configuration",
    );
  }

  const slackConfig = actionConfig.config;

  // TODO: unify automation execution log and failure handling across all message types (not scalable yet)
  // TODO: implement a strategy/policy + registry pattern for different webhook providers.
  if (input.payload.type === "monitor-alert") {
    try {
      await sendSlackMonitorAlert({
        payload: input.payload,
        projectId,
        channelId: slackConfig.channelId,
        automationId: automation.id,
      });
    } catch (error) {
      logger.error("Error executing Slack action", error);
      await monitorAlertDeliveryFailure({
        projectId,
        automation,
        channel: "slack",
      });
    }
    return;
  }

  if (input.payload.type === "project-notification") {
    // Project notifications resolve their AutomationExecution row (so the
    // executions table stays accurate) but never auto-disable — a persistently
    // failing channel keeps recording ERROR rows and stays ACTIVE.
    try {
      await sendSlackProjectNotification({
        payload: input.payload,
        projectId,
        channelId: slackConfig.channelId,
      });
      await prisma.automationExecution.update({
        where: {
          projectId,
          triggerId: automation.trigger.id,
          actionId: automation.action.id,
          id: executionId,
        },
        data: {
          status: ActionExecutionStatus.COMPLETED,
          startedAt: executionStart,
          finishedAt: new Date(),
        },
      });
    } catch (error) {
      logger.error("Error executing Slack action", error);
      await prisma.automationExecution.update({
        where: {
          id: executionId,
          projectId,
          triggerId: automation.trigger.id,
          actionId: automation.action.id,
        },
        data: {
          status: ActionExecutionStatus.ERROR,
          startedAt: executionStart,
          finishedAt: new Date(),
          error: error instanceof Error ? error.message : "Unknown error",
        },
      });
    }
    return;
  }

  try {
    // Build message blocks using predefined formats or custom template
    let blocks: any[] = [];
    let attachments:
      | { color: string; fallback?: string; blocks?: any[] }[]
      | undefined;

    // TODO: Custom templates not supported via the UI yet
    if (slackConfig.messageTemplate) {
      try {
        blocks = JSON.parse(slackConfig.messageTemplate);
        logger.debug(
          `Using custom message template for action ${automation.action.id}`,
        );
      } catch (error) {
        logger.warn(
          `Invalid Slack messageTemplate JSON for action ${automation.action.id}. Using default format`,
          { error: error instanceof Error ? error.message : "Unknown error" },
        );
      }
    }

    // Use predefined message format if no custom template or template failed
    if (blocks.length === 0) {
      const message = SlackMessageBuilder.buildMessage(input.payload);
      blocks = message.blocks;
      attachments = message.attachments;
      logger.debug(
        `Using predefined message format for action ${automation.action.id}`,
      );
    }

    // Get Slack WebClient for project via centralized SlackService
    const client =
      await SlackService.getInstance().getWebClientForProject(projectId);

    // Send message
    const sendResult = await SlackService.getInstance().sendMessage({
      client,
      channelId: slackConfig.channelId,
      blocks,
      attachments,
      text: "Langfuse Notification",
    });

    // Update execution status to completed
    await prisma.automationExecution.update({
      where: {
        projectId,
        triggerId: automation.trigger.id,
        actionId: automation.action.id,
        id: executionId,
      },
      data: {
        status: ActionExecutionStatus.COMPLETED,
        startedAt: executionStart,
        finishedAt: new Date(),
        output: {
          channel: sendResult.channel,
          messageTs: sendResult.messageTs,
        },
      },
    });
  } catch (error) {
    logger.error("Error executing Slack action", error);

    // Get action config for updating in case of failure
    const failureActionConfig = await getActionByIdWithSecrets({
      projectId,
      actionId: automation.action.id,
    });

    if (!failureActionConfig) {
      logger.error("Action config not found for failure handling");
      return;
    }

    // Update execution status and disable trigger
    await prisma.$transaction(async (tx) => {
      // Update execution status
      await tx.automationExecution.update({
        where: {
          id: executionId,
          projectId,
          triggerId: automation.trigger.id,
          actionId: automation.action.id,
        },
        data: {
          status: ActionExecutionStatus.ERROR,
          startedAt: executionStart,
          finishedAt: new Date(),
          error: error instanceof Error ? error.message : "Unknown error",
        },
      });

      // Update trigger to inactive status
      await tx.trigger.update({
        where: { id: automation.trigger.id, projectId },
        data: { status: JobConfigState.INACTIVE },
      });

      // Update action config to store the failing execution ID
      await setActionLastFailingExecutionId({
        tx,
        actionId: automation.action.id,
        projectId,
        executionId,
      });

      logger.warn(
        `Automation ${automation.trigger.id} disabled after 1 failure in project ${projectId}`,
      );
    });

    logger.debug(
      `Slack action failed for action ${automation.action.id} in project ${projectId}`,
    );
  }
}

/**
 * Execute PagerDuty action. Events go only to the fixed PagerDuty hosts of the
 * routing key's region. 429, 5xx and network errors are retried in-process
 * like HTTP actions; a failure that remains counts toward trigger auto-disable.
 */
async function executePagerDutyAction({
  input,
  automation,
}: {
  input: WebhookInput;
  automation: NonNullable<Awaited<ReturnType<typeof getAutomationById>>>;
}) {
  const { projectId, executionId } = input;
  const executionStart = new Date();

  const actionConfig = await getActionByIdWithSecrets({
    projectId,
    actionId: automation.action.id,
  });
  if (!actionConfig || !isPagerDutyActionConfig(actionConfig.config)) {
    throw new InternalServerError("PagerDuty action config not found");
  }
  const config = actionConfig.config;

  try {
    const envelope = buildWebhookOutboundPayload(input);
    const event = await buildPagerDutyDelivery({ input, envelope, config });
    if (!event) {
      logger.info(
        `No PagerDuty event for ${input.payload.type} of automation ${automation.id} in project ${projectId}`,
      );
      return;
    }
    await sendPagerDutyEvent(event, config.region);
  } catch (error) {
    logger.error("Error executing PagerDuty action", error);
    await recordPagerDutyFailure({ input, automation, executionStart, error });
    return;
  }

  if (input.payload.type === "monitor-alert") {
    await resetAutomationFailures({ projectId, automationId: automation.id });
    return;
  }
  await prisma.automationExecution.update({
    where: {
      projectId,
      triggerId: automation.trigger.id,
      actionId: automation.action.id,
      id: executionId,
    },
    data: {
      status: ActionExecutionStatus.COMPLETED,
      startedAt: executionStart,
      finishedAt: new Date(),
    },
  });
}

/** buildPagerDutyDelivery decrypts the routing key and maps the job payload to its PagerDuty event. */
async function buildPagerDutyDelivery({
  input,
  envelope,
  config,
}: {
  input: WebhookInput;
  envelope: WebhookOutboundPayload;
  config: PagerDutyActionConfigWithSecrets;
}): Promise<PagerDutyEvent | undefined> {
  const project = await prisma.project.findUniqueOrThrow({
    where: { id: input.projectId },
    select: { name: true },
  });
  return buildPagerDutyEvent({
    envelope,
    projectId: input.projectId,
    projectName: project.name,
    routingKey: decrypt(config.routingKey),
  });
}

/** sendPagerDutyEvent posts the event to the alert or change endpoint of the key's region. */
async function sendPagerDutyEvent(
  event: PagerDutyEvent,
  region: PagerDutyRegion,
) {
  if (event.kind === "change") {
    await pagerDutyClient.change({ region, event: event.event });
    return;
  }
  await pagerDutyClient.alert({ region, event: event.event });
}

/** recordPagerDutyFailure records a non-retryable delivery failure and auto-disables the trigger like the other action types. */
async function recordPagerDutyFailure({
  input,
  automation,
  executionStart,
  error,
}: {
  input: WebhookInput;
  automation: NonNullable<Awaited<ReturnType<typeof getAutomationById>>>;
  executionStart: Date;
  error: unknown;
}) {
  const { projectId, executionId } = input;
  if (input.payload.type === "monitor-alert") {
    await monitorAlertDeliveryFailure({
      projectId,
      automation,
      channel: "pagerduty",
    });
    return;
  }

  await prisma.$transaction(async (tx) => {
    await tx.automationExecution.update({
      where: {
        id: executionId,
        projectId,
        triggerId: automation.trigger.id,
        actionId: automation.action.id,
      },
      data: {
        status: ActionExecutionStatus.ERROR,
        startedAt: executionStart,
        finishedAt: new Date(),
        error: error instanceof Error ? error.message : "Unknown error",
        output: isBaseError(error) ? { httpStatus: error.httpCode } : undefined,
      },
    });

    // Project notifications record the error but never auto-disable.
    if (input.payload.type === "project-notification") return;

    const consecutiveFailures = await getConsecutiveAutomationFailures({
      automationId: automation.id,
      projectId,
    });
    if (consecutiveFailures >= automationFailureThreshold - 1) {
      await tx.trigger.update({
        where: { id: automation.trigger.id, projectId },
        data: { status: JobConfigState.INACTIVE },
      });
      logger.warn(
        `Automation ${automation.trigger.id} disabled after ${consecutiveFailures} consecutive failures in project ${projectId} (pagerduty)`,
      );
    }
  });
}

/** sendSlackMonitorAlert builds and posts a monitor-alert to Slack, then clears the failure streak. */
async function sendSlackMonitorAlert({
  payload,
  projectId,
  channelId,
  automationId,
}: {
  payload: WebhookInput["payload"];
  projectId: string;
  channelId: string;
  automationId: string;
}) {
  const parsed = MonitorWebhookQueueEventSchema.safeParse(payload);
  if (!parsed.success) {
    throw new InternalServerError(
      `Invalid monitor-alert payload: ${parsed.error.message}`,
    );
  }
  const message = buildMonitorAlertSlackMessage(parsed.data.payload);
  const client =
    await SlackService.getInstance().getWebClientForProject(projectId);
  await SlackService.getInstance().sendMessage({
    client,
    channelId,
    ...message,
  });
  await resetAutomationFailures({ projectId, automationId });
}

/** sendSlackProjectNotification builds and posts a project notification to Slack. */
async function sendSlackProjectNotification({
  payload,
  projectId,
  channelId,
}: {
  payload: WebhookInput["payload"];
  projectId: string;
  channelId: string;
}) {
  const parsed = ProjectNotificationWebhookQueueEventSchema.safeParse(payload);
  if (!parsed.success) {
    throw new InternalServerError(
      `Invalid project notification payload: ${parsed.error.message}`,
    );
  }
  const message = buildProjectNotificationSlackMessage(parsed.data.event);
  const client =
    await SlackService.getInstance().getWebClientForProject(projectId);
  await SlackService.getInstance().sendMessage({
    client,
    channelId,
    ...message,
  });
}

/** monitorAlertDeliveryFailure tracks a failed monitor-alert delivery and auto-disables the trigger past the failure threshold. */
async function monitorAlertDeliveryFailure({
  projectId,
  automation,
  channel,
}: {
  projectId: string;
  automation: NonNullable<Awaited<ReturnType<typeof getAutomationById>>>;
  channel: "slack" | "pagerduty";
}) {
  const count = await incrementAutomationFailure({
    projectId,
    automationId: automation.id,
  });
  if (count >= automationFailureThreshold) {
    await prisma.trigger.update({
      where: { id: automation.trigger.id, projectId },
      data: { status: JobConfigState.INACTIVE },
    });
    await resetAutomationFailures({
      projectId,
      automationId: automation.id,
    });
    logger.warn(
      `Automation ${automation.trigger.id} disabled after ${count} consecutive failures in project ${projectId} (monitor-alert/${channel})`,
    );
  }
}

const setActionLastFailingExecutionId = async ({
  tx,
  actionId,
  projectId,
  executionId,
}: {
  tx: Prisma.TransactionClient;
  actionId: string;
  projectId: string;
  executionId: string;
}) => {
  // The execution config may contain decrypted headers, so patch only this JSON
  // key and leave the stored encrypted config untouched.
  await tx.$executeRaw`
    UPDATE actions
    SET
      config = jsonb_set(
        config,
        '{lastFailingExecutionId}',
        to_jsonb(${executionId}::text),
        true
      ),
      updated_at = NOW()
    WHERE id = ${actionId}
      AND project_id = ${projectId}
  `;
};

/** StaleMonitorAlertReason is why a queued monitor alert was dropped. */
type StaleMonitorAlertReason = "monitor_deleted" | "newer_alert";