Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

7.3 KB
Raw Blame
import { randomUUID } from "node:crypto";
import {
  BaseError,
  LangfuseConflictError,
  LangfuseNotFoundError,
  ServiceUnavailableError,
} from "@langfuse/shared";
import {
  type ApiAccessScope,
  logger,
  recordIncrement,
} from "@langfuse/shared/src/server";
import { env } from "@/src/env.mjs";
import { RateLimitService } from "@/src/features/public-api/server/RateLimitService";
import type {
  PostFeedbackBodyType,
  PostFeedbackResponseType,
} from "@/src/features/public-api/types/feedback";

export type FeedbackSource = "langfuse-mcp" | "public-api";

type FeedbackContext = {
  projectId: string;
  orgId: string;
};

type SlackTextObject = {
  type: "plain_text" | "mrkdwn";
  text: string;
  emoji?: boolean;
};

type SlackBlock =
  | {
      type: "header";
      text: SlackTextObject;
    }
  | {
      type: "section";
      text: SlackTextObject;
    }
  | {
      type: "section";
      fields: SlackTextObject[];
    }
  | {
      type: "divider";
    }
  | {
      type: "context";
      elements: SlackTextObject[];
    };

type SlackPayload = {
  text: string;
  blocks: SlackBlock[];
  unfurl_links: false;
  unfurl_media: false;
};

type FeedbackSlackMessage = SlackPayload;

const SLACK_SECTION_TEXT_LIMIT = 3000;
const SLACK_FIELD_TEXT_LIMIT = 2000;
const FEEDBACK_SLACK_TIMEOUT_MS = 5_000;

const truncateForSlack = (value: string, maxLength: number): string => {
  if (value.length <= maxLength) return value;
  return `${value.slice(0, Math.max(0, maxLength - 14))}\n[truncated]`;
};

const plainText = (value: string, maxLength: number): SlackTextObject => ({
  type: "plain_text",
  text: truncateForSlack(value, maxLength),
  emoji: false,
});

const mrkdwnText = (value: string): SlackTextObject => ({
  type: "mrkdwn",
  text: value,
});

const appendPlainTextSection = (
  blocks: SlackBlock[],
  label: string,
  value: string | undefined,
) => {
  if (!value) return;

  blocks.push(
    {
      type: "section",
      text: mrkdwnText(`*${label}*`),
    },
    {
      type: "section",
      text: plainText(value, SLACK_SECTION_TEXT_LIMIT),
    },
  );
};

const getDataRegion = (): string =>
  env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ?? "self-hosted";

const isHipaaCloudRegion = (): boolean =>
  env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION === "HIPAA";

// Product feedback is intentionally unavailable in HIPAA: Slack delivery is
// not allowed, so the MCP tool and public endpoint should stay hidden rather
// than fail as a misconfigured sink.
export const isProductFeedbackAvailable = (): boolean => !isHipaaCloudRegion();

const feedbackSourceLabel: Record<FeedbackSource, string> = {
  "langfuse-mcp": "Langfuse MCP",
  "public-api": "Public API",
};

const buildFeedbackSlackMessage = ({
  id,
  input,
  source,
  context,
}: {
  id: string;
  input: PostFeedbackBodyType;
  source: FeedbackSource;
  context: FeedbackContext;
}): FeedbackSlackMessage => {
  const blocks: SlackBlock[] = [
    {
      type: "header",
      text: {
        type: "plain_text",
        text: "💬 New Langfuse feedback",
        emoji: true,
      },
    },
    {
      type: "section",
      fields: [
        plainText(
          `📬 SOURCE:\n${feedbackSourceLabel[source]}`,
          SLACK_FIELD_TEXT_LIMIT,
        ),
        plainText(`🎯 TARGET:\n${input.target}`, SLACK_FIELD_TEXT_LIMIT),
        plainText(
          `🧩 TARGET TYPE:\n${input.targetType}`,
          SLACK_FIELD_TEXT_LIMIT,
        ),
        plainText(`🌍 REGION:\n${getDataRegion()}`, SLACK_FIELD_TEXT_LIMIT),
      ],
    },
    { type: "divider" },
  ];

  appendPlainTextSection(blocks, "💬 Feedback:", input.feedback);
  appendPlainTextSection(blocks, "🎯 Goal / use case:", input.goal);
  appendPlainTextSection(blocks, "🔗 Reference URL:", input.referenceUrl);

  blocks.push(
    { type: "divider" },
    {
      type: "context",
      elements: [
        plainText(`🧾 Receipt: ${id}`, SLACK_FIELD_TEXT_LIMIT),
        plainText(`🏢 Org: ${context.orgId}`, SLACK_FIELD_TEXT_LIMIT),
        plainText(`📁 Project: ${context.projectId}`, SLACK_FIELD_TEXT_LIMIT),
      ],
    },
  );

  return {
    text: `New Langfuse feedback · ${feedbackSourceLabel[source]} · ${input.targetType} · ${id}`,
    blocks,
    unfurl_links: false,
    unfurl_media: false,
  };
};

// The HIPAA region must never deliver feedback to Slack, even if a webhook
// were configured there by mistake.
const getConfiguredFeedbackWebhookUrl = (): string | undefined =>
  isHipaaCloudRegion() ? undefined : env.LANGFUSE_FEEDBACK_INTAKE_SLACK_WEBHOOK;

const validateFeedbackWebhookUrl = (webhookUrl: string): string => {
  let parsed: URL;
  try {
    parsed = new URL(webhookUrl);
  } catch {
    throw new ServiceUnavailableError("Feedback Slack sink is misconfigured");
  }

  const requiresHttps = env.NODE_ENV === "production";
  const isInvalidCloudSink =
    env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&
    (parsed.protocol !== "https:" || parsed.hostname !== "hooks.slack.com");
  if ((requiresHttps && parsed.protocol !== "https:") || isInvalidCloudSink) {
    throw new ServiceUnavailableError("Feedback Slack sink is misconfigured");
  }

  return webhookUrl;
};

export const submitFeedback = async ({
  input,
  source,
  scope,
}: {
  input: PostFeedbackBodyType;
  source: FeedbackSource;
  scope: ApiAccessScope;
}): Promise<PostFeedbackResponseType> => {
  if (!isProductFeedbackAvailable()) {
    throw new LangfuseNotFoundError();
  }

  const rateLimitCheck = await RateLimitService.getInstance().rateLimitRequest(
    scope,
    "feedback",
  );
  if (rateLimitCheck?.isRateLimited()) {
    recordIncrement("langfuse.feedback.submission", 1, {
      source,
      outcome: "rate_limited",
    });
    throw new BaseError(
      "TooManyRequestsError",
      429,
      "Feedback rate limit exceeded",
      true,
    );
  }

  const configuredWebhookUrl = getConfiguredFeedbackWebhookUrl();
  if (!configuredWebhookUrl) {
    recordIncrement("langfuse.feedback.submission", 1, {
      source,
      outcome: "sink_unconfigured",
    });
    logger.warn("Feedback intake sink is not configured", {
      source,
      targetType: input.targetType,
      orgId: scope.orgId,
      projectId: scope.projectId,
      region: getDataRegion(),
    });
    throw new LangfuseConflictError(
      "Feedback submission is not configured for this deployment",
    );
  }

  const id = randomUUID();
  const webhookUrl = validateFeedbackWebhookUrl(configuredWebhookUrl);
  const payload = buildFeedbackSlackMessage({
    id,
    input,
    source,
    context: {
      orgId: scope.orgId,
      projectId: scope.projectId ?? "unknown",
    },
  });

  let response: Response;
  try {
    response = await fetch(webhookUrl, {
      method: "POST",
      redirect: "error",
      signal: AbortSignal.timeout(FEEDBACK_SLACK_TIMEOUT_MS),
      headers: {
        "Content-Type": "application/json",
      },
      body: JSON.stringify(payload),
    });
  } catch {
    recordIncrement("langfuse.feedback.submission", 1, {
      source,
      outcome: "sink_failed",
    });
    throw new ServiceUnavailableError("Feedback Slack sink request failed");
  }

  if (!response.ok) {
    recordIncrement("langfuse.feedback.submission", 1, {
      source,
      outcome: "sink_failed",
    });
    throw new ServiceUnavailableError("Feedback Slack sink rejected message");
  }

  recordIncrement("langfuse.feedback.submission", 1, {
    source,
    outcome: "accepted",
  });

  return { id };
};