Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

16.3 KB
Raw Blame
/* eslint-disable no-nested-ternary, @typescript-eslint/no-use-before-define */
import { randomUUID } from "crypto";
import { z } from "zod";

import { env } from "../../env";
import { InvalidRequestError } from "../../errors/InvalidRequestError";
import { LangfuseNotFoundError } from "../../errors/NotFoundError";
import { UnauthorizedError } from "../../errors/UnauthorizedError";
import { type AuthHeaderValidVerificationResultIngestion } from "../auth/types";
import {
  getClickhouseEntityType,
  type IngestionEntityTypes,
} from "../clickhouse/schemaUtils";
import {
  getCurrentSpan,
  instrumentAsync,
  recordDistribution,
  recordIncrement,
} from "../instrumentation";
import { logger } from "../logger";
import { QueueJobs } from "../queues";
import { IngestionQueue } from "../redis/ingestionQueue";
import { redis } from "../redis/redis";
import {
  eventTypes,
  createIngestionEventSchema,
  IngestionEventType,
} from "./types";
import type { IngestionAttribution } from "./ingestionAttribution";
import { StorageService } from "../services/StorageService";
import { createEventUploadStorageService } from "../s3";
import {
  HASH_HEX_LENGTH,
  safeBlobFilenameStem,
  safeBlobKeySegment,
} from "../services/safeBlobKeySegment";
import { buildEventBucketPrefix } from "./eventBucketPath";
import { isTraceIdInSample } from "./sampling";
import {
  createS3ThrottledIngestionError,
  isS3SlowDownError,
  markProjectS3Slowdown,
} from "../redis/s3SlowdownTracking";
import { markProjectIngestFailure } from "../redis/ingestionFailureTracking";

let s3StorageServiceClient: StorageService;

const getS3StorageServiceClient = (bucketName: string): StorageService => {
  if (!s3StorageServiceClient) {
    s3StorageServiceClient = createEventUploadStorageService(bucketName);
  }
  return s3StorageServiceClient;
};

/**
 * Get the delay for the event based on the event type. Uses delay if set, 0 if current UTC timestamp is not between
 * 23:45 and 00:15, and env.LANGFUSE_INGESTION_QUEUE_DELAY_MS otherwise.
 * We need the delay around date boundaries to avoid duplicates for out-of-order processing of events.
 * @param delay - Delay overwrite. Used if non-null.
 */
const getDelay = (delay: number | null, source: "api" | "otel") => {
  if (delay !== null) {
    return delay;
  }
  const now = new Date();
  const hours = now.getUTCHours();
  const minutes = now.getUTCMinutes();

  if ((hours === 23 && minutes >= 45) || (hours === 0 && minutes <= 15)) {
    return env.LANGFUSE_INGESTION_QUEUE_DELAY_MS;
  }

  if (source === "otel") {
    return 0;
  }

  // Use 5s here to avoid duplicate processing on the worker. If the ingestion delay is set to a lower value,
  // we use this instead.
  // Values should be revisited based on a cost/performance trade-off.
  return Math.min(5000, env.LANGFUSE_INGESTION_QUEUE_DELAY_MS);
};

/**
 * Options for event batch processing.
 * @property delay - Delay in ms to wait before processing events in the batch.
 * @property source - Source of the events for metrics tracking (e.g., "otel", "api").
 * @property isLangfuseInternal - Whether the events are being ingested by Langfuse internally (e.g. traces created for prompt experiments).
 * @property forwardToEventsTable - Whether to forward events to the staging events table for batch propagation. If undefined, falls back to environment flags.
 * @property attribution - Request-level ingestion attribution to persist on generated records.
 */
type ProcessEventBatchOptions = {
  delay?: number | null;
  source?: "api" | "otel";
  isLangfuseInternal?: boolean;
  forwardToEventsTable?: boolean;
  attribution: IngestionAttribution;
};

/** processEventBatch validates and queues authorized ingestion events, returning per-event results. */
export const processEventBatch = async (
  input: unknown[],
  authCheck: AuthHeaderValidVerificationResultIngestion,
  options: ProcessEventBatchOptions,
): Promise<{
  successes: { id: string; status: number }[];
  errors: {
    id: string;
    status: number;
    message?: string;
    error?: string;
  }[];
}> => {
  if (input.length === 0) {
    return { successes: [], errors: [] };
  }
  const {
    delay = null,
    source = "api",
    isLangfuseInternal = false,
    forwardToEventsTable,
    attribution,
  } = options;

  // add context of api call to the span
  const currentSpan = getCurrentSpan();
  recordIncrement("langfuse.ingestion.event", input.length, { source });
  recordDistribution("langfuse.ingestion.event_distribution", input.length, {
    source,
  });

  currentSpan?.setAttribute("langfuse.ingestion.batch_size", input.length);
  currentSpan?.setAttribute(
    "langfuse.project.id",
    authCheck.scope.projectId ?? "",
  );
  if (authCheck.scope.orgId)
    currentSpan?.setAttribute("langfuse.org.id", authCheck.scope.orgId);
  if (authCheck.scope.plan)
    currentSpan?.setAttribute("langfuse.org.plan", authCheck.scope.plan);

  /**************
   * VALIDATION *
   **************/
  if (!authCheck.scope.projectId) {
    throw new UnauthorizedError("Missing project ID");
  }

  const validationErrors: { id: string; error: unknown }[] = [];

  const ingestionSchema = createIngestionEventSchema(isLangfuseInternal);
  const batch: z.infer<typeof ingestionSchema>[] = input
    .flatMap((event) => {
      const parsed = ingestionSchema.safeParse(event);
      if (!parsed.success) {
        validationErrors.push({
          id:
            typeof event === "object" && event && "id" in event
              ? typeof event.id === "string"
                ? event.id
                : "unknown"
              : "unknown",
          error: new InvalidRequestError(parsed.error.message),
        });
        return [];
      }
      return [parsed.data];
    })
    .flatMap((event) => {
      if (event.type === eventTypes.SDK_LOG) {
        // Log SDK_LOG events, but remove them from further processing
        logger.info("SDK Log Event", { event });
        return [];
      }
      return [event];
    });

  const sortedBatch = sortBatch(batch);

  // We group events by eventBodyId which allows us to store and process them
  // as one which reduces infra interactions per event. Only used in the S3 case.
  //
  // The dedup struct also caches `entityType` and `bucketPrefix` so the two
  // downstream loops (S3 upload + IngestionQueue enqueue) read a single
  // source-of-truth per id instead of recomputing — that makes the
  // producer/consumer-must-agree invariant structural. The sanitization warn
  // log fires once at the time we resolve the prefix.
  //
  // `String(...)` preserves the prior null → "null" coercion of
  // `authCheck.scope.projectId`. The surrounding function legitimately treats
  // projectId as nullable elsewhere (metric labels, span attributes); we
  // don't narrow it in the type system, and a `null` projectId would land
  // events under an isolated `null/...` path rather than throw.
  const sortedBatchByEventBodyId = sortedBatch.reduce(
    (
      acc: Record<
        string,
        {
          data: IngestionEventType[];
          key: string;
          eventBodyId: string;
          type: (typeof eventTypes)[keyof typeof eventTypes];
          entityType: IngestionEntityTypes;
          bucketPrefix: string;
        }
      >,
      event,
    ) => {
      if (!event.body?.id) {
        return acc;
      }
      const entityType = getClickhouseEntityType(event.type);
      const dedupKey = `${entityType}-${event.body.id}`;
      if (!acc[dedupKey]) {
        const eventBodyId = event.body.id;
        const safeEventBodyId = safeBlobKeySegment(eventBodyId);
        if (safeEventBodyId !== eventBodyId) {
          // Do not log the raw or sanitized ID itself: the prefix can carry
          // PII (litellm encodes provider/model/request metadata in the ID).
          // The 16-hex hash is enough to correlate with the stored object
          // during debugging.
          logger.warn("Sanitized oversized/invalid entity ID for S3 key", {
            projectId: authCheck.scope.projectId,
            entityType,
            originalIdByteLength: Buffer.byteLength(eventBodyId, "utf8"),
            originalIdHash16: safeEventBodyId.slice(-HASH_HEX_LENGTH),
          });
        }
        acc[dedupKey] = {
          data: [],
          // `event.id` becomes a single-segment filename (`<id>.json`) at S3
          // write time and rides the queue payload as `fileKey`. Sanitize
          // here so `/`, `\`, control bytes, and over-budget lengths can't
          // reroute the write or overflow NAME_MAX.
          key: safeBlobFilenameStem(event.id, ".json"),
          type: event.type,
          eventBodyId,
          entityType,
          bucketPrefix: buildEventBucketPrefix({
            projectId: String(authCheck.scope.projectId),
            entityType,
            entityId: eventBodyId,
          }),
        };
      }
      acc[dedupKey].data.push(event);
      return acc;
    },
    {},
  );

  /********************
   * ASYNC PROCESSING *
   ********************/
  let s3UploadErrored = false;
  let s3UploadThrottled = false;
  let s3UploadFailedOtherwise = false;
  await instrumentAsync({ name: "s3-upload-events" }, async () => {
    // S3 Event Upload is blocking, but non-failing.
    // If a promise rejects, we log it below, but do not throw an error.
    // In this case, we upload the full batch into the Redis queue.
    const results = await Promise.allSettled(
      Object.keys(sortedBatchByEventBodyId).map(async (id) => {
        // We upload the event in an array to the S3 bucket grouped by the eventBodyId.
        // That way we batch updates from the same invocation into a single file and reduce
        // write operations on S3.
        const { data, key, bucketPrefix } = sortedBatchByEventBodyId[id];
        const bucketPath = `${bucketPrefix}${key}.json`;
        return getS3StorageServiceClient(
          env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
        ).uploadJson(bucketPath, data);
      }),
    );
    results.forEach((result) => {
      if (result.status === "rejected") {
        s3UploadErrored = true;

        // Check if this is a SlowDown error and mark the project for secondary queue
        if (isS3SlowDownError(result.reason)) {
          s3UploadThrottled = true;
          logger.warn(
            "S3 SlowDown error during upload, marking project for secondary queue",
            {
              projectId: authCheck.scope.projectId,
              error: result.reason,
            },
          );
          markProjectS3Slowdown(authCheck.scope.projectId!).catch(() => {});
          markProjectIngestFailure(authCheck.scope.projectId!, {
            source: "process_event_batch",
            reason: "s3_slowdown",
          });
        } else {
          s3UploadFailedOtherwise = true;
          markProjectIngestFailure(authCheck.scope.projectId!, {
            source: "process_event_batch",
            reason: "s3_upload_error",
          });
          logger.error("Failed to upload event to S3", {
            error: result.reason,
          });
        }
      }
    });
  });

  if (s3UploadThrottled && !s3UploadFailedOtherwise) {
    throw createS3ThrottledIngestionError(source);
  }

  // Send each event individually to IngestionQueue for ClickHouse processing
  if (s3UploadErrored) {
    throw new Error(
      "Failed to upload events to blob storage, aborting event processing",
    );
  }

  if (!redis) {
    throw new Error("Redis not initialized, aborting event processing");
  }

  const projectIdsToSkipS3List =
    env.LANGFUSE_SKIP_S3_LIST_FOR_OBSERVATIONS_PROJECT_IDS?.split(",") ?? [];

  await Promise.all(
    Object.keys(sortedBatchByEventBodyId).map(async (id) => {
      const eventData = sortedBatchByEventBodyId[id];
      const shardingKey = `${authCheck.scope.projectId}-${eventData.eventBodyId}`;
      const queue = IngestionQueue.getInstance({ shardingKey });

      const isDatasetRunItemEvent = eventData.entityType === "dataset_run_item";
      const isObservationEvent = eventData.entityType === "observation";

      const isOtelOrSkipS3Project =
        authCheck.scope.projectId !== null &&
        (source === "otel" ||
          projectIdsToSkipS3List.includes(authCheck.scope.projectId));

      const shouldSkipS3List =
        isDatasetRunItemEvent || (isObservationEvent && isOtelOrSkipS3Project);

      const { isSampled, isSamplingConfigured } = isTraceIdInSample({
        projectId: authCheck.scope.projectId,
        event: eventData.data[0],
      });

      if (!isSampled) {
        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
          projectId: authCheck.scope.projectId ?? "<not set>",
          sampling_decision: "out",
        });

        return;
      }

      if (isSamplingConfigured) {
        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
          projectId: authCheck.scope.projectId ?? "<not set>",
          sampling_decision: "in",
        });
      }

      return queue
        ? queue.add(
            QueueJobs.IngestionJob,
            {
              id: randomUUID(),
              timestamp: new Date(),
              name: QueueJobs.IngestionJob as const,
              payload: {
                data: {
                  type: eventData.type,
                  eventBodyId: eventData.eventBodyId,
                  fileKey: eventData.key,
                  skipS3List: shouldSkipS3List,
                  forwardToEventsTable,
                  bucketPrefix: eventData.bucketPrefix,
                  ingestionApiKey: attribution.ingestionApiKey,
                  ingestionSdkName: attribution.ingestionSdkName,
                  ingestionSdkVersion: attribution.ingestionSdkVersion,
                },
                authCheck: authCheck as {
                  validKey: true;
                  scope: {
                    projectId: string;
                    accessLevel: "project" | "scores";
                  };
                },
              },
            },
            { delay: getDelay(delay, source) },
          )
        : Promise.reject("Failed to instantiate ingestion queue");
    }),
  );

  return aggregateBatchResult(
    validationErrors,
    sortedBatch.map((event) => ({ id: event.id, result: event })),
    authCheck.scope.projectId,
  );
};

/**
 * Sorts a batch of ingestion events. Orders by: updating events last, sorted by timestamp asc.
 */
const sortBatch = (batch: IngestionEventType[]) => {
  const updateEvents: (typeof eventTypes)[keyof typeof eventTypes][] = [
    eventTypes.GENERATION_UPDATE,
    eventTypes.SPAN_UPDATE,
    eventTypes.OBSERVATION_UPDATE, // legacy event type
  ];
  const updates = batch
    .filter((event) => updateEvents.includes(event.type))
    .sort((a, b) => {
      return new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime();
    });
  const others = batch
    .filter((event) => !updateEvents.includes(event.type))
    .sort((a, b) => {
      return new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime();
    });

  // Return the array with non-update events first, followed by update events
  return [...others, ...updates];
};

export const aggregateBatchResult = (
  errors: Array<{ id: string; error: unknown }>,
  results: Array<{ id: string; result: unknown }>,
  projectId?: string,
) => {
  const returnedErrors: {
    id: string;
    status: number;
    message?: string;
    error?: string;
  }[] = [];

  const successes: {
    id: string;
    status: number;
  }[] = [];

  errors.forEach((error) => {
    if (error.error instanceof InvalidRequestError) {
      returnedErrors.push({
        id: error.id,
        status: 400,
        message: "Invalid request data",
        error: error.error.message,
      });
    } else if (error.error instanceof UnauthorizedError) {
      returnedErrors.push({
        id: error.id,
        status: 401,
        message: "Authentication error",
        error: error.error.message,
      });
    } else if (error.error instanceof LangfuseNotFoundError) {
      returnedErrors.push({
        id: error.id,
        status: 404,
        message: "Resource not found",
        error: error.error.message,
      });
    } else {
      returnedErrors.push({
        id: error.id,
        status: 500,
        error: "Internal Server Error",
      });
    }
  });

  if (returnedErrors.length > 0) {
    logger.warn("Error processing events", {
      errors: returnedErrors,
      "langfuse.project.id": projectId,
    });
  }

  results.forEach((result) => {
    successes.push({
      id: result.id,
      status: 201,
    });
  });

  return { successes, errors: returnedErrors };
};