Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

34.8 KB
Raw Blame
import { Job, Processor } from "bullmq";
import type { EarlyOtelBatch } from "@langfuse/native";
import { z } from "zod";
import {
  clickhouseClient,
  createIngestionEventSchema,
  getClickhouseEntityType,
  getCurrentSpan,
  getS3EventStorageClient,
  type IngestionEventType,
  logger,
  markProjectIngestFailure,
  OtelIngestionProcessor,
  processEventBatch,
  QueueName,
  recordDistribution,
  recordHistogram,
  recordIncrement,
  redis,
  SecondaryOtelIngestionQueue,
  TQueueJobTypes,
  traceException,
  compareVersions,
  ResourceSpan,
  type IngestionAttribution,
  UNKNOWN_INGESTION_SDK_VALUE,
  LocalCache,
} from "@langfuse/shared/src/server";
import {
  applyIngestionMasking,
  isIngestionMaskingEnabled,
} from "@langfuse/shared/src/server/ee/ingestionMasking";
import {
  env,
  v4ForceDirectOtelWrite,
  v4WritesToEventsTable,
  v4WritesToLegacyTables,
} from "../env";
import { IngestionService } from "../services/IngestionService";
import { prisma } from "@langfuse/shared/src/db";
import { ClickhouseWriter } from "../services/ClickhouseWriter";
import { logRetryableJobFailure } from "./jobFailureLog";
import { ForbiddenError } from "@langfuse/shared";
import {
  createLegacyOtelMediaTargets,
  processOtelEventMedia,
  type OtelMediaPresence,
} from "../features/otel-media/processOtelMedia";
import { restoreInlineMedia } from "../features/otel-media/resolveExtractedMedia";
import { processOtelEvents } from "../features/otel-ingestion/processOtelEvents";
import {
  parsePreparedOtelBatch,
  prepareOtelBatch,
} from "../features/otel-ingestion/prepareOtelBatch";

/**
 * Legacy media processing follows legacy persistence, independently of
 * whether the same batch also qualifies for a direct events-table write.
 */
export function shouldProcessLegacyOtelMedia(params: {
  mediaUploadEnabled: boolean;
  writesToLegacyTables: boolean;
}): boolean {
  return params.mediaUploadEnabled && params.writesToLegacyTables;
}

/**
 * Check if HTTP headers from the SDK request indicate the batch is eligible
 * for direct event writes.
 *
 * Requirements:
 * - x-langfuse-sdk-name "python" with x-langfuse-sdk-version >= 4.0.0
 * - x-langfuse-sdk-name "javascript" with x-langfuse-sdk-version >= 5.0.0
 * - x-langfuse-ingestion-version === "4" (custom OTel exporter opt-in)
 */
export function checkHeaderBasedDirectWrite(params: {
  sdkName?: string;
  sdkVersion?: string;
  ingestionVersion?: string;
}): boolean {
  const { sdkName, sdkVersion, ingestionVersion } = params;

  // Check x-langfuse-ingestion-version.
  // Values > 4 are rejected at the API route, so anything reaching here is valid.
  const parsed = ingestionVersion ? parseInt(ingestionVersion, 10) : NaN;
  if (!isNaN(parsed) && parsed >= 4) {
    return true;
  }

  // Check Langfuse SDK name + version
  if (!sdkName || !sdkVersion) {
    return false;
  }

  try {
    // compareVersions returns null when current >= minimum (no update needed).
    // Strip pre-release/build metadata so that e.g. 4.0.0-rc.1 qualifies as 4.0.0.
    // Also normalize Python PEP440 shorthand (e.g. 4.0.0b1, 4.0.0rc1) to the core version.
    const baseVersion = extractBaseSdkVersion(sdkVersion);

    if (sdkName === "python") {
      return compareVersions(baseVersion, "v4.0.0") === null;
    }

    if (sdkName === "javascript") {
      return compareVersions(baseVersion, "v5.0.0") === null;
    }
  } catch {
    logger.warn(
      `Failed to parse SDK version from headers: ${sdkName}@${sdkVersion}`,
    );
  }

  return false;
}

export type OtelWritePath =
  | "dual"
  | "direct_header"
  | "direct_env"
  | "direct_scope"
  | "direct_org_cutoff";

/**
 * Whether a batch was exported by a Langfuse SDK rather than a plain OTel
 * pipeline.
 *
 * Either signal is sufficient: the SDKs send `x-langfuse-sdk-name` (normalized
 * to UNKNOWN_INGESTION_SDK_VALUE when absent), and their spans carry a
 * "langfuse"-prefixed instrumentation scope. A third party that sets the
 * header anyway is treated as SDK traffic, which is the conservative direction
 * — it keeps the established write path.
 */
export function isLangfuseSdkTraffic(params: {
  sdkName?: string;
  scopeName?: string | null;
}): boolean {
  const { sdkName, scopeName } = params;

  if (sdkName && sdkName !== UNKNOWN_INGESTION_SDK_VALUE) {
    return true;
  }

  return Boolean(
    scopeName && String(scopeName).toLowerCase().includes("langfuse"),
  );
}

/**
 * Whether any instrumentation scope in the batch identifies a Langfuse SDK.
 *
 * The scope-based version check reads only the first scope (all spans a
 * Langfuse SDK exports share it), but the org-cutoff exclusion must be
 * conservative: a collector-forwarded batch can mix SDK spans with third-party
 * scopes in any order, and one Langfuse scope anywhere is enough to keep the
 * whole batch on its established write path. Scans scope metadata only — no
 * span bodies are touched.
 */
export function batchContainsLangfuseScope(
  resourceSpans: ResourceSpan[],
): boolean {
  // Malformed OTLP shapes reach this code (non-array payloads, non-array
  // scopeSpans and the like); every other resourceSpans consumer guards the
  // same way. Guard per resource so a malformed one reads as scope-less rather
  // than aborting the scan — it must not hide a Langfuse scope on a later
  // resource, nor fail the batch.
  if (!Array.isArray(resourceSpans)) {
    return false;
  }
  return resourceSpans.some(
    (resourceSpan) =>
      Array.isArray(resourceSpan?.scopeSpans) &&
      resourceSpan.scopeSpans.some((scopeSpan) =>
        isLangfuseSdkTraffic({ scopeName: scopeSpan?.scope?.name ?? null }),
      ),
  );
}

/**
 * Whether an organization's signup date puts it past the direct-write cutoff.
 *
 * Organizations created on or after the cutoff are past the point where the v4
 * preview is force-enabled and cannot be switched off, so the events table is
 * the only surface they read. Without `x-langfuse-ingestion-version: 4` their
 * OTLP traffic would take the dual-write path and appear with up to ~10 minutes
 * of delay — a broken first experience for an exporter that is otherwise set up
 * correctly.
 *
 * Deliberately date-based: there is no per-organization or per-project v4
 * column, only a per-user preview flag, so signup date is the only cohort
 * signal available. It mirrors the UI-side rollout boundary but keys on the
 * single organization owning the API key rather than the oldest organization a
 * user belongs to — ingestion has no user context.
 *
 * The rule is Cloud-only, gated at the call site
 * (isProjectOrgPastOtelDirectWriteCutoff): self-hosted deployments move the
 * whole deployment at once via LANGFUSE_MIGRATION_V4_NATIVE_OTEL_BEHAVIOUR.
 */
export function isOrgPastOtelDirectWriteCutoff(params: {
  /** Organization signup date; nullish when it could not be resolved. */
  orgCreatedAt: Date | null | undefined;
  /** ISO date (YYYY-MM-DD), read as midnight UTC. Unset disables the rule. */
  cutoff: string | undefined;
}): boolean {
  const { orgCreatedAt, cutoff } = params;

  if (!cutoff || !orgCreatedAt) {
    return false;
  }

  // An invalid date on either side compares as NaN, which is false for every
  // comparison — unparsable input keeps the pre-cutoff behaviour by itself.
  return orgCreatedAt.getTime() >= Date.parse(`${cutoff}T00:00:00.000Z`);
}

// An organization's signup date never changes, and the project → organization
// mapping changes only through the rare `projects.transfer` mutation — a stale
// entry then self-corrects within the TTL, which is acceptable for a routing
// hint whose destinations both land the data. So the cache only needs to bound
// memory and let deleted projects fall out. A miss costs one indexed Postgres
// read, and only for batches that no other signal already routed (see below).
const ORG_CREATED_AT_CACHE_TTL_MS = 60 * 60 * 1000;
const orgCreatedAtCache = new LocalCache<{ createdAt: Date }>({
  namespace: "otel_org_created_at",
  enabled: true,
  ttlMs: ORG_CREATED_AT_CACHE_TTL_MS,
  max: 10_000,
});

/**
 * Whether the org-created cutoff rule can apply in this deployment at all:
 * it needs a configured date and only runs on Langfuse Cloud. Work that only
 * feeds the rule (the batch scope scan, the signup-date lookup) is skipped
 * while this is false — the shipped default and every self-hosted deployment.
 */
function isOtelDirectWriteOrgCutoffConfigured(): boolean {
  return Boolean(
    env.LANGFUSE_MIGRATION_V4_OTEL_DIRECT_WRITE_ORG_CREATED_CUTOFF &&
    env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
  );
}

/**
 * Resolve whether the project's organization is past the direct-write cutoff.
 *
 * Keyed on project rather than the payload's optional `orgId` so replayed jobs
 * enqueued before that field existed resolve too. Returns false when the
 * project cannot be found, which keeps the batch on its existing path.
 */
async function isProjectOrgPastOtelDirectWriteCutoff(
  projectId: string,
): Promise<boolean> {
  if (!isOtelDirectWriteOrgCutoffConfigured()) {
    return false;
  }
  const cutoff = env.LANGFUSE_MIGRATION_V4_OTEL_DIRECT_WRITE_ORG_CREATED_CUTOFF;

  try {
    const { value } = await orgCreatedAtCache.getOrLoad(projectId, async () => {
      const project = await prisma.project.findUnique({
        where: { id: projectId },
        select: { organization: { select: { createdAt: true } } },
      });

      return {
        value: project
          ? { createdAt: project.organization.createdAt }
          : undefined,
        source: "postgres",
      };
    });

    return isOrgPastOtelDirectWriteCutoff({
      orgCreatedAt: value?.createdAt,
      cutoff,
    });
  } catch (error) {
    // A lookup failure must not fail the batch: fall back to the pre-cutoff
    // path, which is still a correct destination in dual write mode.
    logger.warn(
      `Failed to resolve organization signup date for project ${projectId}; keeping the pre-cutoff OTel write path`,
      error,
    );
    return false;
  }
}

/**
 * Resolve which persistence path a batch takes, and which signal decided it.
 *
 * Precedence is header > env > scope > org cutoff. The org cutoff sits last so
 * the `direct_org_cutoff` label counts only the traffic that rule genuinely
 * moves off the dual path, rather than absorbing batches that a header or a
 * recognized SDK scope would have routed directly anyway.
 *
 * The org cutoff additionally excludes Langfuse SDK traffic. An older SDK has
 * an established dual-write shape — its trace-level attributes surface on the
 * synthetic root observation that only the dual path materializes — so flipping
 * it would change data those users already read. A plain OTel exporter has no
 * such prior expectation, and is the case the cutoff exists for.
 */
export function resolveOtelWritePath(params: {
  headerBasedDirectWrite: boolean;
  envForcesDirect: boolean;
  scopeBasedDirectWrite: boolean;
  orgCutoffForcesDirect: boolean;
  langfuseSdkTraffic: boolean;
}): { useDirectEventWrite: boolean; writePath: OtelWritePath } {
  const {
    headerBasedDirectWrite,
    envForcesDirect,
    scopeBasedDirectWrite,
    orgCutoffForcesDirect,
    langfuseSdkTraffic,
  } = params;

  if (headerBasedDirectWrite) {
    return { useDirectEventWrite: true, writePath: "direct_header" };
  }
  if (envForcesDirect) {
    return { useDirectEventWrite: true, writePath: "direct_env" };
  }
  if (scopeBasedDirectWrite) {
    return { useDirectEventWrite: true, writePath: "direct_scope" };
  }
  if (orgCutoffForcesDirect && !langfuseSdkTraffic) {
    return { useDirectEventWrite: true, writePath: "direct_org_cutoff" };
  }

  return { useDirectEventWrite: false, writePath: "dual" };
}

function extractBaseSdkVersion(sdkVersion: string): string {
  const version = sdkVersion.trim();

  // Standard semver / semver pre-release / build metadata
  if (/^v?\d+\.\d+\.\d+(?:[-+].+)?$/i.test(version)) {
    return version.split(/[-+]/)[0];
  }

  // Python PEP 440 pre-release shorthand: 4.0.0a1, 4.0.0b1, 4.0.0rc1
  const pep440Match = version.match(/^(v?\d+\.\d+\.\d+)(?:a|b|rc)\d+$/i);
  if (pep440Match?.[1]) {
    return pep440Match[1];
  }

  return version;
}

/**
 * SDK information extracted from OTEL resourceSpans.
 */
export type SdkInfo = {
  scopeName: string | null;
  scopeVersion: string | null;
  telemetrySdkLanguage: string | null;
};

/**
 * Extract SDK information from resourceSpans.
 * Gets scope name/version and telemetry SDK language from the OTEL structure.
 */
export function getSdkInfoFromResourceSpans(
  resourceSpans: ResourceSpan,
): SdkInfo {
  try {
    // Get the first scopeSpan (all spans in a batch share the same scope)
    const firstScopeSpan = resourceSpans?.scopeSpans?.[0];
    const scopeName = firstScopeSpan?.scope?.name ?? null;
    const scopeVersion = firstScopeSpan?.scope?.version ?? null;

    // Extract telemetry SDK language from resource attributes
    const resourceAttributes = resourceSpans?.resource?.attributes ?? [];
    const telemetrySdkLanguage =
      resourceAttributes.find((attr) => attr.key === "telemetry.sdk.language")
        ?.value?.stringValue ?? null;

    return { scopeName, scopeVersion, telemetrySdkLanguage };
  } catch (error) {
    logger.warn("Failed to extract SDK info from resourceSpans", error);
    return { scopeName: null, scopeVersion: null, telemetrySdkLanguage: null };
  }
}

/**
 * Check if SDK meets version requirements for direct event writes.
 *
 * Requirements:
 * - Scope name must contain 'langfuse' (case-insensitive)
 * - Python SDK: scope_version >= 3.9.0
 * - JS/JavaScript SDK: scope_version >= 4.4.0
 */
export function checkSdkVersionRequirements(
  sdkInfo: SdkInfo,
  isSdkExperimentBatch: boolean,
): boolean {
  const { scopeName, scopeVersion, telemetrySdkLanguage } = sdkInfo;

  // Must be a Langfuse SDK
  if (!isLangfuseSdkTraffic({ scopeName })) {
    return false;
  }

  if (!scopeVersion || !telemetrySdkLanguage) {
    return false;
  }

  try {
    // Python SDK >= 3.9.0
    if (telemetrySdkLanguage === "python" && isSdkExperimentBatch) {
      const comparison = compareVersions(scopeVersion, "v3.9.0");
      return comparison === null; // null means current >= latest
    }

    // JS/JavaScript SDK >= 4.4.0
    if (
      (telemetrySdkLanguage === "js" ||
        telemetrySdkLanguage === "javascript") &&
      isSdkExperimentBatch
    ) {
      const comparison = compareVersions(scopeVersion, "v4.4.0");
      return comparison === null; // null means current >= latest
    }

    return false;
  } catch (error) {
    logger.warn(
      `Failed to parse SDK version ${scopeVersion} for language ${telemetrySdkLanguage}`,
      error,
    );
    return false;
  }
}

export const otelIngestionQueueProcessorBuilder = (
  enableRedirectToSecondaryQueue: boolean,
): Processor => {
  const projectIdsToRedirectToSecondaryQueue =
    env.LANGFUSE_SECONDARY_OTEL_INGESTION_QUEUE_ENABLED_PROJECT_IDS?.split(
      ",",
    ) ?? [];

  return async (
    job: Job<TQueueJobTypes[QueueName.OtelIngestionQueue]>,
  ): Promise<void> => {
    let earlyBatch: EarlyOtelBatch | undefined;
    let mediaPath: "early" | "reference" | undefined;
    let mediaPresence: OtelMediaPresence | undefined;
    let inputByteLength: number | undefined;
    let inputPreparationDurationMs: number | undefined;
    const span = getCurrentSpan();
    try {
      const projectId = job.data.payload.authCheck.scope.projectId;
      const publicKey = job.data.payload.data.publicKey ?? "";
      const fileKey = job.data.payload.data.fileKey;
      const auth = job.data.payload.authCheck;
      const isLangfuseInternal = job.data.payload.isLangfuseInternal === true;
      const attribution: IngestionAttribution = {
        ingestionApiKey: publicKey,
        ingestionSdkName:
          job.data.payload.sdkName || UNKNOWN_INGESTION_SDK_VALUE,
        ingestionSdkVersion:
          job.data.payload.sdkVersion || UNKNOWN_INGESTION_SDK_VALUE,
      };

      if (span) {
        span.setAttribute("messaging.bullmq.job.input.id", job.data.id);
        span.setAttribute(
          "messaging.bullmq.job.input.projectId",
          job.data.payload.authCheck.scope.projectId,
        );
        span.setAttribute(
          "messaging.bullmq.job.input.fileKey",
          job.data.payload.data.fileKey,
        );
      }
      logger.debug(`Processing ${fileKey} for project ${projectId}`);

      // Check if project should be redirected to secondary queue
      if (
        enableRedirectToSecondaryQueue &&
        projectIdsToRedirectToSecondaryQueue.includes(projectId)
      ) {
        logger.debug(
          `Redirecting otel ingestion event to secondary queue for project ${projectId}`,
        );
        const shardingKey = `${projectId}-${fileKey}`;
        const secondaryQueue = SecondaryOtelIngestionQueue.getInstance({
          shardingKey,
        });
        if (secondaryQueue) {
          await secondaryQueue.add(
            QueueName.OtelIngestionSecondaryQueue,
            job.data,
          );
          // Forwarded to secondary queue; stop processing here.
          return;
        }
      }

      // TODO: Do we need to add these files into the blob_storage_file_log?
      // We could recommend lifecycle rules due to the immutability properties.
      // Otherwise, we'd probably have to upsert one row per generated event further below.
      // Easy change, but needs alignment.

      const earlyMediaEnabled =
        env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED === "true" &&
        (env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS.includes("*") ||
          env.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_PROJECT_IDS.includes(
            projectId,
          ));
      mediaPath = earlyMediaEnabled ? "early" : "reference";
      span?.setAttribute("langfuse.ingestion.otel.media_path", mediaPath);
      // Count selection once per queue attempt, including jobs that later fail.
      recordIncrement("langfuse.ingestion.otel.media_path", 1, {
        path: mediaPath,
      });

      const eventStorage = getS3EventStorageClient(
        env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
      );
      let parsedSpans: ResourceSpan[];
      if (earlyMediaEnabled) {
        const bytes = await eventStorage.downloadBytes(fileKey);
        const preparationStartedAt = performance.now();
        inputByteLength = bytes.byteLength;
        recordHistogram(
          "langfuse.ingestion.s3_file_size_bytes",
          inputByteLength,
          { skippedS3List: "true", otel: "true" },
        );
        const prepared = await prepareOtelBatch({
          bytes: Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength),
          projectId,
          orgId: job.data.payload.authCheck.scope.orgId,
          propagatedHeaders: job.data.payload.propagatedHeaders,
          extractMedia:
            env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED === "true" &&
            Boolean(env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET),
        });
        if ("error" in prepared) {
          logger.warn("Dropping OTEL event due to masking failure", {
            projectId,
            orgId: job.data.payload.authCheck.scope.orgId,
            fileKey,
            error: prepared.error,
            propagatedHeaders: job.data.payload.propagatedHeaders,
          });
          return;
        }
        ({ batch: earlyBatch, resourceSpans: parsedSpans } =
          await parsePreparedOtelBatch(prepared, bytes));
        inputPreparationDurationMs = performance.now() - preparationStartedAt;
      } else {
        // Download file from blob storage
        const resourceSpans = await eventStorage.download(fileKey);
        const preparationStartedAt = performance.now();
        inputByteLength = Buffer.byteLength(resourceSpans, "utf8");

        recordHistogram(
          "langfuse.ingestion.s3_file_size_bytes",
          inputByteLength,
          {
            skippedS3List: "true",
            otel: "true",
          },
        );

        // Parse spans from S3 download
        parsedSpans = JSON.parse(resourceSpans);

        // Apply ingestion masking if enabled (EE feature)
        if (isIngestionMaskingEnabled()) {
          const maskingResult = await applyIngestionMasking({
            data: parsedSpans,
            projectId,
            orgId: job.data.payload.authCheck.scope.orgId,
            propagatedHeaders: job.data.payload.propagatedHeaders,
          });

          if (!maskingResult.success) {
            // Fail-closed: drop event. Emit the S3 location so operators can
            // scan logs to identify which raw payloads need to be replayed via
            // worker/src/scripts/replayIngestionEventsV2 once the upstream
            // masking callback is healthy again.
            logger.warn(`Dropping OTEL event due to masking failure`, {
              projectId,
              orgId: job.data.payload.authCheck.scope.orgId,
              fileKey,
              error: maskingResult.error,
              propagatedHeaders: job.data.payload.propagatedHeaders,
            });
            return;
          }
          parsedSpans = maskingResult.data;
        }
        inputPreparationDurationMs = performance.now() - preparationStartedAt;
      }

      // Generate events via OtelIngestionProcessor
      const processor = new OtelIngestionProcessor({
        projectId,
        publicKey: attribution.ingestionApiKey,
        sdkName: attribution.ingestionSdkName,
        sdkVersion: attribution.ingestionSdkVersion,
        fileKey,
        isLangfuseInternal,
      });
      const events: IngestionEventType[] =
        await processor.processToIngestionEvents(parsedSpans);
      const writesToLegacyTables = v4WritesToLegacyTables(env);
      if (earlyBatch && writesToLegacyTables) {
        await restoreInlineMedia(
          earlyBatch,
          events.map((event) => event.body),
        );
      }

      // Here, we split the events into observations and non-observations.
      // Observations go into the IngestionService directly whereas the non-observations make another run through the processEventBatch method.
      const traces = events.filter(
        (e) => getClickhouseEntityType(e.type) !== "observation",
      );
      // We need to parse each incoming observation through our ingestion schema to make use of its included transformations.
      const ingestionSchema = createIngestionEventSchema(isLangfuseInternal);
      const candidateObservations = events.filter(
        (e) => getClickhouseEntityType(e.type) === "observation",
      );
      let firstParseError: z.ZodError | undefined;
      const observations = candidateObservations
        .map((o) => ingestionSchema.safeParse(o))
        .flatMap((o) => {
          if (!o.success) {
            if (firstParseError === undefined) {
              firstParseError = o.error;
            }
            return [];
          }
          return [o.data];
        });

      // Per-observation parse failures are near-pure noise (customer-sent data
      // that fails our schema). Aggregate to one metric + one sample warn per
      // file instead of one line per observation.
      const parseFailureCount =
        candidateObservations.length - observations.length;
      if (parseFailureCount > 0) {
        recordIncrement(
          "langfuse.ingestion.otel.observation_parse_failure",
          parseFailureCount,
        );
        logger.warn(
          `Failed to parse ${parseFailureCount}/${candidateObservations.length} otel observations for project ${projectId} in ${fileKey}: ${firstParseError}`,
          {
            error: firstParseError,
            fileKey,
            parseFailureCount,
          },
        );
      }

      // In the next row, we only consider observations. The traces will be recorded in processEventBatch.
      recordIncrement("langfuse.ingestion.event", observations.length, {
        source: "otel",
      });
      // Record more stats specific to the Otel processing
      recordDistribution("langfuse.ingestion.otel.trace_count", traces.length);
      recordDistribution(
        "langfuse.ingestion.otel.observation_count",
        observations.length,
      );
      span?.setAttribute("langfuse.ingestion.otel.trace_count", traces.length);
      span?.setAttribute(
        "langfuse.ingestion.otel.observation_count",
        observations.length,
      );

      // Ensure required infra config is present
      if (!redis) throw new Error("Redis not available");
      if (!prisma) throw new Error("Prisma not available");

      const ingestionService = new IngestionService(
        redis,
        prisma,
        ClickhouseWriter.getInstance(),
        clickhouseClient(),
      );

      // Decide whether observations should be processed via new flow (directly to events table)
      // or via the dual write (staging table and batch job to events).
      //
      // Priority 1: HTTP headers from the SDK request (batch-level decision).
      //   - x-langfuse-sdk-name/version: Python >= 4.0.0 or JS >= 5.0.0
      //   - x-langfuse-ingestion-version: "4" (custom OTel exporter opt-in)
      //   When headers qualify, ALL spans in the batch (including third-party scoped) use direct write.
      //
      // Priority 2 (fallback): Per-span OTEL scope inspection (legacy).
      //   - scope.name contains "langfuse", sdk-experiment environment, Python >= 3.9.0 or JS >= 4.4.0
      //
      // Priority 3 (last resort): the owning organization signed up past the
      //   Cloud direct-write cutoff. Non-Langfuse-SDK exports only (see below).
      const headerBasedDirectWrite = checkHeaderBasedDirectWrite({
        sdkName: job.data.payload.sdkName,
        sdkVersion: job.data.payload.sdkVersion,
        ingestionVersion: job.data.payload.ingestionVersion,
      });

      // Priority 0: deployment-level override.
      //   LANGFUSE_MIGRATION_V4_NATIVE_OTEL_BEHAVIOUR=direct forces every batch
      //   onto the direct events_full path regardless of SDK headers/scopes.
      const envForcesDirect = v4ForceDirectOtelWrite(env);

      // Evaluated whenever the higher-precedence header/env signals did not
      // already qualify, so the write_path label can attribute the decision
      // exactly and `direct_org_cutoff` counts only the traffic that rule
      // genuinely moves off the dual path.
      let scopeBasedDirectWrite = false;
      let langfuseSdkTraffic = isLangfuseSdkTraffic({
        sdkName: job.data.payload.sdkName,
      });
      if (!envForcesDirect && !headerBasedDirectWrite) {
        const hasExperimentEnvironment = observations.some((o) => {
          const body = o.body as { environment?: string };
          return body.environment === "sdk-experiment";
        });
        const sdkInfo =
          parsedSpans.length > 0
            ? getSdkInfoFromResourceSpans(parsedSpans[0])
            : {
                scopeName: null,
                scopeVersion: null,
                telemetrySdkLanguage: null,
              };
        scopeBasedDirectWrite = checkSdkVersionRequirements(
          sdkInfo,
          hasExperimentEnvironment,
        );
        // An older SDK that predates the header still identifies itself through
        // its instrumentation scope, so both signals have to be consulted
        // before the org cutoff may flip a batch. Checked across every scope in
        // the batch, not just the first — collector-forwarded batches can mix
        // SDK spans with third-party scopes in any order. The scan only feeds
        // the cutoff rule, so it is skipped while that rule cannot apply.
        langfuseSdkTraffic =
          langfuseSdkTraffic ||
          (isOtelDirectWriteOrgCutoffConfigured() &&
            batchContainsLangfuseScope(parsedSpans));
      }

      // Resolved only when it can still change the outcome — every
      // higher-precedence signal has declined and this is not SDK traffic — so
      // the Postgres lookup stays off the path for batches already routed by a
      // header. resolveOtelWritePath remains authoritative for the rule; this
      // guard is purely an optimization.
      const orgCutoffForcesDirect =
        !headerBasedDirectWrite &&
        !envForcesDirect &&
        !scopeBasedDirectWrite &&
        !langfuseSdkTraffic &&
        (await isProjectOrgPastOtelDirectWriteCutoff(projectId));

      const { useDirectEventWrite, writePath } = resolveOtelWritePath({
        headerBasedDirectWrite,
        envForcesDirect,
        scopeBasedDirectWrite,
        orgCutoffForcesDirect,
        langfuseSdkTraffic,
      });

      span?.setAttribute("langfuse.ingestion.otel.write_path", writePath);
      recordIncrement("langfuse.ingestion.otel.write_path", 1, {
        path: writePath,
      });

      // V4 events_only mode: skip the legacy traces/observations write paths.
      // The direct events_full write below is the sole destination — env
      // validation already guarantees useDirectEventWrite is true here, so
      // observations and traces don't need the mergeAndWrite / IngestionQueue
      // detour that would otherwise populate the legacy tables.
      const skipLegacyWrites = !writesToLegacyTables;
      const shouldProcessLegacyMedia = shouldProcessLegacyOtelMedia({
        mediaUploadEnabled: env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED === "true",
        writesToLegacyTables,
      });

      if (skipLegacyWrites) {
        span?.setAttribute(
          "langfuse.ingestion.otel.skipped_legacy_writes",
          true,
        );
        recordIncrement("langfuse.ingestion.otel.skipped_legacy_writes", 1);
      } else {
        const shouldForwardToEventsTable =
          !useDirectEventWrite && v4WritesToEventsTable(env);

        if (shouldProcessLegacyMedia) {
          // Process the exact normalized representation written by the legacy
          // path. Targets retain body references, so replacements also reach
          // events-table forwarding when that destination is enabled.
          const legacyMediaPresence = await processOtelEventMedia({
            targets: createLegacyOtelMediaTargets(traces.concat(observations)),
            writePath: "legacy",
            earlyBatch,
            mediaPath,
            inputByteLength,
            projectId,
            fileKey,
            mediaBucket: env.LANGFUSE_S3_MEDIA_UPLOAD_BUCKET,
            mediaPrefix: env.LANGFUSE_S3_MEDIA_UPLOAD_PREFIX,
          });
          if (legacyMediaPresence) {
            mediaPresence = combineOtelMediaPresence(
              mediaPresence,
              legacyMediaPresence,
            );
          }
        }

        // Running everything concurrently might be detrimental to the event loop, but has probably
        // the highest possible throughput. Therefore, we start with a Promise.all.
        // If necessary, we may use a for each instead.

        // Process observations via mergeAndWrite
        const observationWritePromise = Promise.all(
          observations.map((observation) =>
            ingestionService.mergeAndWrite({
              eventType: getClickhouseEntityType(observation.type),
              projectId: auth.scope.projectId,
              entityId: observation.body.id || "", // id is always defined for observations
              createdAtTimestamp: new Date(), // Use the current timestamp as event time
              events: [observation],
              forwardToEventsTable: shouldForwardToEventsTable,
              attribution,
            }),
          ),
        );

        // Process traces and observations concurrently
        await Promise.all([
          observationWritePromise,
          processEventBatch(traces, auth, {
            delay: 0,
            source: "otel",
            forwardToEventsTable: shouldForwardToEventsTable,
            attribution,
            isLangfuseInternal,
          }),
        ]);
      }

      // Determine what processing is needed
      const shouldWriteToEventsTable =
        v4WritesToEventsTable(env) && useDirectEventWrite;
      const directMediaPresence = await processOtelEvents({
        processor,
        resourceSpans: parsedSpans,
        ingestionService,
        projectId,
        fileKey,
        shouldWriteToEventsTable,
        earlyBatch,
        mediaPath,
        inputByteLength,
      });
      if (directMediaPresence) {
        mediaPresence = combineOtelMediaPresence(
          mediaPresence,
          directMediaPresence,
        );
      }
    } catch (e) {
      const fileKey = job.data.payload.data.fileKey;
      if (e instanceof ForbiddenError) {
        traceException(e);
        logger.warn(`Failed to parse otel observation: ${e.message}`, {
          error: e,
          fileKey,
        });
        return;
      }

      markProjectIngestFailure(job.data.payload.authCheck.scope.projectId, {
        source: "otel_ingestion_queue",
        reason: "processing_error",
      });

      logRetryableJobFailure({
        message: `Failed job otel ingestion processing for ${job.data.payload.authCheck.scope.projectId}`,
        error: e,
        job,
        fields: { fileKey },
      });
      throw e;
    } finally {
      await earlyBatch?.dispose();
      setOtelInputPreparationSpanAttributes(span, {
        durationMs: inputPreparationDurationMs,
        mediaPath,
        mediaPresence,
      });
      recordOtelInputPreparationMetrics({
        durationMs: inputPreparationDurationMs,
        mediaPath,
        mediaPresence,
        inputByteLength,
      });
    }
  };
};

function combineOtelMediaPresence(
  current: OtelMediaPresence | undefined,
  next: OtelMediaPresence,
): OtelMediaPresence {
  if (current === "present" || next === "present") return "present";
  if (current === "unknown" || next === "unknown") return "unknown";
  return "absent";
}

function setOtelInputPreparationSpanAttributes(
  span: ReturnType<typeof getCurrentSpan>,
  params: {
    durationMs?: number;
    mediaPath?: "early" | "reference";
    mediaPresence?: OtelMediaPresence;
  },
): void {
  if (params.mediaPath) {
    span?.setAttribute(
      "langfuse.ingestion.otel.media_presence",
      params.mediaPresence ?? "unknown",
    );
    if (params.mediaPresence && params.mediaPresence !== "unknown") {
      span?.setAttribute(
        "langfuse.ingestion.otel.media_detected",
        params.mediaPresence === "present",
      );
    }
  }
  if (params.durationMs !== undefined) {
    span?.setAttribute(
      "langfuse.ingestion.otel.input_preparation_duration_ms",
      params.durationMs,
    );
  }
}

function recordOtelInputPreparationMetrics(params: {
  durationMs?: number;
  mediaPath?: "early" | "reference";
  mediaPresence?: OtelMediaPresence;
  inputByteLength?: number;
}): void {
  if (params.durationMs === undefined || !params.mediaPath) return;

  // The duration is captured after download and before event conversion.
  // Media classification comes from the existing downstream detector.
  const tags = {
    media_path: params.mediaPath,
    media_presence: params.mediaPresence ?? "unknown",
  };
  recordDistribution(
    "langfuse.ingestion.otel.input_preparation_duration_ms",
    params.durationMs,
    tags,
  );
  recordIncrement("langfuse.ingestion.otel.input_preparation_sample", 1, tags);
  recordIncrement(
    "langfuse.ingestion.otel.input_preparation_duration_total_ms",
    params.durationMs,
    tags,
  );
  if (params.inputByteLength && params.inputByteLength > 0) {
    recordIncrement(
      "langfuse.ingestion.otel.input_preparation_bytes_total",
      params.inputByteLength,
      tags,
    );
  }
}