Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

70.6 KB
Raw Blame
import { pipeline, Transform, type Readable } from "stream";
import { monitorEventLoopDelay } from "perf_hooks";
import { Job, UnrecoverableError } from "bullmq";
import { prisma } from "@langfuse/shared/src/db";
import {
  QueueName,
  TQueueJobTypes,
  logger,
  StorageService,
  StorageServiceFactory,
  streamTransformations,
  getObservationsForBlobStorageExport,
  getObservationsForBlobStorageExportRaw,
  getObservationsForBlobStorageExportParquet,
  getTracesForBlobStorageExport,
  getTracesForBlobStorageExportParquet,
  getScoresForBlobStorageExport,
  getScoresForBlobStorageExportParquet,
  getEventsForBlobStorageExport,
  getEventsForBlobStorageExportRaw,
  getEventsForBlobStorageExportParquet,
  getCurrentSpan,
  instrumentAsync,
  recordIncrement,
  BlobStorageIntegrationProcessingQueue,
  queryClickhouse,
  QueueJobs,
  enrichObservationWithModelData,
  createModelCache,
  blobStorageEndpointConnectionValidationOptions,
  assertGcsBlobStorageBucketAllowed,
  validateBlobStorageBucketHost,
  validateBlobStorageEndpoint,
  dispatchProjectNotification,
} from "@langfuse/shared/src/server";
import {
  registerInFlightBlobExport,
  unregisterInFlightBlobExport,
  BLOB_TABLE_EXPORT_METRIC,
  type BlobTableExportOutcome,
} from "./inFlightExports";
import {
  BlobExportAbortTracker,
  classifyBlobExportError,
  errorChainText,
} from "./abortClassification";
import { isSigtermReceived } from "../health";
import { TimedGzip, ZLIB_DEFAULT_LEVEL, type GzipStats } from "./gzipStream";
import {
  BLOB_INTEGRATION_DISABLED_METRIC,
  classifyCustomerFault,
  type CustomerFaultReason,
} from "./isCustomerFaultError";
import { isMultipartPartLimitError } from "./partLimitError";
import { isRecordNotFoundError } from "../integrations/prismaErrors";
import { isFinalBullmqAttempt } from "../integrations/bullmqAttempts";
import { ByteCounter, TimedByteCounter } from "./byteCounters";
import { WORKER_HOST_ID } from "../../utils/hostId";
import {
  BlobStorageIntegrationType,
  BlobStorageIntegrationFileType,
  BatchExportFileFormat,
  BlobStorageExportMode,
  OBSERVATION_FIELD_GROUPS_FULL,
  type ObservationFieldGroupFull,
  type ExportFieldGroup,
  getObservationFieldGroups,
  getScoreFieldGroups,
  getTraceFieldGroups,
  isLegacyExportSource,
  isLegacyExporter,
  resolveBlobExportTuning,
  DEFAULT_BLOB_EXPORT_PART_SIZE_BYTES,
} from "@langfuse/shared";
import { decrypt } from "@langfuse/shared/encryption";
// Shared env for the buffered-upload flag (gates part-level upload stats).
import { env as sharedEnv } from "@langfuse/shared/src/env";
import { randomUUID } from "crypto";
import { SpanKind } from "@opentelemetry/api";
import { env } from "../../env";
import { assertExportSourceWritable } from "../exportWriteModeGuard";
import { recordExportVolume } from "../../services/exportVolumeMetric";
import { isExportCaughtUp } from "../../services/exportStalenessMetric";
import {
  buildBlobExportManifest,
  buildBlobExportManifestKey,
  formatBlobExportTimestamp,
  type BlobExportManifestFile,
} from "./manifest";
import {
  buildBlobExportDeprecationNotice,
  buildBlobExportDeprecationNoticeKey,
} from "./deprecationNotice";
import { resolveFirstExportStart } from "./firstExportStart";

const BlobExportFormat = {
  JSON_RAW: "json-raw",
  JSON_GZIP: "json-gzip",
  CSV_RAW: "csv-raw",
  CSV_GZIP: "csv-gzip",
  JSONL_RAW: "jsonl-raw",
  JSONL_GZIP: "jsonl-gzip",
  // ClickHouse-native columnar export; compression is internal to
  // Parquet, so there is no separate raw/gzip split.
  PARQUET: "parquet",
} as const;
type BlobExportFormat =
  (typeof BlobExportFormat)[keyof typeof BlobExportFormat];

// Text formats only; PARQUET is absent because callers branch on parquetEligible first.
const FORMAT_LOOKUP: Partial<
  Record<
    BlobStorageIntegrationFileType,
    { raw: BlobExportFormat; gzip: BlobExportFormat }
  >
> = {
  [BlobStorageIntegrationFileType.JSON]: {
    raw: BlobExportFormat.JSON_RAW,
    gzip: BlobExportFormat.JSON_GZIP,
  },
  [BlobStorageIntegrationFileType.CSV]: {
    raw: BlobExportFormat.CSV_RAW,
    gzip: BlobExportFormat.CSV_GZIP,
  },
  [BlobStorageIntegrationFileType.JSONL]: {
    raw: BlobExportFormat.JSONL_RAW,
    gzip: BlobExportFormat.JSONL_GZIP,
  },
};

function resolveBlobExportFormat(
  fileType: BlobStorageIntegrationFileType,
  compressed: boolean,
): BlobExportFormat {
  const entry = FORMAT_LOOKUP[fileType];
  if (!entry) {
    throw new Error(`No text export format for file type: ${fileType}`);
  }
  return compressed ? entry.gzip : entry.raw;
}

export const BLOB_STORAGE_LAG_BUFFER_MS = 20 * 60 * 1000; // 20-minute lag buffer

// When a catch-up run lands within this distance of the frontier, treat it as
// caught up rather than emitting a tiny trailing chunk. Object keys are truncated
// to whole-second precision (formatBlobExportTimestamp), so a sub-second remainder
// chunk can share a wall-clock second with the full-interval chunk it follows and
// silently overwrite that window's files and manifest. Suppressing the remainder
// removes the collision at its source; the deferred tail is picked up by the next
// scheduled run (its minTimestamp = lastSyncAt already covers it). Must be >= 1000ms
// (the key granularity) and well below any frequencyInterval so no meaningful data
// is deferred.
export const BLOB_STORAGE_REMAINDER_COALESCE_MS = 1000;

// Persisted as lastError when a window exceeds S3's 10,000-part multipart cap.
// Retrying re-queries a retention-shrinking window until a truncated object
// commits as success, so the run stops loud instead of degrading silently.
export const BLOB_EXPORT_PART_LIMIT_ERROR_MESSAGE =
  "Blob storage export stopped: the export window exceeded the storage provider's " +
  "10,000-part multipart upload limit. Increase the upload part size or reduce the " +
  "export frequency so each window fits within the limit.";

export async function* enrichObservationStream(
  stream: AsyncGenerator<Record<string, unknown>>,
  projectId: string,
  modelIdField: string,
  convertLatencyToSeconds: boolean,
  fieldGroups?: ObservationFieldGroupFull[],
  skipEnrichment = false,
): AsyncGenerator<Record<string, unknown>> {
  const { getModel } = createModelCache(projectId);

  // skipEnrichment drops only the model-price lookup; latency + metadata cleanup still run.
  const includeModelId =
    !skipEnrichment && (!fieldGroups || fieldGroups.includes("model"));

  for await (const row of stream) {
    const enriched: Record<string, unknown> = { ...row };

    if (includeModelId) {
      const modelId = row[modelIdField] as string | null | undefined;
      const model = await getModel(modelId);
      const pricing = enrichObservationWithModelData(model);
      enriched.input_price = pricing.inputPrice;
      enriched.output_price = pricing.outputPrice;
      enriched.total_price = pricing.totalPrice;
    }

    // ClickHouse returns {} for Map columns even when not SELECTed — drop it
    // when the metadata group was not requested.
    if (fieldGroups && !fieldGroups.includes("metadata")) {
      delete enriched.metadata;
    }

    if (convertLatencyToSeconds && row.latency !== undefined) {
      const latency = row.latency as number | null;
      enriched.latency = latency != null ? latency / 1000 : null;
    }

    if (convertLatencyToSeconds && row.time_to_first_token !== undefined) {
      const ttft = row.time_to_first_token as number | null;
      enriched.time_to_first_token = ttft != null ? ttft / 1000 : null;
    }

    yield enriched;
  }
}

const getMinTimestampForExport = async (
  projectId: string,
  lastSyncAt: Date | null,
  exportMode: BlobStorageExportMode,
  exportStartDate: Date | null,
): Promise<Date> => {
  // If we have a lastSyncAt, use it (this is for subsequent exports)
  if (lastSyncAt) {
    return lastSyncAt;
  }

  // For a first FULL_HISTORY export, probe ClickHouse for the actual minimum
  // timestamp across every source table. This is the real data minimum, not
  // project createdAt: projects legitimately backfill data that predates the
  // project, and a full-history export must include it.
  let historicalMinTimestampMs: number | null = null;
  if (exportMode === BlobStorageExportMode.FULL_HISTORY) {
    try {
      const result = await queryClickhouse<{ min_timestamp: number | null }>({
        query: `
              SELECT min(toUnixTimestamp(ts)) * 1000 as min_timestamp
              FROM (
                SELECT min(timestamp) as ts
                FROM traces
                WHERE project_id = {projectId: String}

                UNION ALL

                SELECT min(start_time) as ts
                FROM observations
                WHERE project_id = {projectId: String}

                UNION ALL

                SELECT min(timestamp) as ts
                FROM scores
                WHERE project_id = {projectId: String}

                UNION ALL

                SELECT min(start_time) as ts
                FROM events_core
                WHERE project_id = {projectId: String}
                AND is_deleted = 0 -- match the events export query's visibility
              )
              WHERE ts > 0 -- Ignore 0 results (usually empty tables)
            `,
        params: { projectId },
      });

      const minTimestampValue = Number(result[0]?.min_timestamp);
      if (minTimestampValue && minTimestampValue > 0) {
        historicalMinTimestampMs = minTimestampValue;
      }
      logger.info(
        `[BLOB INTEGRATION] ClickHouse min_timestamp for project ${projectId}: ${historicalMinTimestampMs}`,
      );
    } catch (error) {
      logger.error(
        `[BLOB INTEGRATION] Error querying ClickHouse for minimum timestamp for project ${projectId}`,
        error,
      );
      throw new Error(`Failed to fetch minimum timestamp: ${error}`);
    }
  }

  return resolveFirstExportStart({
    exportMode,
    exportStartDate,
    historicalMinTimestampMs,
  });
};

/**
 * Get the frequency interval in milliseconds for a given export frequency.
 * This is used to chunk historic exports into manageable time windows.
 */
const getFrequencyIntervalMs = (frequency: string): number => {
  switch (frequency) {
    case "every_20_minutes":
      return 20 * 60 * 1000; // 20 minutes
    case "hourly":
      return 60 * 60 * 1000; // 1 hour
    case "daily":
      return 24 * 60 * 60 * 1000; // 1 day
    case "weekly":
      return 7 * 24 * 60 * 60 * 1000; // 1 week
    default:
      throw new Error(`Unsupported export frequency: ${frequency}`);
  }
};

const getFileTypeProperties = (fileType: BlobStorageIntegrationFileType) => {
  switch (fileType) {
    case BlobStorageIntegrationFileType.JSON:
      return {
        contentType: "application/json; charset=utf-8",
        extension: "json",
      };
    case BlobStorageIntegrationFileType.CSV:
      return {
        contentType: "text/csv; charset=utf-8",
        extension: "csv",
      };
    case BlobStorageIntegrationFileType.JSONL:
      return {
        contentType: "application/x-ndjson; charset=utf-8",
        extension: "jsonl",
      };
    case BlobStorageIntegrationFileType.PARQUET:
      return {
        contentType: "application/vnd.apache.parquet",
        extension: "parquet",
      };
    default:
      // eslint-disable-next-line no-case-declarations
      const _exhaustiveCheck: never = fileType;
      throw new Error(`Unsupported file type: ${fileType}`);
  }
};

async function* countedStream<T>(
  source: AsyncGenerator<T>,
  stats: { rows: number; sourceWaitMs: number },
): AsyncGenerator<T> {
  let lastYield = performance.now();
  for await (const value of source) {
    stats.sourceWaitMs += performance.now() - lastYield;
    stats.rows++;
    yield value;
    lastYield = performance.now();
  }
}

// Appends a newline to each raw JSONEachRow line and emits it as bytes, so the
// passthrough path produces the same JSONL framing as the standard formatter
// without re-serializing (the line is already JSON from ClickHouse).
const createRawJsonlNewlineTransform = (): Transform =>
  new Transform({
    writableObjectMode: true,
    transform(line: string, _encoding, callback) {
      callback(null, Buffer.from(line + "\n"));
    },
  });

// KMS SSE is not supported for this integration.
type BlobStorageConnectionConfig = {
  bucketName: string;
  endpoint: string | null;
  region: string;
  accessKeyId: string | undefined;
  secretAccessKey: string | undefined;
  forcePathStyle?: boolean;
  type: BlobStorageIntegrationType;
};

const createBlobStorageService = (
  config: BlobStorageConnectionConfig,
): StorageService => {
  const useGoogleCloudStorage =
    config.type === BlobStorageIntegrationType.GOOGLE_CLOUD_STORAGE;
  // GCS: a stored secret is the customer's service account JSON key; none means
  // default credentials (the deployment identity).
  const gcsServiceAccountKey = useGoogleCloudStorage
    ? config.secretAccessKey
    : undefined;
  if (useGoogleCloudStorage && !gcsServiceAccountKey) {
    // Re-checked per run so removing a bucket from the allowlist stops keyless
    // exports that were saved while it was allowed.
    assertGcsBlobStorageBucketAllowed(config.bucketName);
  }
  // The GCS client treats a non-JSON string as a key *file path*; a customer
  // secret must never be read as one.
  if (gcsServiceAccountKey && !gcsServiceAccountKey.trim().startsWith("{")) {
    // Named so classifyCustomerFault disables the integration instead of retrying.
    throw Object.assign(
      new Error("GCS credentials must be a service account JSON key"),
      { name: "InvalidGcsServiceAccountKey" },
    );
  }
  return StorageServiceFactory.getInstance({
    accessKeyId: useGoogleCloudStorage ? undefined : config.accessKeyId,
    secretAccessKey: useGoogleCloudStorage ? undefined : config.secretAccessKey,
    bucketName: config.bucketName,
    endpoint: config.endpoint ?? undefined,
    region: config.region,
    forcePathStyle: config.forcePathStyle ?? false,
    awsSse: undefined,
    awsSseKmsKeyId: undefined,
    useAzureBlob: config.type === BlobStorageIntegrationType.AZURE_BLOB_STORAGE,
    // Undefined → ADC (the deployment's own identity), allowlist-gated above.
    useGoogleCloudStorage,
    googleCloudCredentials: gcsServiceAccountKey,
    useOCIObjectStorage: false, // Not supported in blob storage integration
    connectionValidation: blobStorageEndpointConnectionValidationOptions(),
  });
};

const processBlobStorageExport = async (config: {
  projectId: string;
  minTimestamp: Date;
  maxTimestamp: Date;
  storageService: StorageService;
  prefix?: string;
  type: BlobStorageIntegrationType;
  table: "traces" | "observations" | "scores" | "observations_v2"; // observations_v2 is the events table
  fileType: BlobStorageIntegrationFileType;
  compressed: boolean;
  // zlib level for the gzip step; undefined => zlib default (6). Only relevant
  // when `compressed` is true.
  gzipLevel: number | undefined;
  convertV4LatencyToSeconds: boolean;
  exportFieldGroups?: ExportFieldGroup[];
  rawPassthrough: boolean;
  // undefined concurrency/attempts => backend keeps its native default.
  partSizeBytes: number;
  maxConcurrentParts: number | undefined;
  maxPartAttempts: number | undefined;
  skipEnrichment: boolean;
  bullmqJobId: string | undefined;
  bullmqAttemptsMade: number;
}) => {
  logger.info(
    `[BLOB INTEGRATION] Processing ${config.table} export for project ${config.projectId}`,
  );

  const storageService = config.storageService;

  return await instrumentAsync(
    {
      name: `blob-export-table`,
      spanKind: SpanKind.INTERNAL,
    },
    async (span) => {
      span.setAttribute("blob.table", config.table);
      span.setAttribute("blob.projectId", config.projectId);
      span.setAttribute("blob.compressed", config.compressed);
      span.setAttribute("blob.fileType", config.fileType);
      span.setAttribute(
        "blob.window.minTimestamp",
        config.minTimestamp.toISOString(),
      );
      span.setAttribute(
        "blob.window.maxTimestamp",
        config.maxTimestamp.toISOString(),
      );
      // Identity + host to group concurrent duplicate runs of the same window.
      if (config.bullmqJobId !== undefined) {
        span.setAttribute("messaging.bullmq.job.id", config.bullmqJobId);
      }
      span.setAttribute("job.attemptsMade", config.bullmqAttemptsMade);
      span.setAttribute("host.name", WORKER_HOST_ID);

      // Resolved per-project tuning. Concurrency/attempts omitted when unset
      // (no single value — the backend default applies).
      span.setAttribute("blob.config.partSizeBytes", config.partSizeBytes);
      if (config.maxConcurrentParts !== undefined) {
        span.setAttribute(
          "blob.config.maxConcurrentParts",
          config.maxConcurrentParts,
        );
      }
      if (config.maxPartAttempts !== undefined) {
        span.setAttribute(
          "blob.config.maxPartAttempts",
          config.maxPartAttempts,
        );
      }
      span.setAttribute("blob.config.skipEnrichment", config.skipEnrichment);
      span.setAttribute("blob.config.rawPassthrough", config.rawPassthrough);

      // Event-loop delay during the stream: if it spikes, lock renewal can't
      // fire and the job re-enqueues as stalled. Torn down below.
      const eventLoopDelay = monitorEventLoopDelay({ resolution: 20 });
      eventLoopDelay.enable();

      // In-flight so a SIGTERM abort is loggable distinctly from a stall-timeout.
      const inFlightHandle = registerInFlightBlobExport({
        jobId: config.bullmqJobId,
        projectId: config.projectId,
        table: config.table,
        minTimestamp: config.minTimestamp.toISOString(),
        maxTimestamp: config.maxTimestamp.toISOString(),
        startedAt: Date.now(),
      });

      recordIncrement(BLOB_TABLE_EXPORT_METRIC, 1, {
        outcome: "started" satisfies BlobTableExportOutcome,
        table: config.table,
      });

      // Outside the try so the catch can distinguish a real upload success from
      // a failure.
      let uploadSucceeded = false;

      // Per-stage errors so the catch can name the originating cause instead of
      // the bare "aborted" the pipeline teardown propagates to every stage.
      const abortTracker = new BlobExportAbortTracker();

      try {
        const blobStorageProps = getFileTypeProperties(config.fileType);

        const parquetEligible =
          config.fileType === BlobStorageIntegrationFileType.PARQUET;

        // Raw passthrough is opt-in per project and only valid for
        // JSONL output of the enriched-observation tables — the only formats
        // where ClickHouse FORMAT JSONEachRow bytes map 1:1 to the file. Any
        // other request falls back to the standard path. The integration-level
        // ineligibility warning is emitted once by the dispatcher; here we just
        // select the path per table (scores/traces always use the standard path,
        // so per-table fallback is expected and not worth a warning).
        const passthroughEligible =
          !parquetEligible &&
          config.rawPassthrough &&
          config.fileType === BlobStorageIntegrationFileType.JSONL &&
          (config.table === "observations" ||
            config.table === "observations_v2");

        const exportPath = (() => {
          if (parquetEligible) {
            return "parquet";
          }
          if (passthroughEligible) {
            return "passthrough";
          }
          return "standard";
        })();

        const timestamp = formatBlobExportTimestamp(config.maxTimestamp);
        // Parquet: fixed `.parquet` extension (no `.gz`) and Parquet content type.
        const extension = (() => {
          if (parquetEligible) {
            return "parquet";
          }
          if (config.compressed) {
            return `${blobStorageProps.extension}.gz`;
          }
          return blobStorageProps.extension;
        })();
        const filePath = `${config.prefix ?? ""}${config.projectId}/${config.table}/${timestamp}.${extension}`;
        const uploadContentType = (() => {
          if (parquetEligible) {
            return "application/vnd.apache.parquet";
          }
          if (config.compressed) {
            return "application/gzip";
          }
          return blobStorageProps.contentType;
        })();

        // Each table's query only receives its own groups. Score groups are
        // opt-in, so the empty-selection fallback covers observation groups only;
        // traces export in full unless a trace group is stored.
        const selectedFieldGroups: ExportFieldGroup[] =
          config.exportFieldGroups && config.exportFieldGroups.length > 0
            ? config.exportFieldGroups
            : [...OBSERVATION_FIELD_GROUPS_FULL];
        const observationFieldGroups =
          getObservationFieldGroups(selectedFieldGroups);
        const scoreFieldGroups = getScoreFieldGroups(selectedFieldGroups);
        const traceFieldGroups = getTraceFieldGroups(selectedFieldGroups);

        // blob.path already encodes parquet (no per-table fallback), so no
        // separate blob.config.parquet attribute is needed.
        span.setAttribute("blob.path", exportPath);

        const pipelineCallback = (err: NodeJS.ErrnoException | null) => {
          if (err) {
            // The pipeline source is the ClickHouse read.
            abortTracker.record("ch-read", err);
            logger.error(
              "[BLOB INTEGRATION] Getting data from DB for blob storage integration failed: ",
              err,
            );
          }
        };

        // Source read wait; backpressureMs is set only by the parquet
        // TimedByteCounter and stays 0 on every other path.
        const sourceStats = { rows: 0, sourceWaitMs: 0, backpressureMs: 0 };
        // When enrichment is active, chStats isolates ClickHouse read wait from
        // enrichment CPU. enrichMs = sourceStats.sourceWaitMs - chStats.sourceWaitMs.
        let chStats: { rows: number; sourceWaitMs: number } | null = null;

        // Parquet feeds sourceStats.sourceWaitMs from the piped binary stream so
        // the shared metrics derivation works without a per-row generator.
        const serializedCounter = parquetEligible
          ? new TimedByteCounter(sourceStats)
          : new ByteCounter();
        // No worker-side gzip on the parquet path (compression is internal to
        // Parquet), so the parquet path never allocates a compressed counter.
        const compressedCounter =
          !parquetEligible && config.compressed ? new ByteCounter() : null;
        // Paired with compressedCounter: both exist iff compression is on.
        const gzipStats: GzipStats | null = compressedCounter
          ? {
              level: config.gzipLevel ?? ZLIB_DEFAULT_LEVEL,
              activeMs: 0,
              backpressureMs: 0,
            }
          : null;

        let fileStream: Readable;

        if (parquetEligible) {
          // Stream raw FORMAT Parquet bytes straight to upload — no JS
          // parse/enrich/serialize, no gzip, no row counting (binary has no row
          // boundaries, so sourceStats.rows stays 0). Field-group projection,
          // latency ms→s, and dropped price columns are baked into the SQL. The
          // exception-tag Transform in queryClickhouseExecRaw aborts the upload
          // before commit on a mid-stream failure — same guarantee as passthrough.
          let parquetSource: Readable;
          switch (config.table) {
            case "traces":
              parquetSource = (
                await getTracesForBlobStorageExportParquet(
                  config.projectId,
                  config.minTimestamp,
                  config.maxTimestamp,
                  traceFieldGroups,
                )
              ).stream;
              break;
            case "scores":
              parquetSource = (
                await getScoresForBlobStorageExportParquet(
                  config.projectId,
                  config.minTimestamp,
                  config.maxTimestamp,
                  scoreFieldGroups,
                )
              ).stream;
              break;
            case "observations":
              parquetSource = (
                await getObservationsForBlobStorageExportParquet(
                  config.projectId,
                  config.minTimestamp,
                  config.maxTimestamp,
                  observationFieldGroups,
                )
              ).stream;
              break;
            case "observations_v2": // observations_v2 is the events table
              parquetSource = (
                await getEventsForBlobStorageExportParquet(
                  config.projectId,
                  config.minTimestamp,
                  config.maxTimestamp,
                  observationFieldGroups,
                  config.convertV4LatencyToSeconds,
                )
              ).stream;
              break;
            default:
              throw new Error(`Unsupported table type: ${config.table}`);
          }

          fileStream = pipeline(
            parquetSource,
            serializedCounter,
            pipelineCallback,
          );
        } else if (passthroughEligible) {
          // Stream ClickHouse JSONEachRow row text straight through: skip the
          // per-row JSON.parse, enrichment, and re-serialize. Shaping (latency→s,
          // dropped price columns, field-group selection) is baked into the SQL.
          // The client's own mid-stream exception detection (CH ≥ 25.11) errors
          // the stream on a failed query, which aborts the upload — no committed
          // object and no out-of-band system.query_log check needed.
          const rawRows =
            config.table === "observations"
              ? getObservationsForBlobStorageExportRaw(
                  config.projectId,
                  config.minTimestamp,
                  config.maxTimestamp,
                  observationFieldGroups,
                )
              : getEventsForBlobStorageExportRaw(
                  config.projectId,
                  config.minTimestamp,
                  config.maxTimestamp,
                  observationFieldGroups,
                  config.convertV4LatencyToSeconds,
                );

          const dataStream = countedStream(rawRows, sourceStats);
          const jsonlNewline = createRawJsonlNewlineTransform();

          fileStream =
            compressedCounter && gzipStats
              ? pipeline(
                  dataStream,
                  jsonlNewline,
                  serializedCounter,
                  new TimedGzip(config.gzipLevel, gzipStats),
                  compressedCounter,
                  pipelineCallback,
                )
              : pipeline(
                  dataStream,
                  jsonlNewline,
                  serializedCounter,
                  pipelineCallback,
                );
        } else {
          let rawStream: AsyncGenerator<Record<string, unknown>>;

          switch (config.table) {
            case "traces":
              rawStream = getTracesForBlobStorageExport(
                config.projectId,
                config.minTimestamp,
                config.maxTimestamp,
                traceFieldGroups,
              );
              break;
            case "observations":
              chStats = { rows: 0, sourceWaitMs: 0 };
              rawStream = enrichObservationStream(
                countedStream(
                  getObservationsForBlobStorageExport(
                    config.projectId,
                    config.minTimestamp,
                    config.maxTimestamp,
                    observationFieldGroups,
                  ),
                  chStats,
                ),
                config.projectId,
                "model_id",
                false, // v3 query already returns latency in seconds
                observationFieldGroups,
                config.skipEnrichment,
              );
              break;
            case "scores":
              rawStream = getScoresForBlobStorageExport(
                config.projectId,
                config.minTimestamp,
                config.maxTimestamp,
                scoreFieldGroups,
              );
              break;
            case "observations_v2": // observations_v2 is the events table
              chStats = { rows: 0, sourceWaitMs: 0 };
              rawStream = enrichObservationStream(
                countedStream(
                  getEventsForBlobStorageExport(
                    config.projectId,
                    config.minTimestamp,
                    config.maxTimestamp,
                    observationFieldGroups,
                  ),
                  chStats,
                ),
                config.projectId,
                "model_id",
                config.convertV4LatencyToSeconds,
                observationFieldGroups,
                config.skipEnrichment,
              );
              break;
            default:
              throw new Error(`Unsupported table type: ${config.table}`);
          }

          const dataStream = countedStream(rawStream, sourceStats);
          if (config.fileType === BlobStorageIntegrationFileType.PARQUET) {
            throw new Error(
              `Reached the text-format export path with fileType=PARQUET for project ${config.projectId}; the parquetEligible branch should have handled it`,
            );
          }
          const formatTransform =
            streamTransformations[config.fileType as BatchExportFileFormat]();

          fileStream =
            compressedCounter && gzipStats
              ? pipeline(
                  dataStream,
                  formatTransform,
                  serializedCounter,
                  new TimedGzip(config.gzipLevel, gzipStats),
                  compressedCounter,
                  pipelineCallback,
                )
              : pipeline(
                  dataStream,
                  formatTransform,
                  serializedCounter,
                  pipelineCallback,
                );
        }

        // Upload the file to cloud storage
        // For CSV exports, use larger part size to handle big files
        // 100 MB parts support files up to ~1 TB (100 MB × 10,000 AWS limit)
        // This prevents hitting AWS's 10,000 part limit on large exports
        // Mutable sink the uploader fills live (readable even if upload throws).
        // Only the S3 buffered path populates it; producesUploadStats gates the
        // telemetry so other paths don't report zero-part uploads.
        const uploadStats = {
          partsUploaded: 0,
          partRetries: 0,
          partFailures: 0,
        };
        const producesUploadStats =
          (config.type === BlobStorageIntegrationType.S3 ||
            config.type === BlobStorageIntegrationType.S3_COMPATIBLE) &&
          sharedEnv.LANGFUSE_S3_UPLOAD_ENABLE_BUFFERED === "true";
        let uploadStartMs: number | undefined;
        let uploadDurationMsFinal: number | undefined;
        try {
          uploadStartMs = performance.now();

          try {
            await storageService.uploadFileBuffered({
              fileName: filePath,
              fileType: uploadContentType,
              data: fileStream,
              partSizeBytes: config.partSizeBytes,
              maxConcurrentParts: config.maxConcurrentParts,
              maxPartAttempts: config.maxPartAttempts,
              stats: uploadStats,
            });
          } catch (uploadError) {
            // Attribute to the upload stage; a CH-origin error surfacing here is
            // still classified as ch-error by its preserved cause chain.
            abortTracker.record("upload", uploadError);
            throw uploadError;
          }
          // Record at the upload boundary so a throw in the `finally` below
          // can't miscount a real success as a failure.
          uploadSucceeded = true;
          uploadDurationMsFinal = Math.round(performance.now() - uploadStartMs);
          recordIncrement(BLOB_TABLE_EXPORT_METRIC, 1, {
            outcome: "success" satisfies BlobTableExportOutcome,
            table: config.table,
          });

          const exportFormat = parquetEligible
            ? BlobExportFormat.PARQUET
            : resolveBlobExportFormat(config.fileType, config.compressed);

          // rowCount null on parquet: sourceStats.rows stays 0 for the binary
          // stream and would misreport as an empty file.
          const manifestFile: BlobExportManifestFile = {
            key: filePath,
            table: config.table,
            fileType: parquetEligible
              ? BlobStorageIntegrationFileType.PARQUET
              : config.fileType,
            format: exportFormat,
            compressed: parquetEligible ? false : config.compressed,
            contentType: uploadContentType,
            sizeBytes: compressedCounter
              ? compressedCounter.bytes
              : serializedCounter.bytes,
            rowCount: parquetEligible ? null : sourceStats.rows,
          };
          // Unified export-volume metric: the actual uploaded volume per
          // source (post-gzip size for compressed formats, raw/parquet size
          // otherwise). destination_type (S3 / S3_COMPATIBLE / AZURE_*) splits
          // the egress by data-transfer cost in the Datadog dashboard.
          recordExportVolume({
            integration: "blob_storage",
            bytes: compressedCounter
              ? compressedCounter.bytes
              : serializedCounter.bytes,
            projectId: config.projectId,
            destinationType: config.type,
            source: exportFormat,
            table: config.table,
            path: exportPath,
          });

          const uploadDurationMs = uploadDurationMsFinal;
          // Parquet's source counter sits at the upload boundary, so strip its
          // backpressure to recover pure CH read (no-op elsewhere: gzip isolates
          // CH read in chStats, and backpressureMs is 0 on every other path).
          const chReadMs = Math.round(
            Math.max(
              0,
              (chStats ? chStats.sourceWaitMs : sourceStats.sourceWaitMs) -
                (chStats ? 0 : sourceStats.backpressureMs),
            ),
          );
          const enrichMs = chStats
            ? Math.max(
                0,
                Math.round(sourceStats.sourceWaitMs - chStats.sourceWaitMs),
              )
            : 0;
          const gzipCpuMs = gzipStats
            ? Math.max(
                0,
                Math.round(gzipStats.activeMs - gzipStats.backpressureMs),
              )
            : 0;
          // Measured backpressure (gzip / parquet boundary), else duration residual.
          const uploadWaitMs = (() => {
            if (gzipStats) {
              return Math.round(gzipStats.backpressureMs);
            }
            if (parquetEligible) {
              return Math.round(sourceStats.backpressureMs);
            }
            return Math.max(0, uploadDurationMs - chReadMs - enrichMs);
          })();

          logger.info(
            `[BLOB INTEGRATION] Successfully exported ${config.table} for project ${config.projectId}: ` +
              `jobId=${config.bullmqJobId} attemptsMade=${config.bullmqAttemptsMade} host=${WORKER_HOST_ID} ` +
              `path=${exportPath} ` +
              `rows=${sourceStats.rows} chReadMs=${chReadMs} enrichMs=${enrichMs} ` +
              `gzipCpuMs=${gzipCpuMs} uploadWaitMs=${uploadWaitMs} ` +
              `serializedBytes=${serializedCounter.bytes} uploadDurationMs=${uploadDurationMs} ` +
              // "configured*" = resolved tuning, not necessarily what the backend
              // applied; uploadParts/Retries/Failures below are the actual effects.
              `configuredPartSizeBytes=${config.partSizeBytes} configuredMaxConcurrentParts=${config.maxConcurrentParts ?? "default"} ` +
              `configuredMaxPartAttempts=${config.maxPartAttempts ?? "default"} skipEnrichment=${config.skipEnrichment}` +
              (producesUploadStats
                ? ` uploadParts=${uploadStats.partsUploaded} uploadRetries=${uploadStats.partRetries} uploadFailures=${uploadStats.partFailures}`
                : "") +
              (gzipStats && compressedCounter
                ? ` gzipLevel=${gzipStats.level} compressedBytes=${compressedCounter.bytes}`
                : ""),
          );

          return manifestFile;
        } finally {
          span.setAttribute("blob.rows", sourceStats.rows);
          // Same chReadMs / uploadWaitMs derivation as the success path above.
          const finalChReadMs = Math.round(
            Math.max(
              0,
              (chStats ? chStats.sourceWaitMs : sourceStats.sourceWaitMs) -
                (chStats ? 0 : sourceStats.backpressureMs),
            ),
          );
          const finalEnrichMs = chStats
            ? Math.max(
                0,
                Math.round(sourceStats.sourceWaitMs - chStats.sourceWaitMs),
              )
            : 0;
          span.setAttribute("blob.chReadMs", finalChReadMs);
          span.setAttribute("blob.enrichMs", finalEnrichMs);
          span.setAttribute("blob.serializedBytes", serializedCounter.bytes);
          if (uploadStartMs !== undefined) {
            const totalUploadMs =
              uploadDurationMsFinal ??
              Math.round(performance.now() - uploadStartMs);
            span.setAttribute("blob.uploadDurationMs", totalUploadMs);
            // On failure the values are partial (the upload aborted mid-stream).
            const finalGzipCpuMs = gzipStats
              ? Math.max(
                  0,
                  Math.round(gzipStats.activeMs - gzipStats.backpressureMs),
                )
              : 0;
            const finalUploadWaitMs = (() => {
              if (gzipStats) {
                return Math.round(gzipStats.backpressureMs);
              }
              if (parquetEligible) {
                return Math.round(sourceStats.backpressureMs);
              }
              return Math.max(0, totalUploadMs - finalChReadMs - finalEnrichMs);
            })();
            span.setAttribute("blob.gzipCpuMs", finalGzipCpuMs);
            span.setAttribute("blob.uploadWaitMs", finalUploadWaitMs);
          }
          if (producesUploadStats) {
            span.setAttribute("blob.upload.parts", uploadStats.partsUploaded);
            span.setAttribute("blob.upload.retries", uploadStats.partRetries);
            span.setAttribute("blob.upload.failures", uploadStats.partFailures);
          }
          if (compressedCounter) {
            span.setAttribute("blob.compressedBytes", compressedCounter.bytes);
          }
          if (gzipStats && compressedCounter) {
            const activeMs = Math.round(gzipStats.activeMs);
            const ratio =
              compressedCounter.bytes > 0
                ? serializedCounter.bytes / compressedCounter.bytes
                : 0;
            const pureGzipMs = Math.max(
              0,
              gzipStats.activeMs - gzipStats.backpressureMs,
            );
            const throughputMbPerSec =
              pureGzipMs > 0
                ? serializedCounter.bytes / (pureGzipMs * 1000)
                : 0;

            span.setAttribute("blob.gzip.level", gzipStats.level);
            span.setAttribute("blob.gzip.activeMs", activeMs);
            span.setAttribute("blob.gzip.ratio", Number(ratio.toFixed(3)));
            span.setAttribute(
              "blob.gzip.throughputMbPerSec",
              Number(throughputMbPerSec.toFixed(2)),
            );
          }
        }
      } catch (error) {
        // On SIGTERM, add a concrete shutdown record. It wins origin() only when
        // no earlier concrete fault (e.g. a real CH exception) was recorded.
        if (isSigtermReceived()) {
          abortTracker.record("shutdown", error);
        }
        // Fall back to the propagated error if no stage recorded one.
        const origin = abortTracker.origin() ?? classifyBlobExportError(error);

        span.setAttribute("blob.abortReason", origin.reason);
        span.setAttribute("blob.abortStage", origin.stage);

        // Skip if `success` already fired (a later step threw post-upload).
        if (!uploadSucceeded) {
          recordIncrement(BLOB_TABLE_EXPORT_METRIC, 1, {
            outcome: "failure" satisfies BlobTableExportOutcome,
            abortReason: origin.reason,
            table: config.table,
          });
        }
        logger.error(
          `[BLOB INTEGRATION] Error exporting ${config.table} for project ${config.projectId} ` +
            `(jobId=${config.bullmqJobId} attemptsMade=${config.bullmqAttemptsMade} host=${WORKER_HOST_ID} ` +
            `abortReason=${origin.reason} abortStage=${origin.stage}` +
            `${origin.concrete ? "" : " attribution=best-effort"}): ${origin.chain}`,
          error,
        );
        throw error;
      } finally {
        unregisterInFlightBlobExport(inFlightHandle);

        // ns → ms; the histogram yields NaN/Infinity with zero samples.
        eventLoopDelay.disable();
        const toFiniteMs = (ns: number): number =>
          Number.isFinite(ns) ? ns / 1e6 : 0;
        const maxMs = toFiniteMs(eventLoopDelay.max);
        const p99Ms = toFiniteMs(eventLoopDelay.percentile(99));
        const meanMs = toFiniteMs(eventLoopDelay.mean);
        span.setAttribute("blob.eventLoopDelay.maxMs", Math.round(maxMs));
        span.setAttribute("blob.eventLoopDelay.p99Ms", Math.round(p99Ms));
        span.setAttribute("blob.eventLoopDelay.meanMs", Math.round(meanMs));
      }
    },
  );
};

// Small JSON body, so single-shot uploadFile instead of the buffered multipart
// path the table streams use.
const writeBlobExportManifest = async (params: {
  storageService: StorageService;
  prefix?: string;
  projectId: string;
  exportSource: string;
  minTimestamp: Date;
  maxTimestamp: Date;
  files: BlobExportManifestFile[];
}): Promise<void> => {
  const manifest = buildBlobExportManifest({
    projectId: params.projectId,
    exportSource: params.exportSource,
    minTimestamp: params.minTimestamp,
    maxTimestamp: params.maxTimestamp,
    createdAt: new Date(),
    files: params.files,
  });
  const key = buildBlobExportManifestKey({
    prefix: params.prefix,
    projectId: params.projectId,
    maxTimestamp: params.maxTimestamp,
  });

  await params.storageService.uploadFile({
    fileName: key,
    fileType: "application/json; charset=utf-8",
    // Trailing newline for POSIX-friendly shell tooling.
    data: JSON.stringify(manifest, null, 2) + "\n",
  });

  logger.info(
    `[BLOB INTEGRATION] Wrote run manifest for project ${params.projectId}: ` +
      `key=${key} files=${manifest.files.length} tables=${manifest.tables.join(",")}`,
  );
};

// Drop a plain-text deprecation notice into the export destination
// for legacy-source projects. Best-effort — a failure to write the notice must
// not fail the export run, so it is called after the manifest commit point and
// swallows its own error.
const writeBlobExportDeprecationNotice = async (params: {
  storageService: StorageService;
  prefix?: string;
  projectId: string;
}): Promise<void> => {
  const key = buildBlobExportDeprecationNoticeKey({
    prefix: params.prefix,
    projectId: params.projectId,
  });
  try {
    await params.storageService.uploadFile({
      fileName: key,
      fileType: "text/plain; charset=utf-8",
      data: buildBlobExportDeprecationNotice(),
    });
    logger.info(
      `[BLOB INTEGRATION] Wrote legacy-source deprecation notice for project ${params.projectId}: key=${key}`,
    );
  } catch (error) {
    logger.warn(
      `[BLOB INTEGRATION] Failed to write legacy-source deprecation notice for project ${params.projectId} (key=${key}); export run is unaffected`,
      error,
    );
  }
};

// Counterpart to the writer: once a project migrates off a legacy source (the
// action the notice asks for), a notice left from an earlier run would linger
// with now-false claims. Best-effort delete on the non-legacy path clears it.
// Idempotent — deleting a non-existent key is a no-op — and never fails the run.
const removeBlobExportDeprecationNotice = async (params: {
  storageService: StorageService;
  prefix?: string;
  projectId: string;
}): Promise<void> => {
  const key = buildBlobExportDeprecationNoticeKey({
    prefix: params.prefix,
    projectId: params.projectId,
  });
  try {
    await params.storageService.deleteFiles([key]);
  } catch (error) {
    logger.warn(
      `[BLOB INTEGRATION] Failed to remove stale deprecation notice for project ${params.projectId} (key=${key}); export run is unaffected`,
      error,
    );
  }
};

export const handleBlobStorageIntegrationProjectJob = async (
  job: Job<TQueueJobTypes[QueueName.BlobStorageIntegrationProcessingQueue]>,
) => {
  const { projectId } = job.data.payload;

  const span = getCurrentSpan();
  if (span) {
    span.setAttribute("messaging.bullmq.job.input.jobId", job.data.id);
    span.setAttribute("messaging.bullmq.job.input.projectId", projectId);
    // BullMQ job id (distinct from the payload id above) + attempt count.
    if (job.id !== undefined) {
      span.setAttribute("messaging.bullmq.job.id", job.id);
    }
    span.setAttribute("job.attemptsMade", job.attemptsMade);
    span.setAttribute("host.name", WORKER_HOST_ID);
  }

  logger.info(
    `[BLOB INTEGRATION] Processing blob storage integration for project ${projectId}`,
  );

  const blobStorageIntegration = await prisma.blobStorageIntegration.findUnique(
    {
      where: {
        projectId,
      },
    },
  );

  if (!blobStorageIntegration) {
    logger.warn(
      `[BLOB INTEGRATION] Blob storage integration not found for project ${projectId}`,
    );
    return;
  }
  if (!blobStorageIntegration.enabled) {
    logger.info(
      `[BLOB INTEGRATION] Blob storage integration is disabled for project ${projectId}`,
    );
    await prisma.blobStorageIntegration.updateMany({
      where: { projectId },
      data: { runStartedAt: null },
    });
    return;
  }

  const runStartTime = new Date();
  const { count: claimed } = await prisma.blobStorageIntegration.updateMany({
    where: { projectId },
    data: { runStartedAt: runStartTime },
  });
  if (claimed === 0) {
    logger.info(
      `[BLOB INTEGRATION] Blob storage integration for project ${projectId} was deleted before the run started; dropping obsolete job`,
    );
    return;
  }

  // Sync between lastSyncAt and now - 30 minutes
  // Cap the export to one frequency period to enable chunked historic exports
  const minTimestamp = await getMinTimestampForExport(
    projectId,
    blobStorageIntegration.lastSyncAt,
    blobStorageIntegration.exportMode,
    blobStorageIntegration.exportStartDate,
  );

  logger.info(
    `[BLOB INTEGRATION] Calculated minTimestamp for project ${projectId}: ${minTimestamp}, isValid: ${!isNaN(minTimestamp.getTime())}, getTime: ${minTimestamp.getTime()}, exportMode: ${blobStorageIntegration.exportMode}, lastSyncAt: ${blobStorageIntegration.lastSyncAt}, exportStartDate: ${blobStorageIntegration.exportStartDate}`,
  );

  const now = new Date();
  const uncappedMaxTimestamp = new Date(
    now.getTime() - BLOB_STORAGE_LAG_BUFFER_MS,
  );
  const frequencyIntervalMs = getFrequencyIntervalMs(
    blobStorageIntegration.exportFrequency,
  );

  // Cap maxTimestamp to one frequency period ahead of minTimestamp
  // This ensures large historic exports are broken into manageable chunks
  const maxTimestamp = new Date(
    Math.min(
      minTimestamp.getTime() + frequencyIntervalMs,
      uncappedMaxTimestamp.getTime(),
    ),
  );

  logger.info(
    `[BLOB INTEGRATION] Calculated maxTimestamp for project ${projectId}: ${maxTimestamp}, isValid: ${!isNaN(maxTimestamp.getTime())}, getTime: ${maxTimestamp.getTime()}, frequencyIntervalMs: ${frequencyIntervalMs}`,
  );

  // Skip export if the time window is empty or invalid
  if (minTimestamp >= maxTimestamp) {
    logger.info(
      `[BLOB INTEGRATION] Skipping export for project ${projectId}: time window is empty (min: ${minTimestamp.toISOString()}, max: ${maxTimestamp.toISOString()})`,
    );
    await prisma.blobStorageIntegration.updateMany({
      where: { projectId },
      data: {
        runStartedAt: null,
        nextSyncAt: new Date(now.getTime() + frequencyIntervalMs),
        lastError: null,
        lastErrorAt: null,
        ...(isExportCaughtUp({
          lastSyncAt: blobStorageIntegration.lastSyncAt,
          runStartTime,
        })
          ? { backfill: false }
          : {}),
      },
    });
    return;
  }

  // Legacy-source deprecation is a Cloud policy (isLegacyExporter exempts
  // self-hosted), so the deprecation notice below is Cloud-only too.
  const isCloud = Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);

  try {
    // The catch persists lastError and notifies admins.
    assertExportSourceWritable(
      blobStorageIntegration.exportSource,
      "Select the enriched export source (OBSERVATIONS_V2) in the blob storage integration settings.",
    );

    // Preflight the persisted integration endpoint once per job inside the
    // export error path. StorageService connection-time validation remains the
    // DNS-rebinding defense for each SDK connection.
    if (blobStorageIntegration.endpoint) {
      await validateBlobStorageEndpoint(blobStorageIntegration.endpoint);
      await validateBlobStorageBucketHost(blobStorageIntegration);
    }

    // Process the export based on the integration configuration
    // Convert v4 (events table) latency/time_to_first_token from ms to seconds
    // for integrations created on or after 2026-04-01. Before this date, v4 blob
    // export returned these fields in milliseconds. We preserve that behavior for
    // existing integrations to avoid silently breaking their pipelines.
    const convertV4LatencyToSeconds =
      blobStorageIntegration.createdAt >= new Date("2026-04-01T00:00:00Z");

    // Per-project export tuning (set via DB directly; no UI/tRPC write path).
    // Malformed/absent column resolves to defaults — never throws.
    const { resolved: exportTuning, warnings: exportTuningWarnings } =
      resolveBlobExportTuning(blobStorageIntegration.exportTuning, {
        partSizeBytes: DEFAULT_BLOB_EXPORT_PART_SIZE_BYTES,
      });
    for (const warning of exportTuningWarnings) {
      logger.warn(
        `[BLOB INTEGRATION] exportTuning for project ${projectId}: ${warning}`,
      );
    }

    // One client per run, shared by all table uploads and the manifest write.
    const storageService = createBlobStorageService({
      bucketName: blobStorageIntegration.bucketName,
      endpoint: blobStorageIntegration.endpoint,
      region: blobStorageIntegration.region || "auto",
      accessKeyId: blobStorageIntegration.accessKeyId || undefined,
      secretAccessKey: blobStorageIntegration.secretAccessKey
        ? decrypt(blobStorageIntegration.secretAccessKey)
        : undefined,
      forcePathStyle: blobStorageIntegration.forcePathStyle || undefined,
      type: blobStorageIntegration.type,
    });

    const executionConfig = {
      projectId,
      minTimestamp,
      maxTimestamp,
      storageService,
      prefix: blobStorageIntegration.prefix || undefined,
      type: blobStorageIntegration.type,
      fileType: blobStorageIntegration.fileType,
      compressed: blobStorageIntegration.compressed,
      gzipLevel: exportTuning.gzipLevel,
      convertV4LatencyToSeconds,
      exportFieldGroups:
        blobStorageIntegration.exportFieldGroups as ExportFieldGroup[],
      rawPassthrough: exportTuning.rawPassthrough,
      partSizeBytes: exportTuning.partSizeBytes,
      maxConcurrentParts: exportTuning.maxConcurrentParts,
      maxPartAttempts: exportTuning.maxPartAttempts,
      skipEnrichment: exportTuning.skipEnrichment,
      bullmqJobId: job.id,
      bullmqAttemptsMade: job.attemptsMade,
    };

    // Check if this project should only export traces (legacy behavior via env var)
    const isTraceOnlyProject =
      env.LANGFUSE_BLOB_STORAGE_EXPORT_TRACE_ONLY_PROJECT_IDS.includes(
        projectId,
      );

    // Warn once per run if rawPassthrough is enabled but no table will actually
    // use it. Passthrough only applies to JSONL exports of observations /
    // observations_v2; every non-trace-only export source includes one of those,
    // so the only integration-level ineligibility is a non-JSONL file type or a
    // trace-only project. The per-table fallback for scores/traces is expected
    // dispatch and is intentionally not warned about (avoids ~hourly log noise).
    if (
      exportTuning.rawPassthrough &&
      blobStorageIntegration.fileType !==
        BlobStorageIntegrationFileType.PARQUET &&
      (blobStorageIntegration.fileType !==
        BlobStorageIntegrationFileType.JSONL ||
        isTraceOnlyProject)
    ) {
      logger.warn(
        `[BLOB INTEGRATION] rawPassthrough enabled for project ${projectId} but no eligible table will use it ` +
          `(fileType=${blobStorageIntegration.fileType}, traceOnly=${isTraceOnlyProject}); exporting via the standard path. ` +
          `Passthrough requires JSONL output of observations or observations_v2.`,
      );
    }

    let runFiles: BlobExportManifestFile[];
    if (isTraceOnlyProject) {
      // Only process traces table for projects in the trace-only list (legacy behavior)
      logger.info(
        `[BLOB INTEGRATION] Project ${projectId} is configured for trace-only export via env var, skipping observations, scores, and events`,
      );
      runFiles = [
        await processBlobStorageExport({ ...executionConfig, table: "traces" }),
      ];
    } else {
      // Process tables based on exportSource setting
      const processPromises: Promise<BlobExportManifestFile>[] = [];

      // Always include scores
      processPromises.push(
        processBlobStorageExport({ ...executionConfig, table: "scores" }),
      );

      // Traces and observations - for TRACES_OBSERVATIONS and TRACES_OBSERVATIONS_EVENTS
      if (
        blobStorageIntegration.exportSource === "TRACES_OBSERVATIONS" ||
        blobStorageIntegration.exportSource === "TRACES_OBSERVATIONS_EVENTS"
      ) {
        processPromises.push(
          processBlobStorageExport({ ...executionConfig, table: "traces" }),
          processBlobStorageExport({
            ...executionConfig,
            table: "observations",
          }),
        );
      }

      // Events - for EVENTS and TRACES_OBSERVATIONS_EVENTS
      // events are stored in the observations_v2 directory in blob storage
      if (
        blobStorageIntegration.exportSource === "EVENTS" ||
        blobStorageIntegration.exportSource === "TRACES_OBSERVATIONS_EVENTS"
      ) {
        processPromises.push(
          processBlobStorageExport({
            ...executionConfig,
            table: "observations_v2",
          }),
        );
      }

      runFiles = await Promise.all(processPromises);
    }

    // The manifest is the run's commit point: written strictly after every
    // table upload succeeded. A failure here fails the run, so lastSyncAt does
    // not advance and the retry idempotently overwrites the table files.
    await writeBlobExportManifest({
      storageService,
      prefix: blobStorageIntegration.prefix || undefined,
      projectId,
      exportSource: blobStorageIntegration.exportSource,
      minTimestamp,
      maxTimestamp,
      files: runFiles,
    });

    // Cloud-only v3-deprecation notice; both sides best-effort (never fail the run).
    if (isCloud) {
      if (isLegacyExportSource(blobStorageIntegration.exportSource)) {
        await writeBlobExportDeprecationNotice({
          storageService,
          prefix: blobStorageIntegration.prefix || undefined,
          projectId,
        });
      } else if (
        // Gate cleanup on "old enough to have written a notice": otherwise every
        // enriched-only export adds a needless per-run s3:DeleteObject on the
        // destination, which is write-only for many customers.
        isLegacyExporter(blobStorageIntegration.createdAt, isCloud)
      ) {
        await removeBlobExportDeprecationNotice({
          storageService,
          prefix: blobStorageIntegration.prefix || undefined,
          projectId,
        });
      }
    }

    // Determine if we've caught up with present-day data. A remainder below
    // BLOB_STORAGE_REMAINDER_COALESCE_MS is treated as caught up so we never emit
    // a sub-second trailing chunk that would collide with the full-interval chunk
    // on the same second-precision object key (see the constant's doc comment).
    const remainderMs = uncappedMaxTimestamp.getTime() - maxTimestamp.getTime();
    const caughtUp =
      maxTimestamp.getTime() >= uncappedMaxTimestamp.getTime() ||
      remainderMs < BLOB_STORAGE_REMAINDER_COALESCE_MS;

    let nextSyncAt: Date;
    if (caughtUp) {
      // Normal mode: schedule for the next frequency period
      nextSyncAt = new Date(maxTimestamp.getTime() + frequencyIntervalMs);
      logger.info(
        `[BLOB INTEGRATION] Caught up with exports for project ${projectId}. Next sync at ${nextSyncAt.toISOString()}`,
      );
    } else {
      // Catch-up mode: schedule next chunk immediately
      nextSyncAt = new Date();
      logger.info(
        `[BLOB INTEGRATION] Still catching up for project ${projectId}. Scheduling next chunk immediately (processed up to ${maxTimestamp.toISOString()})`,
      );
    }

    // Update integration after successful processing. count === 0 means the
    // integration was deleted mid-run — skip the catch-up re-enqueue below,
    // the job is obsolete.
    const { count: persisted } = await prisma.blobStorageIntegration.updateMany(
      {
        where: {
          projectId,
        },
        data: {
          lastSyncAt: maxTimestamp,
          nextSyncAt,
          lastError: null,
          lastErrorAt: null,
          runStartedAt: null,
          ...(isExportCaughtUp({ lastSyncAt: maxTimestamp, runStartTime })
            ? { backfill: false }
            : {}),
        },
      },
    );
    if (persisted === 0) {
      logger.info(
        `[BLOB INTEGRATION] Blob storage integration for project ${projectId} was deleted mid-run; dropping obsolete job after successful export`,
      );
      return;
    }

    // If still catching up, immediately queue the next chunk job
    if (!caughtUp) {
      const queue = BlobStorageIntegrationProcessingQueue.getInstance();
      if (queue) {
        const jobId = `${projectId}-${maxTimestamp.toISOString()}`;
        await queue.add(
          QueueJobs.BlobStorageIntegrationProcessingJob,
          {
            id: randomUUID(),
            name: QueueJobs.BlobStorageIntegrationProcessingJob,
            timestamp: new Date(),
            payload: { projectId },
          },
          { jobId, removeOnFail: true },
        );
        logger.info(
          `[BLOB INTEGRATION] Queued next catch-up chunk for project ${projectId} with jobId ${jobId}`,
        );
      }
    }

    logger.info(
      `[BLOB INTEGRATION] Successfully processed blob storage integration for project ${projectId}`,
    );
  } catch (error) {
    // A part-count-limit exhaustion is terminal, not transient: the window is
    // too large for the configured part size and no retry can shrink it safely.
    // Fail loud and stop before BullMQ burns its remaining attempts re-querying
    // ClickHouse against a retention-shrinking window. Runs BEFORE the generic
    // customer-fault/transient handling below.
    if (isMultipartPartLimitError(error)) {
      // Missing-row-safe: a delete mid-run makes the job obsolete. lastSyncAt /
      // nextSyncAt are left untouched so the window is not advanced past a
      // failed export. enabled is left untouched: a too-large window is not a
      // customer-config fault, so this never auto-disables. update (not
      // updateMany) so the notification can read the post-write enabled state.
      let integration;
      try {
        integration = await prisma.blobStorageIntegration.update({
          where: { projectId },
          data: {
            lastError: BLOB_EXPORT_PART_LIMIT_ERROR_MESSAGE,
            lastErrorAt: new Date(),
            runStartedAt: null,
          },
        });
      } catch (persistError) {
        if (isRecordNotFoundError(persistError)) {
          logger.info(
            `[BLOB INTEGRATION] Blob storage integration for project ${projectId} was deleted before the part-limit failure could be recorded; dropping obsolete job`,
          );
          return;
        }
        throw persistError;
      }

      // Cooldown-gated, not bypassed: the integration stays enabled and the
      // watermark does not advance, so every scheduled run re-attempts the same
      // too-large window and re-enters here. The cooldown caps this to one alert
      // per cooldown window instead of one per run. (The disable notification
      // bypasses the cooldown because it is a one-shot terminal event.)
      //
      // Gated on the post-write enabled state, mirroring the generic terminal
      // path's stillEnabled check: a user who disabled the integration mid-run
      // must not get an "export failed" alert for a run they already turned off.
      if (integration.enabled) {
        await notifyBlobStorageExportFailed(projectId, { disabled: false });
      }

      logger.error(
        `[BLOB INTEGRATION] Blob storage export for project ${projectId} exceeded the multipart part-count limit; failing terminally without retry: ${errorChainText(error)}`,
      );
      // UnrecoverableError so BullMQ fails the job now instead of retrying.
      throw new UnrecoverableError(BLOB_EXPORT_PART_LIMIT_ERROR_MESSAGE);
    }

    const errorMessage = extractStorageErrorMessage(error);

    const isFinalAttempt = isFinalBullmqAttempt(job, error);
    // Defined => disable, on the first occurrence: a config fault cannot succeed
    // on a retry, and the failures metric counts every failed attempt.
    const customerFaultReason = classifyCustomerFault(error);

    const outcome = await recordTerminalExportError({
      projectId,
      errorMessage,
      disableReason: customerFaultReason,
    });

    if (outcome.kind === "integration-deleted") {
      return; // obsolete job: complete it rather than fail it
    }

    switch (outcome.kind) {
      case "disabled-by-us":
        // Awaited: the integration is now off, so the scheduler never revisits
        // it and nothing would carry an interrupted dispatch.
        await notifyBlobStorageExportFailed(projectId, { disabled: true });
        return; // resolving is the point; a throw would light the monitor
      case "lost-disable-race":
        return; // the winner sent the terminal email
      case "error-recorded":
        // Skipped once the integration is off: "will retry at the next
        // scheduled export" is no longer true. Not awaited: this path rethrows,
        // so the job is about to fail and be retried regardless.
        if (isFinalAttempt && outcome.stillEnabled) {
          notifyBlobStorageExportFailed(projectId, { disabled: false });
        }
        break;
      case "persist-failed":
        // Nothing was written, so we cannot claim the fault is handled: retry
        // and reclassify. No email — "will retry" is all we could honestly say.
        break;
      // A plain switch over a union is not exhaustiveness-checked; the never
      // assignment below is what makes a missing case fail to compile.
      default: {
        const _exhaustiveCheck: never = outcome;
        throw new Error(
          `Unhandled terminal export outcome: ${JSON.stringify(_exhaustiveCheck)}`,
        );
      }
    }

    const chain = errorChainText(error);
    logger.error(
      `[BLOB INTEGRATION] Error processing blob storage integration for project ${projectId}: ${chain}`,
      error instanceof Error ? { stack: error.stack } : {},
    );
    const rethrown = new Error(chain, { cause: error });
    // Copy the original stack so BullMQ and the queue processor see the real
    // failure site rather than this rethrow line. rethrown.stack starts with
    // the original error's message, which won't match rethrown.message (the
    // full chain), but structured loggers record them as separate fields.
    if (error instanceof Error) rethrown.stack = error.stack;
    throw rethrown;
  }
};

// What the terminal bookkeeping for a failed run did. Each case is a situation
// some earlier revision guarded against, named so the notification decision can
// switch over them instead of recombining flags.
type TerminalExportErrorOutcome =
  | { kind: "integration-deleted" } // deleted mid-run: job is obsolete
  | { kind: "persist-failed" } // write failed, so the row's state is unknown
  // No disable attempted: not a customer fault, or retries remain. stillEnabled
  // false means a concurrent disable or user toggle landed mid-run.
  | { kind: "error-recorded"; stillEnabled: boolean }
  | { kind: "disabled-by-us" }
  | { kind: "lost-disable-race" };

// Persists the failure and, when asked, makes the atomic enabled true→false
// claim. That claim is what keeps the terminal email exactly-once across runs
// racing the same fault (distinct jobIds, so not queue-deduped) and stops it
// claiming a state we failed to write.
async function recordTerminalExportError({
  projectId,
  errorMessage,
  disableReason,
}: {
  projectId: string;
  errorMessage: string;
  disableReason: CustomerFaultReason | undefined;
}): Promise<TerminalExportErrorOutcome> {
  try {
    const updated = await prisma.blobStorageIntegration.update({
      where: { projectId },
      data: {
        lastError: errorMessage,
        lastErrorAt: new Date(),
        runStartedAt: null,
      },
    });

    if (disableReason === undefined) {
      return { kind: "error-recorded", stillEnabled: updated.enabled };
    }

    const { count } = await prisma.blobStorageIntegration.updateMany({
      where: { projectId, enabled: true },
      data: { enabled: false },
    });
    if (count !== 1) return { kind: "lost-disable-race" };

    // Tag by reason so SSRF/abuse disables (ssrf_blocked_endpoint) can be
    // separated from customer misconfig, and a mass-disable regression is
    // visible as a spike in the non-SSRF buckets after rollout.
    recordIncrement(BLOB_INTEGRATION_DISABLED_METRIC, 1, {
      reason: disableReason,
    });
    logger.warn(
      `[BLOB INTEGRATION] Disabled blob storage integration for project ${projectId} after a customer fault (reason=${disableReason}): ${errorMessage}`,
      { blobStorageDisableReason: disableReason, projectId },
    );
    return { kind: "disabled-by-us" };
  } catch (persistError) {
    if (isRecordNotFoundError(persistError)) {
      logger.info(
        `[BLOB INTEGRATION] Blob storage integration for project ${projectId} was deleted mid-run; dropping obsolete job after error: ${errorMessage}`,
      );
      return { kind: "integration-deleted" };
    }
    logger.error(
      `[BLOB INTEGRATION] Failed to persist blob storage error for project ${projectId}`,
      persistError,
    );
    return { kind: "persist-failed" };
  }
}

// Logs and swallows every failure: notification trouble must not turn a
// deliberate resolve back into a job failure. Delivery is therefore best-effort
// even on the awaited path; the persisted lastError is the durable signal.
async function notifyBlobStorageExportFailed(
  projectId: string,
  {
    disabled = false,
    bypassCooldown = disabled,
  }: { disabled?: boolean; bypassCooldown?: boolean } = {},
): Promise<void> {
  try {
    // Called once per exhausted run. The cooldown gates across scheduled
    // runs (the scheduler re-enqueues every frequency period, and each
    // failing run would otherwise email again). The disable notice bypasses
    // it: that is a one-shot terminal event and a cooldown claim could
    // silently drop the single email saying the integration was turned off.
    // Recurring failures stay gated so they alert once per window, not per run.
    if (!bypassCooldown) {
      const cooldownMs =
        env.LANGFUSE_BLOB_STORAGE_FAILURE_NOTIFICATION_COOLDOWN_HOURS *
        60 *
        60 *
        1000;

      // Atomic claim: set timestamp before sending to prevent duplicate emails on concurrent retries.
      // If the email send subsequently fails, the cooldown still applies — the next failure
      // after cooldown expiry will retry the notification.
      const claimed = await prisma.blobStorageIntegration.updateMany({
        where: {
          projectId,
          OR: [
            { lastFailureNotificationSentAt: null },
            {
              lastFailureNotificationSentAt: {
                lt: new Date(Date.now() - cooldownMs),
              },
            },
          ],
        },
        data: { lastFailureNotificationSentAt: new Date() },
      });

      if (claimed.count === 0) {
        logger.info(
          `[BLOB INTEGRATION] Skipping failure notification for project ${projectId}, cooldown still active`,
        );
        return;
      }
    }

    const [project, integration] = await Promise.all([
      prisma.project.findUnique({
        where: { id: projectId },
        select: { name: true },
      }),
      prisma.blobStorageIntegration.findUnique({
        where: { projectId },
        select: { bucketName: true },
      }),
    ]);
    const projectName = project?.name ?? projectId;
    const settingsPath = `/project/${projectId}/settings/integrations/blobstorage`;

    // Route to configured notification channels and admin emails. The
    // cooldown claim above already deduped, so no extra throttle is needed.
    // `disabled` marks the terminal "export turned off" notification, which
    // selects the disabled email/subject variant downstream.
    await dispatchProjectNotification({
      projectId,
      event: {
        eventType: "blob-export-failed",
        severity: "ALERT",
        projectId,
        projectName,
        // The integration is keyed by projectId (1:1); the bucket name is
        // the most useful human label for the failing export destination.
        resourceId: projectId,
        resourceName: integration?.bucketName ?? "Blob storage integration",
        message: disabled
          ? `Blob storage export disabled for project "${projectName}" after a configuration fault.`
          : `Blob storage export failed for project "${projectName}".`,
        url: env.NEXTAUTH_URL
          ? `${env.NEXTAUTH_URL}${settingsPath}`
          : undefined,
        disabled,
      },
    });
  } catch (error) {
    logger.error(
      `[BLOB INTEGRATION] Failed to send failure notification for project ${projectId}`,
      error,
    );
  }
}

function extractStorageErrorMessage(error: unknown): string {
  if (!(error instanceof Error)) return String(error).slice(0, 1000);

  // handleStorageError wraps SDK errors via { cause: sdkError }
  // Unwrap to get the raw SDK message (S3/Azure/GCS)
  const cause = error.cause;
  const message = cause instanceof Error ? cause.message : error.message;

  // GCS returns a <Details> XML element with the real rejection reason
  // (e.g. "Multipart upload is not supported in Rapid storage class.").
  // handleStorageError preserves it as an enumerable property.
  const errorDetails = (error as unknown as { Details?: unknown }).Details;
  const causeDetails = (cause as unknown as { Details?: unknown } | undefined)
    ?.Details;
  const details = (() => {
    if (typeof errorDetails === "string") {
      return errorDetails;
    }
    if (typeof causeDetails === "string") {
      return causeDetails;
    }
    return undefined;
  })();

  const full = details ? `${message} Details: ${details}` : message;
  return full.slice(0, 1000);
}