Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

117 KB
Raw Blame
import {
  expect,
  it,
  describe,
  beforeAll,
  beforeEach,
  afterEach,
  afterAll,
  vi,
} from "vitest";

const originalCloudRegion = vi.hoisted(() => {
  const cloudRegion = process.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
  delete process.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
  return cloudRegion;
});

// Override recordIncrement so the attempt counter is assertable.
const mockRecordIncrement = vi.hoisted(() => vi.fn());
// Stubbable endpoint preflight — the customer-fault tests reject it to drive an
// in-try failure without infra. Defaults to the real impl for other tests.
const mockValidateBlobStorageEndpoint = vi.hoisted(() => vi.fn());
// Spy on the failure notification dispatch so the part-limit test can assert it
// fired (a cooldown-bypassed notification leaves no lastFailureNotificationSentAt
// stamp to observe). Defaults to a no-op so real notification infra isn't needed.
const mockDispatchProjectNotification = vi.hoisted(() => vi.fn());
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
  const actual =
    await importOriginal<typeof import("@langfuse/shared/src/server")>();
  if (mockValidateBlobStorageEndpoint.getMockImplementation() === undefined) {
    mockValidateBlobStorageEndpoint.mockImplementation(
      actual.validateBlobStorageEndpoint,
    );
  }
  if (mockDispatchProjectNotification.getMockImplementation() === undefined) {
    mockDispatchProjectNotification.mockResolvedValue(undefined);
  }
  return {
    ...actual,
    recordIncrement: mockRecordIncrement,
    validateBlobStorageEndpoint: mockValidateBlobStorageEndpoint,
    dispatchProjectNotification: mockDispatchProjectNotification,
  };
});

import { env } from "../env";
import { randomUUID } from "crypto";
import {
  createObservation,
  createObservationsCh,
  createOrgProjectAndApiKey,
  createTraceScore,
  createSessionScore,
  createDatasetRunScore,
  createScoresCh,
  createTrace,
  createTracesCh,
  createEvent,
  createEventsCh,
  StorageService,
  StorageServiceFactory,
  BlobStorageIntegrationProcessingQueue,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import { Job, UnrecoverableError } from "bullmq";
import {
  handleBlobStorageIntegrationProjectJob,
  BLOB_STORAGE_LAG_BUFFER_MS,
  BLOB_STORAGE_REMAINDER_COALESCE_MS,
  BLOB_EXPORT_PART_LIMIT_ERROR_MESSAGE,
} from "../features/blobstorage/handleBlobStorageIntegrationProjectJob";
import { BLOB_INTEGRATION_DISABLED_METRIC } from "../features/blobstorage/isCustomerFaultError";
import {
  BlobStorageIntegrationType,
  BlobStorageIntegrationFileType,
  BLOB_STORAGE_REGION_INVALID_MESSAGE,
  LEGACY_BLOB_EXPORTER_CUTOFF,
} from "@langfuse/shared";
import { encrypt } from "@langfuse/shared/encryption";

// Skip tests that use Azurite in Azure mode due to known Azurite limitations
// with multipart uploads. These tests use MinIO explicitly or are skipped.
// Unfortunately, this is necessary as we don't have a good way to skip empty file uploads
// and at least azurite doesn't handle them gracefully.
const maybeIt = env.LANGFUSE_USE_AZURE_BLOB === "true" ? it.skip : it;
const maybeDescribe =
  process.env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true"
    ? describe
    : describe.skip;

describe("BlobStorageIntegrationProcessingJob", () => {
  let storageService: StorageService;
  let s3StorageService: StorageService;
  let s3Prefix: string | null = null;
  const bucketName = env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET || "";
  const accessKeyId = env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID || "";
  const secretAccessKey = env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY || "";
  const endpoint = env.LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT || undefined;
  const region = env.LANGFUSE_S3_EVENT_UPLOAD_REGION || undefined;
  const minioAccessKeyId = "minio";
  const minioAccessKeySecret = "miniosecret";
  const minioEndpoint = "http://localhost:9090";

  beforeAll(async () => {
    storageService = StorageServiceFactory.getInstance({
      accessKeyId,
      secretAccessKey,
      bucketName,
      endpoint,
      region,
      forcePathStyle: env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
    });
    s3StorageService = StorageServiceFactory.getInstance({
      accessKeyId: minioAccessKeyId,
      secretAccessKey: minioAccessKeySecret,
      bucketName,
      endpoint: minioEndpoint,
      region,
      forcePathStyle: true,
      useAzureBlob: false,
    });
  });

  afterAll(() => {
    if (originalCloudRegion) {
      process.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;
    } else {
      delete process.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
    }
  });

  afterEach(async () => {
    // Clean up all files created during this test
    if (!s3Prefix) return;

    const files = await s3StorageService.listFiles(s3Prefix);

    if (files.length == 0) return;

    await s3StorageService.deleteFiles(files.map((f) => f.file));
    s3Prefix = null;
  });

  // A persisted enriched export source on a deployment that does not write the
  // v4 events table must fail the job loudly instead of silently exporting
  // from unpopulated tables.
  describe("enriched export source guard", () => {
    const originalWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;

    afterEach(() => {
      (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = originalWriteMode;
    });

    it("fails the job and persists lastError when an enriched source runs on legacy", async () => {
      (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = "legacy";
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;

      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId,
          secretAccessKey: encrypt(secretAccessKey),
          region: region ? region : "auto",
          endpoint: endpoint ? endpoint : null,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "daily",
          exportSource: "EVENTS",
          // A past lastSyncAt yields a non-empty export window without
          // requiring ClickHouse data.
          lastSyncAt: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000),
        },
      });

      await expect(
        handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
        } as Job),
      ).rejects.toThrow(/enriched/i);

      const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
        where: { projectId },
      });
      expect(row.lastError).toMatch(/enriched/i);
      expect(row.lastErrorAt).not.toBeNull();

      // Nothing was exported.
      const files = await storageService.listFiles(s3Prefix);
      expect(files.filter((f) => f.file.includes(projectId))).toHaveLength(0);
    });
  });

  // LFE-10148: a persisted legacy export source on an events_only deployment
  // reads the v3 traces/observations tables, which are no longer written — the
  // job must fail loudly instead of exporting stale/empty data.
  describe("legacy export source guard on events_only", () => {
    const originalWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;

    afterEach(() => {
      (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = originalWriteMode;
    });

    it("fails the job and persists lastError when a legacy source runs on events_only", async () => {
      (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;

      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId,
          secretAccessKey: encrypt(secretAccessKey),
          region: region ? region : "auto",
          endpoint: endpoint ? endpoint : null,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "daily",
          exportSource: "TRACES_OBSERVATIONS",
          lastSyncAt: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000),
        },
      });

      await expect(
        handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
        } as Job),
      ).rejects.toThrow(/events_only/i);

      const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
        where: { projectId },
      });
      expect(row.lastError).toMatch(/events_only/i);
      expect(row.lastErrorAt).not.toBeNull();

      // Nothing was exported.
      const files = await storageService.listFiles(s3Prefix);
      expect(files.filter((f) => f.file.includes(projectId))).toHaveLength(0);
    });
  });

  describe("invalid persisted region", () => {
    const originalWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;

    afterEach(() => {
      (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = originalWriteMode;
    });

    it("persists the error before any S3 upload starts", async () => {
      (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";
      const { projectId } = await createOrgProjectAndApiKey();

      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: projectId,
          accessKeyId,
          secretAccessKey: encrypt(secretAccessKey),
          region: "us west-2",
          endpoint: endpoint ?? null,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "daily",
          exportSource: "EVENTS",
          lastSyncAt: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000),
        },
      });

      await expect(
        handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
        } as Job),
      ).rejects.toThrow(BLOB_STORAGE_REGION_INVALID_MESSAGE);

      const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
        where: { projectId },
      });
      expect(row.lastError).toBe(BLOB_STORAGE_REGION_INVALID_MESSAGE);
      expect(row.lastErrorAt).not.toBeNull();
      expect(row.enabled).toBe(true);
    });
  });

  // A classified customer fault disables the integration on its first
  // occurrence and resolves the job; everything else keeps retrying as before.
  describe("customer-fault disable", () => {
    const originalWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;

    afterEach(() => {
      (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = originalWriteMode;
    });

    // Minimal AWS SDK v3 S3 AccessDenied shape (high-confidence customer fault).
    const accessDeniedError = (): Error => {
      const err = new Error("Access Denied");
      err.name = "AccessDenied";
      Object.assign(err, {
        Code: "AccessDenied",
        $metadata: { httpStatusCode: 403 },
      });
      return err;
    };

    // endpoint must be non-null so the handler runs the preflight we stub.
    const createIntegration = async (projectId: string) => {
      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: projectId,
          accessKeyId,
          secretAccessKey: encrypt(secretAccessKey),
          region: region ? region : "auto",
          endpoint: "https://customer-bucket.s3.example.com",
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "daily",
          exportSource: "TRACES_OBSERVATIONS",
          lastSyncAt: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000),
        },
      });
    };

    const runAttempt = (projectId: string, attemptsMade: number) =>
      handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
        attemptsMade,
        opts: { attempts: 5 },
      } as Job);

    // The notification runs fire-and-forget after the handler rejects; give
    // it time to land (or prove it never fires) before negative assertions.
    const settleBackgroundTasks = () =>
      new Promise((resolve) => setTimeout(resolve, 200));

    // Resolving is the point: the queue's failure counter is incremented per
    // failed attempt, so a throw here would light the processing-failures
    // monitor for a fault that no retry can clear.
    it("disables the integration and resolves on the first customer-fault attempt", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      await createIntegration(projectId);
      mockRecordIncrement.mockClear();
      mockValidateBlobStorageEndpoint.mockRejectedValueOnce(
        accessDeniedError(),
      );

      await expect(runAttempt(projectId, 0)).resolves.toBeUndefined();

      const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
        where: { projectId },
      });
      expect(row.enabled).toBe(false);
      expect(row.lastError).toMatch(/access denied/i);
      expect(row.lastErrorAt).not.toBeNull();
      // The "disabled" email bypasses the cooldown, so it must not claim it.
      expect(row.lastFailureNotificationSentAt).toBeNull();
      // Tagged by reason so an SSRF/abuse disable stays separable from a
      // misconfiguration one.
      expect(mockRecordIncrement).toHaveBeenCalledWith(
        BLOB_INTEGRATION_DISABLED_METRIC,
        1,
        { reason: "credentials" },
      );
    });

    it("still disables and resolves when the fault first appears on the final attempt", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      await createIntegration(projectId);
      mockValidateBlobStorageEndpoint.mockRejectedValueOnce(
        accessDeniedError(),
      );

      await expect(runAttempt(projectId, 4)).resolves.toBeUndefined();

      const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
        where: { projectId },
      });
      expect(row.enabled).toBe(false);
      expect(row.lastFailureNotificationSentAt).toBeNull();
    });

    it("does not disable on the final attempt of a non-customer-fault ('other') error", async () => {
      // Enriched source on the legacy write mode => the guard throws a plain
      // Error, which the classifier treats as "other".
      (env as any).LANGFUSE_MIGRATION_V4_WRITE_MODE = "legacy";
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;

      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId,
          secretAccessKey: encrypt(secretAccessKey),
          region: region ? region : "auto",
          endpoint: endpoint ? endpoint : null,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "daily",
          exportSource: "EVENTS",
          lastSyncAt: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000),
        },
      });

      await expect(runAttempt(projectId, 4)).rejects.toThrow(/enriched/i);

      const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
        where: { projectId },
      });
      // "other" errors stay enabled and keep retrying on the next scheduled
      // run — but the run is exhausted, so the cooldown-gated informational
      // notification fires (observable via its atomic cooldown claim).
      expect(row.enabled).toBe(true);
      await vi.waitFor(async () => {
        const notified = await prisma.blobStorageIntegration.findUniqueOrThrow({
          where: { projectId },
        });
        expect(notified.lastFailureNotificationSentAt).not.toBeNull();
      });
    });

    it("suppresses the informational email when the integration was disabled mid-run", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      await createIntegration(projectId);
      // Simulate a concurrent worker's customer-fault disable (or a user
      // toggle) landing mid-run, followed by a non-customer-fault failure:
      // "will retry at the next scheduled export" would be false.
      mockValidateBlobStorageEndpoint.mockImplementationOnce(async () => {
        await prisma.blobStorageIntegration.updateMany({
          where: { projectId },
          data: { enabled: false },
        });
        throw new Error("connection refused");
      });

      await expect(runAttempt(projectId, 4)).rejects.toThrow(
        /connection refused/i,
      );
      await settleBackgroundTasks();

      const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
        where: { projectId },
      });
      expect(row.enabled).toBe(false);
      expect(row.lastError).toMatch(/connection refused/i);
      expect(row.lastFailureNotificationSentAt).toBeNull();
    });

    // A concurrent run (distinct jobId, so not queue-deduped) can win the
    // disable claim first. The loser must stay silent: the winner already sent
    // the terminal "disabled" email, and the loser's only fallback is the
    // informational "will retry at the next scheduled export" variant, which is
    // false for an integration that is now off.
    it("does not notify when a concurrent run won the customer-fault disable claim", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      await createIntegration(projectId);
      mockRecordIncrement.mockClear();
      // Land the winner's disable between the handler's enabled check and the
      // terminal catch, then raise the same fault the winner disabled on.
      mockValidateBlobStorageEndpoint.mockImplementationOnce(async () => {
        await prisma.blobStorageIntegration.updateMany({
          where: { projectId },
          data: { enabled: false },
        });
        throw accessDeniedError();
      });

      await expect(runAttempt(projectId, 4)).resolves.toBeUndefined();
      await settleBackgroundTasks();

      const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
        where: { projectId },
      });
      expect(row.enabled).toBe(false);
      expect(row.lastError).toMatch(/access denied/i);
      // Null proves the loser suppressed its email: the informational fallback
      // would have taken the cooldown claim and stamped this.
      expect(row.lastFailureNotificationSentAt).toBeNull();
      // Only the worker that actually flipped enabled true->false counts it.
      expect(mockRecordIncrement).not.toHaveBeenCalledWith(
        BLOB_INTEGRATION_DISABLED_METRIC,
        expect.anything(),
        expect.anything(),
      );
    });

    // When the failure can't even be written down (Postgres unavailable), the
    // run knows nothing about the row's state, so it must not claim the fault is
    // handled: it stays on the retry path and sends no email. `update` is only
    // called on this terminal path, so spying on it isolates the persistence
    // failure from the export itself.
    it("keeps retrying without notifying when persisting the failure fails", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      await createIntegration(projectId);
      mockRecordIncrement.mockClear();
      mockValidateBlobStorageEndpoint.mockRejectedValueOnce(
        accessDeniedError(),
      );
      const updateSpy = vi
        .spyOn(prisma.blobStorageIntegration, "update")
        .mockRejectedValueOnce(new Error("database unavailable"));

      try {
        await expect(runAttempt(projectId, 4)).rejects.toThrow(
          /access denied/i,
        );
        await settleBackgroundTasks();
      } finally {
        updateSpy.mockRestore();
      }

      const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
        where: { projectId },
      });
      // Nothing was written, so the fault is invisible to the row and the
      // integration keeps running — even though this was a classified fault
      // that would otherwise have disabled it.
      expect(row.enabled).toBe(true);
      expect(row.lastError).toBeNull();
      expect(mockRecordIncrement).not.toHaveBeenCalledWith(
        BLOB_INTEGRATION_DISABLED_METRIC,
        expect.anything(),
        expect.anything(),
      );
      // No email either: its "will retry at the next scheduled export" text is
      // the only thing we can still vouch for, and the rethrow delivers that.
      expect(row.lastFailureNotificationSentAt).toBeNull();
    });
  });

  // A multipart part-count-limit exhaustion is terminal: retrying re-queries a
  // retention-shrinking window until a truncated object commits as success. The
  // run must fail loud (UnrecoverableError, no retry, notification) and stop —
  // without auto-disabling or advancing the export watermark.
  describe("multipart part-limit failure", () => {
    const lastSyncAt = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000);

    const createIntegration = async (projectId: string) => {
      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: projectId,
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region ? region : "auto",
          // endpoint null -> skip the persisted-endpoint preflight; the storage
          // service is mocked anyway.
          endpoint: null,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "daily",
          // Non-enriched source: runs outside the V4-preview guard on every leg.
          exportSource: "TRACES_OBSERVATIONS",
          lastSyncAt,
        },
      });
    };

    // The exact @aws-sdk/lib-storage message, wrapped the way
    // StorageService.handleStorageError wraps the SDK cause.
    const partLimitError = () =>
      new Error("Failed to upload file to S3", {
        cause: new Error(
          "Exceeded 10000 parts in multipart upload to Bucket: b Key: k.",
        ),
      });

    const settleBackgroundTasks = () =>
      new Promise((resolve) => setTimeout(resolve, 200));

    it("fails terminally without retry, persists lastError, notifies, keeps enabled, and does not advance the watermark", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      // No s3Prefix: the storage service is mocked, so nothing is uploaded and
      // the MinIO cleanup in afterEach must not run.
      await createIntegration(projectId);
      mockDispatchProjectNotification.mockClear();

      const getInstanceSpy = vi
        .spyOn(StorageServiceFactory, "getInstance")
        .mockReturnValue({
          uploadFileBuffered: vi.fn().mockRejectedValue(partLimitError()),
        } as unknown as StorageService);

      try {
        // Attempt 0 of 5: a transient error would retry, so rejecting with
        // UnrecoverableError here is the proof that no retry will happen.
        await expect(
          handleBlobStorageIntegrationProjectJob({
            data: { payload: { projectId } },
            attemptsMade: 0,
            opts: { attempts: 5 },
          } as Job),
        ).rejects.toBeInstanceOf(UnrecoverableError);
        await settleBackgroundTasks();
      } finally {
        getInstanceSpy.mockRestore();
      }

      const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
        where: { projectId },
      });
      expect(row.lastError).toBe(BLOB_EXPORT_PART_LIMIT_ERROR_MESSAGE);
      expect(row.lastErrorAt).not.toBeNull();
      // Not a customer-config fault: the integration stays on.
      expect(row.enabled).toBe(true);
      // Watermark untouched so the failed window is not skipped on the next run.
      expect(row.lastSyncAt?.getTime()).toBe(lastSyncAt.getTime());

      // Notification fired, as a failure (not the disabled variant).
      expect(mockDispatchProjectNotification).toHaveBeenCalledWith(
        expect.objectContaining({
          projectId,
          event: expect.objectContaining({
            eventType: "blob-export-failed",
            disabled: false,
          }),
        }),
      );
    });

    // A user toggle (or concurrent disable) landing mid-run leaves the row
    // disabled by the time the terminal catch records the failure. Still fail
    // loud (UnrecoverableError, lastError recorded) but suppress the
    // informational alert, mirroring the generic terminal path's stillEnabled
    // gate: "will retry at the next scheduled export" is false once it is off.
    it("suppresses the failure notification when the integration was disabled mid-run", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      await createIntegration(projectId);
      mockDispatchProjectNotification.mockClear();

      const getInstanceSpy = vi
        .spyOn(StorageServiceFactory, "getInstance")
        .mockReturnValue({
          uploadFileBuffered: vi.fn().mockImplementation(async () => {
            await prisma.blobStorageIntegration.updateMany({
              where: { projectId },
              data: { enabled: false },
            });
            throw partLimitError();
          }),
        } as unknown as StorageService);

      try {
        await expect(
          handleBlobStorageIntegrationProjectJob({
            data: { payload: { projectId } },
            attemptsMade: 0,
            opts: { attempts: 5 },
          } as Job),
        ).rejects.toBeInstanceOf(UnrecoverableError);
        await settleBackgroundTasks();
      } finally {
        getInstanceSpy.mockRestore();
      }

      const row = await prisma.blobStorageIntegration.findUniqueOrThrow({
        where: { projectId },
      });
      // Still terminal: the failure is recorded and the run does not retry.
      expect(row.lastError).toBe(BLOB_EXPORT_PART_LIMIT_ERROR_MESSAGE);
      expect(row.enabled).toBe(false);
      // No alert: the integration is off, so the informational variant would be
      // misleading.
      expect(mockDispatchProjectNotification).not.toHaveBeenCalled();
    });
  });

  // LFE-14894: an integration deleted mid-run makes the job obsolete — it must
  // complete quietly instead of failing on the now-missing row (P2025) and
  // paging on a config that no longer exists.
  describe("integration deleted mid-run", () => {
    // endpoint must be non-null so the handler runs the preflight we stub —
    // that stub is the seam to delete the row after the initial load.
    const createIntegration = async (
      projectId: string,
      overrides: { exportFrequency?: string; lastSyncAt?: Date } = {},
    ) => {
      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: projectId,
          accessKeyId,
          secretAccessKey: encrypt(secretAccessKey),
          region: region ? region : "auto",
          endpoint: "https://customer-bucket.s3.example.com",
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: overrides.exportFrequency ?? "daily",
          exportSource: "TRACES_OBSERVATIONS",
          lastSyncAt:
            overrides.lastSyncAt ??
            new Date(Date.now() - 2 * 24 * 60 * 60 * 1000),
        },
      });
    };

    it("drops the obsolete job instead of failing when the row is gone before the error is persisted", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      await createIntegration(projectId);
      mockRecordIncrement.mockClear();
      // Customer fault on the final attempt — normally disable + notify — but
      // the row vanishes before the catch block can persist anything.
      mockValidateBlobStorageEndpoint.mockImplementationOnce(async () => {
        await prisma.blobStorageIntegration.delete({ where: { projectId } });
        const err = new Error("Access Denied");
        err.name = "AccessDenied";
        Object.assign(err, {
          Code: "AccessDenied",
          $metadata: { httpStatusCode: 403 },
        });
        throw err;
      });

      await expect(
        handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
          attemptsMade: 4,
          opts: { attempts: 5 },
        } as Job),
      ).resolves.toBeUndefined();

      // Give fire-and-forget notification paths time to (not) run.
      await new Promise((resolve) => setTimeout(resolve, 200));
      expect(mockRecordIncrement).not.toHaveBeenCalledWith(
        BLOB_INTEGRATION_DISABLED_METRIC,
        expect.anything(),
        expect.anything(),
      );
    });

    it("drops the obsolete job instead of failing when the row is gone after a successful export", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      // Recent lastSyncAt keeps the window non-empty while staying caught up.
      await createIntegration(projectId, {
        exportFrequency: "hourly",
        lastSyncAt: new Date(
          Date.now() - BLOB_STORAGE_LAG_BUFFER_MS - 30 * 60 * 1000,
        ),
      });
      mockValidateBlobStorageEndpoint.mockImplementationOnce(async () => {
        await prisma.blobStorageIntegration.delete({ where: { projectId } });
      });
      const getInstanceSpy = vi
        .spyOn(StorageServiceFactory, "getInstance")
        .mockReturnValue({
          uploadFile: vi.fn().mockResolvedValue(undefined),
          uploadFileBuffered: vi.fn().mockResolvedValue(undefined),
          deleteFiles: vi.fn().mockResolvedValue(undefined),
          listFiles: vi.fn().mockResolvedValue([]),
        } as unknown as StorageService);

      try {
        await expect(
          handleBlobStorageIntegrationProjectJob({
            data: { payload: { projectId } },
          } as Job),
        ).resolves.toBeUndefined();
      } finally {
        getInstanceSpy.mockRestore();
      }
    });
  });

  it("should not process when blob storage integration is disabled", async () => {
    const { projectId } = await createOrgProjectAndApiKey();
    s3Prefix = projectId;

    // Setup an integration but disabled
    await prisma.blobStorageIntegration.create({
      data: {
        projectId,
        type: BlobStorageIntegrationType.S3,
        bucketName,
        prefix: s3Prefix,
        accessKeyId,
        secretAccessKey: encrypt(secretAccessKey),
        region: region ? region : "auto",
        endpoint: endpoint ? endpoint : null,
        forcePathStyle:
          env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
        enabled: false,
        exportFrequency: "hourly",
        compressed: false,
      },
    });

    // When
    await handleBlobStorageIntegrationProjectJob({
      data: { payload: { projectId } },
    } as Job);

    // Then
    const files = await storageService.listFiles(s3Prefix);
    expect(files.filter((f) => f.file.includes(projectId))).toHaveLength(0);
  });

  maybeDescribe("events table export tests", () => {
    it("should export traces, generations, and scores to S3", async () => {
      // Setup
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      const now = new Date();
      const sessionScoreId = randomUUID();
      const sessionId = randomUUID();
      const datasetRunScoreId = randomUUID();
      const datasetRunId = randomUUID();
      // Set lastSyncAt to 2 hours ago so the chunked export (1 hour window) covers recent data
      const twoHoursAgo = new Date(now.getTime() - 2 * 60 * 60 * 1000);

      // Create integration
      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region ? region : "auto",
          endpoint: minioEndpoint,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "hourly",
          exportSource: "TRACES_OBSERVATIONS_EVENTS",
          // Explicit CSV: this test asserts on text content, and the column
          // default is now PARQUET (binary).
          fileType: BlobStorageIntegrationFileType.CSV,
          nextSyncAt: twoHoursAgo,
          lastSyncAt: twoHoursAgo,
          compressed: false,
        },
      });

      // Create test data within the export window (2 hours ago to 1 hour ago)
      // With 20-min lag buffer, actual window is 2h ago to (1h ago or now-20min, whichever is earlier)
      const traceId = randomUUID();
      const observationId = randomUUID();
      const scoreId = randomUUID();
      const modelId = randomUUID();

      const dataTime = now.getTime() - 90 * 60 * 1000; // 90 minutes before now

      // Create a Model + PricingTier + Prices in Postgres for model enrichment
      await prisma.model.create({
        data: {
          id: modelId,
          projectId,
          modelName: "gpt-4-test",
          matchPattern: "gpt-4-test",
          unit: "TOKENS",
          pricingTiers: {
            create: {
              name: "Standard",
              isDefault: true,
              conditions: [],
              priority: 0,
              prices: {
                createMany: {
                  data: [
                    { modelId, usageType: "input", price: "0.03" },
                    { modelId, usageType: "output", price: "0.06" },
                    { modelId, usageType: "total", price: "0.09" },
                  ],
                },
              },
            },
          },
        },
      });

      // Create event data for events table export
      const event = createEvent({
        project_id: projectId,
        trace_id: traceId,
        type: "GENERATION",
        name: "Test Event",
        start_time: dataTime * 1000, // microseconds
        end_time: (dataTime + 5000) * 1000, // 5s later (microseconds)
        bookmarked: true,
        public: true,
        model_id: modelId,
      });

      // Create trace, observation, score, and event in Clickhouse
      // Data is at 90 minutes ago, which falls within the chunked export window
      await Promise.all([
        createTracesCh([
          createTrace({
            id: traceId,
            project_id: projectId,
            timestamp: dataTime,
            name: "Test Trace",
          }),
        ]),
        createObservationsCh([
          createObservation({
            id: observationId,
            trace_id: traceId,
            project_id: projectId,
            start_time: dataTime,
            end_time: dataTime + 5000, // 5s later
            completion_start_time: dataTime + 1000, // 1s later
            total_cost: 42.5,
            usage_details: { input: 100, output: 200, total: 300 },
            internal_model_id: modelId,
            name: "Test Observation",
          }),
        ]),
        createScoresCh([
          createTraceScore({
            id: scoreId,
            trace_id: traceId,
            project_id: projectId,
            timestamp: dataTime,
            name: "Test Score",
            value: 0.95,
          }),
          createSessionScore({
            id: sessionScoreId,
            session_id: sessionId,
            project_id: projectId,
            timestamp: dataTime,
            name: "Test Session Score",
            value: 0.8,
          }),
          createDatasetRunScore({
            id: datasetRunScoreId,
            dataset_run_id: datasetRunId,
            project_id: projectId,
            timestamp: dataTime,
            name: "Test Dataset Run Score",
            value: 0.7,
          }),
        ]),
        createEventsCh([event]),
      ]);

      // When
      mockRecordIncrement.mockClear();
      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);

      // Then
      // Each of the 4 tables emits a started + success attempt counter (LFE-10407).
      for (const table of [
        "traces",
        "observations",
        "scores",
        "observations_v2",
      ]) {
        for (const outcome of ["started", "success"]) {
          expect(mockRecordIncrement).toHaveBeenCalledWith(
            "langfuse.blobstorage.table_export.count",
            1,
            { outcome, table },
          );
        }
      }

      const files = await s3StorageService.listFiles(s3Prefix);
      const projectFiles = files.filter(
        (f) => f.file.includes(projectId) && !f.file.includes("/manifests/"),
      );

      // Should have 4 files (traces, observations, scores, events)
      expect(projectFiles).toHaveLength(4);

      const manifestFile = files.find((f) => f.file.includes("/manifests/"));
      expect(manifestFile).toBeDefined();
      const manifest = JSON.parse(
        await s3StorageService.download(manifestFile!.file),
      );
      expect(manifest.version).toBe(1);
      expect(manifest.projectId).toBe(projectId);
      expect(manifest.exportSource).toBe("TRACES_OBSERVATIONS_EVENTS");
      expect(new Set(manifest.tables)).toEqual(
        new Set(["traces", "observations", "scores", "observations_v2"]),
      );
      expect(manifest.files).toHaveLength(4);
      expect(
        new Set(manifest.files.map((f: { key: string }) => f.key)),
      ).toEqual(new Set(projectFiles.map((f) => f.file)));
      expect(manifest.window.maxTimestamp).toBe(manifest.maxTimestamp);

      // Check file paths follow the expected pattern
      const traceFile = projectFiles.find((f) => f.file.includes("/traces/"));
      const observationFile = projectFiles.find((f) =>
        f.file.includes("/observations/"),
      );
      const scoreFile = projectFiles.find((f) => f.file.includes("/scores/"));
      const eventFile = projectFiles.find((f) =>
        f.file.includes("/observations_v2/"),
      );

      expect(traceFile).toBeDefined();
      expect(observationFile).toBeDefined();
      expect(scoreFile).toBeDefined();
      expect(eventFile).toBeDefined();

      // Check file contents
      if (traceFile) {
        const content = await s3StorageService.download(traceFile.file);
        expect(content).toContain(traceId);
        expect(content).toContain("Test Trace");
        // Verify new fields: created_at, updated_at
        expect(content).toContain("created_at");
        expect(content).toContain("updated_at");
      }

      if (observationFile) {
        const content = await s3StorageService.download(observationFile.file);
        expect(content).toContain(observationId);
        expect(content).toContain("Test Observation");
        // Verify new fields: total_cost, latency, time_to_first_token
        expect(content).toContain("total_cost");
        expect(content).toContain("latency");
        expect(content).toContain("time_to_first_token");
        // Verify usage_details map contains actual token counts (CSV escapes " as "")
        expect(content).toContain('""input"":100');
        expect(content).toContain('""output"":200');
        // Verify model pricing enrichment
        expect(content).toContain("input_price");
        expect(content).toContain("output_price");
        expect(content).toContain("total_price");
        expect(content).toContain("0.03");
        expect(content).toContain("0.06");
        // Verify newly added native fields
        expect(content).toContain("prompt_id");
        expect(content).toContain("tool_calls");
        expect(content).toContain("tool_definitions");
        expect(content).toContain("usage_pricing_tier_name");
      }

      if (scoreFile) {
        const content = await s3StorageService.download(scoreFile.file);
        expect(content).toContain(scoreId);
        expect(content).toContain("Test Score");
        expect(content).toContain("0.95");
        // Score metadata is opt-in; the column-default field groups omit it
        expect(content.split("\n")[0]).not.toContain("metadata");
        // Verify session_id is exported for session-scoped scores
        expect(content).toContain(sessionScoreId);
        expect(content).toContain(sessionId);
        expect(content).toContain("Test Session Score");
        expect(content).toContain("0.8");
        // Verify dataset_run_id is exported for dataset-run-scoped scores
        expect(content).toContain(datasetRunScoreId);
        expect(content).toContain(datasetRunId);
        expect(content).toContain("Test Dataset Run Score");
        expect(content).toContain("0.7");
        // Verify new fields: created_at, updated_at
        expect(content).toContain("created_at");
        expect(content).toContain("updated_at");
      }

      if (eventFile) {
        const content = await s3StorageService.download(eventFile.file);
        expect(content).toContain(event.span_id);
        expect(content).toContain("Test Event");
        // Verify new fields: bookmarked, public, created_at, updated_at
        expect(content).toContain("bookmarked");
        expect(content).toContain("public");
        expect(content).toContain("created_at");
        expect(content).toContain("updated_at");
        // Verify usage_details map contains actual token counts (CSV escapes " as "")
        expect(content).toContain('""input"":1234');
        expect(content).toContain('""output"":5678');
        // Verify model pricing enrichment with actual values
        expect(content).toContain("input_price");
        expect(content).toContain("output_price");
        expect(content).toContain("total_price");
        expect(content).toContain("0.03");
        expect(content).toContain("0.06");
        // Verify newly added native fields
        expect(content).toContain("tool_calls");
        expect(content).toContain("tool_definitions");
        expect(content).toContain("usage_pricing_tier_name");
      }

      // Check integration lastSyncAt and nextSyncAt are updated
      const updatedIntegration = await prisma.blobStorageIntegration.findUnique(
        {
          where: { projectId },
        },
      );

      if (updatedIntegration?.lastSyncAt && updatedIntegration?.nextSyncAt) {
        expect(updatedIntegration.lastSyncAt.getTime()).toBeGreaterThan(
          twoHoursAgo.getTime(),
        );
        expect(updatedIntegration.nextSyncAt.getTime()).toBeGreaterThan(
          now.getTime(),
        );
      } else {
        expect.fail("Integration should have lastSyncAt and nextSyncAt set");
      }
    });

    it("should exclude columns for deselected exportFieldGroups", async () => {
      // Regression test for two known ClickHouse/enrichment leaks:
      // 1. ClickHouse always returns {} for unselected Map columns (metadata)
      // 2. enrichObservationStream was writing latency:null even when metrics not selected
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      const now = new Date();
      const twoHoursAgo = new Date(now.getTime() - 2 * 60 * 60 * 1000);
      const dataTime = now.getTime() - 90 * 60 * 1000;

      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region ? region : "auto",
          endpoint: minioEndpoint,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "hourly",
          exportSource: "EVENTS",
          exportFieldGroups: ["core", "io"],
          nextSyncAt: twoHoursAgo,
          lastSyncAt: twoHoursAgo,
          compressed: false,
          fileType: BlobStorageIntegrationFileType.JSONL,
        },
      });

      const traceId = randomUUID();
      const event = createEvent({
        project_id: projectId,
        trace_id: traceId,
        type: "GENERATION",
        name: "Test Event",
        start_time: dataTime * 1000,
        end_time: (dataTime + 5000) * 1000,
        metadata: { secret: "should-not-appear" },
        metadata_names: ["secret"],
        metadata_values: ["should-not-appear"],
      });

      await createEventsCh([event]);

      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);

      const files = await s3StorageService.listFiles(s3Prefix);
      const eventFile = files.find((f) => f.file.includes("/observations_v2/"));
      expect(eventFile).toBeDefined();

      if (eventFile) {
        const content = await s3StorageService.download(eventFile.file);
        const row = JSON.parse(content.trim().split("\n")[0]);

        // core + io fields should be present
        expect(row).toHaveProperty("id");
        expect(row).toHaveProperty("trace_id");
        expect(row).toHaveProperty("input");
        expect(row).toHaveProperty("output");

        // metadata group not selected → must not leak even as {}
        expect(row).not.toHaveProperty("metadata");

        // metrics group not selected → latency/time_to_first_token must not appear
        expect(row).not.toHaveProperty("latency");
        expect(row).not.toHaveProperty("time_to_first_token");

        // non-selected groups must not appear
        expect(row).not.toHaveProperty("name");
        expect(row).not.toHaveProperty("level");
        expect(row).not.toHaveProperty("usage_details");
      }
    });

    it.each([
      // observation metadata must not switch on score metadata
      [["core", "metadata"], false, true, false],
      [["core", "score_metadata"], true, false, false],
      [["core", "metadata", "score_metadata"], true, true, false],
      [["core", "score_provenance"], false, false, true],
    ])(
      "should apply field groups %j per table (score metadata: %s, observation metadata: %s, score provenance: %s)",
      async (
        exportFieldGroups,
        expectScoreMetadata,
        expectEventMetadata,
        expectScoreProvenance,
      ) => {
        const { projectId } = await createOrgProjectAndApiKey();
        s3Prefix = projectId;
        const now = new Date();
        const twoHoursAgo = new Date(now.getTime() - 2 * 60 * 60 * 1000);
        const dataTime = now.getTime() - 90 * 60 * 1000;

        await prisma.blobStorageIntegration.create({
          data: {
            projectId,
            type: BlobStorageIntegrationType.S3,
            bucketName,
            prefix: s3Prefix,
            accessKeyId: minioAccessKeyId,
            secretAccessKey: encrypt(minioAccessKeySecret),
            region: region ? region : "auto",
            endpoint: minioEndpoint,
            forcePathStyle:
              env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
            enabled: true,
            exportFrequency: "hourly",
            exportSource: "EVENTS",
            exportFieldGroups,
            nextSyncAt: twoHoursAgo,
            lastSyncAt: twoHoursAgo,
            compressed: false,
            fileType: BlobStorageIntegrationFileType.JSONL,
          },
        });

        const traceId = randomUUID();
        await Promise.all([
          createEventsCh([
            createEvent({
              project_id: projectId,
              trace_id: traceId,
              start_time: dataTime * 1000,
              end_time: (dataTime + 5000) * 1000,
              metadata: { source: "observation" },
              metadata_names: ["source"],
              metadata_values: ["observation"],
            }),
          ]),
          createScoresCh([
            createTraceScore({
              trace_id: traceId,
              project_id: projectId,
              timestamp: dataTime,
              metadata: { source: "score" },
              author_user_id: "author-1",
              config_id: "config-1",
              queue_id: "queue-1",
            }),
          ]),
        ]);

        await handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
        } as Job);

        const files = await s3StorageService.listFiles(s3Prefix);
        const readFirstRow = async (table: string) => {
          const file = files.find((f) => f.file.includes(`/${table}/`));
          expect(file).toBeDefined();
          const content = await s3StorageService.download(file!.file);
          return JSON.parse(content.trim().split("\n")[0]);
        };

        const scoreRow = await readFirstRow("scores");
        expect(scoreRow).toHaveProperty("value");
        if (expectScoreMetadata) {
          expect(scoreRow.metadata).toEqual({ source: "score" });
        } else {
          expect(scoreRow).not.toHaveProperty("metadata");
        }
        if (expectScoreProvenance) {
          expect(scoreRow).toMatchObject({
            author_user_id: "author-1",
            config_id: "config-1",
            queue_id: "queue-1",
          });
          expect(scoreRow).toHaveProperty("execution_trace_id");
        } else {
          expect(scoreRow).not.toHaveProperty("author_user_id");
        }

        const eventRow = await readFirstRow("observations_v2");
        if (expectEventMetadata) {
          expect(eventRow.metadata).toEqual({ source: "observation" });
        } else {
          expect(eventRow).not.toHaveProperty("metadata");
        }
      },
    );

    const TRACE_CORE_COLUMNS = [
      "id",
      "timestamp",
      "name",
      "environment",
      "project_id",
      "user_id",
      "session_id",
      "release",
      "version",
      "public",
      "bookmarked",
      "tags",
      "created_at",
      "updated_at",
    ];

    it.each([
      // no trace group stored: every trace column, in the original order
      [
        ["core", "io", "metadata"],
        [
          "id",
          "timestamp",
          "name",
          "environment",
          "project_id",
          "metadata",
          "user_id",
          "session_id",
          "release",
          "version",
          "public",
          "bookmarked",
          "tags",
          "input",
          "output",
          "created_at",
          "updated_at",
        ],
      ],
      [["core", "io", "metadata", "trace_core"], TRACE_CORE_COLUMNS],
      [
        ["core", "trace_core", "trace_io"],
        [...TRACE_CORE_COLUMNS.slice(0, 12), "input", "output"].concat(
          TRACE_CORE_COLUMNS.slice(12),
        ),
      ],
      // trace_core is implied once any trace group is stored
      [
        ["core", "trace_metadata"],
        [...TRACE_CORE_COLUMNS.slice(0, 5), "metadata"].concat(
          TRACE_CORE_COLUMNS.slice(5),
        ),
      ],
    ])(
      "should export trace columns for field groups %j",
      async (exportFieldGroups, expectedColumns) => {
        const { projectId } = await createOrgProjectAndApiKey();
        s3Prefix = projectId;
        const now = new Date();
        const twoHoursAgo = new Date(now.getTime() - 2 * 60 * 60 * 1000);

        await prisma.blobStorageIntegration.create({
          data: {
            projectId,
            type: BlobStorageIntegrationType.S3,
            bucketName,
            prefix: s3Prefix,
            accessKeyId: minioAccessKeyId,
            secretAccessKey: encrypt(minioAccessKeySecret),
            region: region ? region : "auto",
            endpoint: minioEndpoint,
            forcePathStyle:
              env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
            enabled: true,
            exportFrequency: "hourly",
            exportSource: "TRACES_OBSERVATIONS",
            exportFieldGroups,
            nextSyncAt: twoHoursAgo,
            lastSyncAt: twoHoursAgo,
            compressed: false,
            fileType: BlobStorageIntegrationFileType.JSONL,
          },
        });

        await createTracesCh([
          createTrace({
            project_id: projectId,
            timestamp: now.getTime() - 90 * 60 * 1000,
            input: JSON.stringify({ q: "trace input" }),
            output: JSON.stringify({ a: "trace output" }),
            metadata: { source: "trace" },
          }),
        ]);

        await handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
        } as Job);

        const files = await s3StorageService.listFiles(s3Prefix);
        const traceFile = files.find((f) => f.file.includes("/traces/"));
        expect(traceFile).toBeDefined();
        const content = await s3StorageService.download(traceFile!.file);
        const traceRow = JSON.parse(content.trim().split("\n")[0]);

        expect(Object.keys(traceRow)).toEqual(expectedColumns);
      },
    );

    it("should respect export frequency when setting nextSyncAt", async () => {
      const { projectId } = await createOrgProjectAndApiKey();

      // Setup for weekly export
      const now = new Date();
      const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);

      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: "",
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region,
          endpoint: minioEndpoint,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "weekly",
          lastSyncAt: oneHourAgo,
          compressed: false,
        },
      });

      // Create test data
      const traceId = randomUUID();
      await createTracesCh([
        createTrace({
          id: traceId,
          project_id: projectId,
          timestamp: oneHourAgo.getTime() + 10 * 60 * 1000,
        }),
      ]);

      // When
      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);

      // Then
      const updatedIntegration = await prisma.blobStorageIntegration.findUnique(
        {
          where: { projectId },
        },
      );

      // Should be set to 7 days in the future from maxTimestamp (now - lag buffer)
      const expectedNextSync = new Date(
        now.getTime() - BLOB_STORAGE_LAG_BUFFER_MS + 7 * 24 * 60 * 60 * 1000,
      );

      if (updatedIntegration?.nextSyncAt) {
        // Use a tolerance value in milliseconds instead of numeric precision
        const tolerance = 1000; // 1 second tolerance
        expect(
          Math.abs(
            updatedIntegration.nextSyncAt.getTime() -
              expectedNextSync.getTime(),
          ),
        ).toBeLessThan(tolerance);
      } else {
        expect.fail("Integration should have nextSyncAt set");
      }
    });

    it("should use prefix in file path when specified", async () => {
      // Setup
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = "test-prefix";
      const now = new Date();
      const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);

      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region ? region : "auto",
          endpoint: minioEndpoint,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "daily",
          lastSyncAt: oneHourAgo,
          compressed: false,
        },
      });

      // Create test data
      const traceId = randomUUID();
      await createTracesCh([
        createTrace({
          id: traceId,
          project_id: projectId,
          timestamp: oneHourAgo.getTime() + 10 * 60 * 1000,
        }),
      ]);

      // When
      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);

      // Then
      const files = await storageService.listFiles(s3Prefix);
      const projectFiles = files.filter((f) => f.file.includes(projectId));

      // All files should have the prefix
      expect(projectFiles.every((f) => f.file.startsWith(s3Prefix))).toBe(true);
    });

    it("should handle CSV, JSON, and JSONL file types correctly", async () => {
      // Setup
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = `${projectId}/`;
      const now = new Date();
      const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);

      // Create test data
      const traceId = randomUUID();
      const observationId = randomUUID();
      const scoreId = randomUUID();

      // Create event data for events table export
      const event = createEvent({
        project_id: projectId,
        trace_id: traceId,
        type: "GENERATION",
        name: "Test Event",
        start_time: (now.getTime() - 35 * 60 * 1000) * 1000, // 35 minutes before now (microseconds)
      });

      await Promise.all([
        createTracesCh([
          createTrace({
            id: traceId,
            project_id: projectId,
            timestamp: now.getTime() - 40 * 60 * 1000, // 40 min before now
            name: "Test Trace",
          }),
        ]),
        createObservationsCh([
          createObservation({
            id: observationId,
            trace_id: traceId,
            project_id: projectId,
            start_time: now.getTime() - 35 * 60 * 1000, // 35 minutes before now
            name: "Test Observation",
          }),
        ]),
        createScoresCh([
          createTraceScore({
            id: scoreId,
            trace_id: traceId,
            project_id: projectId,
            timestamp: now.getTime() - 35 * 60 * 1000, // 35 minutes before now
            name: "Test Score",
            value: 0.95,
          }),
        ]),
        createEventsCh([event]),
      ]);

      // Test each file type
      for (const fileType of [
        BlobStorageIntegrationFileType.CSV,
        BlobStorageIntegrationFileType.JSON,
        BlobStorageIntegrationFileType.JSONL,
      ]) {
        const fileTypePrefix = `${fileType.toLowerCase()}-test/`;
        const prefix = `${s3Prefix}${fileTypePrefix}`;

        // Create integration with specific file type
        await prisma.blobStorageIntegration.upsert({
          where: { projectId },
          update: {
            prefix,
            fileType,
            lastSyncAt: oneHourAgo,
          },
          create: {
            projectId,
            type: BlobStorageIntegrationType.S3,
            bucketName,
            prefix,
            accessKeyId: minioAccessKeyId,
            secretAccessKey: encrypt(minioAccessKeySecret),
            region: region ? region : "auto",
            endpoint: minioEndpoint,
            forcePathStyle:
              env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
            enabled: true,
            exportFrequency: "hourly",
            exportSource: "TRACES_OBSERVATIONS_EVENTS",
            fileType,
            lastSyncAt: oneHourAgo,
            compressed: false,
          },
        });

        // Process the integration
        await handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
        } as Job);

        // Get files for this file type
        const files = await s3StorageService.listFiles(prefix);
        const projectFiles = files.filter(
          (f) =>
            f.file.includes(projectId) &&
            f.file.includes(fileTypePrefix) &&
            !f.file.includes("/manifests/"),
        );

        // Should have 4 files (traces, observations, scores, events)
        expect(projectFiles).toHaveLength(4);

        // Check file extensions
        const expectedExtension = fileType.toLowerCase();
        expect(
          projectFiles.every((f) => f.file.endsWith(`.${expectedExtension}`)),
        ).toBe(true);

        // Check file contents for each type
        for (const file of projectFiles) {
          const content = await s3StorageService.download(file.file);

          // Verify content based on file type
          if (file.file.includes("/traces/")) {
            expect(content).toContain(traceId);
            expect(content).toContain("Test Trace");
          } else if (file.file.includes("/observations/")) {
            expect(content).toContain(observationId);
            expect(content).toContain("Test Observation");
          } else if (file.file.includes("/scores/")) {
            expect(content).toContain(scoreId);
            expect(content).toContain("Test Score");
            expect(content).toContain("0.95");
          } else if (file.file.includes("/events/")) {
            expect(content).toContain(event.span_id);
            expect(content).toContain("Test Event");
          }

          // Verify format based on file type
          switch (fileType) {
            case BlobStorageIntegrationFileType.CSV:
              // CSV should have commas and newlines
              expect(content).toContain(",");
              break;
            case BlobStorageIntegrationFileType.JSON:
              // JSON should be parseable and have array brackets
              expect(content.trim().startsWith("[")).toBe(true);
              expect(content.trim().endsWith("]")).toBe(true);
              expect(() => JSON.parse(content)).not.toThrow();
              break;
            case BlobStorageIntegrationFileType.JSONL:
              // JSONL should have newlines and each line should be parseable JSON
              const lines = content.trim().split("\n");
              expect(lines.length).toBeGreaterThan(0);
              // Check first line is valid JSON
              expect(() => JSON.parse(lines[0])).not.toThrow();
              break;
          }
        }
      }
    });
  });

  describe("legacy observations export field groups", () => {
    it("should exclude columns for deselected exportFieldGroups in the legacy observations export", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      const now = new Date();
      const twoHoursAgo = new Date(now.getTime() - 2 * 60 * 60 * 1000);
      const dataTime = now.getTime() - 90 * 60 * 1000;

      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region ? region : "auto",
          endpoint: minioEndpoint,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "hourly",
          exportSource: "TRACES_OBSERVATIONS",
          exportFieldGroups: ["core", "io"],
          nextSyncAt: twoHoursAgo,
          lastSyncAt: twoHoursAgo,
          compressed: false,
          fileType: BlobStorageIntegrationFileType.JSONL,
        },
      });

      const traceId = randomUUID();
      await createObservationsCh([
        createObservation({
          id: randomUUID(),
          trace_id: traceId,
          project_id: projectId,
          start_time: dataTime,
          end_time: dataTime + 5000,
          name: "Legacy Observation",
          metadata: { secret: "should-not-appear" },
          usage_details: { input: 100, output: 200, total: 300 },
        }),
      ]);

      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);

      const files = await s3StorageService.listFiles(s3Prefix);
      const observationFile = files.find((f) =>
        f.file.includes("/observations/"),
      );
      expect(observationFile).toBeDefined();

      if (observationFile) {
        const content = await s3StorageService.download(observationFile.file);
        const row = JSON.parse(content.trim().split("\n")[0]);

        // core + io fields should be present
        expect(row).toHaveProperty("id");
        expect(row).toHaveProperty("trace_id");
        expect(row).toHaveProperty("input");
        expect(row).toHaveProperty("output");

        // metadata group not selected → must not leak
        expect(row).not.toHaveProperty("metadata");

        // metrics group not selected → computed fields must not appear
        expect(row).not.toHaveProperty("latency");
        expect(row).not.toHaveProperty("time_to_first_token");

        // model group not selected → no model id or pricing enrichment
        expect(row).not.toHaveProperty("model_id");
        expect(row).not.toHaveProperty("input_price");

        // other non-selected groups must not appear
        expect(row).not.toHaveProperty("name");
        expect(row).not.toHaveProperty("level");
        expect(row).not.toHaveProperty("usage_details");
      }
    });
  });

  describe("BlobStorageExportMode minTimestamp behavior", () => {
    maybeIt(
      "should export old data for FULL_HISTORY mode when data exists",
      async () => {
        const { projectId } = await createOrgProjectAndApiKey();
        s3Prefix = projectId;

        // Create trace with old timestamp that's far enough in the past
        // but not so old that it might not be found by ClickHouse
        const now = new Date();
        const twoDaysAgo = new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000);
        const oldTrace = createTrace({
          project_id: projectId,
          timestamp: twoDaysAgo.getTime(),
          name: "Old Trace",
        });
        await createTracesCh([oldTrace]);

        // Create integration with FULL_HISTORY mode and no lastSyncAt
        await prisma.blobStorageIntegration.create({
          data: {
            projectId,
            type: BlobStorageIntegrationType.S3,
            bucketName,
            prefix: s3Prefix,
            accessKeyId,
            secretAccessKey: encrypt(secretAccessKey),
            region: region ? region : "auto",
            endpoint: endpoint ? endpoint : null,
            forcePathStyle:
              env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
            enabled: true,
            exportFrequency: "hourly",
            exportMode: "FULL_HISTORY",
            exportStartDate: null,
            lastSyncAt: null, // First export
            compressed: false,
          },
        });

        await handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
        } as Job);

        // If data was found and exported, check the files
        const files = await storageService.listFiles(s3Prefix);
        const projectFiles = files.filter((f) => f.file.includes(projectId));

        // With FULL_HISTORY mode, if the ClickHouse query finds the old data,
        // it should export starting from that timestamp
        if (projectFiles.length > 0) {
          const traceFile = projectFiles.find((f) =>
            f.file.includes("/traces/"),
          );
          expect(traceFile).toBeDefined();

          if (traceFile) {
            const content = await storageService.download(traceFile.file);
            expect(content).toContain(oldTrace.id);
          }
        }

        // Verify integration was updated if export happened
        const updatedIntegration =
          await prisma.blobStorageIntegration.findUnique({
            where: { projectId },
          });

        // If files were exported, lastSyncAt should be set
        if (projectFiles.length > 0) {
          expect(updatedIntegration?.lastSyncAt).toBeDefined();
        }
      },
    );

    it("should use current date for FROM_TODAY mode on first export", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      const now = new Date();
      const yesterday = new Date(now.getTime() - 24 * 60 * 60 * 1000);
      const veryOldTrace = new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000); // 2 days ago

      // Create traces from different time periods
      const oldTrace = createTrace({
        project_id: projectId,
        timestamp: yesterday.getTime(),
        name: "Old Trace",
      });
      const veryOldTraceObj = createTrace({
        project_id: projectId,
        timestamp: veryOldTrace.getTime(),
        name: "Very Old Trace",
      });
      await createTracesCh([oldTrace, veryOldTraceObj]);

      // Create integration with FROM_TODAY mode
      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId,
          secretAccessKey: encrypt(secretAccessKey),
          region: region ? region : "auto",
          endpoint: endpoint ? endpoint : null,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "hourly",
          exportMode: "FROM_TODAY" as any,
          exportStartDate: new Date(), // Use current date
          lastSyncAt: null, // First export
          compressed: false,
        },
      });

      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);

      const files = await storageService.listFiles(s3Prefix);
      const projectFiles = files.filter((f) => f.file.includes(projectId));
      const traceFile = projectFiles.find((f) => f.file.includes("/traces/"));

      // On azure the empty file is not created, for others we proceed to check that it's empty.
      if (traceFile) {
        const content = await storageService.download(traceFile.file);
        // With FROM_TODAY mode and a current exportStartDate, the minTimestamp is set to the provided date (current time)
        // which means only traces within the last 30 minutes would be exported
        // Our test traces are older, so content should be empty or not contain old traces
        expect(content).not.toContain(oldTrace.id);
        expect(content).not.toContain(veryOldTraceObj.id);
      }
    });

    it("should use custom date for FROM_CUSTOM_DATE mode on first export", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      const now = new Date();
      const customDate = new Date(now.getTime() - 12 * 60 * 60 * 1000); // 12 hours ago
      const beforeCustomDate = new Date(customDate.getTime() - 60 * 60 * 1000); // 13 hours ago
      // With chunking, first export covers customDate to customDate + 1 hour
      // So we need data within that first hour window
      const afterCustomDate = new Date(customDate.getTime() + 30 * 60 * 1000); // 30 minutes after custom date

      // Create traces before and after custom date
      const oldTrace = createTrace({
        project_id: projectId,
        timestamp: beforeCustomDate.getTime(),
        name: "Before Custom Date Trace",
      });
      const recentTrace = createTrace({
        project_id: projectId,
        timestamp: afterCustomDate.getTime(),
        name: "After Custom Date Trace",
      });
      await createTracesCh([oldTrace, recentTrace]);

      // Create integration with FROM_CUSTOM_DATE mode
      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region ? region : "auto",
          endpoint: minioEndpoint,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "hourly",
          exportMode: "FROM_CUSTOM_DATE" as any,
          exportStartDate: customDate,
          lastSyncAt: null, // First export
          compressed: false,
        },
      });

      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);

      const files = await s3StorageService.listFiles(s3Prefix);
      const projectFiles = files.filter((f) => f.file.includes(projectId));
      const traceFile = projectFiles.find((f) => f.file.includes("/traces/"));

      expect(traceFile).toBeDefined();

      // Should only include traces from custom date onwards
      if (traceFile) {
        const content = await s3StorageService.download(traceFile.file);
        expect(content).not.toContain(oldTrace.id);
        expect(content).toContain(recentTrace.id);
      }
    });
  });

  describe("Chunked historic exports", () => {
    maybeIt(
      "should cap maxTimestamp to one frequency period ahead for FULL_HISTORY mode",
      async () => {
        const { projectId } = await createOrgProjectAndApiKey();
        s3Prefix = projectId;
        const now = new Date();
        const veryOldTimestamp = new Date(
          now.getTime() - 7 * 24 * 60 * 60 * 1000,
        ); // 7 days ago

        // Create trace from 7 days ago
        const oldTrace = createTrace({
          project_id: projectId,
          timestamp: veryOldTimestamp.getTime(),
          name: "Old Trace",
        });
        await createTracesCh([oldTrace]);

        // Create integration with FULL_HISTORY and hourly frequency (first export)
        await prisma.blobStorageIntegration.create({
          data: {
            projectId,
            type: BlobStorageIntegrationType.S3,
            bucketName,
            prefix: s3Prefix,
            accessKeyId,
            secretAccessKey: encrypt(secretAccessKey),
            region: region ? region : "auto",
            endpoint: endpoint ? endpoint : null,
            forcePathStyle:
              env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
            enabled: true,
            exportFrequency: "hourly",
            exportMode: "FULL_HISTORY",
            exportStartDate: null,
            lastSyncAt: null,
            compressed: false,
          },
        });

        // When
        await handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
        } as Job);

        // Then
        const updatedIntegration =
          await prisma.blobStorageIntegration.findUnique({
            where: { projectId },
          });

        expect(updatedIntegration).toBeDefined();

        // Check if files were exported (meaning data was found)
        const files = await storageService.listFiles(s3Prefix);
        const projectFiles = files.filter((f) => f.file.includes(projectId));

        // If data was found and exported, verify chunking behavior
        if (projectFiles.length > 0 && updatedIntegration?.lastSyncAt) {
          // When ClickHouse finds the old data, it should start from that timestamp
          // and cap the export to 1 hour (frequency interval)
          // lastSyncAt should be capped to 1 hour after the found timestamp
          const minExpectedTime = veryOldTimestamp.getTime();
          const maxExpectedTime = veryOldTimestamp.getTime() + 60 * 60 * 1000; // +1 hour
          const tolerance = 2000; // 2 second tolerance

          expect(
            updatedIntegration.lastSyncAt.getTime(),
          ).toBeGreaterThanOrEqual(minExpectedTime);
          expect(updatedIntegration.lastSyncAt.getTime()).toBeLessThanOrEqual(
            maxExpectedTime + tolerance,
          );
        }
        // If no data was found (fallback to current time), the time window would be invalid
        // and no export would happen, which is acceptable behavior
      },
    );

    it("should immediately schedule next chunk when in catch-up mode", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      const now = new Date();
      const twoDaysAgo = new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000);

      // Create traces over 2 days
      const trace1 = createTrace({
        project_id: projectId,
        timestamp: twoDaysAgo.getTime(),
        name: "Old Trace 1",
      });
      const trace2 = createTrace({
        project_id: projectId,
        timestamp: twoDaysAgo.getTime() + 60 * 60 * 1000, // 1 hour later
        name: "Old Trace 2",
      });
      await createTracesCh([trace1, trace2]);

      // Create integration with hourly frequency starting 2 days ago
      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region ? region : "auto",
          endpoint: minioEndpoint,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "hourly",
          lastSyncAt: twoDaysAgo, // Start from 2 days ago
          compressed: false,
        },
      });

      // When
      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);

      // Then
      const updatedIntegration = await prisma.blobStorageIntegration.findUnique(
        {
          where: { projectId },
        },
      );

      expect(updatedIntegration).toBeDefined();
      if (!updatedIntegration?.nextSyncAt) {
        expect.fail("nextSyncAt should be set");
      }

      // nextSyncAt should be immediate (within a few seconds of now)
      const timeDiff = Math.abs(
        updatedIntegration.nextSyncAt.getTime() - now.getTime(),
      );
      expect(timeDiff).toBeLessThan(5000); // Within 5 seconds
      expect(updatedIntegration.backfill).toBe(true);
    });

    it("keeps the advanced watermark when catch-up enqueue fails", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      const now = new Date();
      const twoDaysAgo = new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000);

      const trace = createTrace({
        project_id: projectId,
        timestamp: twoDaysAgo.getTime(),
        name: "Old Trace",
      });
      await createTracesCh([trace]);

      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region ? region : "auto",
          endpoint: minioEndpoint,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "hourly",
          lastSyncAt: twoDaysAgo,
          compressed: false,
        },
      });

      const add = vi.fn().mockRejectedValue(new Error("redis down"));
      const getInstanceSpy = vi
        .spyOn(BlobStorageIntegrationProcessingQueue, "getInstance")
        .mockReturnValue({ add } as never);

      try {
        await expect(
          handleBlobStorageIntegrationProjectJob({
            data: { payload: { projectId } },
          } as Job),
        ).rejects.toThrow(/redis down/);

        const updatedIntegration =
          await prisma.blobStorageIntegration.findUnique({
            where: { projectId },
          });
        expect(updatedIntegration?.lastSyncAt?.getTime()).toBe(
          twoDaysAgo.getTime() + 60 * 60 * 1000,
        );
        expect(add).toHaveBeenCalled();
      } finally {
        getInstanceSpy.mockRestore();
      }
    });

    it("should coalesce a sub-second remainder instead of emitting a colliding chunk", async () => {
      // Regression for the silent object-key collision: a caught-up run whose
      // full-interval chunk ends only a sub-second before the frontier used to
      // re-enqueue a tiny remainder chunk. Both keys truncate to the same
      // wall-clock second, so the remainder overwrote the full window. Position
      // lastSyncAt so the interval-capped maxTimestamp lands just below the
      // frontier (remainder < BLOB_STORAGE_REMAINDER_COALESCE_MS): the run must
      // be treated as caught up (schedule one interval out), not re-enqueued.
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      const now = new Date();
      const frequencyIntervalMs = 60 * 60 * 1000; // hourly
      // gap = frontier - (minTimestamp + interval). Kept tiny (well under the
      // coalesce threshold) so it stays below threshold even after the handler's
      // own `now` advances a few ms past the test's.
      const gapMs = 50;
      const lastSyncAt = new Date(
        now.getTime() -
          BLOB_STORAGE_LAG_BUFFER_MS -
          frequencyIntervalMs -
          gapMs,
      );

      // A trace inside the full window so a real chunk is exported.
      const trace = createTrace({
        project_id: projectId,
        timestamp: lastSyncAt.getTime() + frequencyIntervalMs / 2,
        name: "Windowed Trace",
      });
      await createTracesCh([trace]);

      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region ? region : "auto",
          endpoint: minioEndpoint,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "hourly",
          lastSyncAt,
          compressed: false,
        },
      });

      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);

      const updatedIntegration = await prisma.blobStorageIntegration.findUnique(
        {
          where: { projectId },
        },
      );

      expect(updatedIntegration).toBeDefined();
      if (!updatedIntegration?.nextSyncAt || !updatedIntegration?.lastSyncAt) {
        expect.fail("nextSyncAt and lastSyncAt should be set");
      }

      // Caught up: nextSyncAt is one interval past the exported boundary, far in
      // the future — not the near-`now` value a catch-up re-enqueue would set.
      expect(
        updatedIntegration.nextSyncAt.getTime() - now.getTime(),
      ).toBeGreaterThan(frequencyIntervalMs / 2);

      // lastSyncAt advances only to the interval-capped boundary; the sub-second
      // tail up to the frontier is deferred to the next scheduled run.
      const frontier = now.getTime() - BLOB_STORAGE_LAG_BUFFER_MS;
      expect(updatedIntegration.lastSyncAt.getTime()).toBeLessThan(frontier);
      expect(frontier - updatedIntegration.lastSyncAt.getTime()).toBeLessThan(
        BLOB_STORAGE_REMAINDER_COALESCE_MS + 2000, // + handler-clock drift tolerance
      );
    });

    it("should schedule normally when caught up", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      const now = new Date();
      const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);

      // Create recent trace
      const trace = createTrace({
        project_id: projectId,
        timestamp: now.getTime() - 40 * 60 * 1000, // 40 minutes ago
        name: "Recent Trace",
      });
      await createTracesCh([trace]);

      // Create integration with lastSyncAt 1 hour ago (within normal range)
      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region ? region : "auto",
          endpoint: minioEndpoint,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "hourly",
          lastSyncAt: oneHourAgo,
          compressed: false,
        },
      });

      // When
      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);

      // Then
      const updatedIntegration = await prisma.blobStorageIntegration.findUnique(
        {
          where: { projectId },
        },
      );

      expect(updatedIntegration).toBeDefined();
      if (!updatedIntegration?.nextSyncAt || !updatedIntegration?.lastSyncAt) {
        expect.fail("nextSyncAt and lastSyncAt should be set");
      }

      // nextSyncAt should be 1 hour after lastSyncAt (normal scheduling)
      const expectedNextSync = new Date(
        updatedIntegration.lastSyncAt.getTime() + 60 * 60 * 1000,
      );
      const tolerance = 1000; // 1 second tolerance

      expect(
        Math.abs(
          updatedIntegration.nextSyncAt.getTime() - expectedNextSync.getTime(),
        ),
      ).toBeLessThan(tolerance);

      // nextSyncAt should be in the future
      expect(updatedIntegration.nextSyncAt.getTime()).toBeGreaterThan(
        now.getTime(),
      );
      expect(updatedIntegration.backfill).toBe(false);
    });
  });

  describe("gzip compression", () => {
    maybeIt(
      "should produce .csv.gz files when compressed is true",
      async () => {
        const { projectId } = await createOrgProjectAndApiKey();
        s3Prefix = `${projectId}/`;
        const now = new Date();
        const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);

        await prisma.blobStorageIntegration.create({
          data: {
            projectId,
            type: BlobStorageIntegrationType.S3,
            bucketName,
            prefix: s3Prefix,
            accessKeyId: minioAccessKeyId,
            secretAccessKey: encrypt(minioAccessKeySecret),
            region: region ? region : "auto",
            endpoint: minioEndpoint,
            forcePathStyle:
              env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
            enabled: true,
            exportFrequency: "hourly",
            fileType: BlobStorageIntegrationFileType.CSV,
            compressed: true,
            lastSyncAt: oneHourAgo,
          },
        });

        const traceId = randomUUID();
        await createTracesCh([
          createTrace({
            id: traceId,
            project_id: projectId,
            timestamp: now.getTime() - 40 * 60 * 1000,
            name: "Compressed Trace",
          }),
        ]);

        await handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
        } as Job);

        const files = await s3StorageService.listFiles(s3Prefix);
        const projectFiles = files.filter(
          (f) => f.file.includes(projectId) && !f.file.includes("/manifests/"),
        );

        expect(projectFiles.length).toBeGreaterThan(0);
        expect(projectFiles.every((f) => f.file.endsWith(".csv.gz"))).toBe(
          true,
        );
      },
    );

    maybeIt(
      "should produce plain .csv files when compressed is false",
      async () => {
        const { projectId } = await createOrgProjectAndApiKey();
        s3Prefix = `${projectId}/`;
        const now = new Date();
        const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);

        await prisma.blobStorageIntegration.create({
          data: {
            projectId,
            type: BlobStorageIntegrationType.S3,
            bucketName,
            prefix: s3Prefix,
            accessKeyId: minioAccessKeyId,
            secretAccessKey: encrypt(minioAccessKeySecret),
            region: region ? region : "auto",
            endpoint: minioEndpoint,
            forcePathStyle:
              env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
            enabled: true,
            exportFrequency: "hourly",
            fileType: BlobStorageIntegrationFileType.CSV,
            compressed: false,
            lastSyncAt: oneHourAgo,
          },
        });

        const traceId = randomUUID();
        await createTracesCh([
          createTrace({
            id: traceId,
            project_id: projectId,
            timestamp: now.getTime() - 40 * 60 * 1000,
            name: "Uncompressed Trace",
          }),
        ]);

        await handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
        } as Job);

        const files = await s3StorageService.listFiles(s3Prefix);
        const projectFiles = files.filter(
          (f) => f.file.includes(projectId) && !f.file.includes("/manifests/"),
        );

        expect(projectFiles.length).toBeGreaterThan(0);
        expect(
          projectFiles.every(
            (f) => f.file.endsWith(".csv") && !f.file.endsWith(".csv.gz"),
          ),
        ).toBe(true);

        // Verify content is plain text (readable)
        const traceFile = projectFiles.find((f) => f.file.includes("/traces/"));
        if (traceFile) {
          const content = await s3StorageService.download(traceFile.file);
          expect(content).toContain(traceId);
          expect(content).toContain("Uncompressed Trace");
        }
      },
    );

    maybeIt(
      "should produce .jsonl.gz files when compressed with JSONL format",
      async () => {
        const { projectId } = await createOrgProjectAndApiKey();
        s3Prefix = `${projectId}/`;
        const now = new Date();
        const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);

        await prisma.blobStorageIntegration.create({
          data: {
            projectId,
            type: BlobStorageIntegrationType.S3,
            bucketName,
            prefix: s3Prefix,
            accessKeyId: minioAccessKeyId,
            secretAccessKey: encrypt(minioAccessKeySecret),
            region: region ? region : "auto",
            endpoint: minioEndpoint,
            forcePathStyle:
              env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
            enabled: true,
            exportFrequency: "hourly",
            fileType: BlobStorageIntegrationFileType.JSONL,
            compressed: true,
            lastSyncAt: oneHourAgo,
          },
        });

        const traceId = randomUUID();
        await createTracesCh([
          createTrace({
            id: traceId,
            project_id: projectId,
            timestamp: now.getTime() - 40 * 60 * 1000,
            name: "JSONL Compressed Trace",
          }),
        ]);

        await handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
        } as Job);

        const files = await s3StorageService.listFiles(s3Prefix);
        const projectFiles = files.filter(
          (f) => f.file.includes(projectId) && !f.file.includes("/manifests/"),
        );

        expect(projectFiles.length).toBeGreaterThan(0);
        expect(projectFiles.every((f) => f.file.endsWith(".jsonl.gz"))).toBe(
          true,
        );
      },
    );
  });

  // Uses the non-enriched TRACES_OBSERVATIONS source so it runs on every CI
  // leg, independent of the V4-preview opt-in.
  describe("run-completion manifest (LFE-10843)", () => {
    maybeIt(
      "writes a manifest listing every file, the window, and per-file format",
      async () => {
        const { projectId } = await createOrgProjectAndApiKey();
        s3Prefix = `${projectId}/`;
        const now = new Date();
        const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);
        const dataTime = now.getTime() - 40 * 60 * 1000;

        await prisma.blobStorageIntegration.create({
          data: {
            projectId,
            type: BlobStorageIntegrationType.S3,
            bucketName,
            prefix: s3Prefix,
            accessKeyId: minioAccessKeyId,
            secretAccessKey: encrypt(minioAccessKeySecret),
            region: region ? region : "auto",
            endpoint: minioEndpoint,
            forcePathStyle:
              env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
            enabled: true,
            exportFrequency: "hourly",
            exportSource: "TRACES_OBSERVATIONS",
            fileType: BlobStorageIntegrationFileType.JSONL,
            compressed: false,
            lastSyncAt: oneHourAgo,
          },
        });

        const traceId = randomUUID();
        await Promise.all([
          createTracesCh([
            createTrace({
              id: traceId,
              project_id: projectId,
              timestamp: dataTime,
              name: "Manifest Trace",
            }),
          ]),
          createObservationsCh([
            createObservation({
              id: randomUUID(),
              trace_id: traceId,
              project_id: projectId,
              start_time: dataTime,
              end_time: dataTime + 5000,
              name: "Manifest Observation",
            }),
          ]),
          createScoresCh([
            createTraceScore({
              id: randomUUID(),
              trace_id: traceId,
              project_id: projectId,
              timestamp: dataTime,
              name: "Manifest Score",
              value: 0.5,
            }),
          ]),
        ]);

        await handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
        } as Job);

        const files = await s3StorageService.listFiles(s3Prefix);
        const tableFiles = files.filter(
          (f) => f.file.includes(projectId) && !f.file.includes("/manifests/"),
        );
        expect(tableFiles).toHaveLength(3); // scores, traces, observations

        const manifestFile = files.find((f) => f.file.includes("/manifests/"));
        expect(manifestFile).toBeDefined();
        expect(manifestFile!.file).toContain(
          `${s3Prefix}${projectId}/manifests/`,
        );
        expect(manifestFile!.file.endsWith(".json")).toBe(true);

        const manifest = JSON.parse(
          await s3StorageService.download(manifestFile!.file),
        );
        expect(manifest.version).toBe(1);
        expect(manifest.projectId).toBe(projectId);
        expect(manifest.exportSource).toBe("TRACES_OBSERVATIONS");
        expect(new Set(manifest.tables)).toEqual(
          new Set(["scores", "traces", "observations"]),
        );

        expect(
          new Set(manifest.files.map((f: { key: string }) => f.key)),
        ).toEqual(new Set(tableFiles.map((f) => f.file)));

        const tracesEntry = manifest.files.find(
          (f: { table: string }) => f.table === "traces",
        );
        expect(tracesEntry.fileType).toBe("JSONL");
        expect(tracesEntry.format).toBe("jsonl-raw");
        expect(tracesEntry.compressed).toBe(false);
        expect(tracesEntry.rowCount).toBe(1);
        expect(typeof tracesEntry.sizeBytes).toBe("number");

        expect(typeof manifest.window.minTimestamp).toBe("string");
        expect(manifest.maxTimestamp).toBe(manifest.window.maxTimestamp);
        expect(typeof manifest.createdAt).toBe("string");
      },
    );
  });

  // LFE-10896: legacy-source projects get a plain-text deprecation notice in
  // their destination; enriched-only (EVENTS) projects do not, and a stale
  // notice is removed once a project migrates off a legacy source. The notice
  // is Cloud-only, so set a cloud region for this suite (the top-level harness
  // clears it) and restore it afterwards.
  describe("legacy-source deprecation notice (LFE-10896)", () => {
    const NOTICE_SUFFIX = "/DEPRECATION_NOTICE.txt";
    const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;

    beforeAll(() => {
      env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
    });
    afterAll(() => {
      env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;
    });

    maybeIt(
      "writes DEPRECATION_NOTICE.txt for a legacy export source",
      async () => {
        const { projectId } = await createOrgProjectAndApiKey();
        s3Prefix = `${projectId}/`;
        const now = new Date();
        const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);
        const dataTime = now.getTime() - 40 * 60 * 1000;

        await prisma.blobStorageIntegration.create({
          data: {
            projectId,
            type: BlobStorageIntegrationType.S3,
            bucketName,
            prefix: s3Prefix,
            accessKeyId: minioAccessKeyId,
            secretAccessKey: encrypt(minioAccessKeySecret),
            region: region ? region : "auto",
            endpoint: minioEndpoint,
            forcePathStyle:
              env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
            enabled: true,
            exportFrequency: "hourly",
            exportSource: "TRACES_OBSERVATIONS",
            fileType: BlobStorageIntegrationFileType.JSONL,
            compressed: false,
            lastSyncAt: oneHourAgo,
          },
        });

        await createTracesCh([
          createTrace({
            id: randomUUID(),
            project_id: projectId,
            timestamp: dataTime,
            name: "Notice Trace",
          }),
        ]);

        await handleBlobStorageIntegrationProjectJob({
          data: { payload: { projectId } },
        } as Job);

        const files = await s3StorageService.listFiles(s3Prefix);
        const noticeFile = files.find((f) => f.file.endsWith(NOTICE_SUFFIX));
        expect(noticeFile).toBeDefined();
        expect(noticeFile!.file).toBe(
          `${s3Prefix}${projectId}${NOTICE_SUFFIX}`,
        );

        const notice = await s3StorageService.download(noticeFile!.file);
        expect(notice).toContain("Traces and observations (legacy)");
        expect(notice).toContain("Enriched observations (recommended)");
        expect(notice).toContain(
          "https://langfuse.com/docs/api-and-data-platform/features/export-to-blob-storage",
        );
      },
    );

    // The EVENTS export reads the enriched events table, which only the
    // V4-preview CI leg provisions — gate these like every other enriched test.
    // isCloud is set by the suite's beforeAll, so the notice logic still runs.
    maybeDescribe("after migrating to the enriched-only source", () => {
      maybeIt(
        "does not write a deprecation notice for the EVENTS source",
        async () => {
          const { projectId } = await createOrgProjectAndApiKey();
          s3Prefix = `${projectId}/`;
          const now = new Date();
          const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);
          const dataTime = now.getTime() - 40 * 60 * 1000;

          await prisma.blobStorageIntegration.create({
            data: {
              projectId,
              type: BlobStorageIntegrationType.S3,
              bucketName,
              prefix: s3Prefix,
              accessKeyId: minioAccessKeyId,
              secretAccessKey: encrypt(minioAccessKeySecret),
              region: region ? region : "auto",
              endpoint: minioEndpoint,
              forcePathStyle:
                env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
              enabled: true,
              exportFrequency: "hourly",
              exportSource: "EVENTS",
              fileType: BlobStorageIntegrationFileType.JSONL,
              compressed: false,
              lastSyncAt: oneHourAgo,
            },
          });

          await createEventsCh([
            createEvent({
              id: randomUUID(),
              project_id: projectId,
              start_time: dataTime,
              name: "No Notice Event",
            }),
          ]);

          await handleBlobStorageIntegrationProjectJob({
            data: { payload: { projectId } },
          } as Job);

          const files = await s3StorageService.listFiles(s3Prefix);
          expect(files.some((f) => f.file.endsWith(NOTICE_SUFFIX))).toBe(false);
        },
      );

      // A notice written while on a legacy source must be cleaned up once the
      // project migrates to the enriched-only source, so it can't linger with
      // now-false claims. Cleanup only runs for integrations old enough to have
      // used a legacy source (pre-exporter-cutoff createdAt).
      maybeIt(
        "removes a stale notice after migrating to the EVENTS source",
        async () => {
          const { projectId } = await createOrgProjectAndApiKey();
          s3Prefix = `${projectId}/`;
          const noticeKey = `${s3Prefix}${projectId}${NOTICE_SUFFIX}`;
          const now = new Date();
          const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);
          const dataTime = now.getTime() - 40 * 60 * 1000;

          // Simulate a notice left behind by an earlier legacy-source run.
          await s3StorageService.uploadFile({
            fileName: noticeKey,
            fileType: "text/plain; charset=utf-8",
            data: "stale notice from a previous legacy run",
          });
          const before = await s3StorageService.listFiles(s3Prefix);
          expect(before.some((f) => f.file === noticeKey)).toBe(true);

          await prisma.blobStorageIntegration.create({
            data: {
              projectId,
              type: BlobStorageIntegrationType.S3,
              bucketName,
              prefix: s3Prefix,
              accessKeyId: minioAccessKeyId,
              secretAccessKey: encrypt(minioAccessKeySecret),
              region: region ? region : "auto",
              endpoint: minioEndpoint,
              forcePathStyle:
                env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
              enabled: true,
              exportFrequency: "hourly",
              exportSource: "EVENTS",
              fileType: BlobStorageIntegrationFileType.JSONL,
              compressed: false,
              lastSyncAt: oneHourAgo,
              // Pre-exporter-cutoff: old enough to have used a legacy source, so
              // cleanup runs. Derived from the live cutoff so an env override
              // can't flip the gate.
              createdAt: new Date(
                LEGACY_BLOB_EXPORTER_CUTOFF.getTime() - 24 * 60 * 60 * 1000,
              ),
            },
          });

          await createEventsCh([
            createEvent({
              id: randomUUID(),
              project_id: projectId,
              start_time: dataTime,
              name: "Migrated Event",
            }),
          ]);

          await handleBlobStorageIntegrationProjectJob({
            data: { payload: { projectId } },
          } as Job);

          const after = await s3StorageService.listFiles(s3Prefix);
          expect(after.some((f) => f.file.endsWith(NOTICE_SUFFIX))).toBe(false);
        },
      );

      // Post-cutoff integrations can never have used a legacy source, so they
      // never wrote a notice — the cleanup delete must be skipped for them to
      // avoid a needless per-run s3:DeleteObject on every enriched-only export.
      maybeIt(
        "skips notice cleanup for a post-cutoff enriched-only integration",
        async () => {
          const { projectId } = await createOrgProjectAndApiKey();
          s3Prefix = `${projectId}/`;
          const noticeKey = `${s3Prefix}${projectId}${NOTICE_SUFFIX}`;
          const now = new Date();
          const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);
          const dataTime = now.getTime() - 40 * 60 * 1000;

          // A stray notice a post-cutoff project could never legitimately have —
          // the gate must leave it untouched (no delete attempted).
          await s3StorageService.uploadFile({
            fileName: noticeKey,
            fileType: "text/plain; charset=utf-8",
            data: "stray notice that must not be touched",
          });

          await prisma.blobStorageIntegration.create({
            data: {
              projectId,
              type: BlobStorageIntegrationType.S3,
              bucketName,
              prefix: s3Prefix,
              accessKeyId: minioAccessKeyId,
              secretAccessKey: encrypt(minioAccessKeySecret),
              region: region ? region : "auto",
              endpoint: minioEndpoint,
              forcePathStyle:
                env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
              enabled: true,
              exportFrequency: "hourly",
              exportSource: "EVENTS",
              fileType: BlobStorageIntegrationFileType.JSONL,
              compressed: false,
              lastSyncAt: oneHourAgo,
              // Post-exporter-cutoff: never could have used a legacy source, so
              // cleanup is skipped and the stray file is left in place. Derived
              // from the live cutoff so an env override can't flip the gate.
              createdAt: new Date(
                LEGACY_BLOB_EXPORTER_CUTOFF.getTime() + 24 * 60 * 60 * 1000,
              ),
            },
          });

          await createEventsCh([
            createEvent({
              id: randomUUID(),
              project_id: projectId,
              start_time: dataTime,
              name: "Post-cutoff Event",
            }),
          ]);

          await handleBlobStorageIntegrationProjectJob({
            data: { payload: { projectId } },
          } as Job);

          const after = await s3StorageService.listFiles(s3Prefix);
          expect(after.some((f) => f.file === noticeKey)).toBe(true);
        },
      );
    });
  });

  // LFE-10402: raw-passthrough streams ClickHouse JSONEachRow bytes straight to
  // gzip → upload, skipping the per-row JS parse/enrich/serialize pipeline. The
  // output must be parsed-equal to the standard path minus the dropped price
  // columns. Passthrough only applies to JSONL exports of observations /
  // observations_v2 and is gated behind the exportTuning.rawPassthrough flag.
  maybeDescribe("raw passthrough export (LFE-10402)", () => {
    const PRICE_COLUMNS = ["input_price", "output_price", "total_price"];
    const stripPrices = (row: Record<string, unknown>) => {
      const copy = { ...row };
      for (const col of PRICE_COLUMNS) delete copy[col];
      return copy;
    };
    const parseJsonl = (content: string): Record<string, unknown>[] =>
      content
        .trim()
        .split("\n")
        .filter(Boolean)
        .map((line) => JSON.parse(line));
    const byId = (rows: Record<string, unknown>[]) =>
      new Map(rows.map((r) => [r.id as string, r]));

    const downloadDir = async (prefix: string, dir: string) => {
      const files = await s3StorageService.listFiles(prefix);
      const file = files.find((f) => f.file.includes(`/${dir}/`));
      expect(file).toBeDefined();
      return parseJsonl(await s3StorageService.download(file!.file));
    };

    const seedModelWithPrices = async (projectId: string, modelId: string) =>
      prisma.model.create({
        data: {
          id: modelId,
          projectId,
          modelName: "gpt-4-passthrough",
          matchPattern: "gpt-4-passthrough",
          unit: "TOKENS",
          pricingTiers: {
            create: {
              name: "Standard",
              isDefault: true,
              conditions: [],
              priority: 0,
              prices: {
                createMany: {
                  data: [
                    { modelId, usageType: "input", price: "0.03" },
                    { modelId, usageType: "output", price: "0.06" },
                    { modelId, usageType: "total", price: "0.09" },
                  ],
                },
              },
            },
          },
        },
      });

    it("produces output parsed-equal to the standard path (minus price columns) for observations + events", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      const now = new Date();
      const twoHoursAgo = new Date(now.getTime() - 2 * 60 * 60 * 1000);
      const dataTime = now.getTime() - 90 * 60 * 1000;
      const modelId = randomUUID();
      const traceId = randomUUID();
      const observationId = randomUUID();
      const eventId = randomUUID();

      await seedModelWithPrices(projectId, modelId);

      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region ? region : "auto",
          endpoint: minioEndpoint,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "hourly",
          exportSource: "TRACES_OBSERVATIONS_EVENTS",
          nextSyncAt: twoHoursAgo,
          lastSyncAt: twoHoursAgo,
          compressed: false,
          fileType: BlobStorageIntegrationFileType.JSONL,
          // default (full) field groups → metadata + model selected in both paths
        },
      });

      await Promise.all([
        createTracesCh([
          createTrace({
            id: traceId,
            project_id: projectId,
            timestamp: dataTime,
            name: "Passthrough Trace",
          }),
        ]),
        createObservationsCh([
          createObservation({
            id: observationId,
            trace_id: traceId,
            project_id: projectId,
            start_time: dataTime,
            end_time: dataTime + 5500, // 5.5s → non-round latency
            completion_start_time: dataTime + 1000,
            total_cost: 42.5,
            usage_details: { input: 100, output: 200, total: 300 },
            internal_model_id: modelId,
            name: "Passthrough Observation",
            metadata: { k: "v" },
          }),
        ]),
        createEventsCh([
          createEvent({
            id: eventId,
            project_id: projectId,
            trace_id: traceId,
            type: "GENERATION",
            name: "Passthrough Event",
            start_time: dataTime * 1000,
            end_time: (dataTime + 5500) * 1000,
            completion_start_time: (dataTime + 1000) * 1000,
            model_id: modelId,
            metadata: { k: "v" },
            metadata_names: ["k"],
            metadata_values: ["v"],
          }),
        ]),
      ]);

      // 1) Standard path
      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);
      const standardObs = byId(await downloadDir(s3Prefix, "observations"));
      const standardEvents = byId(
        await downloadDir(s3Prefix, "observations_v2"),
      );

      // Sanity: the standard path enriches with price columns. Events export
      // is keyed by span_id (not the seeded event id), so grab the single row.
      const standardEventRow = [...standardEvents.values()][0];
      expect(standardObs.get(observationId)).toHaveProperty("input_price");
      expect(standardEventRow).toHaveProperty("total_price");

      // 2) Reset and re-run with rawPassthrough enabled.
      const filesToClear = await s3StorageService.listFiles(s3Prefix);
      await s3StorageService.deleteFiles(filesToClear.map((f) => f.file));
      await prisma.blobStorageIntegration.update({
        where: { projectId },
        data: {
          lastSyncAt: twoHoursAgo,
          nextSyncAt: twoHoursAgo,
          exportTuning: { rawPassthrough: true },
        },
      });

      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);
      const passthroughObs = byId(await downloadDir(s3Prefix, "observations"));
      const passthroughEvents = byId(
        await downloadDir(s3Prefix, "observations_v2"),
      );

      // Passthrough drops the price columns…
      const passthroughEventRow = [...passthroughEvents.values()][0];
      expect(passthroughObs.get(observationId)).not.toHaveProperty(
        "input_price",
      );
      expect(passthroughEventRow).not.toHaveProperty("total_price");

      // …and is otherwise parsed-equal to the standard output.
      expect(passthroughObs.size).toBe(standardObs.size);
      expect(passthroughEvents.size).toBe(standardEvents.size);
      for (const [id, standardRow] of standardObs) {
        expect(passthroughObs.get(id)).toEqual(stripPrices(standardRow));
      }
      for (const [id, standardRow] of standardEvents) {
        expect(passthroughEvents.get(id)).toEqual(stripPrices(standardRow));
      }
    }, 60_000);

    it("falls back to the standard path when fileType is not JSONL", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      const now = new Date();
      const twoHoursAgo = new Date(now.getTime() - 2 * 60 * 60 * 1000);
      const dataTime = now.getTime() - 90 * 60 * 1000;
      const eventId = randomUUID();

      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region ? region : "auto",
          endpoint: minioEndpoint,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "hourly",
          exportSource: "EVENTS",
          nextSyncAt: twoHoursAgo,
          lastSyncAt: twoHoursAgo,
          compressed: false,
          // CSV is ineligible for passthrough → standard path despite the flag
          fileType: BlobStorageIntegrationFileType.CSV,
          exportTuning: { rawPassthrough: true },
        },
      });

      await createEventsCh([
        createEvent({
          id: eventId,
          project_id: projectId,
          trace_id: randomUUID(),
          type: "GENERATION",
          name: "CSV Fallback Event",
          start_time: dataTime * 1000,
          end_time: (dataTime + 5000) * 1000,
        }),
      ]);

      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);

      const eventRows = await s3StorageService.listFiles(s3Prefix);
      const eventFile = eventRows.find((f) =>
        f.file.includes("/observations_v2/"),
      );
      expect(eventFile).toBeDefined();
      const content = await s3StorageService.download(eventFile!.file);
      // CSV header row present → standard (CSV) path ran, not JSONL passthrough.
      // (Events export keys by span_id, so assert on the seeded event name.)
      expect(content.split("\n")[0]).toContain("id");
      expect(content).toContain("CSV Fallback Event");
    }, 30_000);
  });

  maybeDescribe("Parquet export (LFE-10463)", () => {
    // E2e: fileType=PARQUET runs the real handler → MinIO. Parquet magic is
    // ASCII, so it survives the string download at both ends of the body.
    const PARQUET_MAGIC = "PAR1";

    it("exports valid .parquet files for all tables, ignoring compressed, and survives an adversarial trace name", async () => {
      const { projectId } = await createOrgProjectAndApiKey();
      s3Prefix = projectId;
      const now = new Date();
      const twoHoursAgo = new Date(now.getTime() - 2 * 60 * 60 * 1000);
      const dataTime = now.getTime() - 90 * 60 * 1000;
      const traceId = randomUUID();
      const adversarialTraceId = randomUUID();
      const observationId = randomUUID();
      const scoreId = randomUUID();
      const eventId = randomUUID();

      await prisma.blobStorageIntegration.create({
        data: {
          projectId,
          type: BlobStorageIntegrationType.S3,
          bucketName,
          prefix: s3Prefix,
          accessKeyId: minioAccessKeyId,
          secretAccessKey: encrypt(minioAccessKeySecret),
          region: region ? region : "auto",
          endpoint: minioEndpoint,
          forcePathStyle:
            env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
          enabled: true,
          exportFrequency: "hourly",
          exportSource: "TRACES_OBSERVATIONS_EVENTS",
          nextSyncAt: twoHoursAgo,
          lastSyncAt: twoHoursAgo,
          // compressed must be ignored on the parquet path (no .gz suffix).
          compressed: true,
          fileType: BlobStorageIntegrationFileType.PARQUET,
        },
      });

      await Promise.all([
        createTracesCh([
          createTrace({
            id: traceId,
            project_id: projectId,
            timestamp: dataTime,
            name: "Parquet Trace",
          }),
          // Name starting with the exception marker lands in the uncompressed
          // footer min-stat; the per-query-tag scan must not false-positive
          // (footer-DoS regression — export must still succeed).
          createTrace({
            id: adversarialTraceId,
            project_id: projectId,
            timestamp: dataTime,
            name: "\r\n__exception__\r\n adversarial",
          }),
        ]),
        createObservationsCh([
          createObservation({
            id: observationId,
            trace_id: traceId,
            project_id: projectId,
            start_time: dataTime,
            end_time: dataTime + 5000,
            name: "Parquet Observation",
          }),
        ]),
        createScoresCh([
          createTraceScore({
            id: scoreId,
            trace_id: traceId,
            project_id: projectId,
            timestamp: dataTime,
            name: "Parquet Score",
            value: 0.5,
          }),
        ]),
        createEventsCh([
          createEvent({
            id: eventId,
            project_id: projectId,
            trace_id: traceId,
            type: "GENERATION",
            name: "Parquet Event",
            start_time: dataTime * 1000,
            end_time: (dataTime + 5000) * 1000,
          }),
        ]),
      ]);

      await handleBlobStorageIntegrationProjectJob({
        data: { payload: { projectId } },
      } as Job);

      const files = await s3StorageService.listFiles(s3Prefix);
      const projectFiles = files.filter(
        (f) => f.file.includes(projectId) && !f.file.includes("/manifests/"),
      );

      // traces, observations, scores, observations_v2 (events).
      expect(projectFiles).toHaveLength(4);
      for (const f of projectFiles) {
        expect(f.file.endsWith(".parquet")).toBe(true);
        expect(f.file).not.toContain(".gz");
      }

      const manifestFile = files.find((f) => f.file.includes("/manifests/"));
      expect(manifestFile).toBeDefined();
      const manifest = JSON.parse(
        await s3StorageService.download(manifestFile!.file),
      );
      expect(manifest.files).toHaveLength(4);
      for (const f of manifest.files) {
        expect(f.fileType).toBe("PARQUET");
        expect(f.format).toBe("parquet");
        expect(f.compressed).toBe(false);
        expect(f.rowCount).toBeNull();
      }

      // Every object is a valid Parquet file (magic at both ends).
      for (const f of projectFiles) {
        const content = await s3StorageService.download(f.file);
        expect(content.startsWith(PARQUET_MAGIC)).toBe(true);
        expect(content.endsWith(PARQUET_MAGIC)).toBe(true);
      }
    }, 30_000);
  });
});