Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix: allow one active batch export per user and project (#18725)

Niklas Semmlerd07952ca9da9
diff --git a/packages/shared/src/features/batchExport/types.ts b/packages/shared/src/features/batchExport/types.ts
index a71980d16e..b6c6964aac 100644
--- a/packages/shared/src/features/batchExport/types.ts
+++ b/packages/shared/src/features/batchExport/types.ts
@@ -19,6 +19,13 @@ export enum BatchExportStatus {
CANCELLED = "CANCELLED",
}
+/** Interval at which the worker refreshes a running export's updatedAt. */
+export const BATCH_EXPORT_HEARTBEAT_INTERVAL_MS = 5 * 60 * 1000;
+
+/** Age after which an untouched QUEUED/PROCESSING export stops blocking its user's next one. */
+export const BATCH_EXPORT_ABANDONED_AFTER_MS =
+ 3 * BATCH_EXPORT_HEARTBEAT_INTERVAL_MS;
+
export enum BatchExportFileFormat {
JSON = "JSON",
CSV = "CSV",
diff --git a/web/src/__tests__/server/batchExport-trpc.servertest.ts b/web/src/__tests__/server/batchExport-trpc.servertest.ts
index 9a338e48d2..4fe0a602cb 100644
--- a/web/src/__tests__/server/batchExport-trpc.servertest.ts
+++ b/web/src/__tests__/server/batchExport-trpc.servertest.ts
@@ -2,7 +2,11 @@ import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import { appRouter } from "@/src/server/api/root";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
import { prisma, type Role } from "@langfuse/shared/src/db";
-import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
+import {
+ BatchExportQueue,
+ createOrgProjectAndApiKey,
+} from "@langfuse/shared/src/server";
+import { vi } from "vitest";
import { randomUUID } from "crypto";
import type { Session } from "next-auth";
import {
@@ -577,3 +581,116 @@ describe("batchExport tRPC – cancel", () => {
expect(cancelled?.status).toBe(BatchExportStatus.CANCELLED);
});
});
+
+describe("batchExport tRPC – one active export per user and project", () => {
+ afterAll(async () => {
+ await prisma.organization.deleteMany({
+ where: { id: { in: __orgIds } },
+ });
+ });
+
+ const tracesQuery = {
+ tableName: BatchTableNames.Traces,
+ filter: null,
+ orderBy: null,
+ };
+
+ const setup = async () => {
+ const { project, org } = await createOrgProjectAndApiKey();
+ __orgIds.push(org.id);
+ const caller = appRouter.createCaller({
+ ...createInnerTRPCContext({
+ session: makeSession(org.id, org.name, project.id, project.name, {
+ projectRole: "OWNER",
+ }),
+ headers: {},
+ }),
+ prisma,
+ });
+ const createExport = () =>
+ caller.batchExport.create({
+ projectId: project.id,
+ name: "new export",
+ query: tracesQuery,
+ format: BatchExportFileFormat.CSV,
+ });
+ const seedExport = (data: {
+ status: BatchExportStatus;
+ userId?: string;
+ createdAt?: Date;
+ updatedAt?: Date;
+ }) =>
+ prisma.batchExport.create({
+ data: {
+ projectId: project.id,
+ userId: "user-test",
+ name: "existing export",
+ format: BatchExportFileFormat.CSV,
+ query: tracesQuery,
+ ...data,
+ },
+ });
+ return { project, caller, createExport, seedExport };
+ };
+
+ it("rejects a second export and names when the running one started", async () => {
+ const { project, createExport, seedExport } = await setup();
+ await seedExport({
+ status: BatchExportStatus.PROCESSING,
+ createdAt: new Date("2026-01-02T03:04:05Z"),
+ });
+
+ await expect(createExport()).rejects.toMatchObject({
+ code: "CONFLICT",
+ message: expect.stringContaining("2026-01-02 03:04 UTC"),
+ });
+ expect(
+ await prisma.batchExport.count({ where: { projectId: project.id } }),
+ ).toBe(1);
+ });
+
+ it("accepts a new export once the running one is cancelled", async () => {
+ const { project, caller, createExport, seedExport } = await setup();
+ const running = await seedExport({ status: BatchExportStatus.QUEUED });
+
+ await caller.batchExport.cancel({
+ projectId: project.id,
+ batchExportId: running.id,
+ });
+
+ await expect(createExport()).resolves.toBeUndefined();
+ });
+
+ it("fails an export that could not be queued so it does not block a retry", async () => {
+ const { project, createExport } = await setup();
+ const queue = BatchExportQueue.getInstance();
+ const add = vi
+ .spyOn(queue!, "add")
+ .mockRejectedValueOnce(new Error("Redis unavailable"));
+
+ await expect(createExport()).rejects.toMatchObject({
+ code: "INTERNAL_SERVER_ERROR",
+ });
+ add.mockRestore();
+
+ const [failed] = await prisma.batchExport.findMany({
+ where: { projectId: project.id },
+ });
+ expect(failed.status).toBe(BatchExportStatus.FAILED);
+ await expect(createExport()).resolves.toBeUndefined();
+ });
+
+ it("ignores other users' exports and abandoned ones", async () => {
+ const { createExport, seedExport } = await setup();
+ await seedExport({
+ status: BatchExportStatus.PROCESSING,
+ userId: "other-user",
+ });
+ await seedExport({
+ status: BatchExportStatus.PROCESSING,
+ updatedAt: new Date(Date.now() - 2 * 60 * 60 * 1000),
+ });
+
+ await expect(createExport()).resolves.toBeUndefined();
+ });
+});
diff --git a/web/src/features/batch-exports/server/batchExport.ts b/web/src/features/batch-exports/server/batchExport.ts
index 177f442900..ee666a6ab4 100644
--- a/web/src/features/batch-exports/server/batchExport.ts
+++ b/web/src/features/batch-exports/server/batchExport.ts
@@ -13,6 +13,7 @@ import {
protectedProjectProcedure,
} from "@/src/server/api/trpc";
import {
+ BATCH_EXPORT_ABANDONED_AFTER_MS,
BatchExportStatus,
BatchExportTableName,
CreateBatchExportSchema,
@@ -20,7 +21,7 @@ import {
LangfuseNotFoundError,
paginationZod,
} from "@langfuse/shared";
-import { type Prisma } from "@langfuse/shared/src/db";
+import { type Prisma, type PrismaClient } from "@langfuse/shared/src/db";
import {
BatchExportQueue,
logger,
@@ -70,6 +71,10 @@ const assertCanReadAuditLogs = (session: AuthedSession, projectId: string) => {
});
};
+/** Formats a timestamp as "YYYY-MM-DD HH:mm UTC" for server-rendered messages. */
+const formatUtc = (date: Date) =>
+ `${date.toISOString().slice(0, 16).replace("T", " ")} UTC`;
+
const isDownloadWindowExpired = (batchExport: {
finishedAt: Date | null;
expiresAt: Date | null;
@@ -117,6 +122,8 @@ export const batchExportRouter = createTRPCRouter({
logger.info("[BATCH EXPORT] Creating export job", { job: input });
const userId = ctx.session.user.id;
+ await assertNoActiveExport({ prisma: ctx.prisma, projectId, userId });
+
// Create export job
const exportJob = await ctx.prisma.batchExport.create({
data: {
@@ -139,15 +146,10 @@ export const batchExportRouter = createTRPCRouter({
after: exportJob,
});
- // Notify worker
- await BatchExportQueue.getInstance()?.add(QueueJobs.BatchExportJob, {
- id: exportJob.id, // Use the batchExportId to deduplicate when the same job is sent multiple times
- name: QueueJobs.BatchExportJob,
- timestamp: new Date(),
- payload: {
- batchExportId: exportJob.id,
- projectId,
- },
+ await enqueueExportOrMarkFailed({
+ prisma: ctx.prisma,
+ projectId,
+ batchExportId: exportJob.id,
});
} catch (e) {
logger.error("[BATCH EXPORT] Failed to create export job", e);
@@ -346,3 +348,63 @@ export const batchExportRouter = createTRPCRouter({
};
}),
});
+
+/** Rejects a new export while the user still has a queued or running one in the project. */
+const assertNoActiveExport = async ({
+ prisma,
+ projectId,
+ userId,
+}: {
+ prisma: PrismaClient;
+ projectId: string;
+ userId: string;
+}) => {
+ const activeExport = await prisma.batchExport.findFirst({
+ where: {
+ projectId,
+ userId,
+ status: { in: [BatchExportStatus.QUEUED, BatchExportStatus.PROCESSING] },
+ updatedAt: {
+ gte: new Date(Date.now() - BATCH_EXPORT_ABANDONED_AFTER_MS),
+ },
+ },
+ select: { createdAt: true },
+ orderBy: { createdAt: "asc" },
+ });
+ if (activeExport) {
+ throw new TRPCError({
+ code: "CONFLICT",
+ message: `Your export started at ${formatUtc(activeExport.createdAt)} is still running. Wait for it to finish or cancel it under Settings > Exports.`,
+ });
+ }
+};
+
+/** Queues the export job; marks the row FAILED if queueing throws so it does not block a retry. */
+const enqueueExportOrMarkFailed = async ({
+ prisma,
+ projectId,
+ batchExportId,
+}: {
+ prisma: PrismaClient;
+ projectId: string;
+ batchExportId: string;
+}) => {
+ try {
+ await BatchExportQueue.getInstance()?.add(QueueJobs.BatchExportJob, {
+ id: batchExportId, // Use the batchExportId to deduplicate when the same job is sent multiple times
+ name: QueueJobs.BatchExportJob,
+ timestamp: new Date(),
+ payload: { batchExportId, projectId },
+ });
+ } catch (enqueueError) {
+ await prisma.batchExport.update({
+ where: { id: batchExportId, projectId },
+ data: {
+ status: BatchExportStatus.FAILED,
+ finishedAt: new Date(),
+ log: "The export could not be queued. Please try again.",
+ },
+ });
+ throw enqueueError;
+ }
+};
diff --git a/worker/src/__tests__/batchExport.test.ts b/worker/src/__tests__/batchExport.test.ts
index 5813165a15..43a5a04e95 100644
--- a/worker/src/__tests__/batchExport.test.ts
+++ b/worker/src/__tests__/batchExport.test.ts
@@ -18,10 +18,15 @@ import {
} from "@langfuse/shared/src/server";
import {
BatchExportFileFormat,
+ BatchExportStatus,
BatchExportTableName,
DatasetStatus,
} from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
+import {
+ claimExportSlot,
+ touchRunningExport,
+} from "../features/batchExport/handleBatchExportJob";
import { getDatabaseReadStreamPaginated } from "../features/database-read-stream/getDatabaseReadStream";
import { getObservationStream } from "../features/database-read-stream/observation-stream";
import { getTraceStream } from "../features/database-read-stream/trace-stream";
@@ -5332,3 +5337,121 @@ maybeDescribe("getEventsForBlobStorageExport", () => {
expect(rows[0].output).toBe("world");
});
});
+
+describe("batch export per-user concurrency", () => {
+ const createExport = ({
+ projectId,
+ status,
+ userId = "user-1",
+ updatedAt,
+ }: {
+ projectId: string;
+ status: BatchExportStatus;
+ userId?: string;
+ updatedAt?: Date;
+ }) =>
+ prisma.batchExport.create({
+ data: {
+ projectId,
+ userId,
+ name: "export",
+ status,
+ format: BatchExportFileFormat.CSV,
+ query: {},
+ ...(updatedAt ? { updatedAt } : {}),
+ },
+ });
+
+ const claim = (e: { id: string; projectId: string; userId: string }) =>
+ claimExportSlot({
+ projectId: e.projectId,
+ userId: e.userId,
+ batchExportId: e.id,
+ });
+
+ it("lets exactly one of a user's two concurrent exports start", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ const exports = await Promise.all([
+ createExport({ projectId, status: BatchExportStatus.QUEUED }),
+ createExport({ projectId, status: BatchExportStatus.QUEUED }),
+ ]);
+
+ const claims = await Promise.all(exports.map(claim));
+
+ expect(claims.filter(Boolean)).toHaveLength(1);
+ const stored = await prisma.batchExport.findMany({ where: { projectId } });
+ expect(stored.map((e) => e.status).sort()).toEqual([
+ BatchExportStatus.PROCESSING,
+ BatchExportStatus.QUEUED,
+ ]);
+ });
+
+ it("lets different users in a project export at the same time", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ await createExport({ projectId, status: BatchExportStatus.PROCESSING });
+ const queued = await createExport({
+ projectId,
+ status: BatchExportStatus.QUEUED,
+ userId: "user-2",
+ });
+
+ await expect(claim(queued)).resolves.toBe(true);
+ });
+
+ it("ignores the user's exports in other projects and abandoned ones", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ const { projectId: otherProjectId } = await createOrgProjectAndApiKey();
+ await createExport({
+ projectId: otherProjectId,
+ status: BatchExportStatus.PROCESSING,
+ });
+ await createExport({
+ projectId,
+ status: BatchExportStatus.PROCESSING,
+ updatedAt: new Date(Date.now() - 2 * 60 * 60 * 1000),
+ });
+ const queued = await createExport({
+ projectId,
+ status: BatchExportStatus.QUEUED,
+ });
+
+ await expect(claim(queued)).resolves.toBe(true);
+ });
+
+ it("lets a retried export reclaim its own slot", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ const running = await createExport({
+ projectId,
+ status: BatchExportStatus.PROCESSING,
+ });
+
+ await expect(claim(running)).resolves.toBe(true);
+ });
+
+ it("keeps a long-running export holding its slot through the heartbeat", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ const longAgo = new Date(Date.now() - 2 * 60 * 60 * 1000);
+ const running = await createExport({
+ projectId,
+ status: BatchExportStatus.PROCESSING,
+ updatedAt: longAgo,
+ });
+ const cancelled = await createExport({
+ projectId,
+ status: BatchExportStatus.CANCELLED,
+ updatedAt: longAgo,
+ });
+ await touchRunningExport({ projectId, batchExportId: running.id });
+ await touchRunningExport({ projectId, batchExportId: cancelled.id });
+
+ const queued = await createExport({
+ projectId,
+ status: BatchExportStatus.QUEUED,
+ });
+ await expect(claim(queued)).resolves.toBe(false);
+ const untouched = await prisma.batchExport.findUnique({
+ where: { id: cancelled.id },
+ });
+ expect(untouched?.updatedAt).toEqual(longAgo);
+ });
+});
diff --git a/worker/src/__tests__/handleBatchExportJob-comment-filters.unit.test.ts b/worker/src/__tests__/handleBatchExportJob-comment-filters.unit.test.ts
index cb1945f288..d97f720067 100644
--- a/worker/src/__tests__/handleBatchExportJob-comment-filters.unit.test.ts
+++ b/worker/src/__tests__/handleBatchExportJob-comment-filters.unit.test.ts
@@ -13,14 +13,18 @@ const mocks = vi.hoisted(() => ({
updateBatchExport: vi.fn().mockResolvedValue(undefined),
}));
-vi.mock("@langfuse/shared/src/db", () => ({
- prisma: {
+vi.mock("@langfuse/shared/src/db", () => {
+ const prisma = {
+ $executeRaw: vi.fn(),
+ $transaction: (fn: (tx: unknown) => unknown) => fn(prisma),
batchExport: {
+ count: vi.fn().mockResolvedValue(0),
findFirst: mocks.findBatchExport,
update: mocks.updateBatchExport,
},
- },
-}));
+ };
+ return { prisma };
+});
vi.mock("@langfuse/shared/src/server", () => ({
applyCommentFilters: mocks.applyCommentFilters,
diff --git a/worker/src/__tests__/handleBatchExportJob-read-error.unit.test.ts b/worker/src/__tests__/handleBatchExportJob-read-error.unit.test.ts
index f50ef50b94..46b0d4161c 100644
--- a/worker/src/__tests__/handleBatchExportJob-read-error.unit.test.ts
+++ b/worker/src/__tests__/handleBatchExportJob-read-error.unit.test.ts
@@ -17,14 +17,18 @@ const mocks = vi.hoisted(() => ({
getSignedUrl: vi.fn(),
}));
-vi.mock("@langfuse/shared/src/db", () => ({
- prisma: {
+vi.mock("@langfuse/shared/src/db", () => {
+ const prisma = {
+ $executeRaw: vi.fn(),
+ $transaction: (fn: (tx: unknown) => unknown) => fn(prisma),
batchExport: {
+ count: vi.fn().mockResolvedValue(0),
findFirst: mocks.findBatchExport,
update: mocks.updateBatchExport,
},
- },
-}));
+ };
+ return { prisma };
+});
vi.mock("@langfuse/shared/src/server", () => ({
applyCommentFilters: mocks.applyCommentFilters,
diff --git a/worker/src/features/batchExport/handleBatchExportJob.ts b/worker/src/features/batchExport/handleBatchExportJob.ts
index 746c86d35d..685bdce87b 100644
--- a/worker/src/features/batchExport/handleBatchExportJob.ts
+++ b/worker/src/features/batchExport/handleBatchExportJob.ts
@@ -1,6 +1,8 @@
/* eslint-disable no-nested-ternary */
import { pipeline, Transform } from "stream";
import {
+ BATCH_EXPORT_ABANDONED_AFTER_MS,
+ BATCH_EXPORT_HEARTBEAT_INTERVAL_MS,
BatchExportFileFormat,
BatchExportQuerySchema,
BatchExportStatus,
@@ -8,7 +10,7 @@ import {
exportOptions,
LangfuseNotFoundError,
} from "@langfuse/shared";
-import { prisma } from "@langfuse/shared/src/db";
+import { type BatchExport, prisma } from "@langfuse/shared/src/db";
import {
StorageServiceFactory,
sendBatchExportSuccessEmail,
@@ -45,6 +47,16 @@ const BATCH_EXPORT_CLICKHOUSE_SERVICE: PreferredClickhouseService = "ReadOnly";
const isCausedBy = (error: unknown, target: unknown): boolean =>
findInCauseChain(error, (link) => link === target) !== undefined;
+/** Thrown when the user already runs an export in the project, to defer the job. */
+export class BatchExportUserBusyError extends Error {
+ constructor(batchExportId: string) {
+ super(
+ `User already runs a batch export in this project; deferring ${batchExportId}`,
+ );
+ this.name = "BatchExportUserBusyError";
+ }
+}
+
export const handleBatchExportJob = async (
batchExportJob: BatchExportJobType,
) => {
@@ -125,17 +137,36 @@ export const handleBatchExportJob = async (
);
}
- // Set job status to processing
- await prisma.batchExport.update({
- where: {
- id: batchExportId,
+ if (
+ !(await claimExportSlot({
projectId,
- },
- data: {
- status: BatchExportStatus.PROCESSING,
- },
- });
+ userId: jobDetails.userId,
+ batchExportId,
+ }))
+ ) {
+ throw new BatchExportUserBusyError(batchExportId);
+ }
+ const stopHeartbeat = startExportHeartbeat({ projectId, batchExportId });
+ try {
+ await runExport({ projectId, batchExportId, jobDetails, span });
+ } finally {
+ stopHeartbeat();
+ }
+};
+
+/** Streams the export to blob storage, marks it COMPLETED and emails the user. */
+const runExport = async ({
+ projectId,
+ batchExportId,
+ jobDetails,
+ span,
+}: {
+ projectId: string;
+ batchExportId: string;
+ jobDetails: BatchExport;
+ span: ReturnType<typeof getCurrentSpan>;
+}) => {
// Parse query from job
const parsedQuery = BatchExportQuerySchema.safeParse(jobDetails.query);
if (!parsedQuery.success) {
@@ -369,3 +400,70 @@ export const handleBatchExportJob = async (
);
}
};
+
+/** Refreshes a running export's updatedAt until stopped, so it keeps holding its slot. */
+const startExportHeartbeat = (params: {
+ projectId: string;
+ batchExportId: string;
+}) => {
+ const timer = setInterval(() => {
+ touchRunningExport(params).catch((error) =>
+ logger.warn(
+ `[BATCH EXPORT] Heartbeat failed for ${params.batchExportId}`,
+ error,
+ ),
+ );
+ }, BATCH_EXPORT_HEARTBEAT_INTERVAL_MS);
+ timer.unref();
+ return () => clearInterval(timer);
+};
+
+/** Bumps updatedAt of the export if it is still PROCESSING. */
+export const touchRunningExport = ({
+ projectId,
+ batchExportId,
+}: {
+ projectId: string;
+ batchExportId: string;
+}) =>
+ prisma.batchExport.updateMany({
+ where: {
+ id: batchExportId,
+ projectId,
+ status: BatchExportStatus.PROCESSING,
+ },
+ data: { updatedAt: new Date() },
+ });
+
+/** Claims the user's per-project export slot, serialized across workers by an advisory lock. */
+export const claimExportSlot = async ({
+ projectId,
+ userId,
+ batchExportId,
+}: {
+ projectId: string;
+ userId: string;
+ batchExportId: string;
+}): Promise<boolean> =>
+ prisma.$transaction(async (tx) => {
+ await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${`batch-export:${projectId}:${userId}`}, 0))`;
+
+ const running = await tx.batchExport.count({
+ where: {
+ projectId,
+ userId,
+ id: { not: batchExportId },
+ status: BatchExportStatus.PROCESSING,
+ updatedAt: {
+ gte: new Date(Date.now() - BATCH_EXPORT_ABANDONED_AFTER_MS),
+ },
+ },
+ });
+ if (running > 0) return false;
+
+ await tx.batchExport.update({
+ where: { id: batchExportId, projectId },
+ data: { status: BatchExportStatus.PROCESSING },
+ });
+ return true;
+ });
diff --git a/worker/src/queues/__tests__/batchExportQueue.test.ts b/worker/src/queues/__tests__/batchExportQueue.test.ts
index a61e7cf2e3..b23afce889 100644
--- a/worker/src/queues/__tests__/batchExportQueue.test.ts
+++ b/worker/src/queues/__tests__/batchExportQueue.test.ts
@@ -1,4 +1,4 @@
-import { Job } from "bullmq";
+import { DelayedError, Job } from "bullmq";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { BatchExportStatus } from "@langfuse/shared";
import {
@@ -10,6 +10,7 @@ import {
const mocks = vi.hoisted(() => ({
handleBatchExportJob: vi.fn(),
updateBatchExport: vi.fn().mockResolvedValue(undefined),
+ BatchExportUserBusyError: class extends Error {},
}));
vi.mock("@langfuse/shared/src/db", () => ({
@@ -18,6 +19,7 @@ vi.mock("@langfuse/shared/src/db", () => ({
vi.mock("../../features/batchExport/handleBatchExportJob", () => ({
handleBatchExportJob: mocks.handleBatchExportJob,
+ BatchExportUserBusyError: mocks.BatchExportUserBusyError,
}));
import { batchExportQueueProcessor } from "../batchExportQueue";
@@ -80,6 +82,28 @@ describe("batchExportQueueProcessor", () => {
);
});
+ it("defers the job without failing it while the user's other export runs", async () => {
+ vi.useFakeTimers({ now: new Date("2026-01-01T00:00:00Z") });
+ mocks.handleBatchExportJob.mockRejectedValue(
+ new mocks.BatchExportUserBusyError(),
+ );
+ const moveToDelayed = vi.fn().mockResolvedValue(undefined);
+
+ const thrown = await batchExportQueueProcessor({
+ ...job,
+ token: "token-1",
+ moveToDelayed,
+ } as unknown as typeof job).catch((e) => e);
+ vi.useRealTimers();
+
+ expect(thrown).toBeInstanceOf(DelayedError);
+ expect(moveToDelayed).toHaveBeenCalledWith(
+ new Date("2026-01-01T00:00:30Z").getTime(),
+ "token-1",
+ );
+ expect(mocks.updateBatchExport).not.toHaveBeenCalled();
+ });
+
it("rethrows other errors so BullMQ retries them", async () => {
const uploadError = new Error("Failed to upload file to S3 (buffered)");
mocks.handleBatchExportJob.mockRejectedValue(uploadError);
diff --git a/worker/src/queues/batchExportQueue.ts b/worker/src/queues/batchExportQueue.ts
index bf190bd049..5092c6898b 100644
--- a/worker/src/queues/batchExportQueue.ts
+++ b/worker/src/queues/batchExportQueue.ts
@@ -1,4 +1,4 @@
-import { Job, UnrecoverableError } from "bullmq";
+import { DelayedError, Job, UnrecoverableError } from "bullmq";
import {
BaseError,
@@ -13,9 +13,14 @@ import {
logger,
} from "@langfuse/shared/src/server";
import { QueueName, TQueueJobTypes } from "@langfuse/shared/src/server";
-import { handleBatchExportJob } from "../features/batchExport/handleBatchExportJob";
+import {
+ BatchExportUserBusyError,
+ handleBatchExportJob,
+} from "../features/batchExport/handleBatchExportJob";
import { findInCauseChain } from "../features/batchExport/findInCauseChain";
+const USER_BUSY_RETRY_DELAY_MS = 30_000;
+
/**
* Server-side kills of this query alone: TIMEOUT_EXCEEDED and the per-query
* MEMORY_LIMIT_EXCEEDED, in both the older and newer ClickHouse wording.
@@ -41,6 +46,11 @@ export const batchExportQueueProcessor = async (
return true;
} catch (e) {
+ if (e instanceof BatchExportUserBusyError) {
+ logger.info(`[BATCH EXPORT] ${e.message}`);
+ await job.moveToDelayed(Date.now() + USER_BUSY_RETRY_DELAY_MS, job.token);
+ throw new DelayedError();
+ }
if (e instanceof LangfuseNotFoundError) {
logger.warn(
`[BATCH EXPORT] Batch export ${job.data.payload.batchExportId} not found. Job will be skipped.`,