Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): scope batch action reads and writes to project (#18155)

Steffen Reuther80afb820660d
diff --git a/worker/src/__tests__/batchAction.test.ts b/worker/src/__tests__/batchAction.test.ts
index 28e830d768..a103b18ab4 100644
--- a/worker/src/__tests__/batchAction.test.ts
+++ b/worker/src/__tests__/batchAction.test.ts
@@ -1118,6 +1118,85 @@ maybeDescribe("events table batch actions", () => {
expect(updatedBatchAction?.processedCount).toBe(2);
});
+ it("should not add observations to dataset for a batch action from another project", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ const { projectId: otherProjectId } = await createOrgProjectAndApiKey();
+
+ const traceId = uuidv4();
+ await createTracesCh([
+ createTrace({
+ project_id: projectId,
+ id: traceId,
+ timestamp: new Date().getTime(),
+ }),
+ ]);
+ await createEventsCh([
+ createEvent({
+ project_id: projectId,
+ trace_id: traceId,
+ input: { prompt: "Hello" },
+ output: { response: "Hi" },
+ }),
+ ]);
+
+ const dataset = await prisma.dataset.create({
+ data: {
+ id: uuidv4(),
+ projectId,
+ name: uuidv4(),
+ },
+ });
+
+ const otherProjectBatchAction = await prisma.batchAction.create({
+ data: {
+ projectId: otherProjectId,
+ userId: "test-user",
+ actionType: "observation-add-to-dataset",
+ tableName: BatchTableNames.Events,
+ status: BatchActionStatus.Queued,
+ query: { filter: [], orderBy: null },
+ },
+ });
+
+ await expect(
+ handleBatchActionJob({
+ id: uuidv4(),
+ timestamp: new Date(),
+ name: QueueJobs.BatchActionProcessingJob as const,
+ payload: {
+ batchActionId: otherProjectBatchAction.id,
+ projectId,
+ actionId: "observation-add-to-dataset" as const,
+ tableName: BatchTableNames.Events,
+ cutoffCreatedAt: new Date(),
+ query: { filter: [], orderBy: null },
+ config: {
+ datasetId: dataset.id,
+ datasetName: dataset.name,
+ mapping: {
+ input: { mode: "full" as const },
+ expectedOutput: { mode: "full" as const },
+ metadata: { mode: "none" as const },
+ },
+ },
+ type: BatchActionType.Create,
+ },
+ }),
+ ).rejects.toThrow();
+
+ const datasetItems = await prisma.datasetItem.findMany({
+ where: { datasetId: dataset.id },
+ });
+ expect(datasetItems).toHaveLength(0);
+
+ const unchangedBatchAction = await prisma.batchAction.findUniqueOrThrow({
+ where: { id: otherProjectBatchAction.id },
+ });
+ expect(unchangedBatchAction.status).toBe(BatchActionStatus.Queued);
+ expect(unchangedBatchAction.totalCount).toBeNull();
+ expect(unchangedBatchAction.processedCount).toBeNull();
+ });
+
it("should apply jsonSelector mapping when adding events to dataset", async () => {
const { projectId } = await createOrgProjectAndApiKey();
diff --git a/worker/src/__tests__/traceBatchDeleteBatchAction.test.ts b/worker/src/__tests__/traceBatchDeleteBatchAction.test.ts
index a3429d34f5..e46f3228f0 100644
--- a/worker/src/__tests__/traceBatchDeleteBatchAction.test.ts
+++ b/worker/src/__tests__/traceBatchDeleteBatchAction.test.ts
@@ -255,6 +255,7 @@ describe("trace delete batch action worker processor", () => {
let leaseExtensions = 0;
await processTraceDeleteBatchAction({
batchActionId: batchAction.id,
+ projectId,
batchSize: 2,
extendLease: async () => {
leaseExtensions += 1;
@@ -380,6 +381,7 @@ describe("trace delete batch action worker processor", () => {
await processTraceDeleteBatchAction({
batchActionId: batchAction.id,
+ projectId,
batchSize: 2,
});
@@ -476,6 +478,7 @@ describe("trace delete batch action worker processor", () => {
await processTraceDeleteBatchAction({
batchActionId: batchAction.id,
+ projectId,
batchSize: 2,
});
@@ -530,6 +533,7 @@ describe("trace delete batch action worker processor", () => {
const result = await processTraceDeleteBatchAction({
batchActionId: batchAction.id,
+ projectId,
batchSize: 1,
shouldSkipDeletion: async (seenProjectId, traceIds, entityType) => {
skippedDeletion = {
@@ -611,6 +615,7 @@ describe("trace delete batch action worker processor", () => {
await expect(
processTraceDeleteBatchAction({
batchActionId: batchAction.id,
+ projectId,
batchSize: 2,
canCommitProgress: async () => false,
}),
@@ -630,6 +635,58 @@ describe("trace delete batch action worker processor", () => {
});
});
+ it("does not process or update a batch action from another project", async () => {
+ const { projectId } = await createOrgProjectAndApiKey({ plan: "Team" });
+ const { projectId: otherProjectId } = await createOrgProjectAndApiKey({
+ plan: "Team",
+ });
+ const deleteUserId = `delete-user-${randomUUID()}`;
+ const beforeCutoff = new Date(Date.now() - 60_000);
+ const selectedTraceIds = [randomUUID()];
+
+ await createLegacyArtifacts({
+ projectId,
+ traceIds: selectedTraceIds,
+ userId: deleteUserId,
+ timestamp: beforeCutoff,
+ });
+
+ const batchAction = await createTraceDeleteBatchAction({
+ projectId,
+ userId: `user-${randomUUID()}`,
+ useEventsTable: false,
+ cutoffCreatedAt: new Date(),
+ query: traceDeleteQuery(deleteUserId),
+ });
+
+ await expect(
+ processTraceDeleteBatchAction({
+ batchActionId: batchAction.id,
+ projectId: otherProjectId,
+ batchSize: 1,
+ }),
+ ).rejects.toThrow("not found");
+
+ await expectCountsEventually(projectId, selectedTraceIds, {
+ traces: 1,
+ observations: 1,
+ scores: 1,
+ });
+
+ const unchangedBatchAction = await prisma.batchAction.findUniqueOrThrow({
+ where: { id: batchAction.id },
+ });
+ expect(unchangedBatchAction.status).toBe(BatchActionStatus.Queued);
+ expect(unchangedBatchAction.processedCount).toBe(0);
+ expect(unchangedBatchAction.log).toBeNull();
+ expect(
+ TraceDeleteBatchActionConfigSchema.parse(unchangedBatchAction.config),
+ ).toMatchObject({
+ failureCount: 0,
+ inFlightBatch: null,
+ });
+ });
+
it("fails a persisted batch action after ten consecutive failures", async () => {
const { projectId } = await createOrgProjectAndApiKey({ plan: "Team" });
const batchActionId = randomUUID();
@@ -661,6 +718,7 @@ describe("trace delete batch action worker processor", () => {
await expect(
processTraceDeleteBatchAction({
batchActionId,
+ projectId,
batchSize: 1,
}),
).rejects.toThrow();
diff --git a/worker/src/features/batchAction/handleBatchActionJob.ts b/worker/src/features/batchAction/handleBatchActionJob.ts
index 40fc9fc6a3..8a541b7538 100644
--- a/worker/src/features/batchAction/handleBatchActionJob.ts
+++ b/worker/src/features/batchAction/handleBatchActionJob.ts
@@ -619,7 +619,7 @@ export const handleBatchActionJob = async (
}
} catch (error) {
await prisma.batchAction.update({
- where: { id: batchActionId },
+ where: { id: batchActionId, projectId },
data: {
status: BatchActionStatus.Failed,
finishedAt: new Date(),
diff --git a/worker/src/features/batchAction/processAddObservationsToDataset.ts b/worker/src/features/batchAction/processAddObservationsToDataset.ts
index a24d7b86f0..72b8b56719 100644
--- a/worker/src/features/batchAction/processAddObservationsToDataset.ts
+++ b/worker/src/features/batchAction/processAddObservationsToDataset.ts
@@ -146,7 +146,7 @@ export async function processAddObservationsToDataset(params: {
// Update status to PROCESSING
await prisma.batchAction.update({
- where: { id: batchActionId },
+ where: { id: batchActionId, projectId },
data: {
status: BatchActionStatus.Processing,
totalCount: observations.length,
@@ -179,7 +179,7 @@ export async function processAddObservationsToDataset(params: {
// Update progress periodically (every 5 chunks or at the end)
if (i % (CHUNK_SIZE * 5) === 0 || i + CHUNK_SIZE >= observations.length) {
await prisma.batchAction.update({
- where: { id: batchActionId },
+ where: { id: batchActionId, projectId },
data: { processedCount: processed, failedCount: failed },
});
}
@@ -204,7 +204,7 @@ export async function processAddObservationsToDataset(params: {
// Update final status
await prisma.batchAction.update({
- where: { id: batchActionId },
+ where: { id: batchActionId, projectId },
data: {
status: finalStatus,
finishedAt: new Date(),
diff --git a/worker/src/features/batchAction/processTraceDeleteBatchAction.ts b/worker/src/features/batchAction/processTraceDeleteBatchAction.ts
index ae1b59b034..d76da964dc 100644
--- a/worker/src/features/batchAction/processTraceDeleteBatchAction.ts
+++ b/worker/src/features/batchAction/processTraceDeleteBatchAction.ts
@@ -196,13 +196,14 @@ const failTraceDeleteBatchActionState = (
const commitTraceDeleteBatchActionState = async (opts: {
batchActionId: string;
+ projectId: string;
state: TraceDeleteBatchActionState;
canCommitProgress?: CanCommitProgress;
}) => {
await assertCanCommitProgress(opts);
await prisma.batchAction.update({
- where: { id: opts.batchActionId },
+ where: { id: opts.batchActionId, projectId: opts.projectId },
data: {
status: opts.state.status,
processedCount: opts.state.processedCount,
@@ -255,11 +256,12 @@ const getErrorMessage = (error: unknown) =>
const recordTraceDeleteBatchActionFailure = async (opts: {
batchActionId: string;
+ projectId: string;
error: unknown;
canCommitProgress?: CanCommitProgress;
}) => {
const batchAction = await prisma.batchAction.findUnique({
- where: { id: opts.batchActionId },
+ where: { id: opts.batchActionId, projectId: opts.projectId },
});
if (
@@ -278,6 +280,7 @@ const recordTraceDeleteBatchActionFailure = async (opts: {
if (failureCount >= TRACE_DELETE_BATCH_ACTION_MAX_FAILURES) {
await commitTraceDeleteBatchActionState({
batchActionId: opts.batchActionId,
+ projectId: opts.projectId,
canCommitProgress: opts.canCommitProgress,
state: failTraceDeleteBatchActionState(currentState, {
failureCount,
@@ -290,6 +293,7 @@ const recordTraceDeleteBatchActionFailure = async (opts: {
await commitTraceDeleteBatchActionState({
batchActionId: opts.batchActionId,
+ projectId: opts.projectId,
canCommitProgress: opts.canCommitProgress,
state: failTraceDeleteBatchActionState(currentState, {
failureCount,
@@ -300,6 +304,7 @@ const recordTraceDeleteBatchActionFailure = async (opts: {
export const processTraceDeleteBatchAction = async ({
batchActionId,
+ projectId,
batchSize = env.LANGFUSE_DELETE_BATCH_SIZE,
maxBatchesPerRun = TRACE_DELETE_BATCH_ACTION_MAX_BATCHES_PER_RUN,
canCommitProgress,
@@ -307,6 +312,7 @@ export const processTraceDeleteBatchAction = async ({
shouldSkipDeletion = shouldSkipDeletionFor,
}: {
batchActionId: string;
+ projectId: string;
batchSize?: number;
maxBatchesPerRun?: number;
canCommitProgress?: CanCommitProgress;
@@ -320,7 +326,7 @@ export const processTraceDeleteBatchAction = async ({
try {
while (processedBatches < maxBatches) {
const batchAction = await prisma.batchAction.findUnique({
- where: { id: batchActionId },
+ where: { id: batchActionId, projectId },
});
if (!batchAction) {
@@ -365,6 +371,7 @@ export const processTraceDeleteBatchAction = async ({
);
await commitTraceDeleteBatchActionState({
batchActionId,
+ projectId,
canCommitProgress,
state,
});
@@ -377,6 +384,7 @@ export const processTraceDeleteBatchAction = async ({
});
await commitTraceDeleteBatchActionState({
batchActionId,
+ projectId,
canCommitProgress,
state,
});
@@ -384,6 +392,7 @@ export const processTraceDeleteBatchAction = async ({
state = markTraceDeleteBatchActionProcessing(state, config);
await commitTraceDeleteBatchActionState({
batchActionId,
+ projectId,
canCommitProgress,
state,
});
@@ -416,6 +425,7 @@ export const processTraceDeleteBatchAction = async ({
});
await commitTraceDeleteBatchActionState({
batchActionId,
+ projectId,
canCommitProgress,
state,
});
@@ -451,6 +461,7 @@ export const processTraceDeleteBatchAction = async ({
state = completeTraceDeleteBatchActionState(state, processedCount);
await commitTraceDeleteBatchActionState({
batchActionId,
+ projectId,
canCommitProgress,
state,
});
@@ -464,6 +475,7 @@ export const processTraceDeleteBatchAction = async ({
);
await commitTraceDeleteBatchActionState({
batchActionId,
+ projectId,
canCommitProgress,
state,
});
@@ -482,6 +494,7 @@ export const processTraceDeleteBatchAction = async ({
try {
await recordTraceDeleteBatchActionFailure({
batchActionId,
+ projectId,
error,
canCommitProgress,
});
diff --git a/worker/src/features/trace-delete-batch-action-runner/index.ts b/worker/src/features/trace-delete-batch-action-runner/index.ts
index bd7ab4b3b5..1c382eda16 100644
--- a/worker/src/features/trace-delete-batch-action-runner/index.ts
+++ b/worker/src/features/trace-delete-batch-action-runner/index.ts
@@ -149,6 +149,7 @@ export class TraceDeleteBatchActionRunner extends PeriodicExclusiveRunner {
const result = await processTraceDeleteBatchAction({
batchActionId: batchAction.id,
+ projectId: batchAction.projectId,
batchSize: this.batchSize,
maxBatchesPerRun: this.maxBatchesPerRun,
canCommitProgress: () => this.lock.isHeldByCurrentProcess(),