🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): scope batch action reads and writes to project (#18155)
Steffen Reuther80afb820660d
diff --git a/worker/src/__tests__/batchAction.test.ts b/worker/src/__tests__/batchAction.test.tsindex 28e830d768..a103b18ab4 100644--- a/worker/src/__tests__/batchAction.test.ts+++ b/worker/src/__tests__/batchAction.test.ts@@ -1118,6 +1118,85 @@ maybeDescribe("events table batch actions", () => {expect(updatedBatchAction?.processedCount).toBe(2);});+ it("should not add observations to dataset for a batch action from another project", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ const { projectId: otherProjectId } = await createOrgProjectAndApiKey();++ const traceId = uuidv4();+ await createTracesCh([+ createTrace({+ project_id: projectId,+ id: traceId,+ timestamp: new Date().getTime(),+ }),+ ]);+ await createEventsCh([+ createEvent({+ project_id: projectId,+ trace_id: traceId,+ input: { prompt: "Hello" },+ output: { response: "Hi" },+ }),+ ]);++ const dataset = await prisma.dataset.create({+ data: {+ id: uuidv4(),+ projectId,+ name: uuidv4(),+ },+ });++ const otherProjectBatchAction = await prisma.batchAction.create({+ data: {+ projectId: otherProjectId,+ userId: "test-user",+ actionType: "observation-add-to-dataset",+ tableName: BatchTableNames.Events,+ status: BatchActionStatus.Queued,+ query: { filter: [], orderBy: null },+ },+ });++ await expect(+ handleBatchActionJob({+ id: uuidv4(),+ timestamp: new Date(),+ name: QueueJobs.BatchActionProcessingJob as const,+ payload: {+ batchActionId: otherProjectBatchAction.id,+ projectId,+ actionId: "observation-add-to-dataset" as const,+ tableName: BatchTableNames.Events,+ cutoffCreatedAt: new Date(),+ query: { filter: [], orderBy: null },+ config: {+ datasetId: dataset.id,+ datasetName: dataset.name,+ mapping: {+ input: { mode: "full" as const },+ expectedOutput: { mode: "full" as const },+ metadata: { mode: "none" as const },+ },+ },+ type: BatchActionType.Create,+ },+ }),+ ).rejects.toThrow();++ const datasetItems = await prisma.datasetItem.findMany({+ where: { datasetId: dataset.id },+ });+ expect(datasetItems).toHaveLength(0);++ const unchangedBatchAction = await prisma.batchAction.findUniqueOrThrow({+ where: { id: otherProjectBatchAction.id },+ });+ expect(unchangedBatchAction.status).toBe(BatchActionStatus.Queued);+ expect(unchangedBatchAction.totalCount).toBeNull();+ expect(unchangedBatchAction.processedCount).toBeNull();+ });+it("should apply jsonSelector mapping when adding events to dataset", async () => {const { projectId } = await createOrgProjectAndApiKey();diff --git a/worker/src/__tests__/traceBatchDeleteBatchAction.test.ts b/worker/src/__tests__/traceBatchDeleteBatchAction.test.tsindex a3429d34f5..e46f3228f0 100644--- a/worker/src/__tests__/traceBatchDeleteBatchAction.test.ts+++ b/worker/src/__tests__/traceBatchDeleteBatchAction.test.ts@@ -255,6 +255,7 @@ describe("trace delete batch action worker processor", () => {let leaseExtensions = 0;await processTraceDeleteBatchAction({batchActionId: batchAction.id,+ projectId,batchSize: 2,extendLease: async () => {leaseExtensions += 1;@@ -380,6 +381,7 @@ describe("trace delete batch action worker processor", () => {await processTraceDeleteBatchAction({batchActionId: batchAction.id,+ projectId,batchSize: 2,});@@ -476,6 +478,7 @@ describe("trace delete batch action worker processor", () => {await processTraceDeleteBatchAction({batchActionId: batchAction.id,+ projectId,batchSize: 2,});@@ -530,6 +533,7 @@ describe("trace delete batch action worker processor", () => {const result = await processTraceDeleteBatchAction({batchActionId: batchAction.id,+ projectId,batchSize: 1,shouldSkipDeletion: async (seenProjectId, traceIds, entityType) => {skippedDeletion = {@@ -611,6 +615,7 @@ describe("trace delete batch action worker processor", () => {await expect(processTraceDeleteBatchAction({batchActionId: batchAction.id,+ projectId,batchSize: 2,canCommitProgress: async () => false,}),@@ -630,6 +635,58 @@ describe("trace delete batch action worker processor", () => {});});+ it("does not process or update a batch action from another project", async () => {+ const { projectId } = await createOrgProjectAndApiKey({ plan: "Team" });+ const { projectId: otherProjectId } = await createOrgProjectAndApiKey({+ plan: "Team",+ });+ const deleteUserId = `delete-user-${randomUUID()}`;+ const beforeCutoff = new Date(Date.now() - 60_000);+ const selectedTraceIds = [randomUUID()];++ await createLegacyArtifacts({+ projectId,+ traceIds: selectedTraceIds,+ userId: deleteUserId,+ timestamp: beforeCutoff,+ });++ const batchAction = await createTraceDeleteBatchAction({+ projectId,+ userId: `user-${randomUUID()}`,+ useEventsTable: false,+ cutoffCreatedAt: new Date(),+ query: traceDeleteQuery(deleteUserId),+ });++ await expect(+ processTraceDeleteBatchAction({+ batchActionId: batchAction.id,+ projectId: otherProjectId,+ batchSize: 1,+ }),+ ).rejects.toThrow("not found");++ await expectCountsEventually(projectId, selectedTraceIds, {+ traces: 1,+ observations: 1,+ scores: 1,+ });++ const unchangedBatchAction = await prisma.batchAction.findUniqueOrThrow({+ where: { id: batchAction.id },+ });+ expect(unchangedBatchAction.status).toBe(BatchActionStatus.Queued);+ expect(unchangedBatchAction.processedCount).toBe(0);+ expect(unchangedBatchAction.log).toBeNull();+ expect(+ TraceDeleteBatchActionConfigSchema.parse(unchangedBatchAction.config),+ ).toMatchObject({+ failureCount: 0,+ inFlightBatch: null,+ });+ });+it("fails a persisted batch action after ten consecutive failures", async () => {const { projectId } = await createOrgProjectAndApiKey({ plan: "Team" });const batchActionId = randomUUID();@@ -661,6 +718,7 @@ describe("trace delete batch action worker processor", () => {await expect(processTraceDeleteBatchAction({batchActionId,+ projectId,batchSize: 1,}),).rejects.toThrow();diff --git a/worker/src/features/batchAction/handleBatchActionJob.ts b/worker/src/features/batchAction/handleBatchActionJob.tsindex 40fc9fc6a3..8a541b7538 100644--- a/worker/src/features/batchAction/handleBatchActionJob.ts+++ b/worker/src/features/batchAction/handleBatchActionJob.ts@@ -619,7 +619,7 @@ export const handleBatchActionJob = async (}} catch (error) {await prisma.batchAction.update({- where: { id: batchActionId },+ where: { id: batchActionId, projectId },data: {status: BatchActionStatus.Failed,finishedAt: new Date(),diff --git a/worker/src/features/batchAction/processAddObservationsToDataset.ts b/worker/src/features/batchAction/processAddObservationsToDataset.tsindex a24d7b86f0..72b8b56719 100644--- a/worker/src/features/batchAction/processAddObservationsToDataset.ts+++ b/worker/src/features/batchAction/processAddObservationsToDataset.ts@@ -146,7 +146,7 @@ export async function processAddObservationsToDataset(params: {// Update status to PROCESSINGawait prisma.batchAction.update({- where: { id: batchActionId },+ where: { id: batchActionId, projectId },data: {status: BatchActionStatus.Processing,totalCount: observations.length,@@ -179,7 +179,7 @@ export async function processAddObservationsToDataset(params: {// Update progress periodically (every 5 chunks or at the end)if (i % (CHUNK_SIZE * 5) === 0 || i + CHUNK_SIZE >= observations.length) {await prisma.batchAction.update({- where: { id: batchActionId },+ where: { id: batchActionId, projectId },data: { processedCount: processed, failedCount: failed },});}@@ -204,7 +204,7 @@ export async function processAddObservationsToDataset(params: {// Update final statusawait prisma.batchAction.update({- where: { id: batchActionId },+ where: { id: batchActionId, projectId },data: {status: finalStatus,finishedAt: new Date(),diff --git a/worker/src/features/batchAction/processTraceDeleteBatchAction.ts b/worker/src/features/batchAction/processTraceDeleteBatchAction.tsindex ae1b59b034..d76da964dc 100644--- a/worker/src/features/batchAction/processTraceDeleteBatchAction.ts+++ b/worker/src/features/batchAction/processTraceDeleteBatchAction.ts@@ -196,13 +196,14 @@ const failTraceDeleteBatchActionState = (const commitTraceDeleteBatchActionState = async (opts: {batchActionId: string;+ projectId: string;state: TraceDeleteBatchActionState;canCommitProgress?: CanCommitProgress;}) => {await assertCanCommitProgress(opts);await prisma.batchAction.update({- where: { id: opts.batchActionId },+ where: { id: opts.batchActionId, projectId: opts.projectId },data: {status: opts.state.status,processedCount: opts.state.processedCount,@@ -255,11 +256,12 @@ const getErrorMessage = (error: unknown) =>const recordTraceDeleteBatchActionFailure = async (opts: {batchActionId: string;+ projectId: string;error: unknown;canCommitProgress?: CanCommitProgress;}) => {const batchAction = await prisma.batchAction.findUnique({- where: { id: opts.batchActionId },+ where: { id: opts.batchActionId, projectId: opts.projectId },});if (@@ -278,6 +280,7 @@ const recordTraceDeleteBatchActionFailure = async (opts: {if (failureCount >= TRACE_DELETE_BATCH_ACTION_MAX_FAILURES) {await commitTraceDeleteBatchActionState({batchActionId: opts.batchActionId,+ projectId: opts.projectId,canCommitProgress: opts.canCommitProgress,state: failTraceDeleteBatchActionState(currentState, {failureCount,@@ -290,6 +293,7 @@ const recordTraceDeleteBatchActionFailure = async (opts: {await commitTraceDeleteBatchActionState({batchActionId: opts.batchActionId,+ projectId: opts.projectId,canCommitProgress: opts.canCommitProgress,state: failTraceDeleteBatchActionState(currentState, {failureCount,@@ -300,6 +304,7 @@ const recordTraceDeleteBatchActionFailure = async (opts: {export const processTraceDeleteBatchAction = async ({batchActionId,+ projectId,batchSize = env.LANGFUSE_DELETE_BATCH_SIZE,maxBatchesPerRun = TRACE_DELETE_BATCH_ACTION_MAX_BATCHES_PER_RUN,canCommitProgress,@@ -307,6 +312,7 @@ export const processTraceDeleteBatchAction = async ({shouldSkipDeletion = shouldSkipDeletionFor,}: {batchActionId: string;+ projectId: string;batchSize?: number;maxBatchesPerRun?: number;canCommitProgress?: CanCommitProgress;@@ -320,7 +326,7 @@ export const processTraceDeleteBatchAction = async ({try {while (processedBatches < maxBatches) {const batchAction = await prisma.batchAction.findUnique({- where: { id: batchActionId },+ where: { id: batchActionId, projectId },});if (!batchAction) {@@ -365,6 +371,7 @@ export const processTraceDeleteBatchAction = async ({);await commitTraceDeleteBatchActionState({batchActionId,+ projectId,canCommitProgress,state,});@@ -377,6 +384,7 @@ export const processTraceDeleteBatchAction = async ({});await commitTraceDeleteBatchActionState({batchActionId,+ projectId,canCommitProgress,state,});@@ -384,6 +392,7 @@ export const processTraceDeleteBatchAction = async ({state = markTraceDeleteBatchActionProcessing(state, config);await commitTraceDeleteBatchActionState({batchActionId,+ projectId,canCommitProgress,state,});@@ -416,6 +425,7 @@ export const processTraceDeleteBatchAction = async ({});await commitTraceDeleteBatchActionState({batchActionId,+ projectId,canCommitProgress,state,});@@ -451,6 +461,7 @@ export const processTraceDeleteBatchAction = async ({state = completeTraceDeleteBatchActionState(state, processedCount);await commitTraceDeleteBatchActionState({batchActionId,+ projectId,canCommitProgress,state,});@@ -464,6 +475,7 @@ export const processTraceDeleteBatchAction = async ({);await commitTraceDeleteBatchActionState({batchActionId,+ projectId,canCommitProgress,state,});@@ -482,6 +494,7 @@ export const processTraceDeleteBatchAction = async ({try {await recordTraceDeleteBatchActionFailure({batchActionId,+ projectId,error,canCommitProgress,});diff --git a/worker/src/features/trace-delete-batch-action-runner/index.ts b/worker/src/features/trace-delete-batch-action-runner/index.tsindex bd7ab4b3b5..1c382eda16 100644--- a/worker/src/features/trace-delete-batch-action-runner/index.ts+++ b/worker/src/features/trace-delete-batch-action-runner/index.ts@@ -149,6 +149,7 @@ export class TraceDeleteBatchActionRunner extends PeriodicExclusiveRunner {const result = await processTraceDeleteBatchAction({batchActionId: batchAction.id,+ projectId: batchAction.projectId,batchSize: this.batchSize,maxBatchesPerRun: this.maxBatchesPerRun,canCommitProgress: () => this.lock.isHeldByCurrentProcess(),