🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(annotation-queues): validate queue when adding items (#19407)
Steffen Reuther0e0c2029e249
diff --git a/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts b/web/src/__tests__/server/annotation-queue-items-trpc.servertest.tsindex 7067881d8e..b0f62ca32a 100644--- a/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts+++ b/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts@@ -131,6 +131,90 @@ describe("annotation queues trpc", () => {});});+ describe("annotationQueueItems.createMany", () => {+ it.each([false, true])(+ "rejects a queue from another project without writing (isBatchAction: %s)",+ async (isBatchAction) => {+ const setup = await createOrgProjectAndApiKey();+ const otherSetup = await createOrgProjectAndApiKey();+ orgIds.push(setup.org.id, otherSetup.org.id);+ const { caller } = createCallerForProjectRole(setup);+ const otherQueue = await prisma.annotationQueue.create({+ data: {+ name: "Other Queue",+ scoreConfigIds: [],+ projectId: otherSetup.project.id,+ },+ });++ await expect(+ caller.annotationQueueItems.createMany({+ projectId: setup.project.id,+ queueId: otherQueue.id,+ objectIds: [uuidv4()],+ objectType: AnnotationQueueObjectType.TRACE,+ isBatchAction,+ query: isBatchAction ? { filter: [], orderBy: null } : undefined,+ }),+ ).rejects.toMatchObject({ code: "NOT_FOUND" });++ expect(+ await prisma.annotationQueueItem.count({+ where: { queueId: otherQueue.id },+ }),+ ).toBe(0);+ expect(+ await prisma.batchAction.count({+ where: { projectId: setup.project.id },+ }),+ ).toBe(0);+ },+ );+ });++ describe("annotationQueues.byObjectId", () => {+ it("ignores items whose project differs from the queue's project", async () => {+ const setup = await createOrgProjectAndApiKey();+ const otherSetup = await createOrgProjectAndApiKey();+ orgIds.push(setup.org.id, otherSetup.org.id);+ const { caller } = createCallerForProjectRole(setup);+ const queue = await prisma.annotationQueue.create({+ data: {+ name: "Test Queue",+ scoreConfigIds: [],+ projectId: setup.project.id,+ },+ });+ const objectId = uuidv4();+ await prisma.annotationQueueItem.create({+ data: {+ queueId: queue.id,+ objectId,+ objectType: AnnotationQueueObjectType.TRACE,+ projectId: otherSetup.project.id,+ },+ });++ await expect(+ caller.annotationQueues.byObjectId({+ projectId: setup.project.id,+ objectId,+ objectType: AnnotationQueueObjectType.TRACE,+ }),+ ).resolves.toEqual({+ queues: [+ {+ id: queue.id,+ name: queue.name,+ itemId: undefined,+ status: undefined,+ },+ ],+ totalCount: 0,+ });+ });+ });+describe("annotationQueues.create", () => {it("rejects a second queue on cloud:hobby", async () => {const setup = await createOrgProjectAndApiKey();diff --git a/web/src/__tests__/server/traces-batch-delete-trpc.servertest.ts b/web/src/__tests__/server/traces-batch-delete-trpc.servertest.tsindex e7f3cb0fde..9dbc745726 100644--- a/web/src/__tests__/server/traces-batch-delete-trpc.servertest.ts+++ b/web/src/__tests__/server/traces-batch-delete-trpc.servertest.ts@@ -589,11 +589,14 @@ describe("traces.deleteMany batch action", () => {const { projectId, caller } = await createCaller({v4BetaEnabled: true,});+ const queue = await prisma.annotationQueue.create({+ data: { projectId, name: `queue-${randomUUID()}` },+ });await expect(caller.annotationQueueItems.createMany({projectId,- queueId: randomUUID(),+ queueId: queue.id,objectIds: [randomUUID()],objectType: AnnotationQueueObjectType.TRACE,isBatchAction: true,diff --git a/web/src/features/annotation-queues/server/annotationQueueItemsRouter.ts b/web/src/features/annotation-queues/server/annotationQueueItemsRouter.tsindex c7d5eccea2..85ef2a8a8b 100644--- a/web/src/features/annotation-queues/server/annotationQueueItemsRouter.ts+++ b/web/src/features/annotation-queues/server/annotationQueueItemsRouter.ts@@ -297,6 +297,21 @@ export const queueItemRouter = createTRPCRouter({scope: "annotationQueues:CUD",});+ const queue = await ctx.prisma.annotationQueue.findUnique({+ where: {+ id: input.queueId,+ projectId: input.projectId,+ },+ select: {+ name: true,+ id: true,+ },+ });++ if (!queue) {+ throw new LangfuseNotFoundError("Queue not found in project");+ }+let createdCount = 0;if (input.isBatchAction && input.query) {@@ -353,21 +368,10 @@ export const queueItemRouter = createTRPCRouter({}}- const queue = await ctx.prisma.annotationQueue.findUnique({- where: {- id: input.queueId,- projectId: input.projectId,- },- select: {- name: true,- id: true,- },- });-return {createdCount,- queueName: queue?.name,- queueId: queue?.id,+ queueName: queue.name,+ queueId: queue.id,};}),deleteMany: protectedProjectProcedurediff --git a/web/src/features/annotation-queues/server/annotationQueuesRouter.ts b/web/src/features/annotation-queues/server/annotationQueuesRouter.tsindex fc06bdf2ad..be8e5c83f5 100644--- a/web/src/features/annotation-queues/server/annotationQueuesRouter.ts+++ b/web/src/features/annotation-queues/server/annotationQueuesRouter.ts@@ -279,6 +279,7 @@ export const queueRouter = createTRPCRouter({name: true,annotationQueueItem: {where: {+ projectId: input.projectId,objectId: input.objectId,objectType: input.objectType,},diff --git a/worker/src/__tests__/batchAction.test.ts b/worker/src/__tests__/batchAction.test.tsindex a103b18ab4..ef708df521 100644--- a/worker/src/__tests__/batchAction.test.ts+++ b/worker/src/__tests__/batchAction.test.ts@@ -863,6 +863,49 @@ describe("select all test suite", () => {expect(queueItems.every((item) => item.objectType === "TRACE")).toBe(true);});+ it("should not add traces to an annotation queue from another project", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ const { projectId: otherProjectId } = await createOrgProjectAndApiKey();++ const traceId = uuidv4();+ await createTracesCh([+ createTrace({+ project_id: projectId,+ id: traceId,+ timestamp: new Date("2024-01-01").getTime(),+ }),+ ]);++ const otherQueueId = uuidv4();+ await prisma.annotationQueue.create({+ data: {+ id: otherQueueId,+ projectId: otherProjectId,+ name: "other-queue",+ },+ });++ await handleBatchActionJob({+ id: uuidv4(),+ timestamp: new Date(),+ name: QueueJobs.BatchActionProcessingJob as const,+ payload: {+ projectId,+ actionId: "trace-add-to-annotation-queue" as const,+ tableName: BatchExportTableName.Traces,+ cutoffCreatedAt: new Date("2024-01-02"),+ targetId: otherQueueId,+ query: { filter: [], orderBy: { column: "timestamp", order: "DESC" } },+ type: BatchActionType.Create,+ },+ });++ const queueItems = await prisma.annotationQueueItem.findMany({+ where: { queueId: otherQueueId },+ });+ expect(queueItems).toHaveLength(0);+ });+it("should add sessions to annotation queue", async () => {const { projectId } = await createOrgProjectAndApiKey();diff --git a/worker/src/features/batchAction/processAddToQueue.ts b/worker/src/features/batchAction/processAddToQueue.tsindex 505dd67fdd..0a78340c33 100644--- a/worker/src/features/batchAction/processAddToQueue.ts+++ b/worker/src/features/batchAction/processAddToQueue.ts@@ -12,6 +12,18 @@ const addToQueue = async ({objectType: AnnotationQueueObjectType;targetId: string;}) => {+ const queue = await prisma.annotationQueue.findUnique({+ where: { id: targetId, projectId },+ select: { id: true },+ });++ if (!queue) {+ logger.warn(+ `Annotation queue ${targetId} not found in project ${projectId}, skipping`,+ );+ return;+ }+// cannot use prisma `createMany` operation as we do not have unique constraint enforced on schema level// conflict must be handled on query level by reading existing items and filtering out traces that already exist