Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(annotation-queues): validate queue when adding items (#19407)

Steffen Reuther0e0c2029e249
diff --git a/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts b/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts
index 7067881d8e..b0f62ca32a 100644
--- a/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts
+++ b/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts
@@ -131,6 +131,90 @@ describe("annotation queues trpc", () => {
});
});
+ describe("annotationQueueItems.createMany", () => {
+ it.each([false, true])(
+ "rejects a queue from another project without writing (isBatchAction: %s)",
+ async (isBatchAction) => {
+ const setup = await createOrgProjectAndApiKey();
+ const otherSetup = await createOrgProjectAndApiKey();
+ orgIds.push(setup.org.id, otherSetup.org.id);
+ const { caller } = createCallerForProjectRole(setup);
+ const otherQueue = await prisma.annotationQueue.create({
+ data: {
+ name: "Other Queue",
+ scoreConfigIds: [],
+ projectId: otherSetup.project.id,
+ },
+ });
+
+ await expect(
+ caller.annotationQueueItems.createMany({
+ projectId: setup.project.id,
+ queueId: otherQueue.id,
+ objectIds: [uuidv4()],
+ objectType: AnnotationQueueObjectType.TRACE,
+ isBatchAction,
+ query: isBatchAction ? { filter: [], orderBy: null } : undefined,
+ }),
+ ).rejects.toMatchObject({ code: "NOT_FOUND" });
+
+ expect(
+ await prisma.annotationQueueItem.count({
+ where: { queueId: otherQueue.id },
+ }),
+ ).toBe(0);
+ expect(
+ await prisma.batchAction.count({
+ where: { projectId: setup.project.id },
+ }),
+ ).toBe(0);
+ },
+ );
+ });
+
+ describe("annotationQueues.byObjectId", () => {
+ it("ignores items whose project differs from the queue's project", async () => {
+ const setup = await createOrgProjectAndApiKey();
+ const otherSetup = await createOrgProjectAndApiKey();
+ orgIds.push(setup.org.id, otherSetup.org.id);
+ const { caller } = createCallerForProjectRole(setup);
+ const queue = await prisma.annotationQueue.create({
+ data: {
+ name: "Test Queue",
+ scoreConfigIds: [],
+ projectId: setup.project.id,
+ },
+ });
+ const objectId = uuidv4();
+ await prisma.annotationQueueItem.create({
+ data: {
+ queueId: queue.id,
+ objectId,
+ objectType: AnnotationQueueObjectType.TRACE,
+ projectId: otherSetup.project.id,
+ },
+ });
+
+ await expect(
+ caller.annotationQueues.byObjectId({
+ projectId: setup.project.id,
+ objectId,
+ objectType: AnnotationQueueObjectType.TRACE,
+ }),
+ ).resolves.toEqual({
+ queues: [
+ {
+ id: queue.id,
+ name: queue.name,
+ itemId: undefined,
+ status: undefined,
+ },
+ ],
+ totalCount: 0,
+ });
+ });
+ });
+
describe("annotationQueues.create", () => {
it("rejects a second queue on cloud:hobby", async () => {
const setup = await createOrgProjectAndApiKey();
diff --git a/web/src/__tests__/server/traces-batch-delete-trpc.servertest.ts b/web/src/__tests__/server/traces-batch-delete-trpc.servertest.ts
index e7f3cb0fde..9dbc745726 100644
--- a/web/src/__tests__/server/traces-batch-delete-trpc.servertest.ts
+++ b/web/src/__tests__/server/traces-batch-delete-trpc.servertest.ts
@@ -589,11 +589,14 @@ describe("traces.deleteMany batch action", () => {
const { projectId, caller } = await createCaller({
v4BetaEnabled: true,
});
+ const queue = await prisma.annotationQueue.create({
+ data: { projectId, name: `queue-${randomUUID()}` },
+ });
await expect(
caller.annotationQueueItems.createMany({
projectId,
- queueId: randomUUID(),
+ queueId: queue.id,
objectIds: [randomUUID()],
objectType: AnnotationQueueObjectType.TRACE,
isBatchAction: true,
diff --git a/web/src/features/annotation-queues/server/annotationQueueItemsRouter.ts b/web/src/features/annotation-queues/server/annotationQueueItemsRouter.ts
index c7d5eccea2..85ef2a8a8b 100644
--- a/web/src/features/annotation-queues/server/annotationQueueItemsRouter.ts
+++ b/web/src/features/annotation-queues/server/annotationQueueItemsRouter.ts
@@ -297,6 +297,21 @@ export const queueItemRouter = createTRPCRouter({
scope: "annotationQueues:CUD",
});
+ const queue = await ctx.prisma.annotationQueue.findUnique({
+ where: {
+ id: input.queueId,
+ projectId: input.projectId,
+ },
+ select: {
+ name: true,
+ id: true,
+ },
+ });
+
+ if (!queue) {
+ throw new LangfuseNotFoundError("Queue not found in project");
+ }
+
let createdCount = 0;
if (input.isBatchAction && input.query) {
@@ -353,21 +368,10 @@ export const queueItemRouter = createTRPCRouter({
}
}
- const queue = await ctx.prisma.annotationQueue.findUnique({
- where: {
- id: input.queueId,
- projectId: input.projectId,
- },
- select: {
- name: true,
- id: true,
- },
- });
-
return {
createdCount,
- queueName: queue?.name,
- queueId: queue?.id,
+ queueName: queue.name,
+ queueId: queue.id,
};
}),
deleteMany: protectedProjectProcedure
diff --git a/web/src/features/annotation-queues/server/annotationQueuesRouter.ts b/web/src/features/annotation-queues/server/annotationQueuesRouter.ts
index fc06bdf2ad..be8e5c83f5 100644
--- a/web/src/features/annotation-queues/server/annotationQueuesRouter.ts
+++ b/web/src/features/annotation-queues/server/annotationQueuesRouter.ts
@@ -279,6 +279,7 @@ export const queueRouter = createTRPCRouter({
name: true,
annotationQueueItem: {
where: {
+ projectId: input.projectId,
objectId: input.objectId,
objectType: input.objectType,
},
diff --git a/worker/src/__tests__/batchAction.test.ts b/worker/src/__tests__/batchAction.test.ts
index a103b18ab4..ef708df521 100644
--- a/worker/src/__tests__/batchAction.test.ts
+++ b/worker/src/__tests__/batchAction.test.ts
@@ -863,6 +863,49 @@ describe("select all test suite", () => {
expect(queueItems.every((item) => item.objectType === "TRACE")).toBe(true);
});
+ it("should not add traces to an annotation queue from another project", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ const { projectId: otherProjectId } = await createOrgProjectAndApiKey();
+
+ const traceId = uuidv4();
+ await createTracesCh([
+ createTrace({
+ project_id: projectId,
+ id: traceId,
+ timestamp: new Date("2024-01-01").getTime(),
+ }),
+ ]);
+
+ const otherQueueId = uuidv4();
+ await prisma.annotationQueue.create({
+ data: {
+ id: otherQueueId,
+ projectId: otherProjectId,
+ name: "other-queue",
+ },
+ });
+
+ await handleBatchActionJob({
+ id: uuidv4(),
+ timestamp: new Date(),
+ name: QueueJobs.BatchActionProcessingJob as const,
+ payload: {
+ projectId,
+ actionId: "trace-add-to-annotation-queue" as const,
+ tableName: BatchExportTableName.Traces,
+ cutoffCreatedAt: new Date("2024-01-02"),
+ targetId: otherQueueId,
+ query: { filter: [], orderBy: { column: "timestamp", order: "DESC" } },
+ type: BatchActionType.Create,
+ },
+ });
+
+ const queueItems = await prisma.annotationQueueItem.findMany({
+ where: { queueId: otherQueueId },
+ });
+ expect(queueItems).toHaveLength(0);
+ });
+
it("should add sessions to annotation queue", async () => {
const { projectId } = await createOrgProjectAndApiKey();
diff --git a/worker/src/features/batchAction/processAddToQueue.ts b/worker/src/features/batchAction/processAddToQueue.ts
index 505dd67fdd..0a78340c33 100644
--- a/worker/src/features/batchAction/processAddToQueue.ts
+++ b/worker/src/features/batchAction/processAddToQueue.ts
@@ -12,6 +12,18 @@ const addToQueue = async ({
objectType: AnnotationQueueObjectType;
targetId: string;
}) => {
+ const queue = await prisma.annotationQueue.findUnique({
+ where: { id: targetId, projectId },
+ select: { id: true },
+ });
+
+ if (!queue) {
+ logger.warn(
+ `Annotation queue ${targetId} not found in project ${projectId}, skipping`,
+ );
+ return;
+ }
+
// cannot use prisma `createMany` operation as we do not have unique constraint enforced on schema level
// conflict must be handled on query level by reading existing items and filtering out traces that already exist