🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(annotation-queues): serialize hobby queue-limit check on create (#16899)
Max Deichmannd43c9a5c814e
diff --git a/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts b/web/src/__tests__/server/annotation-queue-items-trpc.servertest.tsindex 784f7ad932..fe1a2d07c7 100644--- a/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts+++ b/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts@@ -1,12 +1,15 @@import { appRouter } from "@/src/server/api/root";import { createInnerTRPCContext } from "@/src/server/api/trpc";-import { AnnotationQueueObjectType } from "@langfuse/shared";+import { AnnotationQueueObjectType, type Plan } from "@langfuse/shared";import { prisma } from "@langfuse/shared/src/db";import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";import type { Session } from "next-auth";import { v4 as uuidv4 } from "uuid";-describe("annotationQueueItems trpc", () => {+const HOBBY_LIMIT_MESSAGE =+ /Maximum number of annotation queues reached on Hobby plan/;++describe("annotation queues trpc", () => {const orgIds: string[] = [];type TestSetup = Awaited<ReturnType<typeof createOrgProjectAndApiKey>>;@@ -14,6 +17,7 @@ describe("annotationQueueItems trpc", () => {const createCallerForProjectRole = (setup: TestSetup,projectRole: "ADMIN" | "NONE" = "ADMIN",+ plan: Plan = "cloud:hobby",) => {const session: Session = {expires: "1",@@ -27,7 +31,7 @@ describe("annotationQueueItems trpc", () => {id: setup.org.id,name: setup.org.name,role: "MEMBER",- plan: "cloud:hobby",+ plan,cloudConfig: undefined,metadata: {},aiFeaturesEnabled: false,@@ -75,7 +79,16 @@ describe("annotationQueueItems trpc", () => {});});- describe("typeById", () => {+ const createScoreConfig = (projectId: string) =>+ prisma.scoreConfig.create({+ data: {+ name: `score-${uuidv4().slice(0, 8)}`,+ projectId,+ dataType: "NUMERIC",+ },+ });++ describe("annotationQueueItems.typeById", () => {it("requires annotationQueues:read access", async () => {const setup = await createOrgProjectAndApiKey();orgIds.push(setup.org.id);@@ -123,4 +136,94 @@ describe("annotationQueueItems trpc", () => {).rejects.toThrow("User does not have access to this resource or action");});});++ describe("annotationQueues.create", () => {+ it("rejects a second queue on cloud:hobby", async () => {+ const setup = await createOrgProjectAndApiKey();+ orgIds.push(setup.org.id);+ const { caller } = createCallerForProjectRole(setup);+ const scoreConfig = await createScoreConfig(setup.project.id);++ await caller.annotationQueues.create({+ projectId: setup.project.id,+ name: "first-queue",+ scoreConfigIds: [scoreConfig.id],+ });++ await expect(+ caller.annotationQueues.create({+ projectId: setup.project.id,+ name: "second-queue",+ scoreConfigIds: [scoreConfig.id],+ }),+ ).rejects.toThrow(HOBBY_LIMIT_MESSAGE);++ const queueCount = await prisma.annotationQueue.count({+ where: { projectId: setup.project.id },+ });+ expect(queueCount).toBe(1);+ });++ it("does not admit more than one queue on cloud:hobby under concurrent requests", async () => {+ // Counting outside a transaction lets concurrent requests all observe+ // count = 0 and all insert, so the limit only holds if check+create is+ // serialized. Unique names bypass the (projectId, name) unique index.+ const setup = await createOrgProjectAndApiKey();+ orgIds.push(setup.org.id);+ const { caller } = createCallerForProjectRole(setup);+ const scoreConfig = await createScoreConfig(setup.project.id);++ const results = await Promise.allSettled(+ Array.from({ length: 5 }, (_, i) =>+ caller.annotationQueues.create({+ projectId: setup.project.id,+ name: `hobby-q-${i}`,+ scoreConfigIds: [scoreConfig.id],+ }),+ ),+ );++ const queueCount = await prisma.annotationQueue.count({+ where: { projectId: setup.project.id },+ });+ expect(queueCount).toBe(1);++ const rejected = results.filter((r) => r.status === "rejected");+ expect(results.length - rejected.length).toBe(1);+ for (const result of rejected) {+ // Concurrent losers abort the serializable transaction (P2034) or+ // observe the committed row and hit the hobby limit.+ expect(String(result.reason)).toMatch(+ /Maximum number of annotation queues reached on Hobby plan|Could not create annotation queue, please retry/,+ );+ }+ }, 25_000);++ it("allows more than one queue when the plan is not cloud:hobby", async () => {+ const setup = await createOrgProjectAndApiKey();+ orgIds.push(setup.org.id);+ const { caller } = createCallerForProjectRole(+ setup,+ "ADMIN",+ "cloud:pro",+ );+ const scoreConfig = await createScoreConfig(setup.project.id);++ await caller.annotationQueues.create({+ projectId: setup.project.id,+ name: "pro-queue-1",+ scoreConfigIds: [scoreConfig.id],+ });+ await caller.annotationQueues.create({+ projectId: setup.project.id,+ name: "pro-queue-2",+ scoreConfigIds: [scoreConfig.id],+ });++ const queueCount = await prisma.annotationQueue.count({+ where: { projectId: setup.project.id },+ });+ expect(queueCount).toBe(2);+ });+ });});diff --git a/web/src/features/annotation-queues/server/annotationQueuesRouter.ts b/web/src/features/annotation-queues/server/annotationQueuesRouter.tsindex fe34746546..13e9d8fe83 100644--- a/web/src/features/annotation-queues/server/annotationQueuesRouter.ts+++ b/web/src/features/annotation-queues/server/annotationQueuesRouter.ts@@ -315,44 +315,48 @@ export const queueRouter = createTRPCRouter({);const plan = org?.plan ?? "oss";- if (plan === "cloud:hobby") {- if (- (await ctx.prisma.annotationQueue.count({- where: {- projectId: input.projectId,- },- })) >= 1- ) {- throw new TRPCError({- code: "FORBIDDEN",- message:- "Maximum number of annotation queues reached on Hobby plan.",- });- }- }+ // Counting and inserting must share one serializable snapshot.+ // Otherwise parallel creates with distinct names all observe+ // count = 0 and bypass the hobby limit, which the (projectId, name)+ // unique index cannot prevent.+ const queue = await ctx.prisma.$transaction(+ async (tx) => {+ if (plan === "cloud:hobby") {+ const queueCount = await tx.annotationQueue.count({+ where: { projectId: input.projectId },+ });- const existingQueue = await ctx.prisma.annotationQueue.findFirst({- where: {- projectId: input.projectId,- name: input.name,- },- });+ if (queueCount >= 1) {+ throw new TRPCError({+ code: "FORBIDDEN",+ message:+ "Maximum number of annotation queues reached on Hobby plan.",+ });+ }+ }++ const existingQueue = await tx.annotationQueue.findFirst({+ where: { projectId: input.projectId, name: input.name },+ });- if (existingQueue) {- throw new TRPCError({- code: "CONFLICT",- message: "A queue with this name already exists in the project",- });- }+ if (existingQueue) {+ throw new TRPCError({+ code: "CONFLICT",+ message: "A queue with this name already exists in the project",+ });+ }- const queue = await ctx.prisma.annotationQueue.create({- data: {- name: input.name,- projectId: input.projectId,- description: input.description,- scoreConfigIds: input.scoreConfigIds,+ return tx.annotationQueue.create({+ data: {+ name: input.name,+ projectId: input.projectId,+ description: input.description,+ scoreConfigIds: input.scoreConfigIds,+ },+ });},- });+ { isolationLevel: Prisma.TransactionIsolationLevel.Serializable },+ );await auditLog({session: ctx.session,@@ -368,6 +372,27 @@ export const queueRouter = createTRPCRouter({if (error instanceof TRPCError) {throw error;}++ if (+ error instanceof Prisma.PrismaClientKnownRequestError &&+ error.code === "P2002"+ ) {+ throw new TRPCError({+ code: "CONFLICT",+ message: "A queue with this name already exists in the project",+ });+ }++ if (+ error instanceof Prisma.PrismaClientKnownRequestError &&+ error.code === "P2034"+ ) {+ throw new TRPCError({+ code: "CONFLICT",+ message: "Could not create annotation queue, please retry.",+ });+ }+throw new TRPCError({code: "INTERNAL_SERVER_ERROR",message: "Creating annotation queue failed.",