Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(annotation-queues): serialize hobby queue-limit check on create (#16899)

Max Deichmannd43c9a5c814e
diff --git a/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts b/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts
index 784f7ad932..fe1a2d07c7 100644
--- a/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts
+++ b/web/src/__tests__/server/annotation-queue-items-trpc.servertest.ts
@@ -1,12 +1,15 @@
import { appRouter } from "@/src/server/api/root";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
-import { AnnotationQueueObjectType } from "@langfuse/shared";
+import { AnnotationQueueObjectType, type Plan } from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
import type { Session } from "next-auth";
import { v4 as uuidv4 } from "uuid";
-describe("annotationQueueItems trpc", () => {
+const HOBBY_LIMIT_MESSAGE =
+ /Maximum number of annotation queues reached on Hobby plan/;
+
+describe("annotation queues trpc", () => {
const orgIds: string[] = [];
type TestSetup = Awaited<ReturnType<typeof createOrgProjectAndApiKey>>;
@@ -14,6 +17,7 @@ describe("annotationQueueItems trpc", () => {
const createCallerForProjectRole = (
setup: TestSetup,
projectRole: "ADMIN" | "NONE" = "ADMIN",
+ plan: Plan = "cloud:hobby",
) => {
const session: Session = {
expires: "1",
@@ -27,7 +31,7 @@ describe("annotationQueueItems trpc", () => {
id: setup.org.id,
name: setup.org.name,
role: "MEMBER",
- plan: "cloud:hobby",
+ plan,
cloudConfig: undefined,
metadata: {},
aiFeaturesEnabled: false,
@@ -75,7 +79,16 @@ describe("annotationQueueItems trpc", () => {
});
});
- describe("typeById", () => {
+ const createScoreConfig = (projectId: string) =>
+ prisma.scoreConfig.create({
+ data: {
+ name: `score-${uuidv4().slice(0, 8)}`,
+ projectId,
+ dataType: "NUMERIC",
+ },
+ });
+
+ describe("annotationQueueItems.typeById", () => {
it("requires annotationQueues:read access", async () => {
const setup = await createOrgProjectAndApiKey();
orgIds.push(setup.org.id);
@@ -123,4 +136,94 @@ describe("annotationQueueItems trpc", () => {
).rejects.toThrow("User does not have access to this resource or action");
});
});
+
+ describe("annotationQueues.create", () => {
+ it("rejects a second queue on cloud:hobby", async () => {
+ const setup = await createOrgProjectAndApiKey();
+ orgIds.push(setup.org.id);
+ const { caller } = createCallerForProjectRole(setup);
+ const scoreConfig = await createScoreConfig(setup.project.id);
+
+ await caller.annotationQueues.create({
+ projectId: setup.project.id,
+ name: "first-queue",
+ scoreConfigIds: [scoreConfig.id],
+ });
+
+ await expect(
+ caller.annotationQueues.create({
+ projectId: setup.project.id,
+ name: "second-queue",
+ scoreConfigIds: [scoreConfig.id],
+ }),
+ ).rejects.toThrow(HOBBY_LIMIT_MESSAGE);
+
+ const queueCount = await prisma.annotationQueue.count({
+ where: { projectId: setup.project.id },
+ });
+ expect(queueCount).toBe(1);
+ });
+
+ it("does not admit more than one queue on cloud:hobby under concurrent requests", async () => {
+ // Counting outside a transaction lets concurrent requests all observe
+ // count = 0 and all insert, so the limit only holds if check+create is
+ // serialized. Unique names bypass the (projectId, name) unique index.
+ const setup = await createOrgProjectAndApiKey();
+ orgIds.push(setup.org.id);
+ const { caller } = createCallerForProjectRole(setup);
+ const scoreConfig = await createScoreConfig(setup.project.id);
+
+ const results = await Promise.allSettled(
+ Array.from({ length: 5 }, (_, i) =>
+ caller.annotationQueues.create({
+ projectId: setup.project.id,
+ name: `hobby-q-${i}`,
+ scoreConfigIds: [scoreConfig.id],
+ }),
+ ),
+ );
+
+ const queueCount = await prisma.annotationQueue.count({
+ where: { projectId: setup.project.id },
+ });
+ expect(queueCount).toBe(1);
+
+ const rejected = results.filter((r) => r.status === "rejected");
+ expect(results.length - rejected.length).toBe(1);
+ for (const result of rejected) {
+ // Concurrent losers abort the serializable transaction (P2034) or
+ // observe the committed row and hit the hobby limit.
+ expect(String(result.reason)).toMatch(
+ /Maximum number of annotation queues reached on Hobby plan|Could not create annotation queue, please retry/,
+ );
+ }
+ }, 25_000);
+
+ it("allows more than one queue when the plan is not cloud:hobby", async () => {
+ const setup = await createOrgProjectAndApiKey();
+ orgIds.push(setup.org.id);
+ const { caller } = createCallerForProjectRole(
+ setup,
+ "ADMIN",
+ "cloud:pro",
+ );
+ const scoreConfig = await createScoreConfig(setup.project.id);
+
+ await caller.annotationQueues.create({
+ projectId: setup.project.id,
+ name: "pro-queue-1",
+ scoreConfigIds: [scoreConfig.id],
+ });
+ await caller.annotationQueues.create({
+ projectId: setup.project.id,
+ name: "pro-queue-2",
+ scoreConfigIds: [scoreConfig.id],
+ });
+
+ const queueCount = await prisma.annotationQueue.count({
+ where: { projectId: setup.project.id },
+ });
+ expect(queueCount).toBe(2);
+ });
+ });
});
diff --git a/web/src/features/annotation-queues/server/annotationQueuesRouter.ts b/web/src/features/annotation-queues/server/annotationQueuesRouter.ts
index fe34746546..13e9d8fe83 100644
--- a/web/src/features/annotation-queues/server/annotationQueuesRouter.ts
+++ b/web/src/features/annotation-queues/server/annotationQueuesRouter.ts
@@ -315,44 +315,48 @@ export const queueRouter = createTRPCRouter({
);
const plan = org?.plan ?? "oss";
- if (plan === "cloud:hobby") {
- if (
- (await ctx.prisma.annotationQueue.count({
- where: {
- projectId: input.projectId,
- },
- })) >= 1
- ) {
- throw new TRPCError({
- code: "FORBIDDEN",
- message:
- "Maximum number of annotation queues reached on Hobby plan.",
- });
- }
- }
+ // Counting and inserting must share one serializable snapshot.
+ // Otherwise parallel creates with distinct names all observe
+ // count = 0 and bypass the hobby limit, which the (projectId, name)
+ // unique index cannot prevent.
+ const queue = await ctx.prisma.$transaction(
+ async (tx) => {
+ if (plan === "cloud:hobby") {
+ const queueCount = await tx.annotationQueue.count({
+ where: { projectId: input.projectId },
+ });
- const existingQueue = await ctx.prisma.annotationQueue.findFirst({
- where: {
- projectId: input.projectId,
- name: input.name,
- },
- });
+ if (queueCount >= 1) {
+ throw new TRPCError({
+ code: "FORBIDDEN",
+ message:
+ "Maximum number of annotation queues reached on Hobby plan.",
+ });
+ }
+ }
+
+ const existingQueue = await tx.annotationQueue.findFirst({
+ where: { projectId: input.projectId, name: input.name },
+ });
- if (existingQueue) {
- throw new TRPCError({
- code: "CONFLICT",
- message: "A queue with this name already exists in the project",
- });
- }
+ if (existingQueue) {
+ throw new TRPCError({
+ code: "CONFLICT",
+ message: "A queue with this name already exists in the project",
+ });
+ }
- const queue = await ctx.prisma.annotationQueue.create({
- data: {
- name: input.name,
- projectId: input.projectId,
- description: input.description,
- scoreConfigIds: input.scoreConfigIds,
+ return tx.annotationQueue.create({
+ data: {
+ name: input.name,
+ projectId: input.projectId,
+ description: input.description,
+ scoreConfigIds: input.scoreConfigIds,
+ },
+ });
},
- });
+ { isolationLevel: Prisma.TransactionIsolationLevel.Serializable },
+ );
await auditLog({
session: ctx.session,
@@ -368,6 +372,27 @@ export const queueRouter = createTRPCRouter({
if (error instanceof TRPCError) {
throw error;
}
+
+ if (
+ error instanceof Prisma.PrismaClientKnownRequestError &&
+ error.code === "P2002"
+ ) {
+ throw new TRPCError({
+ code: "CONFLICT",
+ message: "A queue with this name already exists in the project",
+ });
+ }
+
+ if (
+ error instanceof Prisma.PrismaClientKnownRequestError &&
+ error.code === "P2034"
+ ) {
+ throw new TRPCError({
+ code: "CONFLICT",
+ message: "Could not create annotation queue, please retry.",
+ });
+ }
+
throw new TRPCError({
code: "INTERNAL_SERVER_ERROR",
message: "Creating annotation queue failed.",