Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

18.9 KB
Raw Blame
import { env } from "@/src/env.mjs";
import { auditLog } from "@/src/features/audit-logs/auditLog";
import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
import {
  createTRPCRouter,
  protectedProjectProcedure,
} from "@/src/server/api/trpc";
import {
  AnnotationQueueObjectType,
  AnnotationQueueStatus,
  CreateQueueData,
  filterAndValidateDbScoreConfigList,
  isBaseError,
  isPresent,
  LangfuseNotFoundError,
  optionalPaginationZod,
  Prisma,
} from "@langfuse/shared";
import {
  getObservationById,
  getObservationByIdFromEventsTable,
  isPrismaRecordNotFoundError,
  logger,
} from "@langfuse/shared/src/server";
import { TRPCError } from "@trpc/server";
import { z } from "zod";

const QueueData = CreateQueueData.extend({
  scoreConfigIds: CreateQueueData.shape.scoreConfigIds.refine(
    (ids) => new Set(ids).size === ids.length,
    { message: "Score config IDs must be unique" },
  ),
});

export const queueRouter = createTRPCRouter({
  hasAny: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "annotationQueues:read",
      });

      const queue = await ctx.prisma.annotationQueue.findFirst({
        where: {
          projectId: input.projectId,
        },
        select: { id: true },
        take: 1,
      });

      return queue !== null;
    }),
  all: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        ...optionalPaginationZod,
      }),
    )
    .query(async ({ input, ctx }) => {
      try {
        throwIfNoProjectAccess({
          session: ctx.session,
          projectId: input.projectId,
          scope: "annotationQueues:read",
        });

        const [queues, totalCount, scoreConfigs] = await Promise.all([
          ctx.prisma.$queryRaw<
            Array<{
              id: string;
              name: string;
              description?: string | null;
              scoreConfigIds: string[];
              createdAt: string;
              countCompletedItems: number;
              countPendingItems: number;
            }>
          >(Prisma.sql`
          SELECT
            aq.id,
            aq.name,
            aq.description,
            aq.score_config_ids AS "scoreConfigIds",
            aq.created_at AS "createdAt",
            COALESCE(SUM(CASE WHEN aqi.status = 'COMPLETED' THEN 1 ELSE 0 END), 0) AS "countCompletedItems",
            COALESCE(SUM(CASE WHEN aqi.status = 'PENDING' THEN 1 ELSE 0 END), 0) AS "countPendingItems"
          FROM
            annotation_queues aq
          LEFT JOIN
            annotation_queue_items aqi ON aq.id = aqi.queue_id AND aqi.project_id = aq.project_id
          WHERE
            aq.project_id = ${input.projectId}
          GROUP BY
            aq.id, aq.name, aq.description, aq.created_at
          ORDER BY
            aq.created_at DESC
          ${input.limit ? Prisma.sql`LIMIT ${input.limit}` : Prisma.empty}
          ${input.page && input.limit ? Prisma.sql`OFFSET ${input.page * input.limit}` : Prisma.empty}
        `),
          ctx.prisma.annotationQueue.count({
            where: {
              projectId: input.projectId,
            },
          }),
          ctx.prisma.scoreConfig.findMany({
            where: {
              projectId: input.projectId,
            },
            select: {
              id: true,
              name: true,
              dataType: true,
            },
          }),
        ]);

        const userQueueAssignments =
          await ctx.prisma.annotationQueueAssignment.findMany({
            where: {
              userId: ctx.session.user.id,
            },
            select: {
              queueId: true,
            },
          });

        const scoreConfigsById = new Map(
          scoreConfigs.map((config) => [config.id, config]),
        );

        return {
          totalCount,
          queues: queues.map((queue) => ({
            ...queue,
            scoreConfigs: Array.from(new Set(queue.scoreConfigIds))
              .map((id) => scoreConfigsById.get(id))
              .filter(isPresent),
            isCurrentUserAssigned: userQueueAssignments.some(
              (assignment) => assignment.queueId === queue.id,
            ),
          })),
        };
      } catch (error) {
        logger.error(error);
        if (error instanceof TRPCError) {
          throw error;
        }
        throw new TRPCError({
          code: "INTERNAL_SERVER_ERROR",
          message: "Fetching annotation queues failed.",
        });
      }
    }),
  allNamesAndIds: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
      }),
    )
    .query(async ({ input, ctx }) => {
      try {
        const queueNamesAndIds = await ctx.prisma.annotationQueue.findMany({
          where: {
            projectId: input.projectId,
          },
          select: {
            id: true,
            name: true,
          },
        });

        return queueNamesAndIds;
      } catch (error) {
        logger.error(error);
        if (error instanceof TRPCError) {
          throw error;
        }
        throw new TRPCError({
          code: "INTERNAL_SERVER_ERROR",
          message: "Fetching annotation queues failed.",
        });
      }
    }),
  count: protectedProjectProcedure
    .input(z.object({ projectId: z.string() }))
    .query(async ({ input, ctx }) => {
      try {
        return ctx.prisma.annotationQueue.count({
          where: { projectId: input.projectId },
        });
      } catch (error) {
        logger.error(error);
        if (error instanceof TRPCError) {
          throw error;
        }
        throw new TRPCError({
          code: "INTERNAL_SERVER_ERROR",
          message: "Fetching annotation queue count failed.",
        });
      }
    }),
  byId: protectedProjectProcedure
    .input(z.object({ queueId: z.string(), projectId: z.string() }))
    .query(async ({ input, ctx }) => {
      try {
        throwIfNoProjectAccess({
          session: ctx.session,
          projectId: input.projectId,
          scope: "annotationQueues:read",
        });

        const queue = await ctx.prisma.annotationQueue.findUnique({
          where: { id: input.queueId, projectId: input.projectId },
        });

        const configs = await ctx.prisma.scoreConfig.findMany({
          where: {
            projectId: input.projectId,
            id: {
              in: queue?.scoreConfigIds ?? [],
            },
          },
        });

        const scoreConfigsById = new Map(
          filterAndValidateDbScoreConfigList(configs).map((config) => [
            config.id,
            config,
          ]),
        );

        return {
          ...queue,
          scoreConfigs: Array.from(new Set(queue?.scoreConfigIds ?? []))
            .map((id) => scoreConfigsById.get(id))
            .filter(isPresent),
        };
      } catch (error) {
        logger.error(error);
        if (error instanceof TRPCError) {
          throw error;
        }
        throw new TRPCError({
          code: "INTERNAL_SERVER_ERROR",
          message: "Fetching annotation queue failed.",
        });
      }
    }),
  byObjectId: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        objectId: z.string(),
        objectType: z.enum(AnnotationQueueObjectType),
      }),
    )
    .query(async ({ input, ctx }) => {
      try {
        throwIfNoProjectAccess({
          session: ctx.session,
          projectId: input.projectId,
          scope: "annotationQueues:read",
        });

        const queues = await ctx.prisma.annotationQueue.findMany({
          where: {
            projectId: input.projectId,
          },
          select: {
            id: true,
            name: true,
            annotationQueueItem: {
              where: {
                projectId: input.projectId,
                objectId: input.objectId,
                objectType: input.objectType,
              },
              select: {
                queueId: true,
                status: true,
                id: true,
              },
            },
          },
        });

        let totalCount = 0;

        return {
          queues: queues.map((queue) => {
            totalCount += queue.annotationQueueItem.length;
            return {
              id: queue.id,
              name: queue.name,
              itemId: queue.annotationQueueItem[0]?.id, // Safely access the first item's id
              status: queue.annotationQueueItem[0]?.status, // Safely access the first item's status
              // Since there may be multiple queue items in a given queue, but with the same objectId, we select only the first one
              // to simplify the logic and because we are only interested in the first item's details.
            };
          }),
          totalCount,
          // If the given objectId has been added to the same queue more than once, the total count will reflect that, by counting each item (incl duplicates)
        };
      } catch (error) {
        logger.error(error);
        if (error instanceof TRPCError) {
          throw error;
        }
        throw new TRPCError({
          code: "INTERNAL_SERVER_ERROR",
          message: "Fetching annotation queue by objectId failed.",
        });
      }
    }),
  create: protectedProjectProcedure
    .input(
      QueueData.extend({
        projectId: z.string(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      try {
        throwIfNoProjectAccess({
          session: ctx.session,
          projectId: input.projectId,
          scope: "annotationQueues:CUD",
        });

        // gate usage on cloud:hobby
        const org = ctx.session.user.organizations.find((org) =>
          org.projects.some((proj) => proj.id === input.projectId),
        );
        const plan = org?.plan ?? "oss";

        // Counting and inserting must share one serializable snapshot.
        // Otherwise parallel creates with distinct names all observe
        // count = 0 and bypass the hobby limit, which the (projectId, name)
        // unique index cannot prevent.
        const queue = await ctx.prisma.$transaction(
          async (tx) => {
            await validateProjectScoreConfigs(
              tx,
              input.projectId,
              input.scoreConfigIds,
            );

            if (plan === "cloud:hobby") {
              const queueCount = await tx.annotationQueue.count({
                where: { projectId: input.projectId },
              });

              if (queueCount >= 1) {
                throw new TRPCError({
                  code: "FORBIDDEN",
                  message:
                    "Maximum number of annotation queues reached on Hobby plan.",
                });
              }
            }

            const existingQueue = await tx.annotationQueue.findFirst({
              where: { projectId: input.projectId, name: input.name },
            });

            if (existingQueue) {
              throw new TRPCError({
                code: "CONFLICT",
                message: "A queue with this name already exists in the project",
              });
            }

            return tx.annotationQueue.create({
              data: {
                name: input.name,
                projectId: input.projectId,
                description: input.description,
                scoreConfigIds: input.scoreConfigIds,
              },
            });
          },
          { isolationLevel: Prisma.TransactionIsolationLevel.Serializable },
        );

        await auditLog({
          session: ctx.session,
          resourceType: "annotationQueue",
          resourceId: queue.id,
          action: "create",
          after: queue,
        });

        return queue;
      } catch (error) {
        logger.error(error);
        if (error instanceof TRPCError) {
          throw error;
        }

        if (
          error instanceof Prisma.PrismaClientKnownRequestError &&
          error.code === "P2002"
        ) {
          throw new TRPCError({
            code: "CONFLICT",
            message: "A queue with this name already exists in the project",
          });
        }

        if (
          error instanceof Prisma.PrismaClientKnownRequestError &&
          error.code === "P2034"
        ) {
          throw new TRPCError({
            code: "CONFLICT",
            message: "Could not create annotation queue, please retry.",
          });
        }

        throw new TRPCError({
          code: "INTERNAL_SERVER_ERROR",
          message: "Creating annotation queue failed.",
        });
      }
    }),
  update: protectedProjectProcedure
    .input(
      QueueData.partial().extend({
        projectId: z.string(),
        queueId: z.string(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      try {
        throwIfNoProjectAccess({
          session: ctx.session,
          projectId: input.projectId,
          scope: "annotationQueues:CUD",
        });

        const queue = await ctx.prisma.annotationQueue.findFirst({
          where: {
            id: input.queueId,
            projectId: input.projectId,
          },
        });

        if (!queue) {
          throw new LangfuseNotFoundError("Queue not found in project");
        }

        if (input.scoreConfigIds !== undefined) {
          await validateProjectScoreConfigs(
            ctx.prisma,
            input.projectId,
            input.scoreConfigIds,
          );
        }

        if (input.name !== undefined) {
          const existingQueueWithName =
            await ctx.prisma.annotationQueue.findFirst({
              where: {
                projectId: input.projectId,
                name: input.name,
                id: { not: input.queueId },
              },
              select: { id: true },
            });

          if (existingQueueWithName) {
            throw new TRPCError({
              code: "CONFLICT",
              message: "A queue with this name already exists in the project",
            });
          }
        }

        const updatedQueue = await ctx.prisma.annotationQueue.update({
          where: { id: input.queueId, projectId: input.projectId },
          data: {
            name: input.name,
            description: input.description,
            scoreConfigIds: input.scoreConfigIds,
          },
        });

        await auditLog({
          session: ctx.session,
          resourceType: "annotationQueue",
          resourceId: queue.id,
          action: "update",
          before: queue,
          after: updatedQueue,
        });

        return updatedQueue;
      } catch (error) {
        if (error instanceof TRPCError || isBaseError(error)) {
          throw error;
        }

        if (
          error instanceof Prisma.PrismaClientKnownRequestError &&
          error.code === "P2002"
        ) {
          throw new TRPCError({
            code: "CONFLICT",
            message: "A queue with this name already exists in the project",
          });
        }

        logger.error(error);
        throw new TRPCError({
          code: "INTERNAL_SERVER_ERROR",
          message: "Updating annotation queue failed.",
        });
      }
    }),
  delete: protectedProjectProcedure
    .input(z.object({ queueId: z.string(), projectId: z.string() }))
    .mutation(async ({ input, ctx }) => {
      try {
        throwIfNoProjectAccess({
          session: ctx.session,
          projectId: input.projectId,
          scope: "annotationQueues:CUD",
        });
        const queue = await ctx.prisma.annotationQueue.delete({
          where: { id: input.queueId, projectId: input.projectId },
        });

        await auditLog({
          session: ctx.session,
          resourceType: "annotationQueue",
          resourceId: queue.id,
          action: "delete",
          before: queue,
        });

        return queue;
      } catch (error) {
        if (error instanceof TRPCError || isBaseError(error)) {
          throw error;
        }

        if (isPrismaRecordNotFoundError(error)) {
          throw new LangfuseNotFoundError("Queue not found in project");
        }

        logger.error(error);
        throw new TRPCError({
          code: "INTERNAL_SERVER_ERROR",
          message: "Deleting annotation queue failed.",
        });
      }
    }),
  fetchAndLockNext: protectedProjectProcedure
    .input(
      z.object({
        queueId: z.string(),
        projectId: z.string(),
        seenItemIds: z.array(z.string()),
        isBetaEnabled: z.boolean().optional().default(false),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "annotationQueues:CUD",
      });

      const now = new Date();
      const fiveMinutesAgo = new Date(now.getTime() - 5 * 60 * 1000);

      const item = await ctx.prisma.annotationQueueItem.findFirst({
        where: {
          queueId: input.queueId,
          projectId: input.projectId,
          status: AnnotationQueueStatus.PENDING,
          OR: [
            { lockedAt: null },
            { lockedAt: { lt: fiveMinutesAgo } },
            { lockedByUserId: ctx.session.user.id },
          ],
          NOT: {
            id: { in: input.seenItemIds },
          },
        },
        orderBy: {
          createdAt: "asc",
        },
      });

      // Expected behavior, non-error case: all items have been seen AND/OR completed, no more unseen pending items
      if (!item) return null;

      const updatedItem = await ctx.prisma.annotationQueueItem.update({
        where: {
          id: item.id,
          projectId: input.projectId,
        },
        data: {
          lockedAt: now,
          lockedByUserId: ctx.session.user.id,
        },
      });

      const inflatedUpdatedItem = {
        ...updatedItem,
        lockedByUser: { name: ctx.session.user.name },
      };

      if (item.objectType === AnnotationQueueObjectType.OBSERVATION) {
        const clickhouseObservation =
          env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true"
            ? await getObservationByIdFromEventsTable({
                id: item.objectId,
                projectId: input.projectId,
              })
            : // eslint-disable-next-line @typescript-eslint/no-deprecated
              await getObservationById({
                id: item.objectId,
                projectId: input.projectId,
              });
        return {
          ...inflatedUpdatedItem,
          parentTraceId: clickhouseObservation?.traceId,
        };
      }

      return inflatedUpdatedItem;
    }),
});

async function validateProjectScoreConfigs(
  prisma: Pick<Prisma.TransactionClient, "scoreConfig">,
  projectId: string,
  scoreConfigIds: string[],
) {
  const configCount = await prisma.scoreConfig.count({
    where: { projectId, id: { in: scoreConfigIds } },
  });
  if (configCount !== scoreConfigIds.length) {
    throw new TRPCError({
      code: "BAD_REQUEST",
      message: "All score configs must belong to this project.",
    });
  }
}