Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

9.3 KB
Raw Blame
import { auditLog } from "@/src/features/audit-logs/auditLog";
import {
  hasProjectAccess,
  throwIfNoProjectAccess,
} from "@/src/features/rbac/utils/checkProjectAccess";
import {
  createTRPCRouter,
  protectedProjectProcedure,
  type ProjectAuthedContext,
} from "@/src/server/api/trpc";
import { z } from "zod";
import {
  ActivationCostEstimatesSchema,
  CreateEvaluatorSchema,
  DeleteEvaluatorsSchema,
  EvaluatorDefinitionInputSchema,
  EvaluatorIdSchema,
  EvaluatorIdsSchema,
  EvaluatorOptionsSchema,
  EvaluatorVersionsSchema,
  ListEvaluatorGallerySchema,
  ListEvaluatorsSchema,
  SuggestEvaluatorTextSchema,
  UpdateEvaluatorSchema,
} from "./evaluatorTypes";
import { EvaluatorService } from "./evaluatorService";
import { ruleRouter } from "../rules/ruleRouter";
import { getActivationCostEstimates } from "./activationCostService";

const TestEvaluatorSchema = z.object({
  projectId: z.string(),
  evaluatorId: z.string(),
  definition: EvaluatorDefinitionInputSchema,
  observationId: z.string(),
  traceId: z.string(),
  startTime: z.coerce.date(),
});

function serviceForContext(ctx: ProjectAuthedContext) {
  return new EvaluatorService(ctx.prisma, ({ action, evaluatorId }) =>
    auditEvaluator(ctx, action, evaluatorId),
  );
}

function auditEvaluator(
  ctx: ProjectAuthedContext,
  action: "create" | "update" | "delete",
  evaluatorId: string,
) {
  return auditLog({
    session: ctx.session,
    resourceType: "evalTemplate",
    resourceId: evaluatorId,
    action,
  });
}

export const evaluatorRouter = createTRPCRouter({
  rules: ruleRouter,
  list: protectedProjectProcedure
    .input(ListEvaluatorsSchema)
    .query(({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:read",
      });
      return serviceForContext(ctx).list({
        ...input,
        projectId: ctx.session.projectId,
      });
    }),

  listGallery: protectedProjectProcedure
    .input(ListEvaluatorGallerySchema)
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:read",
      });
      const service = serviceForContext(ctx);
      const params = {
        ...input,
        projectId: ctx.session.projectId,
      };
      const [result, totalItems] = await Promise.all([
        service.listCursor(params),
        input.cursor ? undefined : service.count(params),
      ]);

      return {
        ...result,
        totalItems,
      };
    }),

  filterOptions: protectedProjectProcedure
    .input(ListEvaluatorsSchema.pick({ projectId: true }))
    .query(({ ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:read",
      });
      return serviceForContext(ctx).listFilterOptions(ctx.session.projectId);
    }),

  options: protectedProjectProcedure
    .input(EvaluatorOptionsSchema)
    .query(({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:read",
      });
      return serviceForContext(ctx).listOptions({
        ...input,
        projectId: ctx.session.projectId,
      });
    }),

  get: protectedProjectProcedure
    .input(EvaluatorIdSchema)
    .query(({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:read",
      });
      return serviceForContext(ctx).getWithSampleFilter(
        ctx.session.projectId,
        input.evaluatorId,
      );
    }),

  versions: protectedProjectProcedure
    .input(EvaluatorVersionsSchema)
    .query(({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:read",
      });
      return serviceForContext(ctx).listVersions({
        ...input,
        projectId: ctx.session.projectId,
      });
    }),

  executionSummaries: protectedProjectProcedure
    .input(EvaluatorIdsSchema)
    .query(({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evalJobExecution:read",
      });
      return serviceForContext(ctx).getExecutionSummaries({
        ...input,
        projectId: ctx.session.projectId,
      });
    }),

  costByEvaluatorIds: protectedProjectProcedure
    .input(EvaluatorIdsSchema)
    .query(({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evalJobExecution:read",
      });
      return serviceForContext(ctx).getTotalCosts({
        ...input,
        projectId: ctx.session.projectId,
      });
    }),

  activationCostEstimates: protectedProjectProcedure
    .input(ActivationCostEstimatesSchema)
    .mutation(({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evalJobExecution:read",
      });
      const canRunMissingCostTest = hasProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluationRule:CUD",
      });
      const { shouldRunMissingTest, ...estimateInput } = input;
      let missingCostMode: "probe" | "wait" | "skip" = "skip";
      if (canRunMissingCostTest) {
        missingCostMode = shouldRunMissingTest ? "probe" : "wait";
      }
      return getActivationCostEstimates({
        ...estimateInput,
        projectId: ctx.session.projectId,
        orgId: ctx.session.orgId,
        missingCostMode,
        shouldReadFromObservationsTable:
          ctx.session.user.v4BetaEnabled !== true,
      });
    }),

  create: protectedProjectProcedure
    .input(CreateEvaluatorSchema)
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:CUD",
      });
      const service = serviceForContext(ctx);
      return service.create(
        { ...input, projectId: ctx.session.projectId },
        ctx.session.user.id,
      );
    }),

  update: protectedProjectProcedure
    .input(UpdateEvaluatorSchema)
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:CUD",
      });
      const service = serviceForContext(ctx);
      return service.update(
        { ...input, projectId: ctx.session.projectId },
        ctx.session.user.id,
      );
    }),

  reactivate: protectedProjectProcedure
    .input(EvaluatorIdSchema)
    .mutation(({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:CUD",
      });
      return serviceForContext(ctx).reactivate({
        projectId: ctx.session.projectId,
        evaluatorId: input.evaluatorId,
      });
    }),

  delete: protectedProjectProcedure
    .input(EvaluatorIdSchema)
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:CUD",
      });
      const service = serviceForContext(ctx);
      await service.delete(ctx.session.projectId, input.evaluatorId);
      return { success: true };
    }),

  deleteMany: protectedProjectProcedure
    .input(DeleteEvaluatorsSchema)
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:CUD",
      });
      const service = serviceForContext(ctx);
      await service.deleteMany({
        ...input,
        projectId: ctx.session.projectId,
      });
      return { success: true };
    }),

  test: protectedProjectProcedure
    .input(TestEvaluatorSchema)
    .mutation(({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:CUD",
      });
      return serviceForContext(ctx).testEvaluator({
        orgId: ctx.session.orgId,
        projectId: ctx.session.projectId,
        evaluatorId: input.evaluatorId,
        definition: input.definition,
        observationId: input.observationId,
        traceId: input.traceId,
        startTime: input.startTime,
        shouldReadFromObservationsTable:
          ctx.session.user.v4BetaEnabled !== true,
      });
    }),

  suggestName: protectedProjectProcedure
    .input(SuggestEvaluatorTextSchema)
    .mutation(({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:CUD",
      });
      return serviceForContext(ctx).suggestName({
        ...input,
        projectId: ctx.session.projectId,
        userId: ctx.session.user.id,
      });
    }),

  suggestDescription: protectedProjectProcedure
    .input(SuggestEvaluatorTextSchema)
    .mutation(({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: ctx.session.projectId,
        scope: "evaluator:CUD",
      });
      return serviceForContext(ctx).suggestDescription({
        ...input,
        projectId: ctx.session.projectId,
        userId: ctx.session.user.id,
      });
    }),
});