🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): allow read-only cost estimates (#18321)
Tobias Wochinger933541ab9062
diff --git a/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts b/web/src/__tests__/server/evaluator-v2-trpc.servertest.tsindex 942bc91e5a..1b045019de 100644--- a/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts@@ -2,7 +2,7 @@ import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";import { randomUUID } from "node:crypto";import type { Session } from "next-auth";import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";-import { prisma } from "@langfuse/shared/src/db";+import { prisma, type Role } from "@langfuse/shared/src/db";import {createEvent,createEventsCh,@@ -13,6 +13,7 @@ import { createInnerTRPCContext } from "@/src/server/api/trpc";const mocks = vi.hoisted(() => ({getEvaluatorDefinitionPreflightError: vi.fn(async () => null),+ getActivationCostEstimates: vi.fn(async () => []),}));vi.mock(@@ -25,6 +26,13 @@ vi.mock("@/src/features/evals/server/evaluator-preflight", () => ({mocks.getEvaluatorDefinitionPreflightError,}));+vi.mock(+ "@/src/features/evals/v2/server/evaluators/activationCostService",+ () => ({+ getActivationCostEstimates: mocks.getActivationCostEstimates,+ }),+);+const orgIds: string[] = [];let projectId = "";let otherProjectId = "";@@ -32,6 +40,24 @@ let userId = "";let caller: ReturnType<typeof appRouter.createCaller>;let session: Session & { user: NonNullable<Session["user"]> };+function createCallerWithProjectRole(role: Role) {+ const roleSession = {+ ...session,+ user: {+ ...session.user,+ organizations: session.user.organizations.map((organization) => ({+ ...organization,+ projects: organization.projects.map((project) => ({+ ...project,+ role,+ })),+ })),+ },+ };+ const ctx = createInnerTRPCContext({ session: roleSession, headers: {} });+ return appRouter.createCaller({ ...ctx, prisma });+}+const definition = {type: "LLM_AS_JUDGE" as const,promptMessages: [{ role: "user" as const, content: "Judge {{output}}" }],@@ -106,6 +132,41 @@ afterAll(async () => {});describe("evalsV2 tRPC", () => {+ it("allows viewers to estimate costs without running a missing-cost test", async () => {+ const viewerCaller = createCallerWithProjectRole("VIEWER");+ const memberCaller = createCallerWithProjectRole("MEMBER");+ const input = {+ projectId,+ evaluatorIds: ["evaluator-id"],+ filter: [],+ sampling: 1,+ shouldRunMissingTest: true,+ };++ await viewerCaller.evalsV2.activationCostEstimates(input);+ expect(mocks.getActivationCostEstimates).toHaveBeenLastCalledWith(+ expect.objectContaining({ missingCostMode: "skip" }),+ );++ await memberCaller.evalsV2.activationCostEstimates(input);+ expect(mocks.getActivationCostEstimates).toHaveBeenLastCalledWith(+ expect.objectContaining({ missingCostMode: "probe" }),+ );+ });++ it("rejects cost estimates without execution read access", async () => {+ const noAccessCaller = createCallerWithProjectRole("NONE");++ await expect(+ noAccessCaller.evalsV2.activationCostEstimates({+ projectId,+ evaluatorIds: ["evaluator-id"],+ filter: [],+ sampling: 1,+ }),+ ).rejects.toMatchObject({ code: "FORBIDDEN" });+ });+it("supports the evaluator management flow", async () => {const created = await caller.evalsV2.create({projectId,diff --git a/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts b/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.tsindex 94a729bad7..b807412315 100644--- a/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts+++ b/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts@@ -320,13 +320,35 @@ describe("getActivationCostEstimates", () => {filter: [],sampling: 1,shouldReadFromObservationsTable: false,- shouldRunMissingTest: false,+ missingCostMode: "wait",});expect(mocks.testEvaluator).not.toHaveBeenCalled();expect(result[0]?.testRunCostUsd).toBe(0.02);});+ it("does not poll or probe when missing costs are skipped", async () => {+ mocks.getLatestEvaluatorRunCost.mockResolvedValue(null);++ const result = await getActivationCostEstimates({+ orgId: "org-id",+ projectId: "project-id",+ evaluatorIds: ["evaluator-id"],+ filter: [],+ sampling: 1,+ missingCostMode: "skip",+ shouldReadFromObservationsTable: false,+ });++ expect(mocks.getLatestEvaluatorRunCost).toHaveBeenCalledOnce();+ expect(+ mocks.getObservationsWithModelDataFromEventsTable,+ ).not.toHaveBeenCalled();+ expect(mocks.testEvaluator).not.toHaveBeenCalled();+ expect(result[0]?.testRunCostUsd).toBeNull();+ expect(result[0]?.estimatedCostUsd).toBeNull();+ });+it("uses a known test cost when no recent trace is indexed yet", async () => {mocks.getLatestEvaluatorRunCost.mockResolvedValue(null);@@ -337,7 +359,7 @@ describe("getActivationCostEstimates", () => {filter: [],sampling: 0.5,knownTestRunCostUsd: 0.03,- shouldRunMissingTest: false,+ missingCostMode: "wait",shouldReadFromObservationsTable: false,});diff --git a/web/src/features/evals/v2/server/evaluators/activationCostService.ts b/web/src/features/evals/v2/server/evaluators/activationCostService.tsindex 61338b1d81..b6d1cd79ed 100644--- a/web/src/features/evals/v2/server/evaluators/activationCostService.ts+++ b/web/src/features/evals/v2/server/evaluators/activationCostService.ts@@ -29,7 +29,7 @@ export async function getActivationCostEstimates(params: {filter: FilterState;sampling: number;knownTestRunCostUsd?: number;- shouldRunMissingTest?: boolean;+ missingCostMode?: "probe" | "wait" | "skip";shouldReadFromObservationsTable: boolean;timeRange?: {from: Date;@@ -114,7 +114,7 @@ export async function getActivationCostEstimates(params: {(evaluator) => costsByEvaluatorId.get(evaluator.id) === null,);if (- params.shouldRunMissingTest === false &&+ params.missingCostMode === "wait" &&matchingObservations > 0 &&evaluatorsWithoutCost.length > 0) {@@ -127,7 +127,11 @@ export async function getActivationCostEstimates(params: {availableCosts.forEach(({ evaluatorId, cost }) =>costsByEvaluatorId.set(evaluatorId, cost),);- } else if (matchingObservations > 0 && evaluatorsWithoutCost.length > 0) {+ } else if (+ params.missingCostMode !== "skip" &&+ matchingObservations > 0 &&+ evaluatorsWithoutCost.length > 0+ ) {const sample = (await getObservationsWithModelDataFromEventsTable({projectId: params.projectId,diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.tsindex 8b4fef9934..074e94bd1a 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts@@ -1,5 +1,5 @@import { auditLog } from "@/src/features/audit-logs/server";-import { throwIfNoProjectAccess } from "@/src/features/rbac";+import { hasProjectAccess, throwIfNoProjectAccess } from "@/src/features/rbac";import {createTRPCRouter,protectedProjectProcedure,@@ -177,14 +177,25 @@ export const evaluatorRouter = createTRPCRouter({.input(ActivationCostEstimatesSchema).mutation(({ input, ctx }) => {throwIfNoProjectAccess({+ session: ctx.session,+ projectId: ctx.session.projectId,+ scope: "evalJobExecution:read",+ });+ const canRunMissingCostTest = hasProjectAccess({session: ctx.session,projectId: ctx.session.projectId,scope: "evaluationRule:CUD",});+ const { shouldRunMissingTest, ...estimateInput } = input;+ let missingCostMode: "probe" | "wait" | "skip" = "skip";+ if (canRunMissingCostTest) {+ missingCostMode = shouldRunMissingTest ? "probe" : "wait";+ }return getActivationCostEstimates({- ...input,+ ...estimateInput,projectId: ctx.session.projectId,orgId: ctx.session.orgId,+ missingCostMode,shouldReadFromObservationsTable:ctx.session.user.v4BetaEnabled !== true,});