Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): allow read-only cost estimates (#18321)

Tobias Wochinger933541ab9062
diff --git a/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts b/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts
index 942bc91e5a..1b045019de 100644
--- a/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts
+++ b/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts
@@ -2,7 +2,7 @@ import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import { randomUUID } from "node:crypto";
import type { Session } from "next-auth";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
-import { prisma } from "@langfuse/shared/src/db";
+import { prisma, type Role } from "@langfuse/shared/src/db";
import {
createEvent,
createEventsCh,
@@ -13,6 +13,7 @@ import { createInnerTRPCContext } from "@/src/server/api/trpc";
const mocks = vi.hoisted(() => ({
getEvaluatorDefinitionPreflightError: vi.fn(async () => null),
+ getActivationCostEstimates: vi.fn(async () => []),
}));
vi.mock(
@@ -25,6 +26,13 @@ vi.mock("@/src/features/evals/server/evaluator-preflight", () => ({
mocks.getEvaluatorDefinitionPreflightError,
}));
+vi.mock(
+ "@/src/features/evals/v2/server/evaluators/activationCostService",
+ () => ({
+ getActivationCostEstimates: mocks.getActivationCostEstimates,
+ }),
+);
+
const orgIds: string[] = [];
let projectId = "";
let otherProjectId = "";
@@ -32,6 +40,24 @@ let userId = "";
let caller: ReturnType<typeof appRouter.createCaller>;
let session: Session & { user: NonNullable<Session["user"]> };
+function createCallerWithProjectRole(role: Role) {
+ const roleSession = {
+ ...session,
+ user: {
+ ...session.user,
+ organizations: session.user.organizations.map((organization) => ({
+ ...organization,
+ projects: organization.projects.map((project) => ({
+ ...project,
+ role,
+ })),
+ })),
+ },
+ };
+ const ctx = createInnerTRPCContext({ session: roleSession, headers: {} });
+ return appRouter.createCaller({ ...ctx, prisma });
+}
+
const definition = {
type: "LLM_AS_JUDGE" as const,
promptMessages: [{ role: "user" as const, content: "Judge {{output}}" }],
@@ -106,6 +132,41 @@ afterAll(async () => {
});
describe("evalsV2 tRPC", () => {
+ it("allows viewers to estimate costs without running a missing-cost test", async () => {
+ const viewerCaller = createCallerWithProjectRole("VIEWER");
+ const memberCaller = createCallerWithProjectRole("MEMBER");
+ const input = {
+ projectId,
+ evaluatorIds: ["evaluator-id"],
+ filter: [],
+ sampling: 1,
+ shouldRunMissingTest: true,
+ };
+
+ await viewerCaller.evalsV2.activationCostEstimates(input);
+ expect(mocks.getActivationCostEstimates).toHaveBeenLastCalledWith(
+ expect.objectContaining({ missingCostMode: "skip" }),
+ );
+
+ await memberCaller.evalsV2.activationCostEstimates(input);
+ expect(mocks.getActivationCostEstimates).toHaveBeenLastCalledWith(
+ expect.objectContaining({ missingCostMode: "probe" }),
+ );
+ });
+
+ it("rejects cost estimates without execution read access", async () => {
+ const noAccessCaller = createCallerWithProjectRole("NONE");
+
+ await expect(
+ noAccessCaller.evalsV2.activationCostEstimates({
+ projectId,
+ evaluatorIds: ["evaluator-id"],
+ filter: [],
+ sampling: 1,
+ }),
+ ).rejects.toMatchObject({ code: "FORBIDDEN" });
+ });
+
it("supports the evaluator management flow", async () => {
const created = await caller.evalsV2.create({
projectId,
diff --git a/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts b/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts
index 94a729bad7..b807412315 100644
--- a/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts
+++ b/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts
@@ -320,13 +320,35 @@ describe("getActivationCostEstimates", () => {
filter: [],
sampling: 1,
shouldReadFromObservationsTable: false,
- shouldRunMissingTest: false,
+ missingCostMode: "wait",
});
expect(mocks.testEvaluator).not.toHaveBeenCalled();
expect(result[0]?.testRunCostUsd).toBe(0.02);
});
+ it("does not poll or probe when missing costs are skipped", async () => {
+ mocks.getLatestEvaluatorRunCost.mockResolvedValue(null);
+
+ const result = await getActivationCostEstimates({
+ orgId: "org-id",
+ projectId: "project-id",
+ evaluatorIds: ["evaluator-id"],
+ filter: [],
+ sampling: 1,
+ missingCostMode: "skip",
+ shouldReadFromObservationsTable: false,
+ });
+
+ expect(mocks.getLatestEvaluatorRunCost).toHaveBeenCalledOnce();
+ expect(
+ mocks.getObservationsWithModelDataFromEventsTable,
+ ).not.toHaveBeenCalled();
+ expect(mocks.testEvaluator).not.toHaveBeenCalled();
+ expect(result[0]?.testRunCostUsd).toBeNull();
+ expect(result[0]?.estimatedCostUsd).toBeNull();
+ });
+
it("uses a known test cost when no recent trace is indexed yet", async () => {
mocks.getLatestEvaluatorRunCost.mockResolvedValue(null);
@@ -337,7 +359,7 @@ describe("getActivationCostEstimates", () => {
filter: [],
sampling: 0.5,
knownTestRunCostUsd: 0.03,
- shouldRunMissingTest: false,
+ missingCostMode: "wait",
shouldReadFromObservationsTable: false,
});
diff --git a/web/src/features/evals/v2/server/evaluators/activationCostService.ts b/web/src/features/evals/v2/server/evaluators/activationCostService.ts
index 61338b1d81..b6d1cd79ed 100644
--- a/web/src/features/evals/v2/server/evaluators/activationCostService.ts
+++ b/web/src/features/evals/v2/server/evaluators/activationCostService.ts
@@ -29,7 +29,7 @@ export async function getActivationCostEstimates(params: {
filter: FilterState;
sampling: number;
knownTestRunCostUsd?: number;
- shouldRunMissingTest?: boolean;
+ missingCostMode?: "probe" | "wait" | "skip";
shouldReadFromObservationsTable: boolean;
timeRange?: {
from: Date;
@@ -114,7 +114,7 @@ export async function getActivationCostEstimates(params: {
(evaluator) => costsByEvaluatorId.get(evaluator.id) === null,
);
if (
- params.shouldRunMissingTest === false &&
+ params.missingCostMode === "wait" &&
matchingObservations > 0 &&
evaluatorsWithoutCost.length > 0
) {
@@ -127,7 +127,11 @@ export async function getActivationCostEstimates(params: {
availableCosts.forEach(({ evaluatorId, cost }) =>
costsByEvaluatorId.set(evaluatorId, cost),
);
- } else if (matchingObservations > 0 && evaluatorsWithoutCost.length > 0) {
+ } else if (
+ params.missingCostMode !== "skip" &&
+ matchingObservations > 0 &&
+ evaluatorsWithoutCost.length > 0
+ ) {
const sample = (
await getObservationsWithModelDataFromEventsTable({
projectId: params.projectId,
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts
index 8b4fef9934..074e94bd1a 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts
@@ -1,5 +1,5 @@
import { auditLog } from "@/src/features/audit-logs/server";
-import { throwIfNoProjectAccess } from "@/src/features/rbac";
+import { hasProjectAccess, throwIfNoProjectAccess } from "@/src/features/rbac";
import {
createTRPCRouter,
protectedProjectProcedure,
@@ -177,14 +177,25 @@ export const evaluatorRouter = createTRPCRouter({
.input(ActivationCostEstimatesSchema)
.mutation(({ input, ctx }) => {
throwIfNoProjectAccess({
+ session: ctx.session,
+ projectId: ctx.session.projectId,
+ scope: "evalJobExecution:read",
+ });
+ const canRunMissingCostTest = hasProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
scope: "evaluationRule:CUD",
});
+ const { shouldRunMissingTest, ...estimateInput } = input;
+ let missingCostMode: "probe" | "wait" | "skip" = "skip";
+ if (canRunMissingCostTest) {
+ missingCostMode = shouldRunMissingTest ? "probe" : "wait";
+ }
return getActivationCostEstimates({
- ...input,
+ ...estimateInput,
projectId: ctx.session.projectId,
orgId: ctx.session.orgId,
+ missingCostMode,
shouldReadFromObservationsTable:
ctx.session.user.v4BetaEnabled !== true,
});