🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
test(evals): cover evaluator write protections and cross-project checks (#19194)
Tobias Wochinger0449ec2f4ebb
diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.tsindex ea33d8b56b..4338d3e5a3 100644--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts@@ -1317,6 +1317,124 @@ describe("EvaluatorService", () => {);});+ it("rejects every write to a built-in evaluator and leaves it unchanged", async () => {+ const audit = vi.fn(async () => undefined);+ const service = new EvaluatorService(prisma, audit);+ const input = llmInput("Built-in evaluator");+ const builtIn = await service.create(input, null);+ const blockedAt = new Date("2026-01-01T00:00:00.000Z");+ await prisma.evaluator.update({+ where: { id: builtIn.id },+ data: {+ isBuiltIn: true,+ blockedAt,+ blockReason: "DEFAULT_EVAL_MODEL_MISSING",+ blockMessage: "Blocked for test",+ },+ });+ audit.mockClear();+ const changedDefinition = {+ ...input.definition,+ promptMessages: [+ { role: "user" as const, content: "Changed {{output}}" },+ ],+ };++ const writes = [+ () =>+ service.update(+ {+ ...input,+ evaluatorId: builtIn.id,+ name: "Renamed built-in",+ definition: changedDefinition,+ },+ null,+ ),+ () =>+ service.patch(+ { projectId, evaluatorId: builtIn.id, name: "Renamed built-in" },+ null,+ ),+ () =>+ service.upsertByName({ ...input, definition: changedDefinition }, null),+ () => service.reactivate({ projectId, evaluatorId: builtIn.id }),+ () => service.delete(projectId, builtIn.id),+ () => service.deleteMany({ projectId, evaluatorIds: [builtIn.id] }),+ ];+ for (const write of writes) {+ await expect(write()).rejects.toThrow(+ "Built-in evaluators cannot be edited",+ );+ }++ await expect(+ prisma.evaluator.findUnique({+ where: { id: builtIn.id },+ select: {+ name: true,+ blockedAt: true,+ _count: { select: { versions: true } },+ },+ }),+ ).resolves.toEqual({+ name: input.name,+ blockedAt,+ _count: { versions: 1 },+ });+ expect(audit).not.toHaveBeenCalled();+ });++ it("treats FACET evaluators as missing on every write path", async () => {+ const service = createService();+ const blockedAt = new Date("2026-01-01T00:00:00.000Z");+ const facet = await prisma.evaluator.create({+ data: {+ projectId,+ name: "Internal facet",+ type: "FACET",+ blockedAt,+ blockReason: "DEFAULT_EVAL_MODEL_MISSING",+ blockMessage: "Blocked for test",+ versions: { create: { version: 1 } },+ },+ });+ const input = { ...llmInput(facet.name), evaluatorId: facet.id };++ const writes = [+ () => service.update(input, null),+ () =>+ service.patch(+ { projectId, evaluatorId: facet.id, name: "Renamed facet" },+ null,+ ),+ () => service.reactivate({ projectId, evaluatorId: facet.id }),+ () => service.delete(projectId, facet.id),+ () => service.deleteMany({ projectId, evaluatorIds: [facet.id] }),+ ];+ for (const write of writes) {+ await expect(write()).rejects.toThrow("Evaluator not found");+ }+ const upserted = await service.upsertByName(llmInput(facet.name), null);+ expect(upserted.action).toBe("create");+ expect(upserted.evaluator.id).not.toBe(facet.id);++ await expect(+ prisma.evaluator.findUnique({+ where: { id: facet.id },+ select: {+ name: true,+ blockedAt: true,+ _count: { select: { versions: true } },+ },+ }),+ ).resolves.toEqual({+ name: facet.name,+ blockedAt,+ _count: { versions: 1 },+ });+ });+it("returns execution summaries by evaluator id, including evaluators without runs", async () => {const service = createService();const evaluatorIds = [crypto.randomUUID(), crypto.randomUUID()];diff --git a/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts b/web/src/__tests__/server/evaluator-v2-trpc.servertest.tsindex 1b045019de..499feb2c45 100644--- a/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts@@ -6,10 +6,13 @@ import { prisma, type Role } from "@langfuse/shared/src/db";import {createEvent,createEventsCh,+ createObservation,+ createObservationsCh,createOrgProjectAndApiKey,} from "@langfuse/shared/src/server";import { appRouter } from "@/src/server/api/root";import { createInnerTRPCContext } from "@/src/server/api/trpc";+import type * as EvaluatorValidationModule from "@/src/features/evals/v2/server/evaluators/evaluatorValidation";const mocks = vi.hoisted(() => ({getEvaluatorDefinitionPreflightError: vi.fn(async () => null),@@ -18,7 +21,10 @@ const mocks = vi.hoisted(() => ({vi.mock("@/src/features/evals/v2/server/evaluators/evaluatorValidation",- () => ({ assertEvaluatorConfigurationValid: vi.fn() }),+ async (importOriginal) => ({+ ...(await importOriginal<typeof EvaluatorValidationModule>()),+ assertEvaluatorConfigurationValid: vi.fn(),+ }),);vi.mock("@/src/features/evals/server/evaluator-preflight", () => ({@@ -40,11 +46,15 @@ let userId = "";let caller: ReturnType<typeof appRouter.createCaller>;let session: Session & { user: NonNullable<Session["user"]> };-function createCallerWithProjectRole(role: Role) {+function createCallerWithProjectRole(+ role: Role,+ { v4BetaEnabled = false }: { v4BetaEnabled?: boolean } = {},+) {const roleSession = {...session,user: {...session.user,+ v4BetaEnabled,organizations: session.user.organizations.map((organization) => ({...organization,projects: organization.projects.map((project) => ({@@ -167,6 +177,134 @@ describe("evalsV2 tRPC", () => {).rejects.toMatchObject({ code: "FORBIDDEN" });});+ it("rejects execution summaries without read access or for another project", async () => {+ const input = { projectId, evaluatorIds: [randomUUID()] };++ await expect(+ createCallerWithProjectRole("NONE").evalsV2.executionSummaries(input),+ ).rejects.toMatchObject({ code: "FORBIDDEN" });+ await expect(+ caller.evalsV2.executionSummaries({+ ...input,+ projectId: otherProjectId,+ }),+ ).rejects.toMatchObject({ code: "UNAUTHORIZED" });+ });++ it("summarizes only executions from the caller's project", async () => {+ const evaluatorId = randomUUID();+ await createEventsCh([+ createEvent({+ project_id: projectId,+ trace_id: randomUUID(),+ evaluator_id: evaluatorId,+ }),+ createEvent({+ project_id: otherProjectId,+ trace_id: randomUUID(),+ evaluator_id: evaluatorId,+ level: "ERROR",+ }),+ ]);++ await expect(+ caller.evalsV2.executionSummaries({+ projectId,+ evaluatorIds: [evaluatorId],+ }),+ ).resolves.toEqual({ [evaluatorId]: { total: 1, failed: 0 } });+ });++ it("rejects evaluator tests without write access or for another project", async () => {+ const foreignEvaluator = await prisma.evaluator.create({+ data: {+ projectId: otherProjectId,+ name: "Foreign test evaluator",+ type: "LLM_AS_JUDGE",+ versions: { create: { version: 1 } },+ },+ });+ const input = {+ projectId,+ evaluatorId: randomUUID(),+ definition,+ observationId: randomUUID(),+ traceId: randomUUID(),+ startTime: new Date(),+ };++ await expect(+ createCallerWithProjectRole("VIEWER").evalsV2.test(input),+ ).rejects.toMatchObject({ code: "FORBIDDEN" });+ await expect(+ caller.evalsV2.test({ ...input, projectId: otherProjectId }),+ ).rejects.toMatchObject({ code: "UNAUTHORIZED" });+ await expect(+ caller.evalsV2.test({ ...input, evaluatorId: foreignEvaluator.id }),+ ).rejects.toMatchObject({+ code: "NOT_FOUND",+ message: "Evaluator not found",+ });+ });++ it("tests evaluators only against observations from the caller's project", async () => {+ const startTime = new Date();+ const [own, foreign] = [projectId, otherProjectId].map((project) => ({+ projectId: project,+ observationId: randomUUID(),+ traceId: randomUUID(),+ }));+ await Promise.all([+ createObservationsCh(+ [own, foreign].map((target) =>+ createObservation({+ project_id: target.projectId,+ id: target.observationId,+ trace_id: target.traceId,+ start_time: startTime,+ }),+ ),+ ),+ createEventsCh(+ [own, foreign].map((target) =>+ createEvent({+ project_id: target.projectId,+ id: target.observationId,+ span_id: target.observationId,+ trace_id: target.traceId,+ start_time: startTime,+ }),+ ),+ ),+ ]);++ for (const v4BetaEnabled of [false, true]) {+ const memberCaller = createCallerWithProjectRole("MEMBER", {+ v4BetaEnabled,+ });+ const testInput = (target: typeof own) => ({+ projectId,+ evaluatorId: randomUUID(),+ definition,+ observationId: target.observationId,+ traceId: target.traceId,+ startTime,+ });++ // The project has no default model, so a resolved observation ends in an+ // unsuccessful test result rather than an LLM call.+ await expect(+ memberCaller.evalsV2.test(testInput(own)),+ ).resolves.toMatchObject({ success: false });+ await expect(+ memberCaller.evalsV2.test(testInput(foreign)),+ ).rejects.toMatchObject({+ code: "NOT_FOUND",+ message: "Observation not found",+ });+ }+ });+it("supports the evaluator management flow", async () => {const created = await caller.evalsV2.create({projectId,@@ -332,6 +470,104 @@ describe("evalsV2 tRPC", () => {).rejects.toMatchObject({ code: "UNAUTHORIZED" });});+ it("rejects evaluator writes without write access", async () => {+ const evaluator = await caller.evalsV2.create({+ projectId,+ name: "Viewer write target",+ description: null,+ definition,+ });++ for (const write of evaluatorWrites(+ createCallerWithProjectRole("VIEWER"),+ evaluator.id,+ )) {+ await expect(write()).rejects.toMatchObject({ code: "FORBIDDEN" });+ }++ await expect(+ prisma.evaluator.count({ where: { id: evaluator.id } }),+ ).resolves.toBe(1);+ });++ it("rejects writes to built-in evaluators", async () => {+ const builtIn = await caller.evalsV2.create({+ projectId,+ name: "Built-in transport evaluator",+ description: null,+ definition,+ });+ const blockedAt = new Date("2026-01-01T00:00:00.000Z");+ await prisma.evaluator.update({+ where: { id: builtIn.id, projectId },+ data: {+ isBuiltIn: true,+ blockedAt,+ blockReason: "DEFAULT_EVAL_MODEL_MISSING",+ blockMessage: "Blocked for test",+ },+ });++ for (const write of evaluatorWrites(+ createCallerWithProjectRole("MEMBER"),+ builtIn.id,+ )) {+ await expect(write()).rejects.toMatchObject({ code: "BAD_REQUEST" });+ }++ await expect(+ prisma.evaluator.findUnique({+ where: { id: builtIn.id },+ select: {+ name: true,+ blockedAt: true,+ _count: { select: { versions: true } },+ },+ }),+ ).resolves.toEqual({+ name: builtIn.name,+ blockedAt,+ _count: { versions: 1 },+ });+ });++ it("treats FACET evaluators as missing on writes", async () => {+ const blockedAt = new Date("2026-01-01T00:00:00.000Z");+ const facet = await prisma.evaluator.create({+ data: {+ projectId,+ name: "Internal transport facet",+ type: "FACET",+ blockedAt,+ blockReason: "DEFAULT_EVAL_MODEL_MISSING",+ blockMessage: "Blocked for test",+ versions: { create: { version: 1 } },+ },+ });++ for (const write of evaluatorWrites(+ createCallerWithProjectRole("MEMBER"),+ facet.id,+ )) {+ await expect(write()).rejects.toMatchObject({ code: "NOT_FOUND" });+ }++ await expect(+ prisma.evaluator.findUnique({+ where: { id: facet.id },+ select: {+ name: true,+ blockedAt: true,+ _count: { select: { versions: true } },+ },+ }),+ ).resolves.toEqual({+ name: facet.name,+ blockedAt,+ _count: { versions: 1 },+ });+ });+it("returns evaluator costs through the tRPC contract", async () => {const evaluator = await caller.evalsV2.create({projectId,@@ -438,3 +674,23 @@ describe("evalsV2 tRPC", () => {});});});++function evaluatorWrites(+ roleCaller: ReturnType<typeof appRouter.createCaller>,+ evaluatorId: string,+) {+ return [+ () =>+ roleCaller.evalsV2.update({+ projectId,+ evaluatorId,+ name: "Renamed protected evaluator",+ description: null,+ definition,+ }),+ () => roleCaller.evalsV2.reactivate({ projectId, evaluatorId }),+ () => roleCaller.evalsV2.delete({ projectId, evaluatorId }),+ () =>+ roleCaller.evalsV2.deleteMany({ projectId, evaluatorIds: [evaluatorId] }),+ ];+}diff --git a/web/src/__tests__/server/evaluators-public-api.servertest.ts b/web/src/__tests__/server/evaluators-public-api.servertest.tsindex 267047398e..2e96b46cf4 100644--- a/web/src/__tests__/server/evaluators-public-api.servertest.ts+++ b/web/src/__tests__/server/evaluators-public-api.servertest.ts@@ -716,4 +716,77 @@ describe("stable evaluators public API", () => {);expect(response.status).toBe(404);});++ it("rejects patching and deleting a built-in evaluator", async () => {+ const { auth } = await createOrgProjectAndApiKey();+ const evaluator = await createCodeEvaluator("built-in evaluator", auth);+ await prisma.evaluator.update({+ where: { id: evaluator.body.id },+ data: { isBuiltIn: true },+ });++ for (const [method, body] of [+ [+ "PATCH",+ {+ name: "renamed built-in evaluator",+ ...codeDefinition("function evaluate() { return { scores: [1] }; }"),+ },+ ],+ ["DELETE", undefined],+ ] as const) {+ const response = await makeAPICall(+ method,+ `/api/public/v2/evaluators/${evaluator.body.id}`,+ body,+ auth,+ );+ expect(response.status).toBe(400);+ expect(PublicApiError.parse(response.body)).toMatchObject({+ message: "Built-in evaluators cannot be edited",+ });+ }++ await expect(+ prisma.evaluator.findUnique({+ where: { id: evaluator.body.id },+ select: { name: true, _count: { select: { versions: true } } },+ }),+ ).resolves.toEqual({+ name: "built-in evaluator",+ _count: { versions: 1 },+ });+ });++ it("returns 404 when patching or deleting a FACET evaluator", async () => {+ const { auth, project } = await createOrgProjectAndApiKey();+ const facet = await prisma.evaluator.create({+ data: {+ projectId: project.id,+ name: "internal facet",+ type: EvalTemplateType.FACET,+ versions: { create: { version: 1 } },+ },+ });++ for (const [method, body] of [+ ["PATCH", { name: "renamed facet" }],+ ["DELETE", undefined],+ ] as const) {+ const response = await makeAPICall(+ method,+ `/api/public/v2/evaluators/${facet.id}`,+ body,+ auth,+ );+ expect(response.status).toBe(404);+ }++ await expect(+ prisma.evaluator.findUnique({+ where: { id: facet.id },+ select: { name: true },+ }),+ ).resolves.toEqual({ name: "internal facet" });+ });});