Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

test(evals): cover evaluator write protections and cross-project checks (#19194)

Tobias Wochinger0449ec2f4ebb
diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
index ea33d8b56b..4338d3e5a3 100644
--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts
+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
@@ -1317,6 +1317,124 @@ describe("EvaluatorService", () => {
);
});
+ it("rejects every write to a built-in evaluator and leaves it unchanged", async () => {
+ const audit = vi.fn(async () => undefined);
+ const service = new EvaluatorService(prisma, audit);
+ const input = llmInput("Built-in evaluator");
+ const builtIn = await service.create(input, null);
+ const blockedAt = new Date("2026-01-01T00:00:00.000Z");
+ await prisma.evaluator.update({
+ where: { id: builtIn.id },
+ data: {
+ isBuiltIn: true,
+ blockedAt,
+ blockReason: "DEFAULT_EVAL_MODEL_MISSING",
+ blockMessage: "Blocked for test",
+ },
+ });
+ audit.mockClear();
+ const changedDefinition = {
+ ...input.definition,
+ promptMessages: [
+ { role: "user" as const, content: "Changed {{output}}" },
+ ],
+ };
+
+ const writes = [
+ () =>
+ service.update(
+ {
+ ...input,
+ evaluatorId: builtIn.id,
+ name: "Renamed built-in",
+ definition: changedDefinition,
+ },
+ null,
+ ),
+ () =>
+ service.patch(
+ { projectId, evaluatorId: builtIn.id, name: "Renamed built-in" },
+ null,
+ ),
+ () =>
+ service.upsertByName({ ...input, definition: changedDefinition }, null),
+ () => service.reactivate({ projectId, evaluatorId: builtIn.id }),
+ () => service.delete(projectId, builtIn.id),
+ () => service.deleteMany({ projectId, evaluatorIds: [builtIn.id] }),
+ ];
+ for (const write of writes) {
+ await expect(write()).rejects.toThrow(
+ "Built-in evaluators cannot be edited",
+ );
+ }
+
+ await expect(
+ prisma.evaluator.findUnique({
+ where: { id: builtIn.id },
+ select: {
+ name: true,
+ blockedAt: true,
+ _count: { select: { versions: true } },
+ },
+ }),
+ ).resolves.toEqual({
+ name: input.name,
+ blockedAt,
+ _count: { versions: 1 },
+ });
+ expect(audit).not.toHaveBeenCalled();
+ });
+
+ it("treats FACET evaluators as missing on every write path", async () => {
+ const service = createService();
+ const blockedAt = new Date("2026-01-01T00:00:00.000Z");
+ const facet = await prisma.evaluator.create({
+ data: {
+ projectId,
+ name: "Internal facet",
+ type: "FACET",
+ blockedAt,
+ blockReason: "DEFAULT_EVAL_MODEL_MISSING",
+ blockMessage: "Blocked for test",
+ versions: { create: { version: 1 } },
+ },
+ });
+ const input = { ...llmInput(facet.name), evaluatorId: facet.id };
+
+ const writes = [
+ () => service.update(input, null),
+ () =>
+ service.patch(
+ { projectId, evaluatorId: facet.id, name: "Renamed facet" },
+ null,
+ ),
+ () => service.reactivate({ projectId, evaluatorId: facet.id }),
+ () => service.delete(projectId, facet.id),
+ () => service.deleteMany({ projectId, evaluatorIds: [facet.id] }),
+ ];
+ for (const write of writes) {
+ await expect(write()).rejects.toThrow("Evaluator not found");
+ }
+ const upserted = await service.upsertByName(llmInput(facet.name), null);
+ expect(upserted.action).toBe("create");
+ expect(upserted.evaluator.id).not.toBe(facet.id);
+
+ await expect(
+ prisma.evaluator.findUnique({
+ where: { id: facet.id },
+ select: {
+ name: true,
+ blockedAt: true,
+ _count: { select: { versions: true } },
+ },
+ }),
+ ).resolves.toEqual({
+ name: facet.name,
+ blockedAt,
+ _count: { versions: 1 },
+ });
+ });
+
it("returns execution summaries by evaluator id, including evaluators without runs", async () => {
const service = createService();
const evaluatorIds = [crypto.randomUUID(), crypto.randomUUID()];
diff --git a/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts b/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts
index 1b045019de..499feb2c45 100644
--- a/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts
+++ b/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts
@@ -6,10 +6,13 @@ import { prisma, type Role } from "@langfuse/shared/src/db";
import {
createEvent,
createEventsCh,
+ createObservation,
+ createObservationsCh,
createOrgProjectAndApiKey,
} from "@langfuse/shared/src/server";
import { appRouter } from "@/src/server/api/root";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
+import type * as EvaluatorValidationModule from "@/src/features/evals/v2/server/evaluators/evaluatorValidation";
const mocks = vi.hoisted(() => ({
getEvaluatorDefinitionPreflightError: vi.fn(async () => null),
@@ -18,7 +21,10 @@ const mocks = vi.hoisted(() => ({
vi.mock(
"@/src/features/evals/v2/server/evaluators/evaluatorValidation",
- () => ({ assertEvaluatorConfigurationValid: vi.fn() }),
+ async (importOriginal) => ({
+ ...(await importOriginal<typeof EvaluatorValidationModule>()),
+ assertEvaluatorConfigurationValid: vi.fn(),
+ }),
);
vi.mock("@/src/features/evals/server/evaluator-preflight", () => ({
@@ -40,11 +46,15 @@ let userId = "";
let caller: ReturnType<typeof appRouter.createCaller>;
let session: Session & { user: NonNullable<Session["user"]> };
-function createCallerWithProjectRole(role: Role) {
+function createCallerWithProjectRole(
+ role: Role,
+ { v4BetaEnabled = false }: { v4BetaEnabled?: boolean } = {},
+) {
const roleSession = {
...session,
user: {
...session.user,
+ v4BetaEnabled,
organizations: session.user.organizations.map((organization) => ({
...organization,
projects: organization.projects.map((project) => ({
@@ -167,6 +177,134 @@ describe("evalsV2 tRPC", () => {
).rejects.toMatchObject({ code: "FORBIDDEN" });
});
+ it("rejects execution summaries without read access or for another project", async () => {
+ const input = { projectId, evaluatorIds: [randomUUID()] };
+
+ await expect(
+ createCallerWithProjectRole("NONE").evalsV2.executionSummaries(input),
+ ).rejects.toMatchObject({ code: "FORBIDDEN" });
+ await expect(
+ caller.evalsV2.executionSummaries({
+ ...input,
+ projectId: otherProjectId,
+ }),
+ ).rejects.toMatchObject({ code: "UNAUTHORIZED" });
+ });
+
+ it("summarizes only executions from the caller's project", async () => {
+ const evaluatorId = randomUUID();
+ await createEventsCh([
+ createEvent({
+ project_id: projectId,
+ trace_id: randomUUID(),
+ evaluator_id: evaluatorId,
+ }),
+ createEvent({
+ project_id: otherProjectId,
+ trace_id: randomUUID(),
+ evaluator_id: evaluatorId,
+ level: "ERROR",
+ }),
+ ]);
+
+ await expect(
+ caller.evalsV2.executionSummaries({
+ projectId,
+ evaluatorIds: [evaluatorId],
+ }),
+ ).resolves.toEqual({ [evaluatorId]: { total: 1, failed: 0 } });
+ });
+
+ it("rejects evaluator tests without write access or for another project", async () => {
+ const foreignEvaluator = await prisma.evaluator.create({
+ data: {
+ projectId: otherProjectId,
+ name: "Foreign test evaluator",
+ type: "LLM_AS_JUDGE",
+ versions: { create: { version: 1 } },
+ },
+ });
+ const input = {
+ projectId,
+ evaluatorId: randomUUID(),
+ definition,
+ observationId: randomUUID(),
+ traceId: randomUUID(),
+ startTime: new Date(),
+ };
+
+ await expect(
+ createCallerWithProjectRole("VIEWER").evalsV2.test(input),
+ ).rejects.toMatchObject({ code: "FORBIDDEN" });
+ await expect(
+ caller.evalsV2.test({ ...input, projectId: otherProjectId }),
+ ).rejects.toMatchObject({ code: "UNAUTHORIZED" });
+ await expect(
+ caller.evalsV2.test({ ...input, evaluatorId: foreignEvaluator.id }),
+ ).rejects.toMatchObject({
+ code: "NOT_FOUND",
+ message: "Evaluator not found",
+ });
+ });
+
+ it("tests evaluators only against observations from the caller's project", async () => {
+ const startTime = new Date();
+ const [own, foreign] = [projectId, otherProjectId].map((project) => ({
+ projectId: project,
+ observationId: randomUUID(),
+ traceId: randomUUID(),
+ }));
+ await Promise.all([
+ createObservationsCh(
+ [own, foreign].map((target) =>
+ createObservation({
+ project_id: target.projectId,
+ id: target.observationId,
+ trace_id: target.traceId,
+ start_time: startTime,
+ }),
+ ),
+ ),
+ createEventsCh(
+ [own, foreign].map((target) =>
+ createEvent({
+ project_id: target.projectId,
+ id: target.observationId,
+ span_id: target.observationId,
+ trace_id: target.traceId,
+ start_time: startTime,
+ }),
+ ),
+ ),
+ ]);
+
+ for (const v4BetaEnabled of [false, true]) {
+ const memberCaller = createCallerWithProjectRole("MEMBER", {
+ v4BetaEnabled,
+ });
+ const testInput = (target: typeof own) => ({
+ projectId,
+ evaluatorId: randomUUID(),
+ definition,
+ observationId: target.observationId,
+ traceId: target.traceId,
+ startTime,
+ });
+
+ // The project has no default model, so a resolved observation ends in an
+ // unsuccessful test result rather than an LLM call.
+ await expect(
+ memberCaller.evalsV2.test(testInput(own)),
+ ).resolves.toMatchObject({ success: false });
+ await expect(
+ memberCaller.evalsV2.test(testInput(foreign)),
+ ).rejects.toMatchObject({
+ code: "NOT_FOUND",
+ message: "Observation not found",
+ });
+ }
+ });
+
it("supports the evaluator management flow", async () => {
const created = await caller.evalsV2.create({
projectId,
@@ -332,6 +470,104 @@ describe("evalsV2 tRPC", () => {
).rejects.toMatchObject({ code: "UNAUTHORIZED" });
});
+ it("rejects evaluator writes without write access", async () => {
+ const evaluator = await caller.evalsV2.create({
+ projectId,
+ name: "Viewer write target",
+ description: null,
+ definition,
+ });
+
+ for (const write of evaluatorWrites(
+ createCallerWithProjectRole("VIEWER"),
+ evaluator.id,
+ )) {
+ await expect(write()).rejects.toMatchObject({ code: "FORBIDDEN" });
+ }
+
+ await expect(
+ prisma.evaluator.count({ where: { id: evaluator.id } }),
+ ).resolves.toBe(1);
+ });
+
+ it("rejects writes to built-in evaluators", async () => {
+ const builtIn = await caller.evalsV2.create({
+ projectId,
+ name: "Built-in transport evaluator",
+ description: null,
+ definition,
+ });
+ const blockedAt = new Date("2026-01-01T00:00:00.000Z");
+ await prisma.evaluator.update({
+ where: { id: builtIn.id, projectId },
+ data: {
+ isBuiltIn: true,
+ blockedAt,
+ blockReason: "DEFAULT_EVAL_MODEL_MISSING",
+ blockMessage: "Blocked for test",
+ },
+ });
+
+ for (const write of evaluatorWrites(
+ createCallerWithProjectRole("MEMBER"),
+ builtIn.id,
+ )) {
+ await expect(write()).rejects.toMatchObject({ code: "BAD_REQUEST" });
+ }
+
+ await expect(
+ prisma.evaluator.findUnique({
+ where: { id: builtIn.id },
+ select: {
+ name: true,
+ blockedAt: true,
+ _count: { select: { versions: true } },
+ },
+ }),
+ ).resolves.toEqual({
+ name: builtIn.name,
+ blockedAt,
+ _count: { versions: 1 },
+ });
+ });
+
+ it("treats FACET evaluators as missing on writes", async () => {
+ const blockedAt = new Date("2026-01-01T00:00:00.000Z");
+ const facet = await prisma.evaluator.create({
+ data: {
+ projectId,
+ name: "Internal transport facet",
+ type: "FACET",
+ blockedAt,
+ blockReason: "DEFAULT_EVAL_MODEL_MISSING",
+ blockMessage: "Blocked for test",
+ versions: { create: { version: 1 } },
+ },
+ });
+
+ for (const write of evaluatorWrites(
+ createCallerWithProjectRole("MEMBER"),
+ facet.id,
+ )) {
+ await expect(write()).rejects.toMatchObject({ code: "NOT_FOUND" });
+ }
+
+ await expect(
+ prisma.evaluator.findUnique({
+ where: { id: facet.id },
+ select: {
+ name: true,
+ blockedAt: true,
+ _count: { select: { versions: true } },
+ },
+ }),
+ ).resolves.toEqual({
+ name: facet.name,
+ blockedAt,
+ _count: { versions: 1 },
+ });
+ });
+
it("returns evaluator costs through the tRPC contract", async () => {
const evaluator = await caller.evalsV2.create({
projectId,
@@ -438,3 +674,23 @@ describe("evalsV2 tRPC", () => {
});
});
});
+
+function evaluatorWrites(
+ roleCaller: ReturnType<typeof appRouter.createCaller>,
+ evaluatorId: string,
+) {
+ return [
+ () =>
+ roleCaller.evalsV2.update({
+ projectId,
+ evaluatorId,
+ name: "Renamed protected evaluator",
+ description: null,
+ definition,
+ }),
+ () => roleCaller.evalsV2.reactivate({ projectId, evaluatorId }),
+ () => roleCaller.evalsV2.delete({ projectId, evaluatorId }),
+ () =>
+ roleCaller.evalsV2.deleteMany({ projectId, evaluatorIds: [evaluatorId] }),
+ ];
+}
diff --git a/web/src/__tests__/server/evaluators-public-api.servertest.ts b/web/src/__tests__/server/evaluators-public-api.servertest.ts
index 267047398e..2e96b46cf4 100644
--- a/web/src/__tests__/server/evaluators-public-api.servertest.ts
+++ b/web/src/__tests__/server/evaluators-public-api.servertest.ts
@@ -716,4 +716,77 @@ describe("stable evaluators public API", () => {
);
expect(response.status).toBe(404);
});
+
+ it("rejects patching and deleting a built-in evaluator", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const evaluator = await createCodeEvaluator("built-in evaluator", auth);
+ await prisma.evaluator.update({
+ where: { id: evaluator.body.id },
+ data: { isBuiltIn: true },
+ });
+
+ for (const [method, body] of [
+ [
+ "PATCH",
+ {
+ name: "renamed built-in evaluator",
+ ...codeDefinition("function evaluate() { return { scores: [1] }; }"),
+ },
+ ],
+ ["DELETE", undefined],
+ ] as const) {
+ const response = await makeAPICall(
+ method,
+ `/api/public/v2/evaluators/${evaluator.body.id}`,
+ body,
+ auth,
+ );
+ expect(response.status).toBe(400);
+ expect(PublicApiError.parse(response.body)).toMatchObject({
+ message: "Built-in evaluators cannot be edited",
+ });
+ }
+
+ await expect(
+ prisma.evaluator.findUnique({
+ where: { id: evaluator.body.id },
+ select: { name: true, _count: { select: { versions: true } } },
+ }),
+ ).resolves.toEqual({
+ name: "built-in evaluator",
+ _count: { versions: 1 },
+ });
+ });
+
+ it("returns 404 when patching or deleting a FACET evaluator", async () => {
+ const { auth, project } = await createOrgProjectAndApiKey();
+ const facet = await prisma.evaluator.create({
+ data: {
+ projectId: project.id,
+ name: "internal facet",
+ type: EvalTemplateType.FACET,
+ versions: { create: { version: 1 } },
+ },
+ });
+
+ for (const [method, body] of [
+ ["PATCH", { name: "renamed facet" }],
+ ["DELETE", undefined],
+ ] as const) {
+ const response = await makeAPICall(
+ method,
+ `/api/public/v2/evaluators/${facet.id}`,
+ body,
+ auth,
+ );
+ expect(response.status).toBe(404);
+ }
+
+ await expect(
+ prisma.evaluator.findUnique({
+ where: { id: facet.id },
+ select: { name: true },
+ }),
+ ).resolves.toEqual({ name: "internal facet" });
+ });
});