Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

16.8 KB
Raw Blame
import { randomUUID } from "node:crypto";
import { EvalTargetObject, EvalTemplateType } from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
import {
  makeAPICall,
  makeZodVerifiedAPICall,
} from "@/src/__tests__/test-utils";
import {
  createCodeEvaluator,
  createEvaluationRule,
} from "@/src/__tests__/server/stable-evaluation-public-api-test-utils";
import { EvaluatorService } from "@/src/features/evals/v2/server/evaluators/evaluatorService";
import { RuleService } from "@/src/features/evals/v2/server/rules/ruleService";
import {
  DeleteEvaluationRuleResponse,
  EvaluationRule,
  ListEvaluationRulesResponse,
} from "@/src/features/public-api/types/evaluation/evaluationRules";
import { Evaluator } from "@/src/features/public-api/types/evaluation/evaluators";
import { PublicApiError } from "@/src/features/public-api/types/public-api-errors";

const createLegacyRule = (projectId: string, name: string) =>
  prisma.evaluationRule.create({
    data: {
      projectId,
      name,
      targetObject: EvalTargetObject.TRACE,
      status: "INACTIVE",
      filter: [
        {
          type: "string",
          column: "Trace Name",
          key: "legacy-filter-key",
          operator: "=",
          value: "legacy trace",
        },
      ],
      sampling: 1,
      delay: 0,
      timeScope: ["NEW"],
    },
  });

describe("stable evaluation rules public API", () => {
  it("creates evaluation rules", async () => {
    const { auth } = await createOrgProjectAndApiKey();
    const evaluator = await createCodeEvaluator("rule evaluator", auth);
    const datasetId = randomUUID();
    const body = {
      name: "same rule name",
      enabled: false,
      filter: [
        {
          type: "boolean" as const,
          column: "isExperimentItemRootSpan" as const,
          operator: "=" as const,
          value: true,
        },
        {
          type: "stringOptions" as const,
          column: "datasetId" as const,
          operator: "any of" as const,
          value: [datasetId],
        },
        {
          type: "null" as const,
          column: "parentObservationId" as const,
          operator: "is null" as const,
          value: "" as const,
        },
      ],
      evaluatorAssignments: [
        { evaluatorId: evaluator.body.id, variableMapping: null },
      ],
    };

    const first = await makeZodVerifiedAPICall(
      EvaluationRule,
      "POST",
      "/api/public/v2/evaluation-rules",
      body,
      auth,
      201,
    );
    const second = await makeZodVerifiedAPICall(
      EvaluationRule,
      "POST",
      "/api/public/v2/evaluation-rules",
      body,
      auth,
      201,
    );
    expect(first.body.id).not.toBe(second.body.id);
    expect(first.body).toMatchObject({
      createdBy: null,
      sampling: 1,
      evaluatorAssignments: [
        {
          evaluatorId: evaluator.body.id,
          variableMapping: null,
        },
      ],
    });
    expect(first.body).not.toHaveProperty("target");
    expect(first.body.filter).toEqual(
      expect.arrayContaining([
        expect.objectContaining({
          column: "isExperimentItemRootSpan",
          value: true,
        }),
        expect.objectContaining({
          column: "datasetId",
          value: [datasetId],
        }),
        expect.objectContaining({
          type: "null",
          column: "parentObservationId",
          operator: "is null",
          value: "",
        }),
      ]),
    );

    const nullSampling = await makeAPICall(
      "POST",
      "/api/public/v2/evaluation-rules",
      {
        ...body,
        name: "invalid nullable sampling",
        sampling: null,
      },
      auth,
    );
    expect(nullSampling.status).toBe(400);

    const enabledWithoutEvaluator = await makeAPICall(
      "POST",
      "/api/public/v2/evaluation-rules",
      {
        name: "invalid active draft",
        enabled: true,
        sampling: 1,
        filter: [],
        evaluatorAssignments: [],
      },
      auth,
    );
    expect(enabledWithoutEvaluator.status).toBe(400);
  });

  it("gets an evaluation rule", async () => {
    const { auth, projectId } = await createOrgProjectAndApiKey();
    const user = await prisma.user.create({
      data: {
        id: randomUUID(),
        name: "Rule Author",
        email: `${randomUUID()}@example.com`,
      },
    });
    const evaluator = await new EvaluatorService(
      prisma,
      async () => undefined,
    ).create(
      {
        projectId,
        name: "rule author evaluator",
        description: null,
        definition: {
          type: EvalTemplateType.CODE,
          sourceCode: "function evaluate() { return { scores: [] }; }",
          sourceCodeLanguage: "TYPESCRIPT",
        },
      },
      user.id,
    );
    const rule = await new RuleService(prisma, async () => undefined).create(
      {
        projectId,
        name: "authored rule",
        targetObject: EvalTargetObject.EVENT,
        enabled: false,
        sampling: 1,
        filter: [],
        evaluatorAssignments: [
          { evaluatorId: evaluator.id, variableMapping: null },
        ],
      },
      user.id,
    );

    const response = await makeZodVerifiedAPICall(
      EvaluationRule,
      "GET",
      `/api/public/v2/evaluation-rules/${rule.id}`,
      undefined,
      auth,
    );
    expect(response.body.createdBy).toEqual({
      id: user.id,
      name: user.name,
    });

    const legacy = await createLegacyRule(projectId, "legacy get rule");
    await prisma.evaluationRule.update({
      where: { id: legacy.id },
      data: {
        filter: [
          {
            type: "string",
            column: "Trace Name",
            operator: "=",
            value: "legacy trace",
          },
          {
            type: "booleanObject",
            column: "metadata",
            key: "reviewed",
            operator: "=",
            value: true,
          },
          {
            type: "positionInTrace",
            column: "positionInTrace",
            operator: "=",
            key: "root",
          },
        ],
      },
    });
    await prisma.evaluationRuleEvaluatorAssignment.create({
      data: {
        projectId,
        evaluationRuleId: legacy.id,
        evaluatorId: evaluator.id,
        variableMapping: [
          {
            templateVariable: "input",
            langfuseObject: "trace",
            objectName: null,
            selectedColumnId: "input",
            jsonSelector: null,
          },
          {
            templateVariable: "output",
            langfuseObject: "generation",
            objectName: "answer-generation",
            selectedColumnId: "output",
            jsonSelector: "$.answer",
          },
        ],
      },
    });
    const legacyResponse = await makeZodVerifiedAPICall(
      EvaluationRule,
      "GET",
      `/api/public/v2/evaluation-rules/${legacy.id}`,
      undefined,
      auth,
    );
    expect(legacyResponse.body).toMatchObject({
      id: legacy.id,
      filter: [
        { column: "Trace Name", value: "legacy trace" },
        {
          type: "booleanObject",
          column: "metadata",
          key: "reviewed",
          value: true,
        },
        {
          type: "positionInTrace",
          column: "positionInTrace",
          key: "root",
        },
      ],
      evaluatorAssignments: [
        {
          variableMapping: [
            {
              mappingType: "legacy",
              variable: "input",
              langfuseObject: "trace",
              objectName: null,
              source: "input",
            },
            {
              mappingType: "legacy",
              variable: "output",
              langfuseObject: "generation",
              objectName: "answer-generation",
              source: "output",
              jsonPath: "$.answer",
            },
          ],
        },
      ],
    });
  });

  it("returns 404 when getting an invalid evaluation rule ID", async () => {
    const { auth } = await createOrgProjectAndApiKey();
    const missing = await makeAPICall(
      "GET",
      `/api/public/v2/evaluation-rules/${randomUUID()}`,
      undefined,
      auth,
    );
    expect(missing.status).toBe(404);
  });

  it("rejects an LLM evaluator assignment when no variable mapping is configured", async () => {
    const { auth } = await createOrgProjectAndApiKey();
    const evaluator = await makeZodVerifiedAPICall(
      Evaluator,
      "POST",
      "/api/public/v2/evaluators",
      {
        name: "LLM evaluator without default mapping",
        type: "llm_as_judge",
        prompt: [{ role: "user", content: "Judge {{input}}" }],
        outputDefinition: {
          dataType: "NUMERIC",
          minValue: 0,
          maxValue: 1,
        },
      },
      auth,
      201,
    );

    const rule = await makeAPICall(
      "POST",
      "/api/public/v2/evaluation-rules",
      {
        name: "Rule without variable mapping",
        enabled: false,
        evaluatorAssignments: [{ evaluatorId: evaluator.body.id }],
      },
      auth,
    );

    expect(rule.status).toBe(400);
    expect(PublicApiError.parse(rule.body)).toMatchObject({
      code: "invalid_request",
      message: "Missing mappings for evaluator variables: input",
    });
  });

  it("lists evaluation rules", async () => {
    const { auth, projectId } = await createOrgProjectAndApiKey();
    const evaluator = await createCodeEvaluator("listed rule evaluator", auth);
    const modern = await createEvaluationRule({
      name: "listed modern rule",
      evaluatorId: evaluator.body.id,
      auth,
    });
    const legacy = await createLegacyRule(projectId, "legacy listed rule");

    const response = await makeZodVerifiedAPICall(
      ListEvaluationRulesResponse,
      "GET",
      "/api/public/v2/evaluation-rules",
      undefined,
      auth,
    );
    expect(response.body.data).toContainEqual(
      expect.objectContaining({ id: modern.body.id }),
    );
    expect(response.body.data.find(({ id }) => id === legacy.id)).toMatchObject(
      {
        id: legacy.id,
        filter: [
          {
            type: "string",
            column: "Trace Name",
            key: "legacy-filter-key",
            operator: "=",
            value: "legacy trace",
          },
        ],
      },
    );
  });

  it("lists evaluation rules with cursor pagination", async () => {
    const { auth } = await createOrgProjectAndApiKey();
    const evaluator = await createCodeEvaluator(
      "paginated rule evaluator",
      auth,
    );
    const created = await Promise.all([
      createEvaluationRule({
        name: "paginated rule one",
        evaluatorId: evaluator.body.id,
        auth,
      }),
      createEvaluationRule({
        name: "paginated rule two",
        evaluatorId: evaluator.body.id,
        auth,
      }),
    ]);

    const firstPage = await makeZodVerifiedAPICall(
      ListEvaluationRulesResponse,
      "GET",
      "/api/public/v2/evaluation-rules?limit=1",
      undefined,
      auth,
    );
    expect(firstPage.body.data).toHaveLength(1);
    expect(firstPage.body.meta.cursor).toEqual(expect.any(String));

    const secondPage = await makeZodVerifiedAPICall(
      ListEvaluationRulesResponse,
      "GET",
      `/api/public/v2/evaluation-rules?limit=1&cursor=${encodeURIComponent(firstPage.body.meta.cursor!)}`,
      undefined,
      auth,
    );
    expect(secondPage.body.data).toHaveLength(1);
    expect(secondPage.body.meta.cursor).toBeUndefined();
    expect(
      new Set(
        [...firstPage.body.data, ...secondPage.body.data].map(({ id }) => id),
      ),
    ).toEqual(new Set(created.map(({ body: { id } }) => id)));
  });

  it("patches an evaluation rule", async () => {
    const { auth, projectId } = await createOrgProjectAndApiKey();
    const evaluator = await createCodeEvaluator("patched rule evaluator", auth);
    const rule = await createEvaluationRule({
      name: "rule before patch",
      evaluatorId: evaluator.body.id,
      auth,
    });

    const response = await makeZodVerifiedAPICall(
      EvaluationRule,
      "PATCH",
      `/api/public/v2/evaluation-rules/${rule.body.id}`,
      { name: "rule after patch", sampling: 0.5 },
      auth,
    );
    expect(response.body).toMatchObject({
      id: rule.body.id,
      name: "rule after patch",
      sampling: 0.5,
    });

    const contradictory = await makeAPICall(
      "PATCH",
      `/api/public/v2/evaluation-rules/${rule.body.id}`,
      { enabled: true, evaluatorAssignments: [] },
      auth,
    );
    expect(contradictory.status).toBe(400);
    expect(PublicApiError.parse(contradictory.body)).toMatchObject({
      code: "invalid_request",
      message:
        "An enabled evaluation rule requires at least one evaluator assignment",
    });

    const detached = await makeZodVerifiedAPICall(
      EvaluationRule,
      "PATCH",
      `/api/public/v2/evaluation-rules/${rule.body.id}`,
      { evaluatorAssignments: [] },
      auth,
    );
    expect(detached.body).toMatchObject({
      enabled: false,
      evaluatorAssignments: [],
    });

    const enableWithoutAssignments = await makeAPICall(
      "PATCH",
      `/api/public/v2/evaluation-rules/${rule.body.id}`,
      { enabled: true },
      auth,
    );
    expect(enableWithoutAssignments.status).toBe(400);

    const storedExperimentRule = await prisma.evaluationRule.create({
      data: {
        projectId,
        name: "stored experiment rule",
        targetObject: EvalTargetObject.EVENT,
        status: "INACTIVE",
        filter: [
          {
            type: "boolean",
            column: "isExperimentItemRootSpan",
            operator: "=",
            value: true,
          },
        ],
        sampling: 1,
        delay: 0,
        timeScope: ["NEW"],
      },
    });
    const experimentResponse = await makeZodVerifiedAPICall(
      EvaluationRule,
      "PATCH",
      `/api/public/v2/evaluation-rules/${storedExperimentRule.id}`,
      { filter: [] },
      auth,
    );
    expect(experimentResponse.body.filter).toEqual([]);
    await expect(
      prisma.evaluationRule.findUnique({
        where: { id: storedExperimentRule.id },
        select: { targetObject: true, filter: true },
      }),
    ).resolves.toMatchObject({
      targetObject: EvalTargetObject.EVENT,
      filter: [],
    });
  });

  it("matches the existing legacy rule update restrictions", async () => {
    const { auth, projectId } = await createOrgProjectAndApiKey();
    const legacy = await createLegacyRule(projectId, "legacy patch rule");

    const missing = await makeAPICall(
      "PATCH",
      `/api/public/v2/evaluation-rules/${randomUUID()}`,
      { name: "missing rule" },
      auth,
    );
    expect(missing.status).toBe(404);

    const legacyResponse = await makeZodVerifiedAPICall(
      EvaluationRule,
      "PATCH",
      `/api/public/v2/evaluation-rules/${legacy.id}`,
      { enabled: false },
      auth,
    );
    expect(legacyResponse.body).toMatchObject({
      id: legacy.id,
      enabled: false,
    });

    const unsupportedUpdate = await makeAPICall(
      "PATCH",
      `/api/public/v2/evaluation-rules/${legacy.id}`,
      { name: "mutated legacy rule" },
      auth,
    );
    expect(unsupportedUpdate.status).toBe(400);
    await expect(
      prisma.evaluationRule.findUnique({ where: { id: legacy.id } }),
    ).resolves.toMatchObject({
      name: "legacy patch rule",
      status: "INACTIVE",
      targetObject: EvalTargetObject.TRACE,
    });
  });

  it("deletes an evaluation rule", async () => {
    const { auth } = await createOrgProjectAndApiKey();
    const evaluator = await createCodeEvaluator("deleted rule evaluator", auth);
    const rule = await createEvaluationRule({
      name: "deleted rule",
      evaluatorId: evaluator.body.id,
      auth,
    });

    const response = await makeZodVerifiedAPICall(
      DeleteEvaluationRuleResponse,
      "DELETE",
      `/api/public/v2/evaluation-rules/${rule.body.id}`,
      undefined,
      auth,
    );
    expect(response.body.id).toBe(rule.body.id);

    const get = await makeAPICall(
      "GET",
      `/api/public/v2/evaluation-rules/${rule.body.id}`,
      undefined,
      auth,
    );
    expect(get.status).toBe(404);
  });

  it("deletes legacy rules and returns 404 for missing rules", async () => {
    const { auth, projectId } = await createOrgProjectAndApiKey();
    const legacy = await createLegacyRule(projectId, "legacy delete rule");
    const missing = await makeAPICall(
      "DELETE",
      `/api/public/v2/evaluation-rules/${randomUUID()}`,
      undefined,
      auth,
    );
    expect(missing.status).toBe(404);

    const legacyResponse = await makeZodVerifiedAPICall(
      DeleteEvaluationRuleResponse,
      "DELETE",
      `/api/public/v2/evaluation-rules/${legacy.id}`,
      undefined,
      auth,
    );
    expect(legacyResponse.body.id).toBe(legacy.id);
    await expect(
      prisma.evaluationRule.findUnique({ where: { id: legacy.id } }),
    ).resolves.toBeNull();
  });
});