🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): show actionable evaluators in migration view (#16053)
marliessophie5bc4e662634e
diff --git a/web/src/__tests__/server/evals-trpc.servertest.ts b/web/src/__tests__/server/evals-trpc.servertest.tsindex 0965527c75..ab1e6b0414 100644--- a/web/src/__tests__/server/evals-trpc.servertest.ts+++ b/web/src/__tests__/server/evals-trpc.servertest.ts@@ -288,6 +288,61 @@ describe("evals trpc", () => {{ id: inactiveEvaluator.id, displayStatus: "INACTIVE" },]);});++ it("filters evaluator configurations by time scope", async () => {+ const { project, caller } = await prepare();++ const createEvaluator = (scoreName: string, timeScope: string[]) =>+ prisma.jobConfiguration.create({+ data: {+ projectId: project.id,+ jobType: "EVAL",+ scoreName,+ filter: [],+ targetObject: EvalTargetObject.TRACE,+ variableMapping: [],+ sampling: 1,+ delay: 0,+ status: "ACTIVE",+ timeScope,+ },+ });++ const newEvaluator = await createEvaluator("new-score", ["NEW"]);+ const newAndExistingEvaluator = await createEvaluator(+ "new-and-existing-score",+ ["NEW", "EXISTING"],+ );+ const existingEvaluator = await createEvaluator("existing-score", [+ "EXISTING",+ ]);++ const response = await caller.evals.allConfigs({+ projectId: project.id,+ filter: [+ {+ column: "timeScope",+ type: "arrayOptions",+ operator: "any of",+ value: ["NEW"],+ },+ ],+ orderBy: {+ column: "createdAt",+ order: "DESC",+ },+ limit: 10,+ page: 0,+ });++ expect(response.configs.map((config) => config.id)).toEqual(+ expect.arrayContaining([newEvaluator.id, newAndExistingEvaluator.id]),+ );+ expect(response.configs.map((config) => config.id)).not.toContain(+ existingEvaluator.id,+ );+ expect(response.totalCount).toBe(2);+ });});describe("evals.templateNames", () => {diff --git a/web/src/features/evals/components/evaluator-table.tsx b/web/src/features/evals/components/evaluator-table.tsxindex 4dc7e22768..5b101271fb 100644--- a/web/src/features/evals/components/evaluator-table.tsx+++ b/web/src/features/evals/components/evaluator-table.tsx@@ -26,7 +26,10 @@ import TableIdOrName from "@/src/components/table/table-id";import { ExternalLinkIcon, Pen } from "lucide-react";import { usePeekNavigation } from "@/src/components/table/peek/hooks/usePeekNavigation";import { TablePeekViewEvaluatorConfigDetail } from "@/src/components/table/peek/peek-evaluator-config-detail";-import { evalConfigTargetValues } from "@/src/server/api/definitions/evalConfigsTable";+import {+ evalConfigTargetValues,+ evalConfigTimeScopeValues,+} from "@/src/server/api/definitions/evalConfigsTable";import { Button } from "@/src/components/ui/button";import { IconOnlyButton } from "@/src/components/IconOnlyButton";import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";@@ -92,6 +95,7 @@ export default function EvaluatorTable({ projectId }: { projectId: string }) {const newFilterOptions = {status: ["ACTIVE", "PAUSED", "INACTIVE"],target: evalConfigTargetValues,+ timeScope: evalConfigTimeScopeValues,};const queryFilter = useSidebarFilterState(diff --git a/web/src/features/filters/config/evaluators-config.ts b/web/src/features/filters/config/evaluators-config.tsindex cb72328c9c..8de63f1d4e 100644--- a/web/src/features/filters/config/evaluators-config.ts+++ b/web/src/features/filters/config/evaluators-config.ts@@ -21,5 +21,10 @@ export const evaluatorFilterConfig: FilterConfig = {column: "target",label: "Target",},+ {+ type: "categorical" as const,+ column: "timeScope",+ label: "Time Scope",+ },],};diff --git a/web/src/features/v4-migration/evaluatorMigrationUrls.clienttest.ts b/web/src/features/v4-migration/evaluatorMigrationUrls.clienttest.tsnew file mode 100644index 0000000000..d86b032706--- /dev/null+++ b/web/src/features/v4-migration/evaluatorMigrationUrls.clienttest.ts@@ -0,0 +1,35 @@+import { decodeFiltersGeneric } from "@langfuse/shared";+import { describe, expect, it } from "vitest";++import { buildDeprecatedEvaluatorsUrl } from "./evaluatorMigrationUrls";++describe("buildDeprecatedEvaluatorsUrl", () => {+ it("shows actionable active and paused legacy evaluators", () => {+ const url = new URL(+ buildDeprecatedEvaluatorsUrl("project-1"),+ "https://langfuse.local",+ );++ expect(url.pathname).toBe("/project/project-1/evals");+ expect(decodeFiltersGeneric(url.searchParams.get("filter") ?? "")).toEqual([+ {+ column: "status",+ type: "stringOptions",+ operator: "any of",+ value: ["ACTIVE", "PAUSED"],+ },+ {+ column: "target",+ type: "stringOptions",+ operator: "any of",+ value: ["trace", "dataset"],+ },+ {+ column: "timeScope",+ type: "arrayOptions",+ operator: "any of",+ value: ["NEW"],+ },+ ]);+ });+});diff --git a/web/src/features/v4-migration/evaluatorMigrationUrls.ts b/web/src/features/v4-migration/evaluatorMigrationUrls.tsindex 5ceb64e131..9c62569195 100644--- a/web/src/features/v4-migration/evaluatorMigrationUrls.ts+++ b/web/src/features/v4-migration/evaluatorMigrationUrls.ts@@ -5,7 +5,7 @@ const DEPRECATED_EVALUATOR_FILTERS: FilterState = [column: "status",type: "stringOptions",operator: "any of",- value: ["ACTIVE"],+ value: ["ACTIVE", "PAUSED"],},{column: "target",@@ -13,6 +13,12 @@ const DEPRECATED_EVALUATOR_FILTERS: FilterState = [operator: "any of",value: ["trace", "dataset"],},+ {+ column: "timeScope",+ type: "arrayOptions",+ operator: "any of",+ value: ["NEW"],+ },];const MODERN_EVALUATOR_FILTERS: FilterState = [diff --git a/web/src/server/api/definitions/evalConfigsTable.ts b/web/src/server/api/definitions/evalConfigsTable.tsindex eaf525e407..4306f67e1a 100644--- a/web/src/server/api/definitions/evalConfigsTable.ts+++ b/web/src/server/api/definitions/evalConfigsTable.ts@@ -2,6 +2,7 @@ import {EvalTargetObject,type ColumnDefinition,JobConfigState,+ JobTimeScopeZod,} from "@langfuse/shared";export const evalConfigTargetOptions = Object.values(EvalTargetObject).map(@@ -14,6 +15,8 @@ export const evalConfigTargetValues = evalConfigTargetOptions.map((option) => option.value,);+export const evalConfigTimeScopeValues = JobTimeScopeZod.options;+const evaluatorDisplayStatusSql = `CASEWHEN jc."status" = 'INACTIVE' THEN 'INACTIVE'WHEN jc."blocked_at" IS NOT NULL THEN 'PAUSED'@@ -43,6 +46,13 @@ export const evalConfigFilterColumns: ColumnDefinition[] = [internal: 'jc."target_object"',options: evalConfigTargetOptions,},+ {+ name: "Time Scope",+ id: "timeScope",+ type: "arrayOptions",+ internal: 'jc."time_scope"',+ options: evalConfigTimeScopeValues.map((value) => ({ value })),+ },];export const evalConfigsTableCols: ColumnDefinition[] = [@@ -51,6 +61,7 @@ export const evalConfigsTableCols: ColumnDefinition[] = [internal: evaluatorStatusSortRankSql,},evalConfigFilterColumns[1],+ evalConfigFilterColumns[2],{name: "Updated At",id: "updatedAt",