Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): show actionable evaluators in migration view (#16053)

marliessophie5bc4e662634e
diff --git a/web/src/__tests__/server/evals-trpc.servertest.ts b/web/src/__tests__/server/evals-trpc.servertest.ts
index 0965527c75..ab1e6b0414 100644
--- a/web/src/__tests__/server/evals-trpc.servertest.ts
+++ b/web/src/__tests__/server/evals-trpc.servertest.ts
@@ -288,6 +288,61 @@ describe("evals trpc", () => {
{ id: inactiveEvaluator.id, displayStatus: "INACTIVE" },
]);
});
+
+ it("filters evaluator configurations by time scope", async () => {
+ const { project, caller } = await prepare();
+
+ const createEvaluator = (scoreName: string, timeScope: string[]) =>
+ prisma.jobConfiguration.create({
+ data: {
+ projectId: project.id,
+ jobType: "EVAL",
+ scoreName,
+ filter: [],
+ targetObject: EvalTargetObject.TRACE,
+ variableMapping: [],
+ sampling: 1,
+ delay: 0,
+ status: "ACTIVE",
+ timeScope,
+ },
+ });
+
+ const newEvaluator = await createEvaluator("new-score", ["NEW"]);
+ const newAndExistingEvaluator = await createEvaluator(
+ "new-and-existing-score",
+ ["NEW", "EXISTING"],
+ );
+ const existingEvaluator = await createEvaluator("existing-score", [
+ "EXISTING",
+ ]);
+
+ const response = await caller.evals.allConfigs({
+ projectId: project.id,
+ filter: [
+ {
+ column: "timeScope",
+ type: "arrayOptions",
+ operator: "any of",
+ value: ["NEW"],
+ },
+ ],
+ orderBy: {
+ column: "createdAt",
+ order: "DESC",
+ },
+ limit: 10,
+ page: 0,
+ });
+
+ expect(response.configs.map((config) => config.id)).toEqual(
+ expect.arrayContaining([newEvaluator.id, newAndExistingEvaluator.id]),
+ );
+ expect(response.configs.map((config) => config.id)).not.toContain(
+ existingEvaluator.id,
+ );
+ expect(response.totalCount).toBe(2);
+ });
});
describe("evals.templateNames", () => {
diff --git a/web/src/features/evals/components/evaluator-table.tsx b/web/src/features/evals/components/evaluator-table.tsx
index 4dc7e22768..5b101271fb 100644
--- a/web/src/features/evals/components/evaluator-table.tsx
+++ b/web/src/features/evals/components/evaluator-table.tsx
@@ -26,7 +26,10 @@ import TableIdOrName from "@/src/components/table/table-id";
import { ExternalLinkIcon, Pen } from "lucide-react";
import { usePeekNavigation } from "@/src/components/table/peek/hooks/usePeekNavigation";
import { TablePeekViewEvaluatorConfigDetail } from "@/src/components/table/peek/peek-evaluator-config-detail";
-import { evalConfigTargetValues } from "@/src/server/api/definitions/evalConfigsTable";
+import {
+ evalConfigTargetValues,
+ evalConfigTimeScopeValues,
+} from "@/src/server/api/definitions/evalConfigsTable";
import { Button } from "@/src/components/ui/button";
import { IconOnlyButton } from "@/src/components/IconOnlyButton";
import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
@@ -92,6 +95,7 @@ export default function EvaluatorTable({ projectId }: { projectId: string }) {
const newFilterOptions = {
status: ["ACTIVE", "PAUSED", "INACTIVE"],
target: evalConfigTargetValues,
+ timeScope: evalConfigTimeScopeValues,
};
const queryFilter = useSidebarFilterState(
diff --git a/web/src/features/filters/config/evaluators-config.ts b/web/src/features/filters/config/evaluators-config.ts
index cb72328c9c..8de63f1d4e 100644
--- a/web/src/features/filters/config/evaluators-config.ts
+++ b/web/src/features/filters/config/evaluators-config.ts
@@ -21,5 +21,10 @@ export const evaluatorFilterConfig: FilterConfig = {
column: "target",
label: "Target",
},
+ {
+ type: "categorical" as const,
+ column: "timeScope",
+ label: "Time Scope",
+ },
],
};
diff --git a/web/src/features/v4-migration/evaluatorMigrationUrls.clienttest.ts b/web/src/features/v4-migration/evaluatorMigrationUrls.clienttest.ts
new file mode 100644
index 0000000000..d86b032706
--- /dev/null
+++ b/web/src/features/v4-migration/evaluatorMigrationUrls.clienttest.ts
@@ -0,0 +1,35 @@
+import { decodeFiltersGeneric } from "@langfuse/shared";
+import { describe, expect, it } from "vitest";
+
+import { buildDeprecatedEvaluatorsUrl } from "./evaluatorMigrationUrls";
+
+describe("buildDeprecatedEvaluatorsUrl", () => {
+ it("shows actionable active and paused legacy evaluators", () => {
+ const url = new URL(
+ buildDeprecatedEvaluatorsUrl("project-1"),
+ "https://langfuse.local",
+ );
+
+ expect(url.pathname).toBe("/project/project-1/evals");
+ expect(decodeFiltersGeneric(url.searchParams.get("filter") ?? "")).toEqual([
+ {
+ column: "status",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["ACTIVE", "PAUSED"],
+ },
+ {
+ column: "target",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["trace", "dataset"],
+ },
+ {
+ column: "timeScope",
+ type: "arrayOptions",
+ operator: "any of",
+ value: ["NEW"],
+ },
+ ]);
+ });
+});
diff --git a/web/src/features/v4-migration/evaluatorMigrationUrls.ts b/web/src/features/v4-migration/evaluatorMigrationUrls.ts
index 5ceb64e131..9c62569195 100644
--- a/web/src/features/v4-migration/evaluatorMigrationUrls.ts
+++ b/web/src/features/v4-migration/evaluatorMigrationUrls.ts
@@ -5,7 +5,7 @@ const DEPRECATED_EVALUATOR_FILTERS: FilterState = [
column: "status",
type: "stringOptions",
operator: "any of",
- value: ["ACTIVE"],
+ value: ["ACTIVE", "PAUSED"],
},
{
column: "target",
@@ -13,6 +13,12 @@ const DEPRECATED_EVALUATOR_FILTERS: FilterState = [
operator: "any of",
value: ["trace", "dataset"],
},
+ {
+ column: "timeScope",
+ type: "arrayOptions",
+ operator: "any of",
+ value: ["NEW"],
+ },
];
const MODERN_EVALUATOR_FILTERS: FilterState = [
diff --git a/web/src/server/api/definitions/evalConfigsTable.ts b/web/src/server/api/definitions/evalConfigsTable.ts
index eaf525e407..4306f67e1a 100644
--- a/web/src/server/api/definitions/evalConfigsTable.ts
+++ b/web/src/server/api/definitions/evalConfigsTable.ts
@@ -2,6 +2,7 @@ import {
EvalTargetObject,
type ColumnDefinition,
JobConfigState,
+ JobTimeScopeZod,
} from "@langfuse/shared";
export const evalConfigTargetOptions = Object.values(EvalTargetObject).map(
@@ -14,6 +15,8 @@ export const evalConfigTargetValues = evalConfigTargetOptions.map(
(option) => option.value,
);
+export const evalConfigTimeScopeValues = JobTimeScopeZod.options;
+
const evaluatorDisplayStatusSql = `CASE
WHEN jc."status" = 'INACTIVE' THEN 'INACTIVE'
WHEN jc."blocked_at" IS NOT NULL THEN 'PAUSED'
@@ -43,6 +46,13 @@ export const evalConfigFilterColumns: ColumnDefinition[] = [
internal: 'jc."target_object"',
options: evalConfigTargetOptions,
},
+ {
+ name: "Time Scope",
+ id: "timeScope",
+ type: "arrayOptions",
+ internal: 'jc."time_scope"',
+ options: evalConfigTimeScopeValues.map((value) => ({ value })),
+ },
];
export const evalConfigsTableCols: ColumnDefinition[] = [
@@ -51,6 +61,7 @@ export const evalConfigsTableCols: ColumnDefinition[] = [
internal: evaluatorStatusSortRankSql,
},
evalConfigFilterColumns[1],
+ evalConfigFilterColumns[2],
{
name: "Updated At",
id: "updatedAt",