🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): keep rule ordering stable on status changes (#16680)
Tobias Wochinger669deee09a3c
diff --git a/web/src/__tests__/server/evaluation-rule-v2-repository.servertest.ts b/web/src/__tests__/server/evaluation-rule-v2-repository.servertest.tsindex 4a9cc5dfa5..50593135ba 100644--- a/web/src/__tests__/server/evaluation-rule-v2-repository.servertest.ts+++ b/web/src/__tests__/server/evaluation-rule-v2-repository.servertest.ts@@ -160,7 +160,7 @@ describe("evaluation rule v2 repository", () => {});});- it("paginates rules in descending update order", async () => {+ it("uses stable creation order by default and supports explicit sorting", async () => {const [older, newer] = await Promise.all([createRule({ name: "Older" }),createRule({ name: "Newer" }),@@ -168,11 +168,17 @@ describe("evaluation rule v2 repository", () => {await Promise.all([prisma.evaluationRule.update({where: { id: older.id },- data: { updatedAt: new Date("2026-01-01T00:00:00.000Z") },+ data: {+ createdAt: new Date("2026-01-01T00:00:00.000Z"),+ updatedAt: new Date("2026-01-03T00:00:00.000Z"),+ },}),prisma.evaluationRule.update({where: { id: newer.id },- data: { updatedAt: new Date("2026-01-02T00:00:00.000Z") },+ data: {+ createdAt: new Date("2026-01-02T00:00:00.000Z"),+ updatedAt: new Date("2026-01-02T00:00:00.000Z"),+ },}),]);@@ -194,6 +200,23 @@ describe("evaluation rule v2 repository", () => {rules: [expect.objectContaining({ id: older.id })],totalItems: 2,});+ await expect(+ ruleRepository.listRules({+ prisma,+ input: {+ projectId,+ page: 1,+ limit: 2,+ orderBy: { column: "updatedAt", order: "DESC" },+ },+ }),+ ).resolves.toEqual({+ rules: [+ expect.objectContaining({ id: older.id }),+ expect.objectContaining({ id: newer.id }),+ ],+ totalItems: 2,+ });});it("filters rules by name, creator, and enabled state", async () => {diff --git a/web/src/__tests__/server/evaluator-v2-repository.servertest.ts b/web/src/__tests__/server/evaluator-v2-repository.servertest.tsindex 40e52ac220..1f5ef1b294 100644--- a/web/src/__tests__/server/evaluator-v2-repository.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-repository.servertest.ts@@ -262,7 +262,7 @@ describe("evaluator v2 repository", () => {});});- it("paginates evaluators in descending update order", async () => {+ it("preserves default update order and supports explicit creation order", async () => {const [recentlyUpdated, newerButUnchanged] = await Promise.all([createEvaluator(),createEvaluator(),@@ -308,6 +308,22 @@ describe("evaluator v2 repository", () => {evaluators: [expect.objectContaining({ id: newerButUnchanged.id })],totalItems: 2,});++ await expect(+ evaluatorRepository.listEvaluators({+ prisma,+ projectId,+ page: 1,+ limit: 2,+ orderBy: { column: "createdAt", order: "DESC" },+ }),+ ).resolves.toEqual({+ evaluators: [+ expect.objectContaining({ id: newerButUnchanged.id }),+ expect.objectContaining({ id: recentlyUpdated.id }),+ ],+ totalItems: 2,+ });});it("filters evaluators by name, status, type, and creator", async () => {diff --git a/web/src/features/evals/v2/components/Rules/RulesTable/RulesTable.tsx b/web/src/features/evals/v2/components/Rules/RulesTable/RulesTable.tsxindex ccd42eb1ab..7c4fe32147 100644--- a/web/src/features/evals/v2/components/Rules/RulesTable/RulesTable.tsx+++ b/web/src/features/evals/v2/components/Rules/RulesTable/RulesTable.tsx@@ -44,7 +44,7 @@ import {TooltipContent,TooltipTrigger,} from "@/src/components/ui/tooltip";-import { api } from "@/src/utils/api";+import { api, type RouterInputs } from "@/src/utils/api";import { usdFormatter } from "@/src/utils/numbers";import { trpcErrorToast } from "@/src/utils/trpcErrorToast";import {@@ -66,6 +66,7 @@ import {} from "@/src/features/evals/v2/constants/tableFilterColumns";import { omitFilterFacets } from "@/src/features/filters/lib/filter-config";import { createNumberTableColumn } from "@/src/components/design-system/Table/columns/createNumberTableColumn";+import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";import { createUserTableColumn } from "@/src/components/design-system/Table/columns/createUserTableColumn";function RelativeDate({ date }: { date: Date }) {@@ -121,6 +122,10 @@ export function RulesTable({const utils = api.useUtils();const [selectionStore] = useState(createTableSelectionStore);const [pagination, setPagination] = usePaginationState(1, 50);+ const [orderBy, setOrderBy] = useOrderByState({+ column: "createdAt",+ order: "DESC",+ });const [rowHeight, setRowHeight] = useRowHeightLocalStorage("evaluationRulesV2","s",@@ -184,6 +189,9 @@ export function RulesTable({const rules = api.evalsV2.rules.list.useQuery({projectId,...pagination,+ orderBy: orderBy+ ? (orderBy as RouterInputs["evalsV2"]["rules"]["list"]["orderBy"])+ : undefined,search: searchQuery ?? undefined,filter: filterState,});@@ -230,6 +238,7 @@ export function RulesTable({header: "Name",size: 260,isFixedPosition: true,+ enableSorting: true,cell: ({ row }) => {const legacy = isLegacyEvalTarget(row.original.targetObject);const upgradeRequired = requiresLegacyMigrationAction({@@ -268,6 +277,7 @@ export function RulesTable({header: "Enabled",size: 90,enableHiding: true,+ enableSorting: true,cell: ({ row }) => (<RuleActiveSwitchCellrule={row.original}@@ -347,6 +357,7 @@ export function RulesTable({header: "Sampling",size: 100,enableHiding: true,+ enableSorting: true,cell: ({ row }) => `${Math.round(row.original.sampling * 100)}%`,},createUserTableColumn<RuleTableRow>({@@ -364,6 +375,7 @@ export function RulesTable({size: 180,enableHiding: true,defaultHidden: true,+ enableSorting: true,cell: ({ row }) => <RelativeDate date={row.original.createdAt} />,},{@@ -372,6 +384,7 @@ export function RulesTable({header: "Updated at",size: 180,enableHiding: true,+ enableSorting: true,cell: ({ row }) => <RelativeDate date={row.original.updatedAt} />,},{@@ -478,6 +491,11 @@ export function RulesTable({},setColumnOrder,setColumnVisibility,+ setOrderBy: (nextOrderBy) => {+ setOrderBy(nextOrderBy);+ setPagination({ page: 1, limit: pagination.limit });+ selectionActions.clearSelection();+ },},validationContext: {columns,@@ -515,6 +533,7 @@ export function RulesTable({rowHeight={rowHeight}setRowHeight={setRowHeight}filterState={filterState}+ orderByState={orderBy}viewConfig={{tableName: TableViewPresetTableName.EvaluationRules,projectId,@@ -551,6 +570,12 @@ export function RulesTable({columnOrder={columnOrder}onColumnOrderChange={setColumnOrder}rowHeight={rowHeight}+ orderBy={orderBy}+ setOrderBy={(nextOrderBy) => {+ setOrderBy(nextOrderBy);+ setPagination({ page: 1, limit: pagination.limit });+ selectionActions.clearSelection();+ }}pagination={{totalCount: rules.data?.totalItems ?? null,state: {diff --git a/web/src/features/evals/v2/components/Rules/RulesTable/components/RulesTableToolbar/RulesTableToolbar.tsx b/web/src/features/evals/v2/components/Rules/RulesTable/components/RulesTableToolbar/RulesTableToolbar.tsxindex a721efdd49..b15cad7fde 100644--- a/web/src/features/evals/v2/components/Rules/RulesTable/components/RulesTableToolbar/RulesTableToolbar.tsx+++ b/web/src/features/evals/v2/components/Rules/RulesTable/components/RulesTableToolbar/RulesTableToolbar.tsx@@ -8,7 +8,7 @@ import type {import type { ColumnOrderState, VisibilityState } from "@tanstack/react-table";import { type Dispatch, type SetStateAction, type ComponentProps } from "react";import type { RowHeight } from "@/src/components/table/data-table-row-height-switch";-import type { FilterState } from "@langfuse/shared";+import type { FilterState, OrderByState } from "@langfuse/shared";export function RulesTableToolbar({columns,@@ -26,6 +26,7 @@ export function RulesTableToolbar({rowHeight,setRowHeight,filterState,+ orderByState,viewConfig,}: {columns: LangfuseColumnDef<RuleTableRow>[];@@ -43,6 +44,7 @@ export function RulesTableToolbar({rowHeight: RowHeight;setRowHeight: Dispatch<SetStateAction<RowHeight>>;filterState: FilterState;+ orderByState: OrderByState;viewConfig: NonNullable<ComponentProps<typeof DataTableToolbar<RuleTableRow, unknown>>["viewConfig"]>;@@ -55,6 +57,7 @@ export function RulesTableToolbar({<DataTableToolbarcolumns={columns}filterState={filterState}+ orderByState={orderByState}currentSearchQuery={currentQuery}viewConfig={viewConfig}searchConfig={{diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsxindex 8463a546e0..23f66ebc05 100644--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx@@ -1,5 +1,9 @@import { formatDistanceToNowStrict } from "date-fns";-import { TableViewPresetTableName, ZodModelConfig } from "@langfuse/shared";+import {+ type OrderByState,+ TableViewPresetTableName,+ ZodModelConfig,+} from "@langfuse/shared";import { Plus, Trash2 } from "lucide-react";import { useRouter } from "next/router";import { type ComponentProps, useMemo, useState } from "react";@@ -40,7 +44,7 @@ import {createEvaluatorsTableStore,type EvaluatorsTableStore,} from "../store/evaluatorsTableStore";-import { api, type RouterOutputs } from "@/src/utils/api";+import { api, type RouterInputs, type RouterOutputs } from "@/src/utils/api";import { usdFormatter } from "@/src/utils/numbers";import { trpcErrorToast } from "@/src/utils/trpcErrorToast";import {@@ -66,6 +70,7 @@ import {import type { GalleryTemplate } from "../types/templateGallery";import { V4MigrationUpdateRequiredBadge } from "@/src/features/v4-migration/V4MigrationDelayBadge";import { createNumberTableColumn } from "@/src/components/design-system/Table/columns/createNumberTableColumn";+import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";import { createUserTableColumn } from "@/src/components/design-system/Table/columns/createUserTableColumn";type EvaluatorRow = RouterOutputs["evalsV2"]["list"]["evaluators"][number];@@ -170,6 +175,10 @@ export default function EvaluatorsPage() {source: "overview",});const [pagination, setPagination] = usePaginationState(1, 50);+ const [orderBy, setOrderBy] = useOrderByState({+ column: "updatedAt",+ order: "DESC",+ });const [rowHeight, setRowHeight] = useRowHeightLocalStorage("evaluatorsV2","s",@@ -242,6 +251,9 @@ export default function EvaluatorsPage() {{projectId,...pagination,+ orderBy: orderBy+ ? (orderBy as RouterInputs["evalsV2"]["list"]["orderBy"])+ : undefined,search: searchQuery ?? undefined,filter: filterState,},@@ -333,6 +345,7 @@ export default function EvaluatorsPage() {header: "Name",size: 320,isFixedPosition: true,+ enableSorting: true,cell: ({ row }) => (<span className="block truncate font-bold" title={row.original.name}>{row.original.name}@@ -398,6 +411,7 @@ export default function EvaluatorsPage() {header: "Type",size: 160,enableHiding: true,+ enableSorting: true,cell: ({ row }) => <EvaluatorTypeBadge type={row.original.type} />,},createNumberTableColumn<EvaluatorRow>({@@ -449,6 +463,7 @@ export default function EvaluatorsPage() {size: 180,enableHiding: true,defaultHidden: true,+ enableSorting: true,cell: ({ row }) => <RelativeDate date={row.original.createdAt} />,},{@@ -457,6 +472,7 @@ export default function EvaluatorsPage() {header: "Updated at",size: 180,enableHiding: true,+ enableSorting: true,cell: ({ row }) => <RelativeDate date={row.original.updatedAt} />,},{@@ -522,6 +538,11 @@ export default function EvaluatorsPage() {"evaluatorsV2ColumnOrder-v2",columns,);+ const setEvaluatorOrderBy = (nextOrderBy: OrderByState) => {+ setOrderBy(nextOrderBy);+ setPagination({ page: 1, limit: pagination.limit });+ selectionStore.getState().actions.clearSelection();+ };const { isLoading: isViewLoading, ...viewControllers } = useTableViewManager({tableName: TableViewPresetTableName.Evaluators,projectId,@@ -536,6 +557,7 @@ export default function EvaluatorsPage() {},setColumnOrder,setColumnVisibility,+ setOrderBy: setEvaluatorOrderBy,},validationContext: {columns,@@ -632,6 +654,7 @@ export default function EvaluatorsPage() {rowHeight={rowHeight}setRowHeight={setRowHeight}filterState={filterState}+ orderByState={orderBy}currentSearchQuery={searchQuery ?? undefined}viewConfig={{tableName: TableViewPresetTableName.Evaluators,@@ -699,6 +722,8 @@ export default function EvaluatorsPage() {columnOrder={columnOrder}onColumnOrderChange={setColumnOrder}rowHeight={rowHeight}+ orderBy={orderBy}+ setOrderBy={setEvaluatorOrderBy}onRowClick={(row) =>router.push(`/project/${projectId}/evals/${row.id}`)}diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.tsindex 29faa35b23..91782e64bd 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts@@ -7,6 +7,7 @@ import {import type {CreateEvaluatorInput,EvaluatorDefinitionForPersistence,+ EvaluatorListOrderBy,} from "./evaluatorTypes";import { EvaluatorVersionConflictError } from "./evaluatorErrors";import { setRuleStatus } from "../rules/ruleRepository";@@ -249,14 +250,17 @@ export async function listEvaluators(params: {projectId: string;page: number;limit: number;+ orderBy?: EvaluatorListOrderBy;search?: string;filter?: FilterState;}) {const where = await evaluatorWhere(params);+ const orderColumn = params.orderBy?.column ?? "updatedAt";+ const orderDirection = params.orderBy?.order.toLowerCase() ?? "desc";const [evaluators, totalItems, defaultModel] = await Promise.all([params.prisma.evaluator.findMany({where,- orderBy: [{ updatedAt: "desc" }, { id: "desc" }],+ orderBy: [{ [orderColumn]: orderDirection }, { id: "desc" }],skip: (params.page - 1) * params.limit,take: params.limit,include: {diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.tsindex dcb9c01c53..f2b64b7ca6 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts@@ -27,6 +27,7 @@ import {type DeleteEvaluatorsInput,type EvaluatorDefinition,type EvaluatorDefinitionForPersistence,+ type EvaluatorListOrderBy,type EvaluatorVersionCursor,type UpdateEvaluatorInput,encodeEvaluatorVersionCursor,@@ -86,6 +87,7 @@ export class EvaluatorService {projectId: string;page: number;limit: number;+ orderBy?: EvaluatorListOrderBy;search?: string;filter?: FilterState;}) {diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.tsindex 5f546f8d28..1f0a402b69 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts@@ -154,6 +154,12 @@ export const ListEvaluatorsSchema = z.object({projectId: z.string(),page: z.number().int().positive().default(1),limit: paginationLimitZod.optional().default(50),+ orderBy: z+ .object({+ column: z.enum(["name", "type", "createdAt", "updatedAt"]),+ order: z.enum(["ASC", "DESC"]),+ })+ .optional(),search: z.string().trim().max(200).optional(),filter: EvaluatorListFilterSchema,});@@ -189,3 +195,6 @@ export type EvaluatorDefinitionForPersistence =export type CreateEvaluatorInput = z.infer<typeof CreateEvaluatorSchema>;export type UpdateEvaluatorInput = z.infer<typeof UpdateEvaluatorSchema>;export type DeleteEvaluatorsInput = z.infer<typeof DeleteEvaluatorsSchema>;+export type EvaluatorListOrderBy = z.infer<+ typeof ListEvaluatorsSchema+>["orderBy"];diff --git a/web/src/features/evals/v2/server/rules/ruleRepository.ts b/web/src/features/evals/v2/server/rules/ruleRepository.tsindex 2d3ffc9280..dbec658a76 100644--- a/web/src/features/evals/v2/server/rules/ruleRepository.ts+++ b/web/src/features/evals/v2/server/rules/ruleRepository.ts@@ -134,10 +134,16 @@ export async function listRules(params: {input: ListRulesInput;}) {const where = ruleWhere(params.input);+ const requestedOrder = params.input.orderBy;+ const orderColumn =+ requestedOrder?.column === "enabled"+ ? "status"+ : (requestedOrder?.column ?? "createdAt");+ const orderDirection = requestedOrder?.order.toLowerCase() ?? "desc";const [rules, totalItems] = await Promise.all([params.prisma.evaluationRule.findMany({where,- orderBy: [{ updatedAt: "desc" }, { id: "desc" }],+ orderBy: [{ [orderColumn]: orderDirection }, { id: "desc" }],skip: (params.input.page - 1) * params.input.limit,take: params.input.limit,include: ruleInclude,diff --git a/web/src/features/evals/v2/server/rules/ruleTypes.ts b/web/src/features/evals/v2/server/rules/ruleTypes.tsindex ab9d81675e..a687913cd3 100644--- a/web/src/features/evals/v2/server/rules/ruleTypes.ts+++ b/web/src/features/evals/v2/server/rules/ruleTypes.ts@@ -31,6 +31,12 @@ export const ListRulesSchema = z.object({projectId: z.string(),page: z.number().int().positive().default(1),limit: paginationLimitZod.optional().default(50),+ orderBy: z+ .object({+ column: z.enum(["name", "enabled", "sampling", "createdAt", "updatedAt"]),+ order: z.enum(["ASC", "DESC"]),+ })+ .optional(),search: z.string().trim().max(200).optional(),enabled: z.boolean().optional(),targetObjects: z