🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): show and filter evaluator models (#16466)
Tobias Wochinger7d62bce084f2
diff --git a/web/src/__tests__/server/evaluator-v2-repository.servertest.ts b/web/src/__tests__/server/evaluator-v2-repository.servertest.tsindex a573a9f5e7..40e52ac220 100644--- a/web/src/__tests__/server/evaluator-v2-repository.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-repository.servertest.ts@@ -1,5 +1,6 @@import { randomUUID } from "crypto";-import { getCodeEvalVariableMapping } from "@langfuse/shared";+import { getCodeEvalVariableMapping, LLMAdapter } from "@langfuse/shared";+import { encrypt } from "@langfuse/shared/encryption";import { Prisma, prisma } from "@langfuse/shared/src/db";import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";@@ -126,6 +127,33 @@ const createEvaluatorWithThreeVersions = async () => {return evaluator;};+const provisionDefaultEvalModel = async (model: string) => {+ const provider = `openai-${randomUUID()}`;+ const llmApiKey = await prisma.llmApiKeys.create({+ data: {+ projectId,+ provider,+ adapter: LLMAdapter.OpenAI,+ secretKey: encrypt("sk-test"),+ displaySecretKey: "...test",+ baseURL: "https://api.openai.com/v1",+ customModels: [],+ withDefaultModels: true,+ extraHeaders: null,+ extraHeaderKeys: [],+ },+ });+ await prisma.defaultLlmModel.create({+ data: {+ projectId,+ llmApiKeyId: llmApiKey.id,+ provider,+ adapter: LLMAdapter.OpenAI,+ model,+ },+ });+};+beforeAll(async () => {const [first, second, creator] = await Promise.all([createOrgProjectAndApiKey(),@@ -150,6 +178,12 @@ afterEach(async () => {await prisma.evaluator.deleteMany({where: { projectId: { in: [projectId, otherProjectId] } },});+ await prisma.defaultLlmModel.deleteMany({+ where: { projectId: { in: [projectId, otherProjectId] } },+ });+ await prisma.llmApiKeys.deleteMany({+ where: { projectId: { in: [projectId, otherProjectId] } },+ });});afterAll(async () => {@@ -412,6 +446,107 @@ describe("evaluator v2 repository", () => {totalItems: 1,});});++ it("filters evaluators by their effective latest model", async () => {+ await provisionDefaultEvalModel("project-default-model");+ const [explicitMatch, inheritedMatch, historicalOnly, codeEvaluator] =+ await Promise.all([+ createEvaluator({+ name: "Explicit match",+ definition: llmDefinition({ model: "selected-model" }),+ }),+ createEvaluator({+ name: "Inherited match",+ definition: llmDefinition(),+ }),+ createEvaluator({+ name: "Historical only",+ definition: llmDefinition({ model: "selected-model" }),+ }),+ createEvaluator({ name: "Code evaluator" }),+ ]);+ await prisma.$transaction((tx) =>+ evaluatorRepository.appendEvaluatorVersion({+ tx,+ evaluatorId: historicalOnly.id,+ version: 2,+ definition: llmDefinition({ model: "new-model" }),+ createdByUserId: null,+ }),+ );++ await expect(+ evaluatorRepository.listEvaluators({+ prisma,+ projectId,+ page: 1,+ limit: 50,+ filter: [+ {+ column: "model",+ type: "stringOptions",+ operator: "any of",+ value: ["selected-model", "project-default-model"],+ },+ ],+ }),+ ).resolves.toMatchObject({+ evaluators: expect.arrayContaining([+ expect.objectContaining({+ id: explicitMatch.id,+ effectiveModel: "selected-model",+ }),+ expect.objectContaining({+ id: inheritedMatch.id,+ effectiveModel: "project-default-model",+ }),+ ]),+ totalItems: 2,+ });++ await expect(+ evaluatorRepository.listEvaluators({+ prisma,+ projectId,+ page: 1,+ limit: 50,+ filter: [+ {+ column: "model",+ type: "string",+ operator: "contains",+ value: "DEFAULT",+ },+ ],+ }),+ ).resolves.toMatchObject({+ evaluators: [expect.objectContaining({ id: inheritedMatch.id })],+ totalItems: 1,+ });++ await expect(+ evaluatorRepository.listEvaluators({+ prisma,+ projectId,+ page: 1,+ limit: 50,+ filter: [+ {+ column: "model",+ type: "stringOptions",+ operator: "none of",+ value: ["selected-model", "project-default-model"],+ },+ ],+ }),+ ).resolves.toMatchObject({+ evaluators: expect.arrayContaining([+ expect.objectContaining({ id: historicalOnly.id }),+ expect.objectContaining({ id: codeEvaluator.id }),+ ]),+ totalItems: 2,+ });+ });});describe("listEvaluatorFilterOptions", () => {@@ -434,6 +569,36 @@ describe("evaluator v2 repository", () => {).resolves.toEqual({name: ["Alpha evaluator", "Beta evaluator"],creator: ["API", "Evaluator creator"],+ model: [],+ });+ });++ it("returns distinct effective latest models", async () => {+ await provisionDefaultEvalModel("project-default-model");+ const historicalOnly = await createEvaluator({+ definition: llmDefinition({ model: "old-model" }),+ });+ await Promise.all([+ createEvaluator({+ definition: llmDefinition({ model: "explicit-model" }),+ }),+ createEvaluator({ definition: llmDefinition() }),+ createEvaluator(),+ ]);+ await prisma.$transaction((tx) =>+ evaluatorRepository.appendEvaluatorVersion({+ tx,+ evaluatorId: historicalOnly.id,+ version: 2,+ definition: llmDefinition({ model: "explicit-model" }),+ createdByUserId: null,+ }),+ );++ await expect(+ evaluatorRepository.listEvaluatorFilterOptions({ prisma, projectId }),+ ).resolves.toMatchObject({+ model: ["explicit-model", "project-default-model"],});});});diff --git a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.tsindex 26fac4f3a4..c935df44f2 100644--- a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts+++ b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts@@ -19,7 +19,10 @@ vi.mock("@/src/features/evals/server/isCodeEvalEnabled", () => ({}));import { assertEvaluatorConfigurationValid } from "@/src/features/evals/v2/server/evaluators/evaluatorValidation";-import { CreateEvaluatorSchema } from "@/src/features/evals/v2/server/evaluators/evaluatorTypes";+import {+ CreateEvaluatorSchema,+ ListEvaluatorsSchema,+} from "@/src/features/evals/v2/server/evaluators/evaluatorTypes";describe("evaluator configuration validation", () => {beforeEach(() => {@@ -54,6 +57,22 @@ describe("evaluator configuration validation", () => {).toBe(true);});+ it("accepts text filters for evaluator models", () => {+ expect(+ ListEvaluatorsSchema.safeParse({+ projectId: "project-id",+ filter: [+ {+ column: "model",+ type: "string",+ operator: "contains",+ value: "gpt",+ },+ ],+ }).success,+ ).toBe(true);+ });+it("validates deployment capabilities without executing the evaluator", async () => {const codeDefinition = {type: EvalTemplateType.CODE,diff --git a/web/src/components/table/data-table-controls.clienttest.tsx b/web/src/components/table/data-table-controls.clienttest.tsxindex ee1e987f2f..4e0c1f44cc 100644--- a/web/src/components/table/data-table-controls.clienttest.tsx+++ b/web/src/components/table/data-table-controls.clienttest.tsx@@ -34,6 +34,38 @@ beforeAll(() => {});describe("CategoricalFacet", () => {+ it("renders an option suffix after its label", () => {+ render(+ <Accordion type="multiple" value={["model"]}>+ <CategoricalFacet+ label="Model"+ filterKey="model"+ expanded+ loading={false}+ options={["gpt-4.1", "claude-sonnet"]}+ counts={new Map()}+ value={[]}+ onChange={() => {}}+ renderOptionSuffix={(value) =>+ value === "gpt-4.1" ? <span>Project default</span> : null+ }+ isActive={false}+ isDisabled={false}+ onReset={() => {}}+ />+ </Accordion>,+ { wrapper: TooltipProvider },+ );++ const label = screen.getByText("gpt-4.1");+ const suffix = screen.getByText("Project default");+ expect(+ label.compareDocumentPosition(suffix) & Node.DOCUMENT_POSITION_FOLLOWING,+ ).toBeTruthy();+ expect(label).not.toHaveClass("flex-1");+ expect(screen.getByText("claude-sonnet")).toBeInTheDocument();+ });+it("shows selected values even when the backend returns no options", () => {render(<Accordion type="multiple" value={["type"]}>diff --git a/web/src/components/table/data-table-controls.tsx b/web/src/components/table/data-table-controls.tsxindex de7a0af843..e921429eab 100644--- a/web/src/components/table/data-table-controls.tsx+++ b/web/src/components/table/data-table-controls.tsx@@ -534,6 +534,7 @@ export function DataTableControls({onChange={filter.onChange}onOnlyChange={filter.onOnlyChange}renderIcon={filter.renderIcon}+ renderOptionSuffix={filter.renderOptionSuffix}isActive={filter.isActive}onReset={filter.onReset}operator={filter.operator}@@ -1256,6 +1257,7 @@ interface CategoricalFacetProps extends BaseFacetProps {onChange: (values: string[]) => void;onOnlyChange?: (value: string) => void;renderIcon?: (value: string) => React.ReactNode;+ renderOptionSuffix?: (value: string) => React.ReactNode;operator?: "any of" | "all of" | "none of";onOperatorChange?: (operator: "any of" | "all of" | "none of") => void;textFilters?: TextFilterEntry[];@@ -1556,6 +1558,7 @@ export function CategoricalFacet({onChange,onOnlyChange,renderIcon,+ renderOptionSuffix,isActive,isDisabled,disabledReason,@@ -1635,6 +1638,7 @@ export function CategoricalFacet({onChange={onChange}onOnlyChange={onOnlyChange}renderIcon={renderIcon}+ renderOptionSuffix={renderOptionSuffix}operator={operator}onOperatorChange={onOperatorChange}/>@@ -1675,6 +1679,7 @@ function CategoricalSelectContent({onChange,onOnlyChange,renderIcon,+ renderOptionSuffix,operator,onOperatorChange,}: Pick<@@ -1688,6 +1693,7 @@ function CategoricalSelectContent({| "onChange"| "onOnlyChange"| "renderIcon"+ | "renderOptionSuffix"| "operator"| "onOperatorChange">) {@@ -1771,6 +1777,7 @@ function CategoricalSelectContent({id={`${filterKey}-${option}`}label={displayLabel}icon={renderIcon?.(option)}+ suffix={renderOptionSuffix?.(option)}count={counts.get(option) || 0}checked={value.includes(option)}onCheckedChange={(checked) => {@@ -2591,6 +2598,7 @@ interface FilterValueCheckboxProps {id: string;label: string;icon?: React.ReactNode;+ suffix?: React.ReactNode;count: number;checked?: boolean;onCheckedChange?: (checked: boolean) => void;@@ -2603,6 +2611,7 @@ export function FilterValueCheckbox({id,label,icon,+ suffix,count,checked = false,onCheckedChange,@@ -2648,13 +2657,15 @@ export function FilterValueCheckbox({{icon ? <span className="mr-2">{icon}</span> : null}<spanclassName={cn(- "min-w-0 flex-1 truncate text-xs",+ "min-w-0 truncate text-xs",+ !suffix && "flex-1",label === "" && "text-muted-foreground italic",)}title={displayTitle}>{displayLabel}</span>+ {suffix ? <span className="shrink-0 pl-1">{suffix}</span> : null}{/* "Only" or "All" indicator when hovering label. shrink-0 +whitespace-nowrap: appearing may only re-truncate the label —diff --git a/web/src/features/evals/v2/constants/tableFilterColumns.ts b/web/src/features/evals/v2/constants/tableFilterColumns.tsindex ae6cc64c49..593a6a5340 100644--- a/web/src/features/evals/v2/constants/tableFilterColumns.ts+++ b/web/src/features/evals/v2/constants/tableFilterColumns.ts@@ -37,6 +37,13 @@ export const evaluatorTableFilterColumns: ColumnDefinition[] = [internal: "type",options: evaluatorTypeOptions,},+ {+ name: "Model",+ id: "model",+ type: "stringOptions",+ internal: "model",+ options: [],+ },{name: "Creator",id: "creator",@@ -83,6 +90,7 @@ export const evaluatorTableFilterConfig: FilterConfig = {{ type: "categorical", column: "name", label: "Name" },{ type: "categorical", column: "status", label: "Status" },{ type: "categorical", column: "type", label: "Type" },+ { type: "categorical", column: "model", label: "Model" },{ type: "categorical", column: "creator", label: "Creator" },],};diff --git a/web/src/features/evals/v2/hooks/useProjectDefaultModel.clienttest.ts b/web/src/features/evals/v2/hooks/useProjectDefaultModel.clienttest.tsindex 542c3dd3b2..be33229314 100644--- a/web/src/features/evals/v2/hooks/useProjectDefaultModel.clienttest.ts+++ b/web/src/features/evals/v2/hooks/useProjectDefaultModel.clienttest.ts@@ -1,10 +1,16 @@import { act, renderHook } from "@testing-library/react";+import { LLMAdapter } from "@langfuse/shared";import { beforeEach, describe, expect, it, vi } from "vitest";import { useProjectDefaultModel } from "./useProjectDefaultModel";const mocks = vi.hoisted(() => ({refetchConnections: vi.fn(),+ invalidateDefaultModel: vi.fn(),+ invalidateEvaluatorList: vi.fn(),+ invalidateEvaluatorOptions: vi.fn(),+ invalidateEvaluatorFilterOptions: vi.fn(),+ upsertDefaultModel: vi.fn(),}));vi.mock("@/src/env.mjs", () => ({@@ -27,11 +33,12 @@ vi.mock("@/src/utils/api", () => ({api: {useUtils: () => ({defaultLlmModel: {- fetchDefaultModel: { invalidate: vi.fn() },+ fetchDefaultModel: { invalidate: mocks.invalidateDefaultModel },},evalsV2: {- list: { invalidate: vi.fn() },- options: { invalidate: vi.fn() },+ list: { invalidate: mocks.invalidateEvaluatorList },+ options: { invalidate: mocks.invalidateEvaluatorOptions },+ filterOptions: { invalidate: mocks.invalidateEvaluatorFilterOptions },},}),defaultLlmModel: {@@ -39,7 +46,10 @@ vi.mock("@/src/utils/api", () => ({useQuery: () => ({ data: null }),},upsertDefaultModel: {- useMutation: () => ({ isPending: false, mutate: vi.fn() }),+ useMutation: () => ({+ isPending: false,+ mutate: mocks.upsertDefaultModel,+ }),},},llmApiKey: {@@ -60,10 +70,31 @@ vi.mock("@/src/utils/trpcErrorToast", () => ({describe("useProjectDefaultModel", () => {beforeEach(() => {- mocks.refetchConnections.mockReset();+ Object.values(mocks).forEach((mock) => mock.mockReset());vi.spyOn(window, "open").mockImplementation(() => null);});+ it("refreshes model filter options after updating the project default", async () => {+ const { result } = renderHook(() =>+ useProjectDefaultModel({ projectId: "project-1", source: "overview" }),+ );++ act(() =>+ result.current.update.requestUpdate({+ provider: "openai",+ model: "gpt-4.1",+ adapter: LLMAdapter.OpenAI,+ modelParams: {},+ }),+ );+ const mutationOptions = mocks.upsertDefaultModel.mock.calls[0]?.[1];+ await act(async () => mutationOptions?.onSuccess());++ expect(mocks.invalidateEvaluatorFilterOptions).toHaveBeenCalledWith({+ projectId: "project-1",+ });+ });+it("refreshes model connections after returning from provider settings", () => {const { result } = renderHook(() =>useProjectDefaultModel({ projectId: "project-1", source: "editor" }),diff --git a/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts b/web/src/features/evals/v2/hooks/useProjectDefaultModel.tsindex cfc120eb08..bb664d497a 100644--- a/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts+++ b/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts@@ -70,6 +70,7 @@ export function useProjectDefaultModel({utils.defaultLlmModel.fetchDefaultModel.invalidate({ projectId }),utils.evalsV2.list.invalidate({ projectId }),utils.evalsV2.options.invalidate({ projectId }),+ utils.evalsV2.filterOptions.invalidate({ projectId }),]);capture("evaluators:default_model_update", {source,diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsxindex 11ea4d8ee0..20ce9f9dab 100644--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx@@ -15,6 +15,7 @@ import {} from "@/src/components/table/data-table-controls";import { ResizableFilterLayout } from "@/src/components/table/resizable-filter-layout";import type { LangfuseColumnDef } from "@/src/components/table/types";+import { Badge } from "@/src/components/ui/badge";import { Button } from "@/src/components/ui/button";import { EvaluatorsEmptyState } from "../components/EvaluatorsEmptyState/EvaluatorsEmptyState";import { PopoverTrigger } from "@/src/components/ui/popover";@@ -162,6 +163,10 @@ export default function EvaluatorsPage() {const router = useRouter();const capture = usePostHogClientCapture();const projectId = router.query.projectId as string;+ const projectDefaultModel = useProjectDefaultModel({+ projectId,+ source: "overview",+ });const [pagination, setPagination] = usePaginationState(1, 50);const [rowHeight, setRowHeight] = useRowHeightLocalStorage("evaluatorsV2",@@ -185,22 +190,38 @@ export default function EvaluatorsPage() {...evaluatorTableFilterOptions,name: filterOptionsQuery.data?.name ?? [],creator: filterOptionsQuery.data?.creator ?? [],+ model: filterOptionsQuery.data?.model ?? [],}),[filterOptionsQuery.data],);- const queryFilter = useSidebarFilterState(- evaluatorTableFilterConfig,- filterOptions,- {- loading: filterOptionsQuery.isPending,- stateLocation: "urlAndSessionStorage",- sessionFilterContextId: projectId,- onExplicitFilterStateChange: () => {- setPagination({ page: 1, limit: pagination.limit });- selectionStore.getState().actions.clearSelection();- },- },+ const filterConfig = useMemo(+ () => ({+ ...evaluatorTableFilterConfig,+ facets: evaluatorTableFilterConfig.facets.map((facet) =>+ facet.column === "model"+ ? {+ ...facet,+ renderOptionSuffix: (model: string) =>+ model === projectDefaultModel.defaultModel?.model ? (+ <Badge variant="secondary" size="sm">+ Project default+ </Badge>+ ) : null,+ }+ : facet,+ ),+ }),+ [projectDefaultModel.defaultModel?.model],);+ const queryFilter = useSidebarFilterState(filterConfig, filterOptions, {+ loading: filterOptionsQuery.isPending,+ stateLocation: "urlAndSessionStorage",+ sessionFilterContextId: projectId,+ onExplicitFilterStateChange: () => {+ setPagination({ page: 1, limit: pagination.limit });+ selectionStore.getState().actions.clearSelection();+ },+ });const filterState = queryFilter.filterState;const [deleteIds, setDeleteIds] = useState<string[]>([]);const [deleteAll, setDeleteAll] = useState(false);@@ -237,10 +258,6 @@ export default function EvaluatorsPage() {projectId,scope: "evalJob:read",});- const projectDefaultModel = useProjectDefaultModel({- projectId,- source: "overview",- });const defaultModelConnection = projectDefaultModel.connections.find(({ provider }) => provider === projectDefaultModel.defaultModel?.provider,);@@ -395,6 +412,23 @@ export default function EvaluatorsPage() {return cost == null ? "—" : usdFormatter(cost, 2, 4);},},+ {+ accessorKey: "model",+ id: "model",+ header: "Model",+ size: 180,+ enableHiding: true,+ cell: ({ row }) => {+ const model = row.original.effectiveModel;+ return model ? (+ <span className="block truncate" title={model}>+ {model}+ </span>+ ) : (+ "—"+ );+ },+ },{accessorKey: "createdByUser",id: "createdByUser",@@ -511,9 +545,7 @@ export default function EvaluatorsPage() {validationContext: {columns,filterColumnDefinition: evaluatorTableFilterColumns,- expandableFilterColumns: evaluatorTableFilterConfig.facets.map(- (facet) => facet.column,- ),+ expandableFilterColumns: filterConfig.facets.map((facet) => facet.column),},currentFilterState: queryFilter.explicitFilterState,currentExpandedFilters: queryFilter.expanded,@@ -589,9 +621,7 @@ export default function EvaluatorsPage() {onBrowseLibrary={() => setGalleryOpen(true)}/>) : (- <DataTableControlsProvider- tableName={evaluatorTableFilterConfig.tableName}- >+ <DataTableControlsProvider tableName={filterConfig.tableName}><div className="flex h-full min-h-0 w-full flex-1 flex-col overflow-hidden"><EvaluatorsTableToolbarselectionStore={selectionStore}diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.tsindex 6223a1c294..29faa35b23 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts@@ -82,11 +82,100 @@ function versionData(};}-function evaluatorWhere(params: {+type EvaluatorModelFilter = Extract<+ FilterState[number],+ { type: "string" | "stringOptions" }+> & { column: "model" };++function isModelFilter(+ filter: FilterState[number],+): filter is EvaluatorModelFilter {+ return (+ filter.column === "model" &&+ (filter.type === "string" || filter.type === "stringOptions")+ );+}++async function evaluatorIdsMatchingModelFilters(params: {+ prisma: PrismaClient | PrismaTransaction;+ projectId: string;+ filters: EvaluatorModelFilter[];+}) {+ const effectiveModel = Prisma.sql`+ CASE+ WHEN evaluator.type = 'LLM_AS_JUDGE'+ THEN COALESCE(latest_version.model, default_model.model)+ ELSE NULL+ END+ `;+ const predicates = params.filters.map((filter) => {+ const normalizedModel = Prisma.sql`LOWER(${effectiveModel})`;+ if (filter.type === "stringOptions") {+ if (filter.value.length === 0) {+ return filter.operator === "any of"+ ? Prisma.sql`FALSE`+ : Prisma.sql`TRUE`;+ }+ const values = Prisma.join(+ filter.value.map((value) => value.toLowerCase()),+ );+ return filter.operator === "any of"+ ? Prisma.sql`${normalizedModel} IN (${values})`+ : Prisma.sql`(${effectiveModel} IS NULL OR ${normalizedModel} NOT IN (${values}))`;+ }++ const normalizedValue = filter.value.toLowerCase();+ switch (filter.operator) {+ case "=":+ return Prisma.sql`${normalizedModel} = ${normalizedValue}`;+ case "contains":+ return Prisma.sql`STRPOS(${normalizedModel}, ${normalizedValue}) > 0`;+ case "does not contain":+ return Prisma.sql`STRPOS(${normalizedModel}, ${normalizedValue}) = 0`;+ case "starts with":+ return Prisma.sql`LEFT(${normalizedModel}, LENGTH(${normalizedValue})) = ${normalizedValue}`;+ case "ends with":+ return Prisma.sql`RIGHT(${normalizedModel}, LENGTH(${normalizedValue})) = ${normalizedValue}`;+ }+ });+ const matches = await params.prisma.$queryRaw<Array<{ id: string }>>(+ Prisma.sql`+ SELECT evaluator.id+ FROM evaluators AS evaluator+ LEFT JOIN LATERAL (+ SELECT version.model+ FROM evaluator_versions AS version+ WHERE version.evaluator_id = evaluator.id+ ORDER BY version.version DESC+ LIMIT 1+ ) AS latest_version ON TRUE+ LEFT JOIN default_llm_models AS default_model+ ON default_model.project_id = evaluator.project_id+ WHERE evaluator.project_id = ${params.projectId}+ AND ${Prisma.join(predicates, " AND ")}+ `,+ );+ return matches.map(({ id }) => id);+}++async function evaluatorWhere(params: {+ prisma: PrismaClient | PrismaTransaction;projectId: string;search?: string;filter?: FilterState;-}): Prisma.EvaluatorWhereInput {+}): Promise<Prisma.EvaluatorWhereInput> {+ const modelFilters = params.filter?.filter(isModelFilter) ?? [];+ const otherFilters = params.filter?.filter(+ (filter) => !isModelFilter(filter),+ );+ const modelEvaluatorIds =+ modelFilters.length > 0+ ? await evaluatorIdsMatchingModelFilters({+ prisma: params.prisma,+ projectId: params.projectId,+ filters: modelFilters,+ })+ : undefined;const handlers = {name: {string: (filter) => ({ name: stringFilterToPrisma(filter) }),@@ -144,11 +233,12 @@ function evaluatorWhere(params: {return {projectId: params.projectId,+ ...(modelEvaluatorIds ? { id: { in: modelEvaluatorIds } } : {}),...(params.search? { name: { contains: params.search, mode: "insensitive" as const } }: {}),AND: compilePrismaFilters<Prisma.EvaluatorWhereInput>(- params.filter ?? [],+ otherFilters ?? [],handlers,),};@@ -162,8 +252,8 @@ export async function listEvaluators(params: {search?: string;filter?: FilterState;}) {- const where = evaluatorWhere(params);- const [evaluators, totalItems] = await Promise.all([+ const where = await evaluatorWhere(params);+ const [evaluators, totalItems, defaultModel] = await Promise.all([params.prisma.evaluator.findMany({where,orderBy: [{ updatedAt: "desc" }, { id: "desc" }],@@ -186,6 +276,10 @@ export async function listEvaluators(params: {},}),params.prisma.evaluator.count({ where }),+ params.prisma.defaultLlmModel.findUnique({+ where: { projectId: params.projectId },+ select: { model: true },+ }),]);return {evaluators: evaluators.map(({ assignments, ...evaluator }) => ({@@ -196,6 +290,10 @@ export async function listEvaluators(params: {hasActiveRules: assignments.some(({ evaluationRule }) => evaluationRule.status === "ACTIVE",),+ effectiveModel:+ evaluator.type === EvalTemplateType.LLM_AS_JUDGE+ ? (evaluator.versions[0]?.model ?? defaultModel?.model ?? null)+ : null,})),totalItems,};@@ -205,13 +303,24 @@ export async function listEvaluatorFilterOptions(params: {prisma: PrismaClient;projectId: string;}) {- const evaluators = await params.prisma.evaluator.findMany({- where: { projectId: params.projectId },- select: {- name: true,- createdByUser: { select: { name: true, email: true } },- },- });+ const [evaluators, defaultModel] = await Promise.all([+ params.prisma.evaluator.findMany({+ where: { projectId: params.projectId },+ select: {+ name: true,+ type: true,+ createdByUser: { select: { name: true, email: true } },+ versions: {+ ...latestVersion,+ select: { model: true },+ },+ },+ }),+ params.prisma.defaultLlmModel.findUnique({+ where: { projectId: params.projectId },+ select: { model: true },+ }),+ ]);return {name: [...new Set(evaluators.map(({ name }) => name))].sort(),@@ -223,6 +332,15 @@ export async function listEvaluatorFilterOptions(params: {),),].sort(),+ model: [+ ...new Set(+ evaluators.flatMap(({ type, versions }) => {+ if (type !== EvalTemplateType.LLM_AS_JUDGE) return [];+ const model = versions[0]?.model ?? defaultModel?.model;+ return model ? [model] : [];+ }),+ ),+ ].sort(),};}@@ -232,8 +350,9 @@ export async function listEvaluatorIds(params: {search?: string;filter?: FilterState;}) {+ const where = await evaluatorWhere(params);const evaluators = await params.prisma.evaluator.findMany({- where: evaluatorWhere(params),+ where,select: { id: true },});return evaluators.map(({ id }) => id);diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.tsindex 8be868a727..5f546f8d28 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts@@ -110,9 +110,9 @@ const EvaluatorListFilterSchema = z.superRefine((filters, ctx) => {for (const [index, filter] of filters.entries()) {const valid =- ((filter.column === "name" || filter.column === "creator") &&+ (["name", "creator", "model"].includes(filter.column) &&(filter.type === "string" || filter.type === "stringOptions")) ||- ((filter.column === "status" || filter.column === "type") &&+ (["status", "type"].includes(filter.column) &&filter.type === "stringOptions");const validOptions =filter.type !== "stringOptions" ||@@ -126,7 +126,9 @@ const EvaluatorListFilterSchema = zvalue as EvalTemplateType,),)- : filter.column === "name" || filter.column === "creator");+ : filter.column === "name" ||+ filter.column === "creator" ||+ filter.column === "model");if (!valid || !validOptions) {ctx.addIssue({code: "custom",diff --git a/web/src/features/filters/hooks/useSidebarFilterState.tsx b/web/src/features/filters/hooks/useSidebarFilterState.tsxindex 083a87295f..612e75c092 100644--- a/web/src/features/filters/hooks/useSidebarFilterState.tsx+++ b/web/src/features/filters/hooks/useSidebarFilterState.tsx@@ -182,6 +182,8 @@ export interface CategoricalUIFilter extends BaseUIFilter {onOnlyChange?: (value: string) => void;/** Optional function to render an icon next to filter option labels */renderIcon?: (value: string) => React.ReactNode;+ /** Optional content rendered after a filter option label */+ renderOptionSuffix?: (value: string) => React.ReactNode;/*** Current operator of the facet's checkbox filter (arrayOptions AND* stringOptions columns; undefined when no filter is applied):@@ -1901,6 +1903,8 @@ export function useSidebarFilterPresentation(disabledReason: disableState.reason,renderIcon:facet.type === "categorical" ? facet.renderIcon : undefined,+ renderOptionSuffix:+ facet.type === "categorical" ? facet.renderOptionSuffix : undefined,onChange: (values: string[]) => updateFilter(facet.column, values),onOnlyChange: (value: string) => {if (selectedValues.length === 1 && selectedValues.includes(value)) {diff --git a/web/src/features/filters/lib/filter-config.ts b/web/src/features/filters/lib/filter-config.tsindex cfea754d17..846cca01ae 100644--- a/web/src/features/filters/lib/filter-config.ts+++ b/web/src/features/filters/lib/filter-config.ts@@ -17,6 +17,8 @@ interface CategoricalFacet extends BaseFacet {type: "categorical";/** Optional function to render an icon next to filter option labels */renderIcon?: (value: string) => React.ReactNode;+ /** Optional content rendered after a filter option label */+ renderOptionSuffix?: (value: string) => React.ReactNode;/** When true, the sidebar hides the contains/does-not-contain text filter mode for this facet. */disableTextFilter?: boolean;}