Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): show and filter evaluator models (#16466)

Tobias Wochinger7d62bce084f2
diff --git a/web/src/__tests__/server/evaluator-v2-repository.servertest.ts b/web/src/__tests__/server/evaluator-v2-repository.servertest.ts
index a573a9f5e7..40e52ac220 100644
--- a/web/src/__tests__/server/evaluator-v2-repository.servertest.ts
+++ b/web/src/__tests__/server/evaluator-v2-repository.servertest.ts
@@ -1,5 +1,6 @@
import { randomUUID } from "crypto";
-import { getCodeEvalVariableMapping } from "@langfuse/shared";
+import { getCodeEvalVariableMapping, LLMAdapter } from "@langfuse/shared";
+import { encrypt } from "@langfuse/shared/encryption";
import { Prisma, prisma } from "@langfuse/shared/src/db";
import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
@@ -126,6 +127,33 @@ const createEvaluatorWithThreeVersions = async () => {
return evaluator;
};
+const provisionDefaultEvalModel = async (model: string) => {
+ const provider = `openai-${randomUUID()}`;
+ const llmApiKey = await prisma.llmApiKeys.create({
+ data: {
+ projectId,
+ provider,
+ adapter: LLMAdapter.OpenAI,
+ secretKey: encrypt("sk-test"),
+ displaySecretKey: "...test",
+ baseURL: "https://api.openai.com/v1",
+ customModels: [],
+ withDefaultModels: true,
+ extraHeaders: null,
+ extraHeaderKeys: [],
+ },
+ });
+ await prisma.defaultLlmModel.create({
+ data: {
+ projectId,
+ llmApiKeyId: llmApiKey.id,
+ provider,
+ adapter: LLMAdapter.OpenAI,
+ model,
+ },
+ });
+};
+
beforeAll(async () => {
const [first, second, creator] = await Promise.all([
createOrgProjectAndApiKey(),
@@ -150,6 +178,12 @@ afterEach(async () => {
await prisma.evaluator.deleteMany({
where: { projectId: { in: [projectId, otherProjectId] } },
});
+ await prisma.defaultLlmModel.deleteMany({
+ where: { projectId: { in: [projectId, otherProjectId] } },
+ });
+ await prisma.llmApiKeys.deleteMany({
+ where: { projectId: { in: [projectId, otherProjectId] } },
+ });
});
afterAll(async () => {
@@ -412,6 +446,107 @@ describe("evaluator v2 repository", () => {
totalItems: 1,
});
});
+
+ it("filters evaluators by their effective latest model", async () => {
+ await provisionDefaultEvalModel("project-default-model");
+ const [explicitMatch, inheritedMatch, historicalOnly, codeEvaluator] =
+ await Promise.all([
+ createEvaluator({
+ name: "Explicit match",
+ definition: llmDefinition({ model: "selected-model" }),
+ }),
+ createEvaluator({
+ name: "Inherited match",
+ definition: llmDefinition(),
+ }),
+ createEvaluator({
+ name: "Historical only",
+ definition: llmDefinition({ model: "selected-model" }),
+ }),
+ createEvaluator({ name: "Code evaluator" }),
+ ]);
+ await prisma.$transaction((tx) =>
+ evaluatorRepository.appendEvaluatorVersion({
+ tx,
+ evaluatorId: historicalOnly.id,
+ version: 2,
+ definition: llmDefinition({ model: "new-model" }),
+ createdByUserId: null,
+ }),
+ );
+
+ await expect(
+ evaluatorRepository.listEvaluators({
+ prisma,
+ projectId,
+ page: 1,
+ limit: 50,
+ filter: [
+ {
+ column: "model",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["selected-model", "project-default-model"],
+ },
+ ],
+ }),
+ ).resolves.toMatchObject({
+ evaluators: expect.arrayContaining([
+ expect.objectContaining({
+ id: explicitMatch.id,
+ effectiveModel: "selected-model",
+ }),
+ expect.objectContaining({
+ id: inheritedMatch.id,
+ effectiveModel: "project-default-model",
+ }),
+ ]),
+ totalItems: 2,
+ });
+
+ await expect(
+ evaluatorRepository.listEvaluators({
+ prisma,
+ projectId,
+ page: 1,
+ limit: 50,
+ filter: [
+ {
+ column: "model",
+ type: "string",
+ operator: "contains",
+ value: "DEFAULT",
+ },
+ ],
+ }),
+ ).resolves.toMatchObject({
+ evaluators: [expect.objectContaining({ id: inheritedMatch.id })],
+ totalItems: 1,
+ });
+
+ await expect(
+ evaluatorRepository.listEvaluators({
+ prisma,
+ projectId,
+ page: 1,
+ limit: 50,
+ filter: [
+ {
+ column: "model",
+ type: "stringOptions",
+ operator: "none of",
+ value: ["selected-model", "project-default-model"],
+ },
+ ],
+ }),
+ ).resolves.toMatchObject({
+ evaluators: expect.arrayContaining([
+ expect.objectContaining({ id: historicalOnly.id }),
+ expect.objectContaining({ id: codeEvaluator.id }),
+ ]),
+ totalItems: 2,
+ });
+ });
});
describe("listEvaluatorFilterOptions", () => {
@@ -434,6 +569,36 @@ describe("evaluator v2 repository", () => {
).resolves.toEqual({
name: ["Alpha evaluator", "Beta evaluator"],
creator: ["API", "Evaluator creator"],
+ model: [],
+ });
+ });
+
+ it("returns distinct effective latest models", async () => {
+ await provisionDefaultEvalModel("project-default-model");
+ const historicalOnly = await createEvaluator({
+ definition: llmDefinition({ model: "old-model" }),
+ });
+ await Promise.all([
+ createEvaluator({
+ definition: llmDefinition({ model: "explicit-model" }),
+ }),
+ createEvaluator({ definition: llmDefinition() }),
+ createEvaluator(),
+ ]);
+ await prisma.$transaction((tx) =>
+ evaluatorRepository.appendEvaluatorVersion({
+ tx,
+ evaluatorId: historicalOnly.id,
+ version: 2,
+ definition: llmDefinition({ model: "explicit-model" }),
+ createdByUserId: null,
+ }),
+ );
+
+ await expect(
+ evaluatorRepository.listEvaluatorFilterOptions({ prisma, projectId }),
+ ).resolves.toMatchObject({
+ model: ["explicit-model", "project-default-model"],
});
});
});
diff --git a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
index 26fac4f3a4..c935df44f2 100644
--- a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
+++ b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
@@ -19,7 +19,10 @@ vi.mock("@/src/features/evals/server/isCodeEvalEnabled", () => ({
}));
import { assertEvaluatorConfigurationValid } from "@/src/features/evals/v2/server/evaluators/evaluatorValidation";
-import { CreateEvaluatorSchema } from "@/src/features/evals/v2/server/evaluators/evaluatorTypes";
+import {
+ CreateEvaluatorSchema,
+ ListEvaluatorsSchema,
+} from "@/src/features/evals/v2/server/evaluators/evaluatorTypes";
describe("evaluator configuration validation", () => {
beforeEach(() => {
@@ -54,6 +57,22 @@ describe("evaluator configuration validation", () => {
).toBe(true);
});
+ it("accepts text filters for evaluator models", () => {
+ expect(
+ ListEvaluatorsSchema.safeParse({
+ projectId: "project-id",
+ filter: [
+ {
+ column: "model",
+ type: "string",
+ operator: "contains",
+ value: "gpt",
+ },
+ ],
+ }).success,
+ ).toBe(true);
+ });
+
it("validates deployment capabilities without executing the evaluator", async () => {
const codeDefinition = {
type: EvalTemplateType.CODE,
diff --git a/web/src/components/table/data-table-controls.clienttest.tsx b/web/src/components/table/data-table-controls.clienttest.tsx
index ee1e987f2f..4e0c1f44cc 100644
--- a/web/src/components/table/data-table-controls.clienttest.tsx
+++ b/web/src/components/table/data-table-controls.clienttest.tsx
@@ -34,6 +34,38 @@ beforeAll(() => {
});
describe("CategoricalFacet", () => {
+ it("renders an option suffix after its label", () => {
+ render(
+ <Accordion type="multiple" value={["model"]}>
+ <CategoricalFacet
+ label="Model"
+ filterKey="model"
+ expanded
+ loading={false}
+ options={["gpt-4.1", "claude-sonnet"]}
+ counts={new Map()}
+ value={[]}
+ onChange={() => {}}
+ renderOptionSuffix={(value) =>
+ value === "gpt-4.1" ? <span>Project default</span> : null
+ }
+ isActive={false}
+ isDisabled={false}
+ onReset={() => {}}
+ />
+ </Accordion>,
+ { wrapper: TooltipProvider },
+ );
+
+ const label = screen.getByText("gpt-4.1");
+ const suffix = screen.getByText("Project default");
+ expect(
+ label.compareDocumentPosition(suffix) & Node.DOCUMENT_POSITION_FOLLOWING,
+ ).toBeTruthy();
+ expect(label).not.toHaveClass("flex-1");
+ expect(screen.getByText("claude-sonnet")).toBeInTheDocument();
+ });
+
it("shows selected values even when the backend returns no options", () => {
render(
<Accordion type="multiple" value={["type"]}>
diff --git a/web/src/components/table/data-table-controls.tsx b/web/src/components/table/data-table-controls.tsx
index de7a0af843..e921429eab 100644
--- a/web/src/components/table/data-table-controls.tsx
+++ b/web/src/components/table/data-table-controls.tsx
@@ -534,6 +534,7 @@ export function DataTableControls({
onChange={filter.onChange}
onOnlyChange={filter.onOnlyChange}
renderIcon={filter.renderIcon}
+ renderOptionSuffix={filter.renderOptionSuffix}
isActive={filter.isActive}
onReset={filter.onReset}
operator={filter.operator}
@@ -1256,6 +1257,7 @@ interface CategoricalFacetProps extends BaseFacetProps {
onChange: (values: string[]) => void;
onOnlyChange?: (value: string) => void;
renderIcon?: (value: string) => React.ReactNode;
+ renderOptionSuffix?: (value: string) => React.ReactNode;
operator?: "any of" | "all of" | "none of";
onOperatorChange?: (operator: "any of" | "all of" | "none of") => void;
textFilters?: TextFilterEntry[];
@@ -1556,6 +1558,7 @@ export function CategoricalFacet({
onChange,
onOnlyChange,
renderIcon,
+ renderOptionSuffix,
isActive,
isDisabled,
disabledReason,
@@ -1635,6 +1638,7 @@ export function CategoricalFacet({
onChange={onChange}
onOnlyChange={onOnlyChange}
renderIcon={renderIcon}
+ renderOptionSuffix={renderOptionSuffix}
operator={operator}
onOperatorChange={onOperatorChange}
/>
@@ -1675,6 +1679,7 @@ function CategoricalSelectContent({
onChange,
onOnlyChange,
renderIcon,
+ renderOptionSuffix,
operator,
onOperatorChange,
}: Pick<
@@ -1688,6 +1693,7 @@ function CategoricalSelectContent({
| "onChange"
| "onOnlyChange"
| "renderIcon"
+ | "renderOptionSuffix"
| "operator"
| "onOperatorChange"
>) {
@@ -1771,6 +1777,7 @@ function CategoricalSelectContent({
id={`${filterKey}-${option}`}
label={displayLabel}
icon={renderIcon?.(option)}
+ suffix={renderOptionSuffix?.(option)}
count={counts.get(option) || 0}
checked={value.includes(option)}
onCheckedChange={(checked) => {
@@ -2591,6 +2598,7 @@ interface FilterValueCheckboxProps {
id: string;
label: string;
icon?: React.ReactNode;
+ suffix?: React.ReactNode;
count: number;
checked?: boolean;
onCheckedChange?: (checked: boolean) => void;
@@ -2603,6 +2611,7 @@ export function FilterValueCheckbox({
id,
label,
icon,
+ suffix,
count,
checked = false,
onCheckedChange,
@@ -2648,13 +2657,15 @@ export function FilterValueCheckbox({
{icon ? <span className="mr-2">{icon}</span> : null}
<span
className={cn(
- "min-w-0 flex-1 truncate text-xs",
+ "min-w-0 truncate text-xs",
+ !suffix && "flex-1",
label === "" && "text-muted-foreground italic",
)}
title={displayTitle}
>
{displayLabel}
</span>
+ {suffix ? <span className="shrink-0 pl-1">{suffix}</span> : null}
{/* "Only" or "All" indicator when hovering label. shrink-0 +
whitespace-nowrap: appearing may only re-truncate the label —
diff --git a/web/src/features/evals/v2/constants/tableFilterColumns.ts b/web/src/features/evals/v2/constants/tableFilterColumns.ts
index ae6cc64c49..593a6a5340 100644
--- a/web/src/features/evals/v2/constants/tableFilterColumns.ts
+++ b/web/src/features/evals/v2/constants/tableFilterColumns.ts
@@ -37,6 +37,13 @@ export const evaluatorTableFilterColumns: ColumnDefinition[] = [
internal: "type",
options: evaluatorTypeOptions,
},
+ {
+ name: "Model",
+ id: "model",
+ type: "stringOptions",
+ internal: "model",
+ options: [],
+ },
{
name: "Creator",
id: "creator",
@@ -83,6 +90,7 @@ export const evaluatorTableFilterConfig: FilterConfig = {
{ type: "categorical", column: "name", label: "Name" },
{ type: "categorical", column: "status", label: "Status" },
{ type: "categorical", column: "type", label: "Type" },
+ { type: "categorical", column: "model", label: "Model" },
{ type: "categorical", column: "creator", label: "Creator" },
],
};
diff --git a/web/src/features/evals/v2/hooks/useProjectDefaultModel.clienttest.ts b/web/src/features/evals/v2/hooks/useProjectDefaultModel.clienttest.ts
index 542c3dd3b2..be33229314 100644
--- a/web/src/features/evals/v2/hooks/useProjectDefaultModel.clienttest.ts
+++ b/web/src/features/evals/v2/hooks/useProjectDefaultModel.clienttest.ts
@@ -1,10 +1,16 @@
import { act, renderHook } from "@testing-library/react";
+import { LLMAdapter } from "@langfuse/shared";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { useProjectDefaultModel } from "./useProjectDefaultModel";
const mocks = vi.hoisted(() => ({
refetchConnections: vi.fn(),
+ invalidateDefaultModel: vi.fn(),
+ invalidateEvaluatorList: vi.fn(),
+ invalidateEvaluatorOptions: vi.fn(),
+ invalidateEvaluatorFilterOptions: vi.fn(),
+ upsertDefaultModel: vi.fn(),
}));
vi.mock("@/src/env.mjs", () => ({
@@ -27,11 +33,12 @@ vi.mock("@/src/utils/api", () => ({
api: {
useUtils: () => ({
defaultLlmModel: {
- fetchDefaultModel: { invalidate: vi.fn() },
+ fetchDefaultModel: { invalidate: mocks.invalidateDefaultModel },
},
evalsV2: {
- list: { invalidate: vi.fn() },
- options: { invalidate: vi.fn() },
+ list: { invalidate: mocks.invalidateEvaluatorList },
+ options: { invalidate: mocks.invalidateEvaluatorOptions },
+ filterOptions: { invalidate: mocks.invalidateEvaluatorFilterOptions },
},
}),
defaultLlmModel: {
@@ -39,7 +46,10 @@ vi.mock("@/src/utils/api", () => ({
useQuery: () => ({ data: null }),
},
upsertDefaultModel: {
- useMutation: () => ({ isPending: false, mutate: vi.fn() }),
+ useMutation: () => ({
+ isPending: false,
+ mutate: mocks.upsertDefaultModel,
+ }),
},
},
llmApiKey: {
@@ -60,10 +70,31 @@ vi.mock("@/src/utils/trpcErrorToast", () => ({
describe("useProjectDefaultModel", () => {
beforeEach(() => {
- mocks.refetchConnections.mockReset();
+ Object.values(mocks).forEach((mock) => mock.mockReset());
vi.spyOn(window, "open").mockImplementation(() => null);
});
+ it("refreshes model filter options after updating the project default", async () => {
+ const { result } = renderHook(() =>
+ useProjectDefaultModel({ projectId: "project-1", source: "overview" }),
+ );
+
+ act(() =>
+ result.current.update.requestUpdate({
+ provider: "openai",
+ model: "gpt-4.1",
+ adapter: LLMAdapter.OpenAI,
+ modelParams: {},
+ }),
+ );
+ const mutationOptions = mocks.upsertDefaultModel.mock.calls[0]?.[1];
+ await act(async () => mutationOptions?.onSuccess());
+
+ expect(mocks.invalidateEvaluatorFilterOptions).toHaveBeenCalledWith({
+ projectId: "project-1",
+ });
+ });
+
it("refreshes model connections after returning from provider settings", () => {
const { result } = renderHook(() =>
useProjectDefaultModel({ projectId: "project-1", source: "editor" }),
diff --git a/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts b/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts
index cfc120eb08..bb664d497a 100644
--- a/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts
+++ b/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts
@@ -70,6 +70,7 @@ export function useProjectDefaultModel({
utils.defaultLlmModel.fetchDefaultModel.invalidate({ projectId }),
utils.evalsV2.list.invalidate({ projectId }),
utils.evalsV2.options.invalidate({ projectId }),
+ utils.evalsV2.filterOptions.invalidate({ projectId }),
]);
capture("evaluators:default_model_update", {
source,
diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
index 11ea4d8ee0..20ce9f9dab 100644
--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
@@ -15,6 +15,7 @@ import {
} from "@/src/components/table/data-table-controls";
import { ResizableFilterLayout } from "@/src/components/table/resizable-filter-layout";
import type { LangfuseColumnDef } from "@/src/components/table/types";
+import { Badge } from "@/src/components/ui/badge";
import { Button } from "@/src/components/ui/button";
import { EvaluatorsEmptyState } from "../components/EvaluatorsEmptyState/EvaluatorsEmptyState";
import { PopoverTrigger } from "@/src/components/ui/popover";
@@ -162,6 +163,10 @@ export default function EvaluatorsPage() {
const router = useRouter();
const capture = usePostHogClientCapture();
const projectId = router.query.projectId as string;
+ const projectDefaultModel = useProjectDefaultModel({
+ projectId,
+ source: "overview",
+ });
const [pagination, setPagination] = usePaginationState(1, 50);
const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
"evaluatorsV2",
@@ -185,22 +190,38 @@ export default function EvaluatorsPage() {
...evaluatorTableFilterOptions,
name: filterOptionsQuery.data?.name ?? [],
creator: filterOptionsQuery.data?.creator ?? [],
+ model: filterOptionsQuery.data?.model ?? [],
}),
[filterOptionsQuery.data],
);
- const queryFilter = useSidebarFilterState(
- evaluatorTableFilterConfig,
- filterOptions,
- {
- loading: filterOptionsQuery.isPending,
- stateLocation: "urlAndSessionStorage",
- sessionFilterContextId: projectId,
- onExplicitFilterStateChange: () => {
- setPagination({ page: 1, limit: pagination.limit });
- selectionStore.getState().actions.clearSelection();
- },
- },
+ const filterConfig = useMemo(
+ () => ({
+ ...evaluatorTableFilterConfig,
+ facets: evaluatorTableFilterConfig.facets.map((facet) =>
+ facet.column === "model"
+ ? {
+ ...facet,
+ renderOptionSuffix: (model: string) =>
+ model === projectDefaultModel.defaultModel?.model ? (
+ <Badge variant="secondary" size="sm">
+ Project default
+ </Badge>
+ ) : null,
+ }
+ : facet,
+ ),
+ }),
+ [projectDefaultModel.defaultModel?.model],
);
+ const queryFilter = useSidebarFilterState(filterConfig, filterOptions, {
+ loading: filterOptionsQuery.isPending,
+ stateLocation: "urlAndSessionStorage",
+ sessionFilterContextId: projectId,
+ onExplicitFilterStateChange: () => {
+ setPagination({ page: 1, limit: pagination.limit });
+ selectionStore.getState().actions.clearSelection();
+ },
+ });
const filterState = queryFilter.filterState;
const [deleteIds, setDeleteIds] = useState<string[]>([]);
const [deleteAll, setDeleteAll] = useState(false);
@@ -237,10 +258,6 @@ export default function EvaluatorsPage() {
projectId,
scope: "evalJob:read",
});
- const projectDefaultModel = useProjectDefaultModel({
- projectId,
- source: "overview",
- });
const defaultModelConnection = projectDefaultModel.connections.find(
({ provider }) => provider === projectDefaultModel.defaultModel?.provider,
);
@@ -395,6 +412,23 @@ export default function EvaluatorsPage() {
return cost == null ? "—" : usdFormatter(cost, 2, 4);
},
},
+ {
+ accessorKey: "model",
+ id: "model",
+ header: "Model",
+ size: 180,
+ enableHiding: true,
+ cell: ({ row }) => {
+ const model = row.original.effectiveModel;
+ return model ? (
+ <span className="block truncate" title={model}>
+ {model}
+ </span>
+ ) : (
+ "—"
+ );
+ },
+ },
{
accessorKey: "createdByUser",
id: "createdByUser",
@@ -511,9 +545,7 @@ export default function EvaluatorsPage() {
validationContext: {
columns,
filterColumnDefinition: evaluatorTableFilterColumns,
- expandableFilterColumns: evaluatorTableFilterConfig.facets.map(
- (facet) => facet.column,
- ),
+ expandableFilterColumns: filterConfig.facets.map((facet) => facet.column),
},
currentFilterState: queryFilter.explicitFilterState,
currentExpandedFilters: queryFilter.expanded,
@@ -589,9 +621,7 @@ export default function EvaluatorsPage() {
onBrowseLibrary={() => setGalleryOpen(true)}
/>
) : (
- <DataTableControlsProvider
- tableName={evaluatorTableFilterConfig.tableName}
- >
+ <DataTableControlsProvider tableName={filterConfig.tableName}>
<div className="flex h-full min-h-0 w-full flex-1 flex-col overflow-hidden">
<EvaluatorsTableToolbar
selectionStore={selectionStore}
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
index 6223a1c294..29faa35b23 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
@@ -82,11 +82,100 @@ function versionData(
};
}
-function evaluatorWhere(params: {
+type EvaluatorModelFilter = Extract<
+ FilterState[number],
+ { type: "string" | "stringOptions" }
+> & { column: "model" };
+
+function isModelFilter(
+ filter: FilterState[number],
+): filter is EvaluatorModelFilter {
+ return (
+ filter.column === "model" &&
+ (filter.type === "string" || filter.type === "stringOptions")
+ );
+}
+
+async function evaluatorIdsMatchingModelFilters(params: {
+ prisma: PrismaClient | PrismaTransaction;
+ projectId: string;
+ filters: EvaluatorModelFilter[];
+}) {
+ const effectiveModel = Prisma.sql`
+ CASE
+ WHEN evaluator.type = 'LLM_AS_JUDGE'
+ THEN COALESCE(latest_version.model, default_model.model)
+ ELSE NULL
+ END
+ `;
+ const predicates = params.filters.map((filter) => {
+ const normalizedModel = Prisma.sql`LOWER(${effectiveModel})`;
+ if (filter.type === "stringOptions") {
+ if (filter.value.length === 0) {
+ return filter.operator === "any of"
+ ? Prisma.sql`FALSE`
+ : Prisma.sql`TRUE`;
+ }
+ const values = Prisma.join(
+ filter.value.map((value) => value.toLowerCase()),
+ );
+ return filter.operator === "any of"
+ ? Prisma.sql`${normalizedModel} IN (${values})`
+ : Prisma.sql`(${effectiveModel} IS NULL OR ${normalizedModel} NOT IN (${values}))`;
+ }
+
+ const normalizedValue = filter.value.toLowerCase();
+ switch (filter.operator) {
+ case "=":
+ return Prisma.sql`${normalizedModel} = ${normalizedValue}`;
+ case "contains":
+ return Prisma.sql`STRPOS(${normalizedModel}, ${normalizedValue}) > 0`;
+ case "does not contain":
+ return Prisma.sql`STRPOS(${normalizedModel}, ${normalizedValue}) = 0`;
+ case "starts with":
+ return Prisma.sql`LEFT(${normalizedModel}, LENGTH(${normalizedValue})) = ${normalizedValue}`;
+ case "ends with":
+ return Prisma.sql`RIGHT(${normalizedModel}, LENGTH(${normalizedValue})) = ${normalizedValue}`;
+ }
+ });
+ const matches = await params.prisma.$queryRaw<Array<{ id: string }>>(
+ Prisma.sql`
+ SELECT evaluator.id
+ FROM evaluators AS evaluator
+ LEFT JOIN LATERAL (
+ SELECT version.model
+ FROM evaluator_versions AS version
+ WHERE version.evaluator_id = evaluator.id
+ ORDER BY version.version DESC
+ LIMIT 1
+ ) AS latest_version ON TRUE
+ LEFT JOIN default_llm_models AS default_model
+ ON default_model.project_id = evaluator.project_id
+ WHERE evaluator.project_id = ${params.projectId}
+ AND ${Prisma.join(predicates, " AND ")}
+ `,
+ );
+ return matches.map(({ id }) => id);
+}
+
+async function evaluatorWhere(params: {
+ prisma: PrismaClient | PrismaTransaction;
projectId: string;
search?: string;
filter?: FilterState;
-}): Prisma.EvaluatorWhereInput {
+}): Promise<Prisma.EvaluatorWhereInput> {
+ const modelFilters = params.filter?.filter(isModelFilter) ?? [];
+ const otherFilters = params.filter?.filter(
+ (filter) => !isModelFilter(filter),
+ );
+ const modelEvaluatorIds =
+ modelFilters.length > 0
+ ? await evaluatorIdsMatchingModelFilters({
+ prisma: params.prisma,
+ projectId: params.projectId,
+ filters: modelFilters,
+ })
+ : undefined;
const handlers = {
name: {
string: (filter) => ({ name: stringFilterToPrisma(filter) }),
@@ -144,11 +233,12 @@ function evaluatorWhere(params: {
return {
projectId: params.projectId,
+ ...(modelEvaluatorIds ? { id: { in: modelEvaluatorIds } } : {}),
...(params.search
? { name: { contains: params.search, mode: "insensitive" as const } }
: {}),
AND: compilePrismaFilters<Prisma.EvaluatorWhereInput>(
- params.filter ?? [],
+ otherFilters ?? [],
handlers,
),
};
@@ -162,8 +252,8 @@ export async function listEvaluators(params: {
search?: string;
filter?: FilterState;
}) {
- const where = evaluatorWhere(params);
- const [evaluators, totalItems] = await Promise.all([
+ const where = await evaluatorWhere(params);
+ const [evaluators, totalItems, defaultModel] = await Promise.all([
params.prisma.evaluator.findMany({
where,
orderBy: [{ updatedAt: "desc" }, { id: "desc" }],
@@ -186,6 +276,10 @@ export async function listEvaluators(params: {
},
}),
params.prisma.evaluator.count({ where }),
+ params.prisma.defaultLlmModel.findUnique({
+ where: { projectId: params.projectId },
+ select: { model: true },
+ }),
]);
return {
evaluators: evaluators.map(({ assignments, ...evaluator }) => ({
@@ -196,6 +290,10 @@ export async function listEvaluators(params: {
hasActiveRules: assignments.some(
({ evaluationRule }) => evaluationRule.status === "ACTIVE",
),
+ effectiveModel:
+ evaluator.type === EvalTemplateType.LLM_AS_JUDGE
+ ? (evaluator.versions[0]?.model ?? defaultModel?.model ?? null)
+ : null,
})),
totalItems,
};
@@ -205,13 +303,24 @@ export async function listEvaluatorFilterOptions(params: {
prisma: PrismaClient;
projectId: string;
}) {
- const evaluators = await params.prisma.evaluator.findMany({
- where: { projectId: params.projectId },
- select: {
- name: true,
- createdByUser: { select: { name: true, email: true } },
- },
- });
+ const [evaluators, defaultModel] = await Promise.all([
+ params.prisma.evaluator.findMany({
+ where: { projectId: params.projectId },
+ select: {
+ name: true,
+ type: true,
+ createdByUser: { select: { name: true, email: true } },
+ versions: {
+ ...latestVersion,
+ select: { model: true },
+ },
+ },
+ }),
+ params.prisma.defaultLlmModel.findUnique({
+ where: { projectId: params.projectId },
+ select: { model: true },
+ }),
+ ]);
return {
name: [...new Set(evaluators.map(({ name }) => name))].sort(),
@@ -223,6 +332,15 @@ export async function listEvaluatorFilterOptions(params: {
),
),
].sort(),
+ model: [
+ ...new Set(
+ evaluators.flatMap(({ type, versions }) => {
+ if (type !== EvalTemplateType.LLM_AS_JUDGE) return [];
+ const model = versions[0]?.model ?? defaultModel?.model;
+ return model ? [model] : [];
+ }),
+ ),
+ ].sort(),
};
}
@@ -232,8 +350,9 @@ export async function listEvaluatorIds(params: {
search?: string;
filter?: FilterState;
}) {
+ const where = await evaluatorWhere(params);
const evaluators = await params.prisma.evaluator.findMany({
- where: evaluatorWhere(params),
+ where,
select: { id: true },
});
return evaluators.map(({ id }) => id);
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts
index 8be868a727..5f546f8d28 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts
@@ -110,9 +110,9 @@ const EvaluatorListFilterSchema = z
.superRefine((filters, ctx) => {
for (const [index, filter] of filters.entries()) {
const valid =
- ((filter.column === "name" || filter.column === "creator") &&
+ (["name", "creator", "model"].includes(filter.column) &&
(filter.type === "string" || filter.type === "stringOptions")) ||
- ((filter.column === "status" || filter.column === "type") &&
+ (["status", "type"].includes(filter.column) &&
filter.type === "stringOptions");
const validOptions =
filter.type !== "stringOptions" ||
@@ -126,7 +126,9 @@ const EvaluatorListFilterSchema = z
value as EvalTemplateType,
),
)
- : filter.column === "name" || filter.column === "creator");
+ : filter.column === "name" ||
+ filter.column === "creator" ||
+ filter.column === "model");
if (!valid || !validOptions) {
ctx.addIssue({
code: "custom",
diff --git a/web/src/features/filters/hooks/useSidebarFilterState.tsx b/web/src/features/filters/hooks/useSidebarFilterState.tsx
index 083a87295f..612e75c092 100644
--- a/web/src/features/filters/hooks/useSidebarFilterState.tsx
+++ b/web/src/features/filters/hooks/useSidebarFilterState.tsx
@@ -182,6 +182,8 @@ export interface CategoricalUIFilter extends BaseUIFilter {
onOnlyChange?: (value: string) => void;
/** Optional function to render an icon next to filter option labels */
renderIcon?: (value: string) => React.ReactNode;
+ /** Optional content rendered after a filter option label */
+ renderOptionSuffix?: (value: string) => React.ReactNode;
/**
* Current operator of the facet's checkbox filter (arrayOptions AND
* stringOptions columns; undefined when no filter is applied):
@@ -1901,6 +1903,8 @@ export function useSidebarFilterPresentation(
disabledReason: disableState.reason,
renderIcon:
facet.type === "categorical" ? facet.renderIcon : undefined,
+ renderOptionSuffix:
+ facet.type === "categorical" ? facet.renderOptionSuffix : undefined,
onChange: (values: string[]) => updateFilter(facet.column, values),
onOnlyChange: (value: string) => {
if (selectedValues.length === 1 && selectedValues.includes(value)) {
diff --git a/web/src/features/filters/lib/filter-config.ts b/web/src/features/filters/lib/filter-config.ts
index cfea754d17..846cca01ae 100644
--- a/web/src/features/filters/lib/filter-config.ts
+++ b/web/src/features/filters/lib/filter-config.ts
@@ -17,6 +17,8 @@ interface CategoricalFacet extends BaseFacet {
type: "categorical";
/** Optional function to render an icon next to filter option labels */
renderIcon?: (value: string) => React.ReactNode;
+ /** Optional content rendered after a filter option label */
+ renderOptionSuffix?: (value: string) => React.ReactNode;
/** When true, the sidebar hides the contains/does-not-contain text filter mode for this facet. */
disableTextFilter?: boolean;
}