🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable no-nested-ternary, @typescript-eslint/no-deprecated -- Legacy UI usage retained during migration to design-system components. */
import { type UseFormReturn, useForm } from "react-hook-form";
import { Alert } from "@/src/components/design-system/Alert/Alert";
import { Input } from "@/src/components/ui/input";
import { Button } from "@/src/components/ui/button";
import {
Form,
FormControl,
FormDescription,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/src/components/ui/form";
import { zodResolver } from "@hookform/resolvers/zod";
import { Tabs } from "@/src/components/design-system/Tabs/Tabs";
import { Badge } from "@/src/components/design-system/Badge/Badge";
import {
tracesTableColsWithOptions,
singleFilterList,
availableTraceEvalVariables,
datasetFormFilterColsWithOptions,
observationEvalFilterColsWithOptions,
experimentEvalFilterColsWithOptions,
type availableDatasetEvalVariables,
JobConfigState,
validateEvaluatorFiltersForTarget,
evalTraceTableCols,
type EvalTemplate,
EvalTemplateSourceCodeLanguage,
variableMapping,
observationVariableMapping,
EvalTargetObject,
EvalTargetObjectSchema,
getCodeEvalVariableMapping,
} from "@langfuse/shared";
import { z } from "zod";
import { useEffect, useMemo, useState, memo, Suspense, lazy } from "react";
import { api } from "@/src/utils/api";
import { InlineFilterBuilder } from "@/src/features/filters/components/filter-builder";
import { useRouter } from "next/router";
import { TRPCClientError } from "@trpc/client";
import { reportError } from "@/src/utils/reportError";
import { Slider } from "@/src/components/ui/Slider";
import { Card } from "@/src/components/ui/card";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
import { Checkbox } from "@/src/components/design-system/Checkbox/Checkbox";
import { Switch } from "@/src/components/design-system/Switch/Switch";
import {
evalConfigFormSchema,
getActiveJsonPathCompatibilityWarning,
type EvalFormType,
RETIRED_TRACE_FILTER_COLUMNS,
getTargetDisplayName,
inferDefaultMapping,
type LangfuseObject,
} from "@/src/features/evals/utils/evaluator-form-utils";
import { validateAndTransformVariableMapping } from "@/src/features/evals/utils/variable-mapping-validation";
import { useVariableMappingSync } from "@/src/features/evals/hooks/useVariableMappingSync";
import { ExecutionCountTooltip } from "@/src/features/evals/components/ExecutionCountTooltip";
import {
getDateFromOption,
type TableDateRange,
} from "@/src/utils/date-range-utils";
import { type PartialConfig } from "@/src/features/evals/types";
import { type EvalCapabilities } from "@/src/features/evals/hooks/useEvalCapabilities";
import {
EvalVersionCallout,
getEvalVersionCalloutContent,
} from "@/src/features/evals/components/EvalVersionCallout";
import { Skeleton } from "@/src/components/ui/skeleton";
import {
Dialog,
DialogBody,
DialogContent,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/src/components/ui/dialog";
import {
Tooltip,
TooltipTrigger,
TooltipContent,
} from "@/src/components/ui/tooltip";
import {
BetweenHorizonalStart,
CircleDot,
AlertTriangle,
FlaskConical,
InfoIcon,
ListTree,
ExternalLink,
} from "@langfuse/icons";
import Link from "next/link";
import {
isDatasetTarget,
isEventTarget,
isExperimentTarget,
isLegacyEvalTarget,
isTraceTarget,
shouldShowLegacyTracePreview,
} from "@/src/features/evals/utils/typeHelpers";
import {
useUserFacingTarget,
useEvaluatorTargetState,
} from "@/src/features/evals/hooks/useEvaluatorTarget";
import {
DEFAULT_OBSERVATION_FILTER,
DEFAULT_TRACE_FILTER,
} from "@/src/features/evals/utils/evaluator-constants";
import { useEvalConfigFilterOptions } from "@/src/features/evals/hooks/useEvalConfigFilterOptions";
import { VariableMappingCard } from "@/src/features/evals/components/VariableMappingCard";
import { useReadPath } from "@/src/features/events/hooks/useReadPath";
import { useIsCodeEvalEnabled } from "@/src/features/evals/hooks/useIsCodeEvalEnabled";
import {
isCodeEvalTemplate,
resolveCodeEvalTarget,
} from "@/src/features/evals/utils/code-eval-template-utils";
import { CodeEvalTestRunCard } from "@/src/features/evals/components/CodeEvalTestRunCard";
import { getExperimentEvalPreviewFilters } from "@/src/features/evals/utils/experiment-eval-preview-utils";
import { cn } from "@/src/utils/tailwind";
import { PeekTableStateProvider } from "@/src/components/table/peek/contexts/PeekTableStateProvider";
// Lazy load tables
const TracesTable = lazy(() => import("@/src/features/traces/TracesTable"));
const ObservationsTable = lazy(
() => import("@/src/features/tracing-tables/observations/ObservationsTable"),
);
const EventsTable = lazy(
() => import("@/src/features/events/components/EventsTable"),
);
const TracesPreview = memo(
({
projectId,
filterState,
}: {
projectId: string;
filterState: z.infer<typeof singleFilterList>[number][];
}) => {
const dateRange = useMemo(() => {
return {
from: getDateFromOption({
filterSource: "TABLE",
option: "last1Day",
}),
} as TableDateRange;
}, []);
return (
<>
<div className="flex flex-col items-start gap-1">
<span className="text-sm leading-none font-bold">
Preview sample matched traces
</span>
<FormDescription>
Sample over the last 24 hours that match these filters
</FormDescription>
</div>
<div className="mb-4 flex max-h-[30dvh] w-full flex-col overflow-hidden border-r border-b border-l">
<Suspense fallback={<Skeleton className="h-[30dvh] w-full" />}>
{/* Match peek tables: preview state stays local and never touches the URL. */}
<PeekTableStateProvider>
<TracesTable
projectId={projectId}
hideControls
externalFilterState={filterState}
externalDateRange={dateRange}
limitRows={10}
/>
</PeekTableStateProvider>
</Suspense>
</div>
</>
);
},
);
TracesPreview.displayName = "TracesPreview";
const ObservationsPreview = memo(
({
projectId,
filterState,
isNewCompatible,
compatibilityCheckWasPerformed,
}: {
projectId: string;
filterState: z.infer<typeof singleFilterList>[number][];
isNewCompatible: boolean;
compatibilityCheckWasPerformed: boolean;
}) => {
const { isV4 } = useReadPath();
const dateRange = useMemo(() => {
return {
from: getDateFromOption({
filterSource: "TABLE",
option: "last1Day",
}),
} as TableDateRange;
}, []);
// Show upgrade message only when SDK check was performed and user is not on OTEL SDK
const showSdkUpgradeMessage =
compatibilityCheckWasPerformed && !isNewCompatible;
return (
<>
<div className="flex flex-col items-start gap-1">
<FormDescription>
Sample over the last 24 hours that match filters
</FormDescription>
</div>
<div className="mb-4 flex max-h-[30dvh] w-full flex-col overflow-hidden border-r border-b border-l">
<Suspense fallback={<Skeleton className="h-[30dvh] w-full" />}>
{showSdkUpgradeMessage ? (
<div className="flex h-[30dvh] flex-col items-center justify-center gap-2 border-t p-4 text-center">
<AlertTriangle size="xl" __rawClassName="text-dark-yellow " />
<div className="flex flex-col gap-1">
<span className="text-foreground font-bold">
Please verify your SDK version
</span>
<span className="text-muted-foreground max-w-md text-sm">
We did not find any data ingested with langfuse
OTEL-compatible SDKs in the last 7 days. Observation-level
evaluators require JS SDK v4+ or Python SDK v3+. You can
still configure this evaluator now—it will start running
once you upgrade.{" "}
<a
href="https://langfuse.com/docs/observability/sdk/upgrade-path"
target="_blank"
rel="noopener noreferrer"
className="text-dark-blue font-bold hover:opacity-80"
>
Learn more
</a>
.
</span>
</div>
</div>
) : (
// Keep the evaluator preview isolated from the parent route's table state.
<PeekTableStateProvider>
{isV4 ? (
<EventsTable
projectId={projectId}
showSearchBar={false}
showToolbar={false}
showFilterSidebar={false}
showPeekPanel={false}
externalFilterState={filterState}
externalDateRange={dateRange}
limitRows={10}
/>
) : (
<ObservationsTable
projectId={projectId}
hideControls
externalFilterState={filterState}
externalDateRange={dateRange}
limitRows={10}
/>
)}
</PeekTableStateProvider>
)}
</Suspense>
</div>
</>
);
},
);
ObservationsPreview.displayName = "ObservationsPreview";
function getCodeEvalSourceLanguageLabel(
sourceCodeLanguage: EvalTemplate["sourceCodeLanguage"],
) {
return sourceCodeLanguage === EvalTemplateSourceCodeLanguage.PYTHON
? "Python"
: "TypeScript";
}
function CodeEvalSourceLink({
projectId,
evalTemplate,
}: {
projectId: string;
evalTemplate: EvalTemplate;
}) {
const sourceCodeLanguage =
evalTemplate.sourceCodeLanguage ??
EvalTemplateSourceCodeLanguage.TYPESCRIPT;
const languageLabel = getCodeEvalSourceLanguageLabel(sourceCodeLanguage);
const editButton = evalTemplate.projectId ? (
<Button asChild variant="outline">
<Link
href={`/project/${projectId}/evals/templates/${evalTemplate.id}?mode=edit`}
target="_blank"
rel="noopener noreferrer"
>
Edit source code
<ExternalLink size="base" color="default" __rawClassName="ml-1" />
</Link>
</Button>
) : (
<Button
variant="outline"
disabled
title="Only user-managed templates can be edited"
>
Edit source code
<ExternalLink size="base" color="default" __rawClassName="ml-1" />
</Button>
);
return (
<div className="bg-muted/20 flex max-w-[500px] items-center justify-between gap-3 rounded-md border px-3 py-2">
<span className="text-muted-foreground text-sm">{languageLabel}</span>
{editButton}
</div>
);
}
const EMPTY_FILTER_STATE: z.infer<typeof singleFilterList>[number][] = [];
export const InnerEvaluatorForm = (props: {
projectId: string;
evalTemplate: EvalTemplate;
useDialog: boolean;
disabled?: boolean;
existingEvaluator?: PartialConfig;
onFormSuccess?: () => void;
shouldWrapVariables?: boolean;
mode?: "create" | "edit";
hideTargetSection?: boolean;
preventRedirect?: boolean;
preprocessFormValues?: (values: any) => any;
hideAdvancedSettings?: boolean;
hideTargetSelection?: boolean;
hidePreviewTable?: boolean;
hideRootObservationFilter?: boolean;
showPreviewTargetBadge?: boolean;
evalCapabilities: EvalCapabilities;
defaultRunOnLive?: boolean;
defaultTarget?: EvalTargetObject;
renderFooter?: (params: {
isLoading: boolean;
isSaveDisabled: boolean;
}) => React.ReactNode;
oldConfigId?: string;
sourceRuleAction?: "mark-inactive" | "delete";
}) => {
const capture = usePostHogClientCapture();
const router = useRouter();
const [showTraceConfirmDialog, setShowTraceConfirmDialog] = useState(false);
const { isV4 } = useReadPath();
const { enabled: isCodeEvalEnabled } = useIsCodeEvalEnabled();
const isCodeEvalConfig =
isCodeEvalEnabled && isCodeEvalTemplate(props.evalTemplate);
// Destructure eval capabilities passed from parent
const { allowLegacy } = props.evalCapabilities;
// Existing legacy evaluators must keep rendering their (read-only) legacy
// target UI in edit mode even when new legacy setups are not allowed.
const showLegacyTargetOptions =
allowLegacy ||
(props.mode === "edit" &&
isLegacyEvalTarget(props.existingEvaluator?.targetObject ?? ""));
// Custom hooks for managing evaluator state
const {
userFacingTarget,
setUserFacingTarget,
useOtelDataForExperiment,
setUseOtelDataForExperiment,
} = useUserFacingTarget(props.existingEvaluator?.targetObject);
const {
traceFilterOptions,
observationEvalFilterOptions,
experimentEvalFilterOptions,
datasetFilterOptions,
} = useEvalConfigFilterOptions({
projectId: props.projectId,
useEventsTable: isV4,
includeLegacyTraceOptions: showLegacyTargetOptions,
});
const targetState = useEvaluatorTargetState();
// Check if existing trace evaluator has invalid filters (e.g., score filters added by bug ff4b03c0b)
const hasInvalidTraceFilters = useMemo(() => {
if (
!props.existingEvaluator?.filter ||
props.existingEvaluator.targetObject !== EvalTargetObject.TRACE
) {
return false;
}
const validation = validateEvaluatorFiltersForTarget({
targetObject: EvalTargetObject.TRACE,
filter: props.existingEvaluator.filter,
});
return !validation.isValid;
}, [props.existingEvaluator?.filter, props.existingEvaluator?.targetObject]);
const defaultTargetResult = EvalTargetObjectSchema.safeParse(
props.existingEvaluator?.targetObject ??
props.defaultTarget ??
EvalTargetObject.EVENT,
);
const parsedDefaultTarget = defaultTargetResult.success
? defaultTargetResult.data
: EvalTargetObject.EVENT;
const defaultTarget = isCodeEvalConfig
? resolveCodeEvalTarget(parsedDefaultTarget)
: parsedDefaultTarget;
const getDefaultMapping = () => {
if (isCodeEvalConfig) {
return getCodeEvalVariableMapping();
}
if (props.existingEvaluator?.variableMapping) {
return isEventTarget(props.existingEvaluator.targetObject) ||
isExperimentTarget(props.existingEvaluator.targetObject)
? z
.array(observationVariableMapping)
.parse(props.existingEvaluator.variableMapping)
: z
.array(variableMapping)
.parse(props.existingEvaluator.variableMapping);
}
return z.array(variableMapping).parse(
props.evalTemplate
? props.evalTemplate.vars.map((v) => ({
templateVariable: v,
langfuseObject: "trace" as const,
objectName: null,
selectedColumnId: "input",
jsonSelector: null,
}))
: [],
);
};
const form = useForm({
resolver: zodResolver(evalConfigFormSchema),
disabled: props.disabled,
defaultValues: {
scoreName:
props.existingEvaluator?.scoreName ?? `${props.evalTemplate.name}`,
target: defaultTarget,
filter: props.existingEvaluator?.filter
? singleFilterList.parse(props.existingEvaluator.filter)
: defaultTarget === EvalTargetObject.TRACE
? // For new trace evaluators, exclude internal environments by default
DEFAULT_TRACE_FILTER
: defaultTarget === EvalTargetObject.EVENT
? // For new observation evaluators, default to GENERATION type
DEFAULT_OBSERVATION_FILTER
: [],
mapping: getDefaultMapping(),
sampling: props.existingEvaluator?.sampling
? props.existingEvaluator.sampling.toNumber()
: 1,
delay: props.existingEvaluator?.delay
? props.existingEvaluator.delay / 1000
: 30,
timeScope: (props.existingEvaluator?.timeScope ?? ["NEW"]).filter(
(option): option is "NEW" | "EXISTING" =>
["NEW", "EXISTING"].includes(option),
),
runOnLive: props.existingEvaluator
? props.existingEvaluator.status === "ACTIVE"
: (props.defaultRunOnLive ?? true),
},
}) as UseFormReturn<EvalFormType>;
const currentMapping = form.watch("mapping") ?? [];
const hasJsonPathCompatibilityError = currentMapping.some((mappingRow) =>
Boolean(getActiveJsonPathCompatibilityWarning(mappingRow)),
);
const syncStatus = useVariableMappingSync({
templateVars: isCodeEvalConfig ? [] : props.evalTemplate?.vars,
currentMapping: currentMapping,
});
useEffect(() => {
if (!props.evalTemplate) return;
if (isCodeEvalConfig) return;
const mapping = form.getValues("mapping");
if (mapping.length === 0 && props.evalTemplate.vars.length > 0) {
// Initialize mapping for new evaluators (only if there are vars to map)
const target = form.getValues("target");
form.setValue(
"mapping",
props.evalTemplate.vars.map((v) => ({
templateVariable: v,
langfuseObject: isLegacyEvalTarget(target)
? ("trace" as const)
: undefined,
...inferDefaultMapping(v),
})),
);
form.setValue("scoreName", `${props.evalTemplate.name}`);
} else if (
props.existingEvaluator &&
!props.disabled &&
!syncStatus.inSync
) {
// Reconcile mapping when edit mode is enabled
const target = form.getValues("target");
// Keep mappings for unchanged variables
const preservedMappings = mapping.filter((m) =>
syncStatus.unchanged.includes(m.templateVariable),
);
// Add mappings for new variables
const newMappings = syncStatus.added.map((varName) => ({
templateVariable: varName,
langfuseObject: isLegacyEvalTarget(target)
? ("trace" as const)
: undefined,
...inferDefaultMapping(varName),
}));
// Combine and update form
form.setValue("mapping", [...preservedMappings, ...newMappings]);
}
}, [
form,
props.evalTemplate,
props.disabled,
props.existingEvaluator,
syncStatus,
isCodeEvalConfig,
]);
const utils = api.useUtils();
// No onError override: the react-query default (handleTrpcError) owns
// classification, Sentry capture, and the standard error toast.
const createJobMutation = api.evals.createJob.useMutation({
onSuccess: () => utils.models.invalidate(),
});
const updateJobMutation = api.evals.updateEvalJob.useMutation({
onSuccess: () => utils.evals.invalidate(),
});
const [availableVariables, setAvailableVariables] = useState<
typeof availableTraceEvalVariables | typeof availableDatasetEvalVariables
>(() =>
targetState.getAvailableVariables(
props.existingEvaluator?.targetObject ?? EvalTargetObject.EVENT,
),
);
const watchedTarget = form.watch("target");
const watchedScoreName = form.watch("scoreName");
const watchedFilter = form.watch("filter") ?? EMPTY_FILTER_STATE;
const shouldShowExperimentEventsPreview =
isExperimentTarget(watchedTarget) && isV4;
const shouldShowEventsPreview =
isEventTarget(watchedTarget) || shouldShowExperimentEventsPreview;
const previewTableVisible = !props.disabled && !props.hidePreviewTable;
const previewAlreadyShowsSdkWarning =
previewTableVisible && shouldShowEventsPreview;
const evalVersionCalloutContent = getEvalVersionCalloutContent(
watchedTarget,
props.evalCapabilities,
);
const eventsPreviewFilterState = useMemo(
() =>
shouldShowExperimentEventsPreview
? getExperimentEvalPreviewFilters(watchedFilter)
: watchedFilter,
[shouldShowExperimentEventsPreview, watchedFilter],
);
// Clear mapping error if user switches away from trace target
useEffect(() => {
if (
!isTraceTarget(watchedTarget) &&
form.formState.errors.mapping?.type === "manual"
) {
form.clearErrors("mapping");
}
}, [watchedTarget, form]);
function onSubmit(values: z.infer<typeof evalConfigFormSchema>) {
capture(
props.mode === "edit"
? "eval_config:update"
: "eval_config:new_form_submit",
);
// Apply preprocessFormValues if it exists
if (props.preprocessFormValues) {
values = props.preprocessFormValues(values);
}
const validatedFilter = singleFilterList.safeParse(values.filter);
if (
props.existingEvaluator?.timeScope.includes("EXISTING") &&
props.mode === "edit" &&
!values.timeScope.includes("EXISTING")
) {
form.setError("timeScope", {
type: "manual",
message:
"The evaluator ran on existing traces already. This cannot be changed anymore.",
});
return;
}
if (form.getValues("timeScope").length === 0) {
form.setError("timeScope", {
type: "manual",
message: "Please select at least one.",
});
return;
}
if (validatedFilter.success === false) {
form.setError("filter", {
type: "manual",
message: "Please fill out all filter fields",
});
return;
}
if (
isCodeEvalConfig &&
!isEventTarget(values.target) &&
!isExperimentTarget(values.target)
) {
form.setError("target", {
type: "manual",
message: "Code evaluators can only run on observations or experiments.",
});
return;
}
// Block NEW trace-level evals that target observations
if (
!isCodeEvalConfig &&
props.mode !== "edit" &&
isTraceTarget(values.target) &&
values.mapping.some(
(m) => m.langfuseObject && m.langfuseObject !== "trace",
)
) {
form.setError("mapping", {
type: "manual",
message:
"Trace-level evaluators targeting observations are no longer supported. Please use observation-level evaluators or target trace IO instead.",
});
return;
}
const validatedVarMapping = isCodeEvalConfig
? {
success: true as const,
data: getCodeEvalVariableMapping(),
}
: validateAndTransformVariableMapping(
values.mapping,
values.target as EvalTargetObject,
);
if (!validatedVarMapping.success) {
form.setError("mapping", {
type: "manual",
message: validatedVarMapping.error,
});
return;
}
const delay = values.delay * 1000; // convert to ms
const sampling = values.sampling;
const mapping = validatedVarMapping.data;
const filter = validatedFilter.data;
const scoreName = values.scoreName;
// For modern targets, derive status from runOnLive
const isModern = !isLegacyEvalTarget(values.target);
const status = (() => {
if (isModern) {
if (values.runOnLive) {
return JobConfigState.ACTIVE;
}
return JobConfigState.INACTIVE;
}
return undefined;
})();
(props.mode === "edit" && props.existingEvaluator?.id
? updateJobMutation.mutateAsync({
projectId: props.projectId,
evalConfigId: props.existingEvaluator.id,
config: {
delay,
filter,
variableMapping: mapping,
sampling,
scoreName,
timeScope: isModern ? ["NEW"] : values.timeScope,
...(status ? { status } : {}),
},
})
: createJobMutation.mutateAsync({
projectId: props.projectId,
target: values.target,
evalTemplateId: props.evalTemplate.id,
scoreName,
filter,
mapping,
sampling,
delay,
timeScope: isModern ? ["NEW"] : values.timeScope,
...(status ? { status } : {}),
...(props.oldConfigId
? {
sourceRuleId: props.oldConfigId,
sourceRuleAction:
props.sourceRuleAction ?? ("mark-inactive" as const),
}
: {}),
})
)
.then(() => {
props.onFormSuccess?.();
if (props.mode !== "edit" && !props.preventRedirect) {
router.push(`/project/${props.projectId}/evals/legacy`);
// Don't reset form when redirecting - it will unmount anyway
} else {
// Only reset form when NOT redirecting
form.reset();
}
})
.catch((error) => {
// Mutation failures were already classified + toasted by the
// react-query default onError; only post-success errors
// (onFormSuccess, router.push) need reporting here. reportError
// captures with an area tag and warns — console.error would mint a
// second, unclassified Sentry event via captureConsoleIntegration.
if (!(error instanceof TRPCClientError)) {
reportError(error, {
area: "evals",
extra: { context: "evaluator-form-post-submit" },
});
}
});
}
function handleAndResolveTarget(value: string) {
const newUserFacingTarget = value as
| "trace"
| "event"
| "offline-experiment";
if (newUserFacingTarget === userFacingTarget) {
return;
}
// Show dialog when clicking trace if user has no legacy evals
if (
newUserFacingTarget === "trace" &&
!props.evalCapabilities.hasLegacyEvals &&
props.mode !== "edit"
) {
setShowTraceConfirmDialog(true);
return;
}
// Update user-facing target
setUserFacingTarget(newUserFacingTarget);
// Determine the actual target based on selection
let actualTarget: EvalTargetObject;
if (newUserFacingTarget === "trace") {
actualTarget = EvalTargetObject.TRACE;
} else if (newUserFacingTarget === "event") {
actualTarget = EvalTargetObject.EVENT;
} else {
// offline-experiment: code evaluators always use the observation-backed experiment target
actualTarget = isCodeEvalConfig
? EvalTargetObject.EXPERIMENT
: useOtelDataForExperiment
? EvalTargetObject.EXPERIMENT
: EvalTargetObject.DATASET;
}
// Transform variable mapping for new target type
const currentMapping = form.getValues("mapping");
const newMapping = isCodeEvalConfig
? getCodeEvalVariableMapping()
: targetState.transformMapping(currentMapping, actualTarget);
// Update form state with target-appropriate default filters
form.setValue(
"filter",
actualTarget === EvalTargetObject.TRACE
? DEFAULT_TRACE_FILTER
: actualTarget === EvalTargetObject.EVENT
? DEFAULT_OBSERVATION_FILTER
: [],
);
form.setValue("mapping", newMapping);
form.setValue("runOnLive", props.defaultRunOnLive ?? true);
setAvailableVariables(targetState.getAvailableVariables(actualTarget));
return actualTarget;
}
const shouldShowCodeEvalTestPanel =
isCodeEvalConfig &&
!props.disabled &&
(isEventTarget(watchedTarget) ||
(isExperimentTarget(watchedTarget) && isV4));
const shouldShowCodeEvalSourceLinkInSettingsCard =
isCodeEvalConfig &&
!props.disabled &&
isExperimentTarget(watchedTarget) &&
!isV4;
const formBody = (
<div
className={cn(
"grid gap-4",
shouldShowCodeEvalTestPanel &&
"xl:grid-cols-[minmax(0,2fr)_minmax(320px,1fr)] xl:items-start",
)}
>
<div className={cn(shouldShowCodeEvalTestPanel && "xl:col-span-2")}>
<FormField
control={form.control}
name="scoreName"
render={({ field }) => (
<FormItem>
<FormLabel>Generated Score Name</FormLabel>
<FormControl>
<Input {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
</div>
{!props.hideTargetSection && (
<Card className="flex max-w-full flex-col gap-2 overflow-y-auto p-4">
{hasInvalidTraceFilters && (
<Alert variant="destructive" icon={AlertTriangle}>
<Alert.Title>Unsupported filter detected</Alert.Title>
<Alert.Description>
This evaluator has a filter that is not supported for
trace-level evaluators. It is effectively paused. Please remove
all filters and re-add them from scratch to resume execution.
</Alert.Description>
</Alert>
)}
<div className="flex flex-col gap-4">
{!props.hideTargetSelection && (
<FormField
control={form.control}
name="target"
render={({ field }) => (
<FormItem>
<FormLabel>
Run on{" "}
{props.mode === "edit" && (
<Tooltip>
<TooltipTrigger>
<InfoIcon size="sm" color="muted" />
</TooltipTrigger>
<TooltipContent className="max-w-[200px] p-2">
<span className="leading-4">
An evaluator's target data may only be
configured at creation.
</span>
</TooltipContent>
</Tooltip>
)}
</FormLabel>
<FormControl>
<Tabs
value={userFacingTarget}
onValueChange={(value) => {
const actualTarget = handleAndResolveTarget(value);
if (actualTarget) {
field.onChange(actualTarget);
}
}}
>
<Tabs.List
variant="inset"
size="md"
layout="packed"
gap="lg"
>
<span className="min-w-[100px]">
<Tabs.Trigger
value="event"
disabled={props.disabled || props.mode === "edit"}
icon={CircleDot}
label="Observations"
/>
</span>
{showLegacyTargetOptions && (
<span className="min-w-[100px]">
<Tabs.Trigger
value="trace"
disabled={
props.disabled || props.mode === "edit"
}
icon={ListTree}
>
Traces
<Badge color="filled" text="Legacy" />
</Tabs.Trigger>
</span>
)}
<span className="min-w-[100px]">
<Tabs.Trigger
value="offline-experiment"
disabled={props.disabled || props.mode === "edit"}
icon={FlaskConical}
label="Experiments"
/>
</span>
</Tabs.List>
</Tabs>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
)}
{/* Second tab bar for experiment data source selection */}
{!props.hideTargetSelection &&
userFacingTarget === "offline-experiment" &&
showLegacyTargetOptions && (
<div className="flex flex-col gap-2">
<FormLabel className="text-sm">Experiment Method</FormLabel>
<Tabs
value={useOtelDataForExperiment ? "otel" : "non-otel"}
onValueChange={(value) => {
// Don't allow changes in edit mode or disabled mode
if (props.mode === "edit" || props.disabled) {
return;
}
const useOtel = value === "otel";
setUseOtelDataForExperiment(useOtel);
// Update the actual form target: only use EXPERIMENT if beta is enabled
const actualTarget = useOtel
? EvalTargetObject.EXPERIMENT
: EvalTargetObject.DATASET;
form.setValue("target", actualTarget);
// Transform variable mapping for new target type
const currentMapping = form.getValues("mapping");
const newMapping = targetState.transformMapping(
currentMapping,
actualTarget,
);
// Update form state
form.setValue("filter", []);
form.setValue("mapping", newMapping);
setAvailableVariables(
targetState.getAvailableVariables(actualTarget),
);
}}
>
<Tabs.List
variant="inset"
size="md"
layout="packed"
gap="lg"
>
<span className="min-w-[100px]">
<Tabs.Trigger
value="otel"
disabled={props.mode === "edit" || props.disabled}
icon={FlaskConical}
label="Experiment Runner SDK"
/>
</span>
<span className="min-w-[100px]">
<Tabs.Trigger
value="non-otel"
disabled={props.mode === "edit" || props.disabled}
icon={BetweenHorizonalStart}
>
Low-level SDK methods
<Badge color="filled" text="Legacy" />
</Tabs.Trigger>
</span>
</Tabs.List>
</Tabs>
</div>
)}
{!props.hideTargetSelection &&
props.mode !== "edit" &&
!props.disabled &&
!previewAlreadyShowsSdkWarning &&
evalVersionCalloutContent && (
<EvalVersionCallout content={evalVersionCalloutContent} />
)}
{!props.hideAdvancedSettings &&
isLegacyEvalTarget(form.watch("target")) && (
<FormField
control={form.control}
name="timeScope"
render={({ field }) => (
<FormItem className="flex-1">
<FormLabel>Evaluate</FormLabel>
<FormControl>
<div className="flex flex-col gap-2">
<div className="flex space-x-2">
<Checkbox
id="newObjects"
checked={field.value.includes("NEW")}
onCheckedChange={(checked) => {
const newValue = checked
? [...field.value, "NEW"]
: field.value.filter((v) => v !== "NEW");
field.onChange(newValue);
}}
disabled={props.disabled}
/>
<div className="grid gap-1.5 leading-none">
<label
htmlFor="newObjects"
className="text-sm leading-none font-bold peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
>
New {getTargetDisplayName(form.watch("target"))}
</label>
</div>
</div>
<div className="flex space-x-2">
<Checkbox
id="existingObjects"
checked={field.value.includes("EXISTING")}
onCheckedChange={(checked) => {
const newValue = checked
? [...field.value, "EXISTING"]
: field.value.filter((v) => v !== "EXISTING");
field.onChange(newValue);
}}
disabled={
props.disabled ||
(props.mode === "edit" &&
field.value.includes("EXISTING"))
}
/>
<div className="flex items-center gap-1.5 leading-none">
<label
htmlFor="existingObjects"
className="text-sm leading-none font-bold peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
>
Existing{" "}
{getTargetDisplayName(form.watch("target"))}
</label>
{field.value.includes("EXISTING") &&
!props.disabled &&
(props.mode === "edit" ? (
<Tooltip>
<TooltipTrigger>
<InfoIcon size="sm" color="muted" />
</TooltipTrigger>
<TooltipContent className="max-w-[300px] p-2">
<span className="leading-4">
This evaluator has already run on
existing{" "}
{getTargetDisplayName(
form.watch("target"),
)}{" "}
once. Set up a new evaluator to re-run
on existing{" "}
{getTargetDisplayName(
form.watch("target"),
)}
.
</span>
</TooltipContent>
</Tooltip>
) : (
<ExecutionCountTooltip
projectId={props.projectId}
item={form.watch("target")}
filter={form.watch("filter")}
/>
))}
</div>
</div>
</div>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
)}
{/* Run on Live toggle for modern (non-legacy) targets */}
{!props.hideAdvancedSettings &&
!isLegacyEvalTarget(form.watch("target")) && (
<FormField
control={form.control}
name="runOnLive"
render={({ field }) => {
const target = form.watch("target");
return (
<div className="flex max-w-4xl flex-col gap-2">
<FormItem className="flex items-center justify-between rounded-lg border p-3">
<div className="space-y-0.5">
<FormLabel>
{isEventTarget(target)
? "Run on live incoming observations"
: "Run on new experiments"}
</FormLabel>
<FormDescription>
Automatically evaluate new incoming{" "}
{getTargetDisplayName(target)}.
</FormDescription>
</div>
<FormControl>
<Switch
checked={field.value}
onCheckedChange={field.onChange}
disabled={props.disabled}
/>
</FormControl>
</FormItem>
{!field.value && isEventTarget(target) && (
<p className="text-muted-foreground text-xs">
This evaluator can still be used for batched
evaluation of historic observations.{" "}
<a
href="https://langfuse.com/docs/evaluation/evaluation-methods/llm-as-a-judge"
target="_blank"
rel="noopener noreferrer"
className="text-dark-blue hover:opacity-80"
>
Read the docs
</a>
</p>
)}
</div>
);
}}
/>
)}
{(isLegacyEvalTarget(form.watch("target")) ||
form.watch("runOnLive")) && (
<>
<FormField
control={form.control}
name="filter"
render={({ field }) => {
const target = form.watch("target");
// Get appropriate columns based on target type
const getFilterColumns = () => {
if (isEventTarget(target)) {
// Event evaluators - use observation columns
return observationEvalFilterColsWithOptions(
observationEvalFilterOptions,
).filter(
(column) =>
!(
props.hideRootObservationFilter &&
column.id === "isRootObservation"
),
);
} else if (isTraceTarget(target)) {
return tracesTableColsWithOptions(
traceFilterOptions,
evalTraceTableCols,
);
} else if (isExperimentTarget(target)) {
// Experiment evaluators - only dataset filter
return experimentEvalFilterColsWithOptions(
experimentEvalFilterOptions,
);
}
// dataset (legacy non-OTEL experiments)
return datasetFormFilterColsWithOptions(
datasetFilterOptions,
);
};
const hasFilters = field.value && field.value.length > 0;
return (
<FormItem>
<FormLabel>Filter</FormLabel>
<FormControl>
<div className="max-w-[500px]">
{props.disabled && !hasFilters ? (
<p className="text-muted-foreground text-xs">
All {getTargetDisplayName(target)} will be
evaluated
</p>
) : (
<InlineFilterBuilder
key={target}
columnIdentifier={
isDatasetTarget(target) ||
isTraceTarget(target)
? "name"
: "id"
}
columns={getFilterColumns()}
filterState={field.value ?? []}
onChange={(
value: z.infer<
typeof singleFilterList
>[number][],
) => {
field.onChange(value);
if (router.query.traceId) {
const { traceId, ...otherParams } =
router.query;
router.replace(
{
pathname: router.pathname,
query: otherParams,
},
undefined,
{ shallow: true },
);
}
}}
disabled={props.disabled}
columnsWithCustomSelect={
isTraceTarget(target)
? ["traceTags", "traceName"]
: isEventTarget(target)
? ["tags", "name", "calledToolNames"]
: undefined
}
columnsHiddenUnlessSelected={
isTraceTarget(target)
? RETIRED_TRACE_FILTER_COLUMNS
: undefined
}
/>
)}
</div>
</FormControl>
{!props.disabled && !hasFilters && (
<div className="flex max-w-[500px] gap-1">
<AlertTriangle
size="base"
__rawClassName="text-dark-yellow "
/>
<div className="text-dark-yellow text-sm [&_p]:leading-relaxed">
No filters set. This evaluator will run on all{" "}
{getTargetDisplayName(target)}.
</div>
</div>
)}
<FormMessage />
</FormItem>
);
}}
/>
{/* Preview based on target type */}
{previewTableVisible && (
<>
{/* The traces preview reads the legacy traces table, which
is not the v4 user's experience — never show it there. */}
{shouldShowLegacyTracePreview(
form.watch("target"),
isV4,
) && (
<TracesPreview
projectId={props.projectId}
filterState={watchedFilter}
/>
)}
{shouldShowEventsPreview && (
<ObservationsPreview
projectId={props.projectId}
filterState={eventsPreviewFilterState}
isNewCompatible={props.evalCapabilities.isNewCompatible}
compatibilityCheckWasPerformed={
props.evalCapabilities.compatibilityCheckWasPerformed
}
/>
)}
</>
)}
{!props.hideAdvancedSettings && (
<>
<FormField
control={form.control}
name="sampling"
render={({ field }) => (
<FormItem>
<FormLabel>Sampling</FormLabel>
<FormControl>
<div className="max-w-[500px]">
<Slider
disabled={props.disabled}
min={0}
max={1}
step={0.0001}
value={[field.value]}
onValueChange={(value) =>
field.onChange(value[0])
}
showInput={true}
displayAsPercentage={true}
/>
</div>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
{isLegacyEvalTarget(form.watch("target")) && (
<FormField
control={form.control}
name="delay"
render={({ field }) => (
<FormItem>
<FormLabel>Delay (seconds)</FormLabel>
<FormControl>
<Input {...field} type="number" min={0} />
</FormControl>
<FormDescription>
Time between first Trace/Dataset run event and
evaluation execution to ensure all data is
available
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
)}
</>
)}
{shouldShowCodeEvalSourceLinkInSettingsCard ? (
<CodeEvalSourceLink
projectId={props.projectId}
evalTemplate={props.evalTemplate}
/>
) : null}
</>
)}
</div>
</Card>
)}
{shouldShowCodeEvalTestPanel ? (
<CodeEvalTestRunCard
projectId={props.projectId}
evalTemplate={props.evalTemplate}
target={
isEventTarget(watchedTarget)
? EvalTargetObject.EVENT
: EvalTargetObject.EXPERIMENT
}
scoreName={watchedScoreName}
enableExecutionTracePeek={!props.existingEvaluator}
/>
) : isCodeEvalConfig ? null : (
<VariableMappingCard
projectId={props.projectId}
availableVariables={availableVariables}
evalTemplate={props.evalTemplate}
form={form}
disabled={props.disabled}
shouldWrapVariables={props.shouldWrapVariables}
hideAdvancedSettings={props.hideAdvancedSettings}
oldConfigId={props.oldConfigId}
isNewCompatible={props.evalCapabilities.isNewCompatible}
compatibilityCheckWasPerformed={
props.evalCapabilities.compatibilityCheckWasPerformed
}
showPreviewTargetBadge={props.showPreviewTargetBadge}
/>
)}
</div>
);
const mutationIsLoading =
createJobMutation.isPending || updateJobMutation.isPending;
const formFooter = props.renderFooter ? (
props.renderFooter({
isLoading: mutationIsLoading,
isSaveDisabled: hasJsonPathCompatibilityError,
})
) : (
<div className="flex w-full flex-col items-end gap-4">
{!props.disabled ? (
<Button
type="submit"
loading={mutationIsLoading}
disabled={hasJsonPathCompatibilityError}
className="mt-3 max-w-fit"
>
{props.mode === "edit" ? "Update" : "Execute"}
</Button>
) : null}
</div>
);
return (
<>
<Form {...form}>
<form
onSubmit={(e) => {
e.stopPropagation(); // Prevent event bubbling to parent forms
form.handleSubmit(onSubmit)(e);
}}
onKeyDown={(e) => {
if (e.key === "Enter" && e.target instanceof HTMLInputElement) {
e.preventDefault();
}
}}
className="flex w-full flex-col gap-4"
>
{props.useDialog ? <DialogBody>{formBody}</DialogBody> : formBody}
{formFooter &&
(props.useDialog ? (
<DialogFooter>{formFooter}</DialogFooter>
) : (
<div className="mt-4 flex flex-row justify-end">{formFooter}</div>
))}
</form>
</Form>
<Dialog
open={showTraceConfirmDialog}
onOpenChange={setShowTraceConfirmDialog}
>
<DialogContent>
<DialogHeader>
<DialogTitle>You selected a legacy evaluator</DialogTitle>
</DialogHeader>
<DialogBody className="text-sm">
We strongly recommend using observation evaluators. Trace evaluators
will be deprecated in the future. Only proceed if you are sure you
cannot upgrade your SDK version now.
</DialogBody>
<DialogFooter>
<Button
variant="outline"
onClick={() => setShowTraceConfirmDialog(false)}
>
Cancel
</Button>
<Button
onClick={() => {
setShowTraceConfirmDialog(false);
setUserFacingTarget("trace");
// Update form and mapping
const actualTarget = "trace";
const langfuseObject: LangfuseObject = "trace";
const newMapping = form.getValues("mapping").map((field) => ({
...field,
langfuseObject,
}));
form.setValue("filter", DEFAULT_TRACE_FILTER);
form.setValue("mapping", newMapping);
setAvailableVariables(availableTraceEvalVariables);
form.setValue("target", actualTarget);
}}
>
Continue
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
</>
);
};