🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable @typescript-eslint/no-deprecated -- Legacy UI usage retained during migration to design-system components. */
import { DatasetRunsTable } from "@/src/features/datasets/components/DatasetRunsTable";
import { api } from "@/src/utils/api";
import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
import { useState, useCallback } from "react";
import { FlaskConical, MoreVertical } from "@langfuse/icons";
import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
import {
Dialog,
DialogContent,
DialogHeader,
DialogTitle,
DialogTrigger,
} from "@/src/components/ui/dialog";
import { Button } from "@/src/components/ui/button";
import { CreateExperimentsForm } from "@/src/features/experiments/components/CreateExperimentsForm";
import { useEvaluatorDefaults } from "@/src/features/experiments/hooks/useEvaluatorDefaults";
import { useExperimentEvaluatorData } from "@/src/features/experiments/hooks/useExperimentEvaluatorData";
import { useExperimentAccess } from "@/src/features/experiments/hooks/useExperimentAccess";
import { singleRunToExperimentsUrl } from "@/src/features/experiments/utils/experimentUrlTranslation";
import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
import { DatasetAnalytics } from "@/src/features/datasets/components/DatasetAnalytics";
import { RESOURCE_METRICS } from "@/src/features/dashboard/lib/score-analytics-utils";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
import Page from "@/src/components/layouts/Page";
import {
getDatasetTabs,
DATASET_TABS,
} from "@/src/features/navigation/utils/dataset-tabs";
import { TemplateSelector } from "@/src/features/evals/components/TemplateSelector";
import {
EvaluatorForm,
useEvaluatorFormTemplate,
} from "@/src/features/evals/components/EvaluatorForm";
import useLocalStorage from "@/src/components/useLocalStorage";
import { getDatasetBreadcrumb } from "@/src/features/datasets/utils/getDatasetBreadcrumb";
import { default as ExperimentsTable } from "@/src/features/experiments/components/table/ExperimentsTable";
import { Skeleton } from "@/src/components/ui/skeleton";
import { RouteParamsPendingFallback } from "@/src/components/RouteParamsPendingFallback/RouteParamsPendingFallback";
import { useReadyRouteParams } from "@/src/hooks/useReadyRouteParams";
import { DatasetActionMenu } from "@/src/features/datasets/components/DatasetActionMenu";
export default function DatasetExperimentsPage() {
const route = useReadyRouteParams(["projectId", "datasetId"]);
if (!route.ready) return <RouteParamsPendingFallback />;
return (
<DatasetExperimentsView
projectId={route.params.projectId}
datasetId={route.params.datasetId}
/>
);
}
function DatasetExperimentsView({
projectId,
datasetId,
}: {
projectId: string;
datasetId: string;
}) {
const capture = usePostHogClientCapture();
const utils = api.useUtils();
const [isCreateExperimentDialogOpen, setIsCreateExperimentDialogOpen] =
useState(false);
const [selectedMetrics, setSelectedMetrics] = useLocalStorage<string[]>(
`${projectId}-${datasetId}-chart-metrics`,
RESOURCE_METRICS.map((metric) => metric.key),
);
const [scoreOptions, setScoreOptions] = useState<
{
key: string;
value: string;
}[]
>([]);
const dataset = api.datasets.byId.useQuery(
{
datasetId,
projectId,
},
{ enabled: Boolean(projectId) && Boolean(datasetId) },
);
const hasReadAccess = useHasProjectAccess({
projectId,
scope: "evalJobExecution:read",
});
const hasExperimentWriteAccess = useHasProjectAccess({
projectId,
scope: "promptExperiments:CUD",
});
const { isExperimentsBetaActive, isInitializing } = useExperimentAccess();
const handleExperimentSuccess = async (data?: {
success: boolean;
datasetId: string;
runId: string;
runName: string;
}) => {
setIsCreateExperimentDialogOpen(false);
if (!data) return;
if (isExperimentsBetaActive) {
utils.experiments.all.invalidate();
utils.experiments.countAll.invalidate();
// The empty-window fallback is its own query, and a new run belongs in
// it: without this it keeps serving its cached list.
utils.experiments.mostRecent.invalidate();
} else {
utils.datasets.runsByDatasetId.invalidate();
utils.datasets.baseRunDataByDatasetId.invalidate();
}
showSuccessToast({
operation: "experiment.trigger",
title: "Experiment triggered successfully",
description: "Waiting for experiment to complete...",
link: {
text: "View experiment",
href: isExperimentsBetaActive
? singleRunToExperimentsUrl(projectId, data.runId)
: `/project/${projectId}/datasets/${data.datasetId}/compare?runs=${data.runId}`,
},
});
};
const hasEvaluationRuleReadAccess = useHasProjectAccess({
projectId,
scope: "evaluationRule:read",
});
const hasEvaluationRuleWriteAccess = useHasProjectAccess({
projectId,
scope: "evaluationRule:CUD",
});
const hasEvaluatorReadAccess = useHasProjectAccess({
projectId,
scope: "evaluator:read",
});
const evalTemplates = api.evals.latestTemplates.useQuery(
{ projectId },
{
enabled:
!isExperimentsBetaActive &&
hasEvaluatorReadAccess &&
Boolean(projectId),
},
);
const evaluators = api.evals.jobConfigsByTarget.useQuery(
{ projectId, targetObject: ["dataset", "experiment"] },
{
enabled:
!isExperimentsBetaActive && hasEvaluationRuleReadAccess && !!datasetId,
},
);
const { createDefaultEvaluator } = useEvaluatorDefaults();
const {
selectedEvaluatorData,
showEvaluatorForm,
handleConfigureEvaluator,
handleCloseEvaluatorForm,
handleEvaluatorSuccess,
handleSelectEvaluator,
} = useExperimentEvaluatorData({
datasetId,
createDefaultEvaluator,
evaluatorsData: evaluators.data,
evalTemplatesData: evalTemplates.data,
refetchEvaluators: evaluators.refetch,
});
const evalTemplate = useEvaluatorFormTemplate({
evalTemplates: evalTemplates.data?.templates ?? [],
evalTemplate: selectedEvaluatorData?.evaluator.evalTemplate,
templateId: selectedEvaluatorData?.templateId,
});
// Callback for preprocessing evaluator form values
// For experiment evaluators, we only run on new data (not historic)
const preprocessFormValues = useCallback((values: any) => values, []);
const breadcrumb = getDatasetBreadcrumb(
projectId,
datasetId,
dataset.data?.name,
);
if (isInitializing) {
return <Skeleton className="h-full w-full" />;
}
if (isExperimentsBetaActive) {
return (
<Page
headerProps={{
title: dataset.data?.name ?? "",
itemType: "DATASET",
breadcrumb,
tabsProps: {
tabs: getDatasetTabs(projectId, datasetId),
activeTab: DATASET_TABS.EXPERIMENTS,
},
actionButtonsRight: (
<Dialog
open={isCreateExperimentDialogOpen}
onOpenChange={setIsCreateExperimentDialogOpen}
>
<DialogTrigger asChild disabled={!hasExperimentWriteAccess}>
<Button
disabled={!hasExperimentWriteAccess}
onClick={() => capture("dataset_run:new_form_open")}
>
<FlaskConical size="base" color="default" />
<span className="ml-2 hidden md:block">Run experiment</span>
</Button>
</DialogTrigger>
<DialogContent className="max-h-[90vh] max-w-3xl overflow-y-auto">
<CreateExperimentsForm
key={`create-experiment-form-${datasetId}`}
projectId={projectId as string}
setFormOpen={setIsCreateExperimentDialogOpen}
defaultValues={{
datasetId,
}}
handleExperimentSuccess={handleExperimentSuccess}
showSDKRunInfoPage
/>
</DialogContent>
</Dialog>
),
}}
>
<ExperimentsTable
projectId={projectId}
fixedFilter={[
{
column: "experimentDatasetId",
type: "stringOptions",
operator: "any of",
value: [datasetId],
},
]}
sessionFilterContextId={`dataset-${datasetId}`}
showControlsInPageHeader
/>
</Page>
);
}
return (
<Page
headerProps={{
title: dataset.data?.name ?? "",
itemType: "DATASET",
breadcrumb,
help: dataset.data?.description
? {
description: dataset.data.description,
}
: undefined,
tabsProps: {
tabs: getDatasetTabs(projectId, datasetId),
activeTab: DATASET_TABS.EXPERIMENTS,
},
actionButtonsRight: (
<>
<Dialog
open={isCreateExperimentDialogOpen}
onOpenChange={setIsCreateExperimentDialogOpen}
>
<DialogTrigger asChild disabled={!hasExperimentWriteAccess}>
<Button
disabled={!hasExperimentWriteAccess}
onClick={() => capture("dataset_run:new_form_open")}
>
<FlaskConical size="base" color="default" />
<span className="ml-2 hidden md:block">Run experiment</span>
</Button>
</DialogTrigger>
<DialogContent className="max-h-[90vh] max-w-3xl overflow-y-auto">
<CreateExperimentsForm
key={`create-experiment-form-${datasetId}`}
projectId={projectId as string}
setFormOpen={setIsCreateExperimentDialogOpen}
defaultValues={{
datasetId,
}}
handleExperimentSuccess={handleExperimentSuccess}
showSDKRunInfoPage
/>
</DialogContent>
</Dialog>
{hasEvaluationRuleReadAccess && hasEvaluatorReadAccess ? (
<div className="w-fit">
<TemplateSelector
projectId={projectId}
datasetId={datasetId}
evalTemplates={evalTemplates.data?.templates ?? []}
onConfigureTemplate={handleConfigureEvaluator}
onSelectEvaluator={handleSelectEvaluator}
disabled={!hasEvaluationRuleWriteAccess}
/>
</div>
) : null}
<DatasetAnalytics
key="dataset-analytics"
scoreOptions={scoreOptions}
selectedMetrics={selectedMetrics}
setSelectedMetrics={setSelectedMetrics}
/>
<DetailPageNav
currentId={datasetId}
path={(entry) =>
`/project/${projectId}/datasets/${entry.id}/experiments`
}
listKey="datasets"
/>
<DatasetActionMenu
projectId={projectId}
datasetId={datasetId}
datasetName={dataset.data?.name ?? ""}
datasetDescription={dataset.data?.description ?? undefined}
datasetMetadata={dataset.data?.metadata}
datasetInputSchema={dataset.data?.inputSchema ?? undefined}
datasetExpectedOutputSchema={
dataset.data?.expectedOutputSchema ?? undefined
}
manageEvaluatorsHref={
hasReadAccess
? `/project/${projectId}/evals?target=dataset`
: undefined
}
>
{({ getTriggerProps }) => (
<Button
variant="outline"
size="icon"
aria-label="Dataset actions"
{...getTriggerProps()}
>
<MoreVertical size="base" color="default" />
</Button>
)}
</DatasetActionMenu>
</>
),
}}
>
<DatasetRunsTable
projectId={projectId}
datasetId={datasetId}
selectedMetrics={selectedMetrics}
setScoreOptions={setScoreOptions}
/>
{/* Dialog for configuring evaluators */}
{selectedEvaluatorData && evalTemplate && (
<Dialog
open={showEvaluatorForm}
onOpenChange={(open) => {
if (!open) {
handleCloseEvaluatorForm();
}
}}
>
<DialogContent className="max-h-[90vh] max-w-(--breakpoint-md) overflow-y-auto">
<DialogHeader>
<DialogTitle>
{selectedEvaluatorData.evaluator.id ? "Edit" : "Configure"}{" "}
Evaluator
</DialogTitle>
</DialogHeader>
<EvaluatorForm
useDialog={true}
projectId={projectId}
evalTemplate={evalTemplate}
existingEvaluator={selectedEvaluatorData.evaluator}
mode={selectedEvaluatorData.evaluator.id ? "edit" : "create"}
hideTargetSection={!selectedEvaluatorData.evaluator.id}
onFormSuccess={handleEvaluatorSuccess}
preprocessFormValues={preprocessFormValues}
/>
</DialogContent>
</Dialog>
)}
</Page>
);
}