Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): track the evaluator onboarding funnel in PostHog (#18075)

Steffen Reuther0a2bff14b9e3
diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx b/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx
index f49a895d40..b4d0c01902 100644
--- a/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx
+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx
@@ -425,7 +425,16 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {
<div className="flex items-center gap-2">
{step !== "confirm" ? (
- <CreateEvaluatorButton href={createEvaluatorHref} />
+ <CreateEvaluatorButton
+ href={createEvaluatorHref}
+ onClick={() => {
+ // The legacy (v3) editor is not part of the onboarding funnel.
+ if (forceV3Experience) return;
+ capture("eval:onboarding_started", {
+ entryPoint: "batch_evaluation",
+ });
+ }}
+ />
) : null}
{showMappingEditor ? (
<MappingRunButton
@@ -460,11 +469,18 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {
);
}
-function CreateEvaluatorButton({ href }: { href: string }) {
+function CreateEvaluatorButton({
+ href,
+ onClick,
+}: {
+ href: string;
+ onClick: () => void;
+}) {
return (
<Button variant="secondary" className="gap-1.5" asChild>
<Link
href={href}
+ onClick={onClick}
target="_blank"
rel="noreferrer"
aria-label="Create new Evaluator (opens in a new tab)"
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx
index cb41ec2918..01aaddf4dc 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx
@@ -11,12 +11,19 @@ import {
import { EvaluatorGalleryView } from "@/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView";
import type { GalleryTemplate } from "@/src/features/evals/v2/types/templateGallery";
import { prepareEvaluatorGallery } from "@/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery";
-import { EVALUATOR_GALLERY_ALL_SECTION_KEY } from "@/src/features/evals/v2/constants/evaluatorGallery";
+import {
+ EVALUATOR_GALLERY_ALL_SECTION_KEY,
+ EVALUATOR_GALLERY_RECOMMENDED_SECTION_KEY,
+} from "@/src/features/evals/v2/constants/evaluatorGallery";
import useLocalStorage from "@/src/components/useLocalStorage";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import { getEvaluatorCreationAnalyticsProperties } from "@/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties";
+import { getTemplateSelectionAnalyticsProperties } from "@/src/features/evals/v2/fns/templateGallery/getTemplateSelectionAnalyticsProperties";
import { api } from "@/src/utils/api";
+// Search reports once the user pauses typing, so one query is one event.
+const SEARCH_CAPTURE_DELAY_MS = 1000;
+
export function EvaluatorGalleryDialog({
projectId,
open,
@@ -39,6 +46,10 @@ export function EvaluatorGalleryDialog({
new Set(),
);
const searchInputRef = useRef<HTMLInputElement | null>(null);
+ const pendingSearchCapture = useRef<{
+ timeout: ReturnType<typeof setTimeout>;
+ queryLength: number;
+ } | null>(null);
const scrollContainerRef = useRef<HTMLDivElement | null>(null);
const [bannerDismissed, setBannerDismissed] = useLocalStorage(
"evaluatorGallery:decisionModelBannerDismissed:v1",
@@ -81,7 +92,34 @@ export function EvaluatorGalleryDialog({
projectEvaluators.data?.pages[0]?.totalItems ?? customTemplates.length,
search,
});
+ const flushSearchCapture = () => {
+ const pending = pendingSearchCapture.current;
+ if (!pending) return;
+ clearTimeout(pending.timeout);
+ pendingSearchCapture.current = null;
+ capture("eval:onboarding_gallery_searched", {
+ queryLength: pending.queryLength,
+ });
+ };
+ const handleSearchChange = (nextSearch: string) => {
+ setSearch(nextSearch);
+ const queryLength = nextSearch.trim().length;
+ if (queryLength === 0) {
+ flushSearchCapture();
+ return;
+ }
+ if (pendingSearchCapture.current) {
+ clearTimeout(pendingSearchCapture.current.timeout);
+ }
+ pendingSearchCapture.current = {
+ queryLength,
+ timeout: setTimeout(flushSearchCapture, SEARCH_CAPTURE_DELAY_MS),
+ };
+ };
const selectSection = (key: string) => {
+ if (key !== activeSection) {
+ capture("eval:onboarding_gallery_section_selected", { sectionKey: key });
+ }
setActiveSection(key);
scrollContainerRef.current?.scrollTo({ top: 0 });
};
@@ -93,7 +131,26 @@ export function EvaluatorGalleryDialog({
return next;
});
};
- const handleSelectTemplate = (template: GalleryTemplate) => {
+ const handleSelectTemplate = (
+ template: GalleryTemplate,
+ sectionKey: string,
+ ) => {
+ flushSearchCapture();
+ capture(
+ "eval:onboarding_step_completed",
+ sectionKey === EVALUATOR_GALLERY_RECOMMENDED_SECTION_KEY
+ ? {
+ stepName: "suggestion_selected",
+ surface: "gallery",
+ ...getTemplateSelectionAnalyticsProperties(template),
+ }
+ : {
+ stepName: "template_selected",
+ surface: "gallery",
+ sectionKey,
+ ...getTemplateSelectionAnalyticsProperties(template),
+ },
+ );
const evaluatorType =
template.source === "managed" ? template.evaluator.type : template.type;
const creationSource =
@@ -110,6 +167,12 @@ export function EvaluatorGalleryDialog({
onSelectTemplate(template);
};
const handleCreateFromScratch = (evaluatorType: EvalTemplateType) => {
+ flushSearchCapture();
+ capture("eval:onboarding_step_completed", {
+ stepName: "new_from_scratch_selected",
+ surface: "gallery",
+ evaluatorType,
+ });
capture(
"evaluators:gallery_creation_source_select",
getEvaluatorCreationAnalyticsProperties({
@@ -141,7 +204,7 @@ export function EvaluatorGalleryDialog({
</DialogHeader>
<EvaluatorGalleryView
search={search}
- onSearchChange={setSearch}
+ onSearchChange={handleSearchChange}
searchInputRef={searchInputRef}
navigationItems={navigationItems}
activeSection={activeSection}
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx
index f8f975933a..9a00f2a2c3 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx
@@ -66,7 +66,7 @@ export function EvaluatorGalleryView({
sections: GallerySection[];
expandedSections: ReadonlySet<string>;
onExpandedChange: (key: string, expanded: boolean) => void;
- onSelectTemplate: (template: GalleryTemplate) => void;
+ onSelectTemplate: (template: GalleryTemplate, sectionKey: string) => void;
onCreateFromScratch: (type: EvalTemplateType) => void;
scrollContainerRef?: RefObject<HTMLDivElement | null>;
isLoading: boolean;
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.tsx
index 96a5c086bd..8aa7605734 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.tsx
@@ -27,11 +27,13 @@ export function EvaluatorGallerySection({
section: GallerySection;
expanded: boolean;
onExpandedChange?: (expanded: boolean) => void;
- onSelectTemplate: (template: GalleryTemplate) => void;
+ onSelectTemplate: (template: GalleryTemplate, sectionKey: string) => void;
}) {
const isRecommended =
section.key === EVALUATOR_GALLERY_RECOMMENDED_SECTION_KEY;
const isSafety = section.key === EVALUATOR_GALLERY_SAFETY_SECTION_KEY;
+ const selectTemplate = (template: GalleryTemplate) =>
+ onSelectTemplate(template, section.key);
const shownTemplates = expanded
? section.templates
: section.templates.slice(0, EVALUATOR_GALLERY_PREVIEW_SIZE);
@@ -48,7 +50,7 @@ export function EvaluatorGallerySection({
<EvaluatorRecommendedCard
key={getGalleryTemplateId(template)}
template={template}
- onSelect={onSelectTemplate}
+ onSelect={selectTemplate}
/>
))}
</EvaluatorRecommendedCards>
@@ -73,7 +75,7 @@ export function EvaluatorGallerySection({
<EvaluatorTemplateRow
key={getGalleryTemplateId(template)}
template={template}
- onSelect={onSelectTemplate}
+ onSelect={selectTemplate}
/>
))}
</div>
diff --git a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/SampleObservationSelectorContainer/SampleObservationSelectorContainer.tsx b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/SampleObservationSelectorContainer/SampleObservationSelectorContainer.tsx
index 46561f6f8e..d692c5a5d8 100644
--- a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/SampleObservationSelectorContainer/SampleObservationSelectorContainer.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/SampleObservationSelectorContainer/SampleObservationSelectorContainer.tsx
@@ -1,5 +1,10 @@
import { useStore } from "zustand";
+import type { FilterState } from "@langfuse/shared";
+import useLocalStorage from "@/src/components/useLocalStorage";
+import { EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY } from "@/src/features/evals/v2/constants/evaluatorFilterExperience";
+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";
+import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";
import { EvaluatorSampleObservationSelector } from "@/src/features/evals/v2/components/Evaluators/Testing/components/EvaluatorSampleObservationSelector/EvaluatorSampleObservationSelector";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
@@ -24,6 +29,26 @@ export function SampleObservationSelectorContainer({
);
const filterState = useStore(store, (state) => state.sampleFilter);
const actions = store.getState().actions;
+ const onboardingAnalytics = useEvalOnboardingAnalytics();
+ const [filterExperience] = useLocalStorage<EvaluatorFilterExperience>(
+ EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY,
+ "query",
+ );
+ const changeFilterState = (nextFilterState: FilterState) => {
+ // The search bar can re-commit an unchanged filter; only a real change is
+ // a step.
+ const hasChanged =
+ JSON.stringify(nextFilterState) !==
+ JSON.stringify(store.getState().sampleFilter);
+ actions.setSampleFilter(nextFilterState);
+ if (hasChanged) {
+ onboardingAnalytics?.completeStep({
+ stepName: "evaluator_filter_changed",
+ filterExperience,
+ filterCount: nextFilterState.length,
+ });
+ }
+ };
return (
<EvaluatorSampleObservationSelector
@@ -31,7 +56,7 @@ export function SampleObservationSelectorContainer({
timeRange={timeRange}
selectedObservationId={selectedObservationId}
filterState={filterState}
- onFilterStateChange={actions.setSampleFilter}
+ onFilterStateChange={changeFilterState}
onSelect={actions.setSelectedObservation}
onOpenTrace={onOpenTrace}
/>
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
index c23ac7c2d1..9cce3c1a9d 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
@@ -29,6 +29,7 @@ import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFil
import { EvaluatorSavedRuleFilterPreview } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedRuleFilterPreview";
import { EvaluatorBackfillSettings } from "@/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings";
import { useEvaluatorSavedBackfill } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill";
+import type { createEvalOnboardingAnalytics } from "@/src/features/evals/v2/fns/createEvalOnboardingAnalytics";
type Rule = RouterOutputs["evalsV2"]["rules"]["list"]["rules"][number];
type DialogPhase = "saved" | "closing-saved" | "create-rule" | "closed";
@@ -46,11 +47,13 @@ type EvaluatorSavedDialogEvaluator = {
export function EvaluatorSavedDialogContainer({
projectId,
evaluator,
+ onboardingAnalytics,
onDismiss,
onFinish,
}: {
projectId: string;
evaluator: EvaluatorSavedDialogEvaluator;
+ onboardingAnalytics: ReturnType<typeof createEvalOnboardingAnalytics> | null;
onDismiss: () => Promise<void>;
onFinish: () => Promise<void>;
}) {
@@ -72,6 +75,7 @@ export function EvaluatorSavedDialogContainer({
const estimateRequestId = useRef(0);
const initialEstimateRequested = useRef(false);
const createRuleHandoffPending = useRef(false);
+ const ruleCreatedInEditor = useRef(false);
const hasRequestedMissingCostTest = useRef(
evaluator.hasCompletedTestCall ?? false,
);
@@ -190,6 +194,11 @@ export function EvaluatorSavedDialogContainer({
{ filter: currentRule.filter, sampling: currentRule.sampling },
backfill.executionRange,
);
+ onboardingAnalytics?.completeOnboarding({
+ executionPath: "existing_rule",
+ hasBackfill: backfill.enabled,
+ samplingPercent: Math.round(currentRule.sampling * 100),
+ });
await invalidateRuleQueries();
await finish();
};
@@ -220,6 +229,11 @@ export function EvaluatorSavedDialogContainer({
{ filter: supportedRuleFilters, sampling },
backfill.executionRange,
);
+ onboardingAnalytics?.completeOnboarding({
+ executionPath: "test_filters",
+ hasBackfill: backfill.enabled,
+ samplingPercent: Math.round(sampling * 100),
+ });
await invalidateRuleQueries();
await finish();
};
@@ -340,6 +354,11 @@ export function EvaluatorSavedDialogContainer({
};
const handleModeChange = (nextMode: EvaluatorSavedMode) => {
+ if (nextMode !== mode) {
+ onboardingAnalytics?.track("eval:onboarding_scope_changed", {
+ scope: nextMode === "test-filters" ? "test_filters" : "different_scope",
+ });
+ }
const modeChangeRequestId = ++estimateRequestId.current;
setActivationOpen(false);
setIsEstimating(false);
@@ -435,6 +454,7 @@ export function EvaluatorSavedDialogContainer({
action: "new_rule",
hasBackfill: false,
});
+ onboardingAnalytics?.track("eval:onboarding_create_rule_opened", {});
openCreateRule();
}
};
@@ -518,9 +538,12 @@ export function EvaluatorSavedDialogContainer({
maxAllowedItems={backfill.allowedItems}
matchingObservations={backfill.matchingObservations}
isEstimating={backfill.isEstimating}
- onEnabledChange={(enabled) =>
- backfill.setEnabled(enabled, currentBackfillScope)
- }
+ onEnabledChange={(enabled) => {
+ onboardingAnalytics?.track("eval:onboarding_historic_eval_toggled", {
+ isEnabled: enabled,
+ });
+ backfill.setEnabled(enabled, currentBackfillScope);
+ }}
onWindowChange={(window) =>
backfill.updateWindow(window, currentBackfillScope)
}
@@ -558,6 +581,11 @@ export function EvaluatorSavedDialogContainer({
? (sampling) => {
setTestFilterSampling(sampling);
setActivationSampling(sampling);
+ onboardingAnalytics?.track(
+ "eval:onboarding_sampling_changed",
+ {},
+ { onceKey: "sampling_changed" },
+ );
}
: null
}
@@ -602,11 +630,17 @@ export function EvaluatorSavedDialogContainer({
onModeChange={handleModeChange}
onOpenAutoFocus={requestInitialEstimate}
onDismiss={() => {
+ onboardingAnalytics?.track("eval:onboarding_execution_skipped", {
+ method: "dialog_dismissed",
+ });
createRuleHandoffPending.current = false;
setDialogPhase("closed");
onDismiss().catch(trpcErrorToast);
}}
onSecondaryAction={() => {
+ onboardingAnalytics?.track("eval:onboarding_execution_skipped", {
+ method: "skip_button",
+ });
setDialogPhase("closed");
finish().catch(trpcErrorToast);
}}
@@ -622,10 +656,22 @@ export function EvaluatorSavedDialogContainer({
open
onOpenChange={(open) => {
if (!open) {
+ if (!ruleCreatedInEditor.current) {
+ onboardingAnalytics?.track("eval:onboarding_execution_skipped", {
+ method: "rule_editor_closed",
+ });
+ }
setDialogPhase("closed");
finish().catch(() => undefined);
}
}}
+ onCreated={() => {
+ ruleCreatedInEditor.current = true;
+ onboardingAnalytics?.completeOnboarding({
+ executionPath: "new_rule",
+ hasBackfill: false,
+ });
+ }}
initialEvaluator={{ ...evaluator, initialVariableMapping: null }}
successNotification="none"
/>
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx
index 9ac4068592..fcd335ffc5 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx
@@ -1,6 +1,6 @@
import { useStore } from "zustand";
import { useShallow } from "zustand/react/shallow";
-import type { LLMAdapter } from "@langfuse/shared";
+import type { EvalTemplateType, LLMAdapter } from "@langfuse/shared";
import { DefinitionStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/DefinitionStep";
import { CodeEditor } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/CodeEditor/CodeEditor";
@@ -12,6 +12,7 @@ import { PromptEditor } from "@/src/features/evals/v2/components/Evaluators/Eval
import { ScoreOutputEditor } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ScoreOutputEditor/ScoreOutputEditor";
import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";
import type { ProjectDefaultModelConfig } from "@/src/features/evals/v2/types/ProjectDefaultModelConfig";
import type { CodeEvalValidationResult } from "@/src/features/evals/utils/code-eval-template-validation";
@@ -40,6 +41,7 @@ export function DefinitionStepContainer({
onSetProjectDefault: (model: ProjectDefaultModelConfig) => void;
codeValidationResult: CodeEvalValidationResult | null;
}) {
+ const onboardingAnalytics = useEvalOnboardingAnalytics();
const state = useStore(
store,
useShallow((state) => ({
@@ -48,11 +50,20 @@ export function DefinitionStepContainer({
actions: state.actions,
})),
);
+ const changeType = (type: EvalTemplateType) => {
+ const previousEvaluatorType = store.getState().type;
+ state.actions.setType(type);
+ if (type !== previousEvaluatorType) {
+ onboardingAnalytics?.track("eval:onboarding_evaluator_type_changed", {
+ previousEvaluatorType,
+ });
+ }
+ };
const stepProps = {
open: state.open,
onOpenChange: (open: boolean) => onStepOpenChange(1, open),
- onTypeChange: state.actions.setType,
+ onTypeChange: changeType,
isEditing,
};
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx
index 310c9d0432..450993f1b5 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx
@@ -12,6 +12,7 @@ import { JudgeModelConfigurationDialog } from "@/src/features/evals/v2/component
import type { ProjectDefaultModelConfig } from "@/src/features/evals/v2/types/ProjectDefaultModelConfig";
import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";
export function ModelSelector({
projectId,
@@ -33,6 +34,14 @@ export function ModelSelector({
onSetProjectDefault: (model: ProjectDefaultModelConfig) => void;
}) {
const [configurationOpen, setConfigurationOpen] = useState(false);
+ const onboardingAnalytics = useEvalOnboardingAnalytics();
+ const trackModelChange = (modelMode: "default" | "custom") => {
+ onboardingAnalytics?.track("eval:onboarding_model_changed", { modelMode });
+ onboardingAnalytics?.completeStep({
+ stepName: "model_selected",
+ modelMode,
+ });
+ };
const state = useStore(
store,
useShallow((state) => ({
@@ -59,13 +68,28 @@ export function ModelSelector({
<>
<JudgeModelPicker
open={state.open}
- onOpenChange={state.actions.setModelPickerOpen}
+ onOpenChange={(open) => {
+ if (open && !state.open) {
+ onboardingAnalytics?.track(
+ "eval:onboarding_model_picker_opened",
+ {},
+ );
+ }
+ state.actions.setModelPickerOpen(open);
+ }}
mode={state.mode}
defaultModel={defaultModel}
providerGroups={providerGroups}
selectedModel={state.selectedModel}
- onModeChange={state.actions.setModelMode}
- onSelectCustom={state.actions.selectModel}
+ onModeChange={(mode) => {
+ const hasChanged = mode !== state.mode;
+ state.actions.setModelMode(mode);
+ if (hasChanged) trackModelChange(mode);
+ }}
+ onSelectCustom={(model) => {
+ state.actions.selectModel(model);
+ trackModelChange("custom");
+ }}
onConfigureProviders={onConfigureProviders}
onConfigureModel={() => setConfigurationOpen(true)}
hasModelConfiguration={
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx
index 9278390356..a9b318e8de 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx
@@ -58,6 +58,7 @@ import {
import { useEvaluatorSetupSample } from "@/src/features/evals/v2/hooks/useEvaluatorSetupSample";
import { useCopyToClipboard } from "@/src/hooks/useCopyToClipboard";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";
import { cn } from "@/src/utils/tailwind";
import type { EvaluatorPromptMessage } from "@langfuse/shared";
@@ -107,6 +108,7 @@ export function PromptEditorContent({
promptPreviewEnabled: state.promptPreviewEnabled,
sampleObject,
});
+ const onboardingAnalytics = useEvalOnboardingAnalytics();
const [activeMessageId, setActiveMessageId] = useState<string | null>(null);
const activeMessageIndex = activeMessageId
? state.promptMessageIds.indexOf(activeMessageId)
@@ -130,6 +132,9 @@ export function PromptEditorContent({
const toIndex = state.promptMessageIds.indexOf(String(over.id));
if (fromIndex < 0 || toIndex < 0) return;
state.actions.reorderPromptMessage(fromIndex, toIndex);
+ onboardingAnalytics?.track("eval:onboarding_prompt_modified", {
+ modification: "messages_reordered",
+ });
};
return (
@@ -161,9 +166,26 @@ export function PromptEditorContent({
sampleObject,
})}
previewEnabled={state.promptPreviewEnabled}
- onPreviewEnabledChange={state.actions.setPromptPreviewEnabled}
- onChange={(next) => state.actions.setPromptMessage(index, next)}
- onRemove={() => state.actions.removePromptMessage(index)}
+ onPreviewEnabledChange={(isEnabled) => {
+ state.actions.setPromptPreviewEnabled(isEnabled);
+ onboardingAnalytics?.track("eval:onboarding_preview_toggled", {
+ isEnabled,
+ });
+ }}
+ onChange={(next) => {
+ state.actions.setPromptMessage(index, next);
+ onboardingAnalytics?.track(
+ "eval:onboarding_prompt_modified",
+ { modification: "message_edited" },
+ { onceKey: "prompt_message_edited" },
+ );
+ }}
+ onRemove={() => {
+ state.actions.removePromptMessage(index);
+ onboardingAnalytics?.track("eval:onboarding_prompt_modified", {
+ modification: "message_removed",
+ });
+ }}
/>
))}
</SortableContext>
@@ -175,6 +197,9 @@ export function PromptEditorContent({
onClick={() => {
state.actions.setPromptPreviewEnabled(false);
state.actions.addPromptMessage();
+ onboardingAnalytics?.track("eval:onboarding_prompt_modified", {
+ modification: "message_added",
+ });
}}
>
<Plus className="h-3.5 w-3.5 shrink-0" />
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ScoreOutputEditor/ScoreOutputEditor.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ScoreOutputEditor/ScoreOutputEditor.tsx
index f8e8e50224..e9d82053e7 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ScoreOutputEditor/ScoreOutputEditor.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ScoreOutputEditor/ScoreOutputEditor.tsx
@@ -2,16 +2,23 @@ import { useStore } from "zustand";
import { ScoreOutputConfiguration } from "@/src/features/evals/v2/components/Evaluators/Judges/ScoreOutputConfiguration/ScoreOutputConfiguration";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";
export function ScoreOutputEditor({ store }: { store: EvaluatorSetupStore }) {
const scoreOutput = useStore(store, (state) => state.scoreOutput);
const setScoreOutput = store.getState().actions.setScoreOutput;
+ const onboardingAnalytics = useEvalOnboardingAnalytics();
return (
<ScoreOutputConfiguration
mode="editable"
state={scoreOutput}
- onChange={setScoreOutput}
+ onChange={(nextScoreOutput) => {
+ setScoreOutput(nextScoreOutput);
+ onboardingAnalytics?.completeStep({
+ stepName: "score_output_modified",
+ });
+ }}
/>
);
}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx
index 75d389c22d..7e9255e0aa 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx
@@ -6,6 +6,7 @@ import type { AIAssistedInput } from "@/src/components/ui/ai-assisted-input";
import { NameStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep";
import { getEvaluatorNameStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";
export function NameStepContainer({
store,
@@ -20,6 +21,7 @@ export function NameStepContainer({
>["aiAssistance"];
onStepOpenChange: (step: number, open: boolean) => void;
}) {
+ const onboardingAnalytics = useEvalOnboardingAnalytics();
const state = useStore(
store,
useShallow((state) => {
@@ -45,7 +47,12 @@ export function NameStepContainer({
open={state.open}
onOpenChange={(open) => onStepOpenChange(state.step, open)}
name={state.name}
- onNameChange={state.actions.setName}
+ onNameChange={(name) => {
+ state.actions.setName(name);
+ onboardingAnalytics?.completeStep({
+ stepName: "evaluator_name_updated",
+ });
+ }}
description={state.description}
onDescriptionChange={state.actions.setDescription}
nameAIAssistance={nameAIAssistance}
diff --git a/web/src/features/evals/v2/components/EvaluatorsEmptyState/EvaluatorsEmptyState.tsx b/web/src/features/evals/v2/components/EvaluatorsEmptyState/EvaluatorsEmptyState.tsx
index 2c01834f3c..fc51d6d04b 100644
--- a/web/src/features/evals/v2/components/EvaluatorsEmptyState/EvaluatorsEmptyState.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorsEmptyState/EvaluatorsEmptyState.tsx
@@ -7,6 +7,7 @@ import { DETECT_TOPICS_ASSISTANT_PROMPT } from "@/src/features/evals/v2/constant
import { EvaluatorsEmptyStateView } from "./components/EvaluatorsEmptyStateView/EvaluatorsEmptyStateView";
import { prepareEvaluatorEmptyState } from "@/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState";
import type { GalleryTemplate } from "@/src/features/evals/v2/types/templateGallery";
+import { getTemplateSelectionAnalyticsProperties } from "@/src/features/evals/v2/fns/templateGallery/getTemplateSelectionAnalyticsProperties";
export function EvaluatorsEmptyState({
onSelectTemplate,
@@ -29,6 +30,15 @@ export function EvaluatorsEmptyState({
templateKey: template.key,
});
}
+ capture("eval:onboarding_started", {
+ entryPoint: "empty_state_starting_point",
+ hasExistingEvaluators: false,
+ });
+ capture("eval:onboarding_step_completed", {
+ stepName: "suggestion_selected",
+ surface: "empty_state",
+ ...getTemplateSelectionAnalyticsProperties(template),
+ });
onSelectTemplate(template);
};
@@ -62,6 +72,10 @@ export function EvaluatorsEmptyState({
}
onBrowseLibrary={() => {
capture("evaluators:empty_state_browse_library");
+ capture("eval:onboarding_started", {
+ entryPoint: "empty_state_browse_templates",
+ hasExistingEvaluators: false,
+ });
onBrowseLibrary();
}}
/>
diff --git a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/CreateRuleDialog.tsx b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/CreateRuleDialog.tsx
index cd83589264..5674d87e0d 100644
--- a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/CreateRuleDialog.tsx
+++ b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/CreateRuleDialog.tsx
@@ -20,6 +20,7 @@ export function CreateRuleDialog({
initialFilter,
targetObject,
successNotification,
+ onCreated,
}: {
projectId: string;
open: boolean;
@@ -29,6 +30,7 @@ export function CreateRuleDialog({
initialFilter?: FilterState;
targetObject?: Extract<EvalTargetObject, "event" | "experiment">;
successNotification: "toast" | "none";
+ onCreated?: () => void;
}) {
const [evaluatorSearch, setEvaluatorSearch] = useState("");
const [evaluatorSearchQuery, setEvaluatorSearchQuery] = useState("");
@@ -98,6 +100,7 @@ export function CreateRuleDialog({
targetObject={targetObject}
evaluatorSearch={evaluatorSearch}
successNotification={successNotification}
+ onCreated={onCreated}
onEvaluatorSearchChange={(value) => {
setEvaluatorSearch(value);
debouncedEvaluatorSearch(value);
diff --git a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx
index f89fd1ae83..1adaed79bf 100644
--- a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx
+++ b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx
@@ -37,6 +37,7 @@ export function CreateRuleDialogContent({
targetObject,
evaluatorSearch,
successNotification,
+ onCreated,
onEvaluatorSearchChange,
}: {
projectId: string;
@@ -49,6 +50,7 @@ export function CreateRuleDialogContent({
targetObject?: Extract<EvalTargetObject, "event" | "experiment">;
evaluatorSearch: string;
successNotification: "toast" | "none";
+ onCreated?: () => void;
onEvaluatorSearchChange: (search: string) => void;
}) {
const capture = usePostHogClientCapture();
@@ -119,6 +121,7 @@ export function CreateRuleDialogContent({
samplingPercent: Math.round(draft.sampling * 100),
isEnabled: true,
});
+ onCreated?.();
if (successNotification === "toast") {
showSuccessToast({
title: "Rule created",
diff --git a/web/src/features/evals/v2/components/VariableMapping/components/EditableVariableMapping/EditableVariableMapping.tsx b/web/src/features/evals/v2/components/VariableMapping/components/EditableVariableMapping/EditableVariableMapping.tsx
index ec278fead7..06c22dd066 100644
--- a/web/src/features/evals/v2/components/VariableMapping/components/EditableVariableMapping/EditableVariableMapping.tsx
+++ b/web/src/features/evals/v2/components/VariableMapping/components/EditableVariableMapping/EditableVariableMapping.tsx
@@ -33,6 +33,7 @@ import {
experimentTargetEvalVariableColumns,
} from "@langfuse/shared";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";
const TOOL_CALLS_COLUMN_ID = "toolCalls";
@@ -293,6 +294,7 @@ function VariableMappingRow({
onDelete?: () => void;
}) {
const capture = usePostHogClientCapture();
+ const onboardingAnalytics = useEvalOnboardingAnalytics();
const segments = useMemo(
() =>
fieldState.jsonSelector
@@ -343,6 +345,10 @@ function VariableMappingRow({
capture("evaluators:variable_mapping_configured", {
method: "tree",
});
+ onboardingAnalytics?.completeStep({
+ stepName: "variable_mapping_updated",
+ method: "tree",
+ });
}
onChange({
selectedColumnId: columnId,
@@ -357,6 +363,10 @@ function VariableMappingRow({
capture("evaluators:variable_mapping_configured", {
method: "json_path",
});
+ onboardingAnalytics?.completeStep({
+ stepName: "variable_mapping_updated",
+ method: "json_path",
+ });
}
onChange({
selectedColumnId: fieldState.selectedColumnId,
diff --git a/web/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext.tsx b/web/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext.tsx
new file mode 100644
index 0000000000..c2297db547
--- /dev/null
+++ b/web/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext.tsx
@@ -0,0 +1,27 @@
+import { createContext, useContext, type ReactNode } from "react";
+import type { createEvalOnboardingAnalytics } from "@/src/features/evals/v2/fns/createEvalOnboardingAnalytics";
+
+type EvalOnboardingAnalytics = ReturnType<typeof createEvalOnboardingAnalytics>;
+
+// Null outside an evaluator creation flow (editing, rule editor), so shared
+// editor components only report onboarding events while creating.
+const EvalOnboardingAnalyticsContext =
+ createContext<EvalOnboardingAnalytics | null>(null);
+
+export function EvalOnboardingAnalyticsProvider({
+ value,
+ children,
+}: {
+ value: EvalOnboardingAnalytics | null;
+ children: ReactNode;
+}) {
+ return (
+ <EvalOnboardingAnalyticsContext.Provider value={value}>
+ {children}
+ </EvalOnboardingAnalyticsContext.Provider>
+ );
+}
+
+export function useEvalOnboardingAnalytics() {
+ return useContext(EvalOnboardingAnalyticsContext);
+}
diff --git a/web/src/features/evals/v2/fns/createEvalOnboardingAnalytics.clienttest.ts b/web/src/features/evals/v2/fns/createEvalOnboardingAnalytics.clienttest.ts
new file mode 100644
index 0000000000..e3f0744aa2
--- /dev/null
+++ b/web/src/features/evals/v2/fns/createEvalOnboardingAnalytics.clienttest.ts
@@ -0,0 +1,88 @@
+import type { EvalTemplateType } from "@langfuse/shared";
+import { describe, expect, it, vi } from "vitest";
+
+import { createEvalOnboardingAnalytics } from "./createEvalOnboardingAnalytics";
+
+function setup() {
+ const capture = vi.fn();
+ let evaluatorType: EvalTemplateType = "LLM_AS_JUDGE";
+ const analytics = createEvalOnboardingAnalytics({
+ capture,
+ getEvaluatorType: () => evaluatorType,
+ });
+ return {
+ analytics,
+ capture,
+ setEvaluatorType: (type: EvalTemplateType) => {
+ evaluatorType = type;
+ },
+ };
+}
+
+describe("createEvalOnboardingAnalytics", () => {
+ it("emits each step once, stamped with the evaluator type at the time", () => {
+ const { analytics, capture, setEvaluatorType } = setup();
+
+ analytics.completeStep({ stepName: "evaluator_name_updated" });
+ setEvaluatorType("CODE");
+ analytics.completeStep({ stepName: "evaluator_name_updated" });
+ analytics.completeStep({ stepName: "evaluator_tested" });
+
+ expect(capture.mock.calls).toEqual([
+ [
+ "eval:onboarding_step_completed",
+ { stepName: "evaluator_name_updated", evaluatorType: "LLM_AS_JUDGE" },
+ ],
+ [
+ "eval:onboarding_step_completed",
+ { stepName: "evaluator_tested", evaluatorType: "CODE" },
+ ],
+ ]);
+ });
+
+ it("fires interaction events every time unless they share a once key", () => {
+ const { analytics, capture } = setup();
+
+ analytics.track("eval:onboarding_model_picker_opened", {});
+ analytics.track("eval:onboarding_model_picker_opened", {});
+ analytics.track(
+ "eval:onboarding_sampling_changed",
+ {},
+ { onceKey: "sampling_changed" },
+ );
+ analytics.track(
+ "eval:onboarding_sampling_changed",
+ {},
+ { onceKey: "sampling_changed" },
+ );
+
+ expect(capture.mock.calls.map(([event]) => event)).toEqual([
+ "eval:onboarding_model_picker_opened",
+ "eval:onboarding_model_picker_opened",
+ "eval:onboarding_sampling_changed",
+ ]);
+ });
+
+ it("completes onboarding once with the execute step", () => {
+ const { analytics, capture } = setup();
+ const execution = {
+ executionPath: "test_filters" as const,
+ hasBackfill: true,
+ samplingPercent: 50,
+ };
+
+ analytics.completeOnboarding(execution);
+ analytics.completeOnboarding(execution);
+
+ expect(capture.mock.calls).toEqual([
+ [
+ "eval:onboarding_step_completed",
+ { stepName: "execute", ...execution, evaluatorType: "LLM_AS_JUDGE" },
+ ],
+ [
+ "eval:onboarding_completed",
+ { ...execution, evaluatorType: "LLM_AS_JUDGE" },
+ ],
+ ]);
+ });
+});
diff --git a/web/src/features/evals/v2/fns/createEvalOnboardingAnalytics.ts b/web/src/features/evals/v2/fns/createEvalOnboardingAnalytics.ts
new file mode 100644
index 0000000000..f891ca96a7
--- /dev/null
+++ b/web/src/features/evals/v2/fns/createEvalOnboardingAnalytics.ts
@@ -0,0 +1,75 @@
+import type { EvalTemplateType } from "@langfuse/shared";
+import type { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import type { EvalOnboardingEventMap } from "@/src/features/evals/v2/types/evalOnboardingAnalytics";
+
+type Capture = ReturnType<typeof usePostHogClientCapture>;
+type WithoutEvaluatorType<T> = T extends unknown
+ ? Omit<T, "evaluatorType">
+ : never;
+type OnboardingStep = EvalOnboardingEventMap["eval:onboarding_step_completed"];
+type ExecuteStep = Extract<OnboardingStep, { stepName: "execute" }>;
+// Gallery events fire before the setup page exists and are captured directly.
+type SetupEventName = Exclude<
+ keyof EvalOnboardingEventMap,
+ | "eval:onboarding_started"
+ | "eval:onboarding_step_completed"
+ | "eval:onboarding_completed"
+ | "eval:onboarding_gallery_searched"
+ | "eval:onboarding_gallery_section_selected"
+>;
+
+/**
+ * Analytics for one evaluator setup attempt. Every event carries the evaluator
+ * type at the moment of the action; each step is emitted at most once so the
+ * funnel counts reaching a step, not how often it was repeated.
+ */
+export function createEvalOnboardingAnalytics({
+ capture,
+ getEvaluatorType,
+}: {
+ capture: Capture;
+ getEvaluatorType: () => EvalTemplateType;
+}) {
+ const emittedKeys = new Set<string>();
+ const claim = (key: string) => {
+ if (emittedKeys.has(key)) return false;
+ emittedKeys.add(key);
+ return true;
+ };
+
+ const completeStep = (step: WithoutEvaluatorType<OnboardingStep>) => {
+ if (!claim(`step:${step.stepName}`)) return;
+ capture("eval:onboarding_step_completed", {
+ ...step,
+ evaluatorType: getEvaluatorType(),
+ } as OnboardingStep);
+ };
+
+ return {
+ track<E extends SetupEventName>(
+ event: E,
+ properties: WithoutEvaluatorType<EvalOnboardingEventMap[E]>,
+ options?: { onceKey?: string },
+ ) {
+ if (options?.onceKey && !claim(options.onceKey)) return;
+ // `properties` is checked against the event map by this method's
+ // signature; TypeScript cannot relate the generic index back to the
+ // registry's property map, so the hand-off is widened here.
+ (capture as (event: E, properties: object) => void)(event, {
+ ...properties,
+ evaluatorType: getEvaluatorType(),
+ });
+ },
+ completeStep,
+ completeOnboarding(
+ execution: Omit<ExecuteStep, "stepName" | "evaluatorType">,
+ ) {
+ if (!claim("completed")) return;
+ completeStep({ stepName: "execute", ...execution });
+ capture("eval:onboarding_completed", {
+ ...execution,
+ evaluatorType: getEvaluatorType(),
+ });
+ },
+ };
+}
diff --git a/web/src/features/evals/v2/fns/templateGallery/getTemplateSelectionAnalyticsProperties.ts b/web/src/features/evals/v2/fns/templateGallery/getTemplateSelectionAnalyticsProperties.ts
new file mode 100644
index 0000000000..8e34245312
--- /dev/null
+++ b/web/src/features/evals/v2/fns/templateGallery/getTemplateSelectionAnalyticsProperties.ts
@@ -0,0 +1,15 @@
+import type { GalleryTemplate } from "@/src/features/evals/v2/types/templateGallery";
+
+// Managed template keys are Langfuse-owned; custom (project) templates are
+// reported only as custom, never by id or name.
+export function getTemplateSelectionAnalyticsProperties(
+ template: GalleryTemplate,
+) {
+ return template.source === "managed"
+ ? {
+ evaluatorType: template.evaluator.type,
+ managedTemplateKey: template.key,
+ isCustomTemplate: false,
+ }
+ : { evaluatorType: template.type, isCustomTemplate: true };
+}
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
index 2979bbfc0f..95e7ec2035 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
@@ -67,6 +67,8 @@ import {
type EvaluatorCreationSource,
} from "@/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties";
import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";
+import { createEvalOnboardingAnalytics } from "@/src/features/evals/v2/fns/createEvalOnboardingAnalytics";
+import { EvalOnboardingAnalyticsProvider } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";
type InitialEvaluator = {
id: string;
@@ -225,6 +227,14 @@ export function EvaluatorSetupPage(
mode: props.mode,
}),
);
+ const [onboardingAnalytics] = useState(() =>
+ props.mode === "create"
+ ? createEvalOnboardingAnalytics({
+ capture,
+ getEvaluatorType: () => evaluatorSetupStore.getState().type,
+ })
+ : null,
+ );
useEffect(() => {
evaluatorSetupStore
.getState()
@@ -369,6 +379,9 @@ export function EvaluatorSetupPage(
onSuccess: (result) => {
setTestResult(result);
if ("executionTraceId" in result) setHasCompletedTestCall(true);
+ if ("success" in result && result.success) {
+ onboardingAnalytics?.completeStep({ stepName: "evaluator_tested" });
+ }
setLastTestRunCostUsd(
"estimatedCostUsd" in result &&
typeof result.estimatedCostUsd === "number"
@@ -566,6 +579,10 @@ export function EvaluatorSetupPage(
definition,
});
hasCreatedRef.current = true;
+ onboardingAnalytics?.completeStep({
+ stepName: "evaluator_saved",
+ isBlocked: !shouldOfferRuleAttachment(evaluator),
+ });
capture("evaluators:create", {
...getEvaluatorCreationAnalyticsProperties({
evaluatorType: state.type,
@@ -674,7 +691,13 @@ export function EvaluatorSetupPage(
providerGroups={projectDefaultModel.providerGroups}
providerAdapters={projectDefaultModel.providerAdapters}
canSetProjectDefault={projectDefaultModel.canUpdate}
- onConfigureProviders={projectDefaultModel.openProviderSettings}
+ onConfigureProviders={() => {
+ onboardingAnalytics?.track(
+ "eval:onboarding_llm_connection_tab_opened",
+ {},
+ );
+ projectDefaultModel.openProviderSettings();
+ }}
onSetProjectDefault={projectDefaultModel.update.requestUpdate}
codeValidationResult={
codeValidation.isPending ? null : codeValidation.validationResult
@@ -687,8 +710,13 @@ export function EvaluatorSetupPage(
? { state: "generating" }
: {
state: "idle",
- onGenerate: () =>
- requestNameSuggestion(true).catch(trpcErrorToast),
+ onGenerate: () => {
+ onboardingAnalytics?.track(
+ "eval:onboarding_ai_generate_requested",
+ { field: "name" },
+ );
+ requestNameSuggestion(true).catch(trpcErrorToast);
+ },
}
}
descriptionAIAssistance={
@@ -700,6 +728,10 @@ export function EvaluatorSetupPage(
state: "idle",
// requestDescriptionSuggestion reports its own failures.
onGenerate: () => {
+ onboardingAnalytics?.track(
+ "eval:onboarding_ai_generate_requested",
+ { field: "description" },
+ );
requestDescriptionSuggestion();
},
}
@@ -717,6 +749,10 @@ export function EvaluatorSetupPage(
projectId={projectId}
timeRange={absoluteTimeRange}
onOpenTrace={(observation) => {
+ onboardingAnalytics?.track(
+ "eval:onboarding_sample_observation_previewed",
+ {},
+ );
if (observation.traceId) {
sampleTracePeekNavigation.openPeek(observation.traceId);
}
@@ -786,81 +822,84 @@ export function EvaluatorSetupPage(
) : undefined,
}}
>
- <div className="flex min-h-0 flex-1 flex-col">
- <TableHeaderControls
- timeRange={timeRange}
- setTimeRange={setTimeRange}
- />
- {initialEvaluator?.blockedAt ? (
- <div className="mx-3 mt-3">
- <EvaluatorBlockedBanner
- projectId={projectId}
- blockedAt={initialEvaluator.blockedAt}
- blockReason={initialEvaluator.blockReason}
- blockMessage={initialEvaluator.blockMessage}
- canReactivate={canReactivate}
- reactivationPending={reactivate.isPending}
- onReactivate={() => {
- capture("evaluators:reactivate", {
- blockReason:
- initialEvaluator.blockReason ?? "EVAL_MODEL_CONFIG_INVALID",
- });
- reactivate.mutate({
- projectId,
- evaluatorId: initialEvaluator.id,
- });
- }}
- />
- </div>
- ) : null}
- {isMobile ? (
- <div className="min-h-0 flex-1 overflow-y-auto">
- <div>{evaluatorEditor}</div>
- <div className="border-t [&>aside]:h-auto">
- {evaluatorTestPanel}
+ <EvalOnboardingAnalyticsProvider value={onboardingAnalytics}>
+ <div className="flex min-h-0 flex-1 flex-col">
+ <TableHeaderControls
+ timeRange={timeRange}
+ setTimeRange={setTimeRange}
+ />
+ {initialEvaluator?.blockedAt ? (
+ <div className="mx-3 mt-3">
+ <EvaluatorBlockedBanner
+ projectId={projectId}
+ blockedAt={initialEvaluator.blockedAt}
+ blockReason={initialEvaluator.blockReason}
+ blockMessage={initialEvaluator.blockMessage}
+ canReactivate={canReactivate}
+ reactivationPending={reactivate.isPending}
+ onReactivate={() => {
+ capture("evaluators:reactivate", {
+ blockReason:
+ initialEvaluator.blockReason ??
+ "EVAL_MODEL_CONFIG_INVALID",
+ });
+ reactivate.mutate({
+ projectId,
+ evaluatorId: initialEvaluator.id,
+ });
+ }}
+ />
</div>
- </div>
- ) : (
- <ResizableSplitLayout
- className="h-auto min-h-0 flex-1"
- primaryContent={evaluatorEditor}
- secondaryContent={evaluatorTestPanel}
- open={testPanelOpen}
- defaultPrimarySize={60}
- defaultSecondarySize={40}
- minPrimarySize={30}
- minSecondarySize="360px"
- collapsedSecondarySize="48px"
- onOpenChange={
- evaluatorSetupStore.getState().actions.setTestPanelOpen
+ ) : null}
+ {isMobile ? (
+ <div className="min-h-0 flex-1 overflow-y-auto">
+ <div>{evaluatorEditor}</div>
+ <div className="border-t [&>aside]:h-auto">
+ {evaluatorTestPanel}
+ </div>
+ </div>
+ ) : (
+ <ResizableSplitLayout
+ className="h-auto min-h-0 flex-1"
+ primaryContent={evaluatorEditor}
+ secondaryContent={evaluatorTestPanel}
+ open={testPanelOpen}
+ defaultPrimarySize={60}
+ defaultSecondarySize={40}
+ minPrimarySize={30}
+ minSecondarySize="360px"
+ collapsedSecondarySize="48px"
+ onOpenChange={
+ evaluatorSetupStore.getState().actions.setTestPanelOpen
+ }
+ persistId="evaluator-test-panel"
+ />
+ )}
+ <EvaluatorSetupFooter
+ store={evaluatorSetupStore}
+ initialSnapshot={initialSnapshot.current}
+ isEditing={Boolean(initialEvaluator)}
+ isSaving={
+ saveInFlight ||
+ create.isPending ||
+ update.isPending ||
+ suggestName.isPending ||
+ suggestDescription.isPending
+ }
+ nameAIAssistanceAvailable={nameAIAssistanceAvailable}
+ codeValidation={
+ codeDraft.type === "CODE"
+ ? {
+ isValid: codeValidation.isValid,
+ isPending: codeValidation.isPending,
+ }
+ : null
}
- persistId="evaluator-test-panel"
+ onClose={requestClose}
+ onSave={save}
/>
- )}
- <EvaluatorSetupFooter
- store={evaluatorSetupStore}
- initialSnapshot={initialSnapshot.current}
- isEditing={Boolean(initialEvaluator)}
- isSaving={
- saveInFlight ||
- create.isPending ||
- update.isPending ||
- suggestName.isPending ||
- suggestDescription.isPending
- }
- nameAIAssistanceAvailable={nameAIAssistanceAvailable}
- codeValidation={
- codeDraft.type === "CODE"
- ? {
- isValid: codeValidation.isValid,
- isPending: codeValidation.isPending,
- }
- : null
- }
- onClose={requestClose}
- onSave={save}
- />
- </div>
+ </div>
+ </EvalOnboardingAnalyticsProvider>
{initialEvaluator ? (
<EvaluatorVersionHistorySheet
open={historyOpen}
@@ -934,6 +973,7 @@ export function EvaluatorSetupPage(
<EvaluatorSavedDialogContainer
projectId={projectId}
evaluator={savedEvaluator}
+ onboardingAnalytics={onboardingAnalytics}
onDismiss={async () => {
await router.push(
`/project/${projectId}/evals/${savedEvaluator.id}`,
diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
index a1581da11b..3f9effe33f 100644
--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
@@ -287,6 +287,13 @@ export default function EvaluatorsPage() {
evaluators.data.totalItems === 0 &&
!searchQuery &&
filterState.length === 0;
+ const openGalleryFromNewEvaluatorButton = () => {
+ capture("eval:onboarding_started", {
+ entryPoint: "new_evaluator_button",
+ hasExistingEvaluators: evaluators.isSuccess ? !showOnboarding : undefined,
+ });
+ setGalleryOpen(true);
+ };
const hasExecutionReadAccess = useHasProjectAccess({
projectId,
scope: "evalJobExecution:read",
@@ -527,11 +534,15 @@ export default function EvaluatorsPage() {
onEdit={() =>
router.push(`/project/${projectId}/evals/${row.original.id}`)
}
- onClone={() =>
- router.push(
+ onClone={() => {
+ capture("eval:onboarding_started", {
+ entryPoint: "clone_evaluator",
+ hasExistingEvaluators: true,
+ });
+ return router.push(
`/project/${projectId}/evals/new?evaluatorId=${encodeURIComponent(row.original.id)}`,
- )
- }
+ );
+ }}
onDelete={() => setDeleteIds([row.original.id])}
/>
</div>
@@ -539,6 +550,7 @@ export default function EvaluatorsPage() {
},
],
[
+ capture,
costs.data,
costs.isPending,
hasExecutionReadAccess,
@@ -620,7 +632,7 @@ export default function EvaluatorsPage() {
"Create reusable evaluator definitions and test them before activation.",
},
mobileActionButtons: showOnboarding ? (
- <Button size="sm" onClick={() => setGalleryOpen(true)}>
+ <Button size="sm" onClick={openGalleryFromNewEvaluatorButton}>
<Plus className="mr-2 h-4 w-4" />
New evaluator
</Button>
@@ -678,7 +690,7 @@ export default function EvaluatorsPage() {
{...evaluatorAlerts}
/>
)}
- <Button onClick={() => setGalleryOpen(true)}>
+ <Button onClick={openGalleryFromNewEvaluatorButton}>
<Plus className="mr-2 h-4 w-4" />
New evaluator
</Button>
diff --git a/web/src/features/evals/v2/types/evalOnboardingAnalytics.ts b/web/src/features/evals/v2/types/evalOnboardingAnalytics.ts
new file mode 100644
index 0000000000..9407ad55c5
--- /dev/null
+++ b/web/src/features/evals/v2/types/evalOnboardingAnalytics.ts
@@ -0,0 +1,109 @@
+import type { EvalTemplateType } from "@langfuse/shared";
+import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";
+
+// Funnel for creating an evaluator: gallery -> setup page -> saved dialog.
+// Metadata only — enums, counts, lengths, booleans and Langfuse-owned template
+// or section keys. Never search text, prompt content, evaluator/rule names,
+// filter values or custom template ids.
+
+type EvaluatorContext = { evaluatorType: EvalTemplateType };
+
+type ExecutionPath = "test_filters" | "existing_rule" | "new_rule";
+
+type Execution = {
+ executionPath: ExecutionPath;
+ hasBackfill: boolean;
+ // Unknown for a rule created in the rule editor.
+ samplingPercent?: number;
+};
+
+type TemplateSelection = EvaluatorContext & {
+ managedTemplateKey?: string;
+ isCustomTemplate: boolean;
+};
+
+type OnboardingStep =
+ | (EvaluatorContext & {
+ stepName: "new_from_scratch_selected";
+ surface: "gallery";
+ })
+ | (TemplateSelection & {
+ stepName: "suggestion_selected";
+ surface: "gallery" | "empty_state";
+ })
+ | (TemplateSelection & {
+ stepName: "template_selected";
+ surface: "gallery";
+ sectionKey: string;
+ })
+ | (EvaluatorContext & {
+ stepName: "model_selected";
+ modelMode: "default" | "custom";
+ })
+ | (EvaluatorContext & { stepName: "score_output_modified" })
+ | (EvaluatorContext & {
+ stepName: "variable_mapping_updated";
+ method: "tree" | "json_path";
+ })
+ | (EvaluatorContext & { stepName: "evaluator_name_updated" })
+ | (EvaluatorContext & {
+ stepName: "evaluator_filter_changed";
+ filterExperience: EvaluatorFilterExperience;
+ filterCount: number;
+ })
+ | (EvaluatorContext & { stepName: "evaluator_tested" })
+ | (EvaluatorContext & { stepName: "evaluator_saved"; isBlocked: boolean })
+ | (EvaluatorContext & Execution & { stepName: "execute" });
+
+export type EvalOnboardingEventMap = {
+ "eval:onboarding_started": {
+ entryPoint:
+ | "new_evaluator_button"
+ | "empty_state_starting_point"
+ | "empty_state_browse_templates"
+ | "clone_evaluator"
+ | "batch_evaluation";
+ hasExistingEvaluators?: boolean;
+ };
+ // Fires at most once per step and setup attempt; the interaction events
+ // below fire on every action.
+ "eval:onboarding_step_completed": OnboardingStep;
+ "eval:onboarding_completed": EvaluatorContext & Execution;
+ "eval:onboarding_gallery_searched": { queryLength: number };
+ "eval:onboarding_gallery_section_selected": { sectionKey: string };
+ "eval:onboarding_evaluator_type_changed": EvaluatorContext & {
+ previousEvaluatorType: EvalTemplateType;
+ };
+ "eval:onboarding_preview_toggled": EvaluatorContext & { isEnabled: boolean };
+ "eval:onboarding_sample_observation_previewed": EvaluatorContext;
+ // `message_edited` (content or role, typed) fires once per setup attempt;
+ // adding, removing and reordering messages fire on every action.
+ "eval:onboarding_prompt_modified": EvaluatorContext & {
+ modification:
+ | "message_edited"
+ | "message_added"
+ | "message_removed"
+ | "messages_reordered";
+ };
+ "eval:onboarding_llm_connection_tab_opened": EvaluatorContext;
+ "eval:onboarding_model_picker_opened": EvaluatorContext;
+ "eval:onboarding_model_changed": EvaluatorContext & {
+ modelMode: "default" | "custom";
+ };
+ "eval:onboarding_ai_generate_requested": EvaluatorContext & {
+ field: "name" | "description";
+ };
+ // Once per setup attempt: the slider reports every drag tick. The final
+ // value is on `eval:onboarding_completed`.
+ "eval:onboarding_sampling_changed": EvaluatorContext;
+ "eval:onboarding_historic_eval_toggled": EvaluatorContext & {
+ isEnabled: boolean;
+ };
+ "eval:onboarding_scope_changed": EvaluatorContext & {
+ scope: "test_filters" | "different_scope";
+ };
+ "eval:onboarding_create_rule_opened": EvaluatorContext;
+ "eval:onboarding_execution_skipped": EvaluatorContext & {
+ method: "skip_button" | "dialog_dismissed" | "rule_editor_closed";
+ };
+};
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
index 9510cfabf3..d617cf22a1 100644
--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts
+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
@@ -2,6 +2,7 @@ import { type CaptureResult, type CaptureOptions } from "posthog-js";
import { usePostHog } from "posthog-js/react";
import { useCallback } from "react";
import type { AnnotationEventMap } from "@/src/features/scores/lib/annotationAnalytics";
+import type { EvalOnboardingEventMap } from "@/src/features/evals/v2/types/evalOnboardingAnalytics";
export const V4_BETA_ENABLED_POSTHOG_PROPERTY = "v4BetaEnabled";
@@ -219,6 +220,29 @@ const events = {
"detach_evaluator",
"filter_reused",
],
+ // Evaluator creation funnel (gallery -> setup page -> saved dialog). Props
+ // are typed in EvalOnboardingEventMap: metadata only, never search text,
+ // prompt content, names or filter values.
+ eval: [
+ "onboarding_started",
+ "onboarding_step_completed",
+ "onboarding_completed",
+ "onboarding_gallery_searched",
+ "onboarding_gallery_section_selected",
+ "onboarding_evaluator_type_changed",
+ "onboarding_preview_toggled",
+ "onboarding_sample_observation_previewed",
+ "onboarding_prompt_modified",
+ "onboarding_llm_connection_tab_opened",
+ "onboarding_model_picker_opened",
+ "onboarding_model_changed",
+ "onboarding_ai_generate_requested",
+ "onboarding_sampling_changed",
+ "onboarding_historic_eval_toggled",
+ "onboarding_scope_changed",
+ "onboarding_create_rule_opened",
+ "onboarding_execution_skipped",
+ ],
// One-shot batch evaluation from the events / experiments tables.
// Counts and enums only — never mapping contents or observation payloads.
batch_eval: ["run"],
@@ -466,11 +490,10 @@ type EventName = {
[Resource in keyof typeof events]: `${Resource}:${(typeof events)[Resource][number]}`;
}[keyof typeof events];
-type EventProperties = AnnotationEventMap & {
- [E in Exclude<EventName, keyof AnnotationEventMap>]: Record<
- string,
- any
- > | null;
+type TypedEventMap = AnnotationEventMap & EvalOnboardingEventMap;
+
+type EventProperties = TypedEventMap & {
+ [E in Exclude<EventName, keyof TypedEventMap>]: Record<string, any> | null;
};
export const usePostHogClientCapture = () => {