🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): track the evaluator onboarding funnel in PostHog (#18075)
Steffen Reuther0a2bff14b9e3
diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx b/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsxindex f49a895d40..b4d0c01902 100644--- a/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx@@ -425,7 +425,16 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {<div className="flex items-center gap-2">{step !== "confirm" ? (- <CreateEvaluatorButton href={createEvaluatorHref} />+ <CreateEvaluatorButton+ href={createEvaluatorHref}+ onClick={() => {+ // The legacy (v3) editor is not part of the onboarding funnel.+ if (forceV3Experience) return;+ capture("eval:onboarding_started", {+ entryPoint: "batch_evaluation",+ });+ }}+ />) : null}{showMappingEditor ? (<MappingRunButton@@ -460,11 +469,18 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {);}-function CreateEvaluatorButton({ href }: { href: string }) {+function CreateEvaluatorButton({+ href,+ onClick,+}: {+ href: string;+ onClick: () => void;+}) {return (<Button variant="secondary" className="gap-1.5" asChild><Linkhref={href}+ onClick={onClick}target="_blank"rel="noreferrer"aria-label="Create new Evaluator (opens in a new tab)"diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsxindex cb41ec2918..01aaddf4dc 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx@@ -11,12 +11,19 @@ import {import { EvaluatorGalleryView } from "@/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView";import type { GalleryTemplate } from "@/src/features/evals/v2/types/templateGallery";import { prepareEvaluatorGallery } from "@/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery";-import { EVALUATOR_GALLERY_ALL_SECTION_KEY } from "@/src/features/evals/v2/constants/evaluatorGallery";+import {+ EVALUATOR_GALLERY_ALL_SECTION_KEY,+ EVALUATOR_GALLERY_RECOMMENDED_SECTION_KEY,+} from "@/src/features/evals/v2/constants/evaluatorGallery";import useLocalStorage from "@/src/components/useLocalStorage";import { usePostHogClientCapture } from "@/src/features/posthog-analytics";import { getEvaluatorCreationAnalyticsProperties } from "@/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties";+import { getTemplateSelectionAnalyticsProperties } from "@/src/features/evals/v2/fns/templateGallery/getTemplateSelectionAnalyticsProperties";import { api } from "@/src/utils/api";+// Search reports once the user pauses typing, so one query is one event.+const SEARCH_CAPTURE_DELAY_MS = 1000;+export function EvaluatorGalleryDialog({projectId,open,@@ -39,6 +46,10 @@ export function EvaluatorGalleryDialog({new Set(),);const searchInputRef = useRef<HTMLInputElement | null>(null);+ const pendingSearchCapture = useRef<{+ timeout: ReturnType<typeof setTimeout>;+ queryLength: number;+ } | null>(null);const scrollContainerRef = useRef<HTMLDivElement | null>(null);const [bannerDismissed, setBannerDismissed] = useLocalStorage("evaluatorGallery:decisionModelBannerDismissed:v1",@@ -81,7 +92,34 @@ export function EvaluatorGalleryDialog({projectEvaluators.data?.pages[0]?.totalItems ?? customTemplates.length,search,});+ const flushSearchCapture = () => {+ const pending = pendingSearchCapture.current;+ if (!pending) return;+ clearTimeout(pending.timeout);+ pendingSearchCapture.current = null;+ capture("eval:onboarding_gallery_searched", {+ queryLength: pending.queryLength,+ });+ };+ const handleSearchChange = (nextSearch: string) => {+ setSearch(nextSearch);+ const queryLength = nextSearch.trim().length;+ if (queryLength === 0) {+ flushSearchCapture();+ return;+ }+ if (pendingSearchCapture.current) {+ clearTimeout(pendingSearchCapture.current.timeout);+ }+ pendingSearchCapture.current = {+ queryLength,+ timeout: setTimeout(flushSearchCapture, SEARCH_CAPTURE_DELAY_MS),+ };+ };const selectSection = (key: string) => {+ if (key !== activeSection) {+ capture("eval:onboarding_gallery_section_selected", { sectionKey: key });+ }setActiveSection(key);scrollContainerRef.current?.scrollTo({ top: 0 });};@@ -93,7 +131,26 @@ export function EvaluatorGalleryDialog({return next;});};- const handleSelectTemplate = (template: GalleryTemplate) => {+ const handleSelectTemplate = (+ template: GalleryTemplate,+ sectionKey: string,+ ) => {+ flushSearchCapture();+ capture(+ "eval:onboarding_step_completed",+ sectionKey === EVALUATOR_GALLERY_RECOMMENDED_SECTION_KEY+ ? {+ stepName: "suggestion_selected",+ surface: "gallery",+ ...getTemplateSelectionAnalyticsProperties(template),+ }+ : {+ stepName: "template_selected",+ surface: "gallery",+ sectionKey,+ ...getTemplateSelectionAnalyticsProperties(template),+ },+ );const evaluatorType =template.source === "managed" ? template.evaluator.type : template.type;const creationSource =@@ -110,6 +167,12 @@ export function EvaluatorGalleryDialog({onSelectTemplate(template);};const handleCreateFromScratch = (evaluatorType: EvalTemplateType) => {+ flushSearchCapture();+ capture("eval:onboarding_step_completed", {+ stepName: "new_from_scratch_selected",+ surface: "gallery",+ evaluatorType,+ });capture("evaluators:gallery_creation_source_select",getEvaluatorCreationAnalyticsProperties({@@ -141,7 +204,7 @@ export function EvaluatorGalleryDialog({</DialogHeader><EvaluatorGalleryViewsearch={search}- onSearchChange={setSearch}+ onSearchChange={handleSearchChange}searchInputRef={searchInputRef}navigationItems={navigationItems}activeSection={activeSection}diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsxindex f8f975933a..9a00f2a2c3 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx@@ -66,7 +66,7 @@ export function EvaluatorGalleryView({sections: GallerySection[];expandedSections: ReadonlySet<string>;onExpandedChange: (key: string, expanded: boolean) => void;- onSelectTemplate: (template: GalleryTemplate) => void;+ onSelectTemplate: (template: GalleryTemplate, sectionKey: string) => void;onCreateFromScratch: (type: EvalTemplateType) => void;scrollContainerRef?: RefObject<HTMLDivElement | null>;isLoading: boolean;diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.tsxindex 96a5c086bd..8aa7605734 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.tsx@@ -27,11 +27,13 @@ export function EvaluatorGallerySection({section: GallerySection;expanded: boolean;onExpandedChange?: (expanded: boolean) => void;- onSelectTemplate: (template: GalleryTemplate) => void;+ onSelectTemplate: (template: GalleryTemplate, sectionKey: string) => void;}) {const isRecommended =section.key === EVALUATOR_GALLERY_RECOMMENDED_SECTION_KEY;const isSafety = section.key === EVALUATOR_GALLERY_SAFETY_SECTION_KEY;+ const selectTemplate = (template: GalleryTemplate) =>+ onSelectTemplate(template, section.key);const shownTemplates = expanded? section.templates: section.templates.slice(0, EVALUATOR_GALLERY_PREVIEW_SIZE);@@ -48,7 +50,7 @@ export function EvaluatorGallerySection({<EvaluatorRecommendedCardkey={getGalleryTemplateId(template)}template={template}- onSelect={onSelectTemplate}+ onSelect={selectTemplate}/>))}</EvaluatorRecommendedCards>@@ -73,7 +75,7 @@ export function EvaluatorGallerySection({<EvaluatorTemplateRowkey={getGalleryTemplateId(template)}template={template}- onSelect={onSelectTemplate}+ onSelect={selectTemplate}/>))}</div>diff --git a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/SampleObservationSelectorContainer/SampleObservationSelectorContainer.tsx b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/SampleObservationSelectorContainer/SampleObservationSelectorContainer.tsxindex 46561f6f8e..d692c5a5d8 100644--- a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/SampleObservationSelectorContainer/SampleObservationSelectorContainer.tsx+++ b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/SampleObservationSelectorContainer/SampleObservationSelectorContainer.tsx@@ -1,5 +1,10 @@import { useStore } from "zustand";+import type { FilterState } from "@langfuse/shared";+import useLocalStorage from "@/src/components/useLocalStorage";+import { EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY } from "@/src/features/evals/v2/constants/evaluatorFilterExperience";+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";+import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";import { EvaluatorSampleObservationSelector } from "@/src/features/evals/v2/components/Evaluators/Testing/components/EvaluatorSampleObservationSelector/EvaluatorSampleObservationSelector";import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";@@ -24,6 +29,26 @@ export function SampleObservationSelectorContainer({);const filterState = useStore(store, (state) => state.sampleFilter);const actions = store.getState().actions;+ const onboardingAnalytics = useEvalOnboardingAnalytics();+ const [filterExperience] = useLocalStorage<EvaluatorFilterExperience>(+ EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY,+ "query",+ );+ const changeFilterState = (nextFilterState: FilterState) => {+ // The search bar can re-commit an unchanged filter; only a real change is+ // a step.+ const hasChanged =+ JSON.stringify(nextFilterState) !==+ JSON.stringify(store.getState().sampleFilter);+ actions.setSampleFilter(nextFilterState);+ if (hasChanged) {+ onboardingAnalytics?.completeStep({+ stepName: "evaluator_filter_changed",+ filterExperience,+ filterCount: nextFilterState.length,+ });+ }+ };return (<EvaluatorSampleObservationSelector@@ -31,7 +56,7 @@ export function SampleObservationSelectorContainer({timeRange={timeRange}selectedObservationId={selectedObservationId}filterState={filterState}- onFilterStateChange={actions.setSampleFilter}+ onFilterStateChange={changeFilterState}onSelect={actions.setSelectedObservation}onOpenTrace={onOpenTrace}/>diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsxindex c23ac7c2d1..9cce3c1a9d 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx@@ -29,6 +29,7 @@ import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilimport { EvaluatorSavedRuleFilterPreview } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedRuleFilterPreview";import { EvaluatorBackfillSettings } from "@/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings";import { useEvaluatorSavedBackfill } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill";+import type { createEvalOnboardingAnalytics } from "@/src/features/evals/v2/fns/createEvalOnboardingAnalytics";type Rule = RouterOutputs["evalsV2"]["rules"]["list"]["rules"][number];type DialogPhase = "saved" | "closing-saved" | "create-rule" | "closed";@@ -46,11 +47,13 @@ type EvaluatorSavedDialogEvaluator = {export function EvaluatorSavedDialogContainer({projectId,evaluator,+ onboardingAnalytics,onDismiss,onFinish,}: {projectId: string;evaluator: EvaluatorSavedDialogEvaluator;+ onboardingAnalytics: ReturnType<typeof createEvalOnboardingAnalytics> | null;onDismiss: () => Promise<void>;onFinish: () => Promise<void>;}) {@@ -72,6 +75,7 @@ export function EvaluatorSavedDialogContainer({const estimateRequestId = useRef(0);const initialEstimateRequested = useRef(false);const createRuleHandoffPending = useRef(false);+ const ruleCreatedInEditor = useRef(false);const hasRequestedMissingCostTest = useRef(evaluator.hasCompletedTestCall ?? false,);@@ -190,6 +194,11 @@ export function EvaluatorSavedDialogContainer({{ filter: currentRule.filter, sampling: currentRule.sampling },backfill.executionRange,);+ onboardingAnalytics?.completeOnboarding({+ executionPath: "existing_rule",+ hasBackfill: backfill.enabled,+ samplingPercent: Math.round(currentRule.sampling * 100),+ });await invalidateRuleQueries();await finish();};@@ -220,6 +229,11 @@ export function EvaluatorSavedDialogContainer({{ filter: supportedRuleFilters, sampling },backfill.executionRange,);+ onboardingAnalytics?.completeOnboarding({+ executionPath: "test_filters",+ hasBackfill: backfill.enabled,+ samplingPercent: Math.round(sampling * 100),+ });await invalidateRuleQueries();await finish();};@@ -340,6 +354,11 @@ export function EvaluatorSavedDialogContainer({};const handleModeChange = (nextMode: EvaluatorSavedMode) => {+ if (nextMode !== mode) {+ onboardingAnalytics?.track("eval:onboarding_scope_changed", {+ scope: nextMode === "test-filters" ? "test_filters" : "different_scope",+ });+ }const modeChangeRequestId = ++estimateRequestId.current;setActivationOpen(false);setIsEstimating(false);@@ -435,6 +454,7 @@ export function EvaluatorSavedDialogContainer({action: "new_rule",hasBackfill: false,});+ onboardingAnalytics?.track("eval:onboarding_create_rule_opened", {});openCreateRule();}};@@ -518,9 +538,12 @@ export function EvaluatorSavedDialogContainer({maxAllowedItems={backfill.allowedItems}matchingObservations={backfill.matchingObservations}isEstimating={backfill.isEstimating}- onEnabledChange={(enabled) =>- backfill.setEnabled(enabled, currentBackfillScope)- }+ onEnabledChange={(enabled) => {+ onboardingAnalytics?.track("eval:onboarding_historic_eval_toggled", {+ isEnabled: enabled,+ });+ backfill.setEnabled(enabled, currentBackfillScope);+ }}onWindowChange={(window) =>backfill.updateWindow(window, currentBackfillScope)}@@ -558,6 +581,11 @@ export function EvaluatorSavedDialogContainer({? (sampling) => {setTestFilterSampling(sampling);setActivationSampling(sampling);+ onboardingAnalytics?.track(+ "eval:onboarding_sampling_changed",+ {},+ { onceKey: "sampling_changed" },+ );}: null}@@ -602,11 +630,17 @@ export function EvaluatorSavedDialogContainer({onModeChange={handleModeChange}onOpenAutoFocus={requestInitialEstimate}onDismiss={() => {+ onboardingAnalytics?.track("eval:onboarding_execution_skipped", {+ method: "dialog_dismissed",+ });createRuleHandoffPending.current = false;setDialogPhase("closed");onDismiss().catch(trpcErrorToast);}}onSecondaryAction={() => {+ onboardingAnalytics?.track("eval:onboarding_execution_skipped", {+ method: "skip_button",+ });setDialogPhase("closed");finish().catch(trpcErrorToast);}}@@ -622,10 +656,22 @@ export function EvaluatorSavedDialogContainer({openonOpenChange={(open) => {if (!open) {+ if (!ruleCreatedInEditor.current) {+ onboardingAnalytics?.track("eval:onboarding_execution_skipped", {+ method: "rule_editor_closed",+ });+ }setDialogPhase("closed");finish().catch(() => undefined);}}}+ onCreated={() => {+ ruleCreatedInEditor.current = true;+ onboardingAnalytics?.completeOnboarding({+ executionPath: "new_rule",+ hasBackfill: false,+ });+ }}initialEvaluator={{ ...evaluator, initialVariableMapping: null }}successNotification="none"/>diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsxindex 9ac4068592..fcd335ffc5 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx@@ -1,6 +1,6 @@import { useStore } from "zustand";import { useShallow } from "zustand/react/shallow";-import type { LLMAdapter } from "@langfuse/shared";+import type { EvalTemplateType, LLMAdapter } from "@langfuse/shared";import { DefinitionStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/DefinitionStep";import { CodeEditor } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/CodeEditor/CodeEditor";@@ -12,6 +12,7 @@ import { PromptEditor } from "@/src/features/evals/v2/components/Evaluators/Evalimport { ScoreOutputEditor } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ScoreOutputEditor/ScoreOutputEditor";import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";import type { ProjectDefaultModelConfig } from "@/src/features/evals/v2/types/ProjectDefaultModelConfig";import type { CodeEvalValidationResult } from "@/src/features/evals/utils/code-eval-template-validation";@@ -40,6 +41,7 @@ export function DefinitionStepContainer({onSetProjectDefault: (model: ProjectDefaultModelConfig) => void;codeValidationResult: CodeEvalValidationResult | null;}) {+ const onboardingAnalytics = useEvalOnboardingAnalytics();const state = useStore(store,useShallow((state) => ({@@ -48,11 +50,20 @@ export function DefinitionStepContainer({actions: state.actions,})),);+ const changeType = (type: EvalTemplateType) => {+ const previousEvaluatorType = store.getState().type;+ state.actions.setType(type);+ if (type !== previousEvaluatorType) {+ onboardingAnalytics?.track("eval:onboarding_evaluator_type_changed", {+ previousEvaluatorType,+ });+ }+ };const stepProps = {open: state.open,onOpenChange: (open: boolean) => onStepOpenChange(1, open),- onTypeChange: state.actions.setType,+ onTypeChange: changeType,isEditing,};diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsxindex 310c9d0432..450993f1b5 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx@@ -12,6 +12,7 @@ import { JudgeModelConfigurationDialog } from "@/src/features/evals/v2/componentimport type { ProjectDefaultModelConfig } from "@/src/features/evals/v2/types/ProjectDefaultModelConfig";import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";export function ModelSelector({projectId,@@ -33,6 +34,14 @@ export function ModelSelector({onSetProjectDefault: (model: ProjectDefaultModelConfig) => void;}) {const [configurationOpen, setConfigurationOpen] = useState(false);+ const onboardingAnalytics = useEvalOnboardingAnalytics();+ const trackModelChange = (modelMode: "default" | "custom") => {+ onboardingAnalytics?.track("eval:onboarding_model_changed", { modelMode });+ onboardingAnalytics?.completeStep({+ stepName: "model_selected",+ modelMode,+ });+ };const state = useStore(store,useShallow((state) => ({@@ -59,13 +68,28 @@ export function ModelSelector({<><JudgeModelPickeropen={state.open}- onOpenChange={state.actions.setModelPickerOpen}+ onOpenChange={(open) => {+ if (open && !state.open) {+ onboardingAnalytics?.track(+ "eval:onboarding_model_picker_opened",+ {},+ );+ }+ state.actions.setModelPickerOpen(open);+ }}mode={state.mode}defaultModel={defaultModel}providerGroups={providerGroups}selectedModel={state.selectedModel}- onModeChange={state.actions.setModelMode}- onSelectCustom={state.actions.selectModel}+ onModeChange={(mode) => {+ const hasChanged = mode !== state.mode;+ state.actions.setModelMode(mode);+ if (hasChanged) trackModelChange(mode);+ }}+ onSelectCustom={(model) => {+ state.actions.selectModel(model);+ trackModelChange("custom");+ }}onConfigureProviders={onConfigureProviders}onConfigureModel={() => setConfigurationOpen(true)}hasModelConfiguration={diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsxindex 9278390356..a9b318e8de 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx@@ -58,6 +58,7 @@ import {import { useEvaluatorSetupSample } from "@/src/features/evals/v2/hooks/useEvaluatorSetupSample";import { useCopyToClipboard } from "@/src/hooks/useCopyToClipboard";import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";import { cn } from "@/src/utils/tailwind";import type { EvaluatorPromptMessage } from "@langfuse/shared";@@ -107,6 +108,7 @@ export function PromptEditorContent({promptPreviewEnabled: state.promptPreviewEnabled,sampleObject,});+ const onboardingAnalytics = useEvalOnboardingAnalytics();const [activeMessageId, setActiveMessageId] = useState<string | null>(null);const activeMessageIndex = activeMessageId? state.promptMessageIds.indexOf(activeMessageId)@@ -130,6 +132,9 @@ export function PromptEditorContent({const toIndex = state.promptMessageIds.indexOf(String(over.id));if (fromIndex < 0 || toIndex < 0) return;state.actions.reorderPromptMessage(fromIndex, toIndex);+ onboardingAnalytics?.track("eval:onboarding_prompt_modified", {+ modification: "messages_reordered",+ });};return (@@ -161,9 +166,26 @@ export function PromptEditorContent({sampleObject,})}previewEnabled={state.promptPreviewEnabled}- onPreviewEnabledChange={state.actions.setPromptPreviewEnabled}- onChange={(next) => state.actions.setPromptMessage(index, next)}- onRemove={() => state.actions.removePromptMessage(index)}+ onPreviewEnabledChange={(isEnabled) => {+ state.actions.setPromptPreviewEnabled(isEnabled);+ onboardingAnalytics?.track("eval:onboarding_preview_toggled", {+ isEnabled,+ });+ }}+ onChange={(next) => {+ state.actions.setPromptMessage(index, next);+ onboardingAnalytics?.track(+ "eval:onboarding_prompt_modified",+ { modification: "message_edited" },+ { onceKey: "prompt_message_edited" },+ );+ }}+ onRemove={() => {+ state.actions.removePromptMessage(index);+ onboardingAnalytics?.track("eval:onboarding_prompt_modified", {+ modification: "message_removed",+ });+ }}/>))}</SortableContext>@@ -175,6 +197,9 @@ export function PromptEditorContent({onClick={() => {state.actions.setPromptPreviewEnabled(false);state.actions.addPromptMessage();+ onboardingAnalytics?.track("eval:onboarding_prompt_modified", {+ modification: "message_added",+ });}}><Plus className="h-3.5 w-3.5 shrink-0" />diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ScoreOutputEditor/ScoreOutputEditor.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ScoreOutputEditor/ScoreOutputEditor.tsxindex f8e8e50224..e9d82053e7 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ScoreOutputEditor/ScoreOutputEditor.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ScoreOutputEditor/ScoreOutputEditor.tsx@@ -2,16 +2,23 @@ import { useStore } from "zustand";import { ScoreOutputConfiguration } from "@/src/features/evals/v2/components/Evaluators/Judges/ScoreOutputConfiguration/ScoreOutputConfiguration";import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";export function ScoreOutputEditor({ store }: { store: EvaluatorSetupStore }) {const scoreOutput = useStore(store, (state) => state.scoreOutput);const setScoreOutput = store.getState().actions.setScoreOutput;+ const onboardingAnalytics = useEvalOnboardingAnalytics();return (<ScoreOutputConfigurationmode="editable"state={scoreOutput}- onChange={setScoreOutput}+ onChange={(nextScoreOutput) => {+ setScoreOutput(nextScoreOutput);+ onboardingAnalytics?.completeStep({+ stepName: "score_output_modified",+ });+ }}/>);}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsxindex 75d389c22d..7e9255e0aa 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx@@ -6,6 +6,7 @@ import type { AIAssistedInput } from "@/src/components/ui/ai-assisted-input";import { NameStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep";import { getEvaluatorNameStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps";import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";export function NameStepContainer({store,@@ -20,6 +21,7 @@ export function NameStepContainer({>["aiAssistance"];onStepOpenChange: (step: number, open: boolean) => void;}) {+ const onboardingAnalytics = useEvalOnboardingAnalytics();const state = useStore(store,useShallow((state) => {@@ -45,7 +47,12 @@ export function NameStepContainer({open={state.open}onOpenChange={(open) => onStepOpenChange(state.step, open)}name={state.name}- onNameChange={state.actions.setName}+ onNameChange={(name) => {+ state.actions.setName(name);+ onboardingAnalytics?.completeStep({+ stepName: "evaluator_name_updated",+ });+ }}description={state.description}onDescriptionChange={state.actions.setDescription}nameAIAssistance={nameAIAssistance}diff --git a/web/src/features/evals/v2/components/EvaluatorsEmptyState/EvaluatorsEmptyState.tsx b/web/src/features/evals/v2/components/EvaluatorsEmptyState/EvaluatorsEmptyState.tsxindex 2c01834f3c..fc51d6d04b 100644--- a/web/src/features/evals/v2/components/EvaluatorsEmptyState/EvaluatorsEmptyState.tsx+++ b/web/src/features/evals/v2/components/EvaluatorsEmptyState/EvaluatorsEmptyState.tsx@@ -7,6 +7,7 @@ import { DETECT_TOPICS_ASSISTANT_PROMPT } from "@/src/features/evals/v2/constantimport { EvaluatorsEmptyStateView } from "./components/EvaluatorsEmptyStateView/EvaluatorsEmptyStateView";import { prepareEvaluatorEmptyState } from "@/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState";import type { GalleryTemplate } from "@/src/features/evals/v2/types/templateGallery";+import { getTemplateSelectionAnalyticsProperties } from "@/src/features/evals/v2/fns/templateGallery/getTemplateSelectionAnalyticsProperties";export function EvaluatorsEmptyState({onSelectTemplate,@@ -29,6 +30,15 @@ export function EvaluatorsEmptyState({templateKey: template.key,});}+ capture("eval:onboarding_started", {+ entryPoint: "empty_state_starting_point",+ hasExistingEvaluators: false,+ });+ capture("eval:onboarding_step_completed", {+ stepName: "suggestion_selected",+ surface: "empty_state",+ ...getTemplateSelectionAnalyticsProperties(template),+ });onSelectTemplate(template);};@@ -62,6 +72,10 @@ export function EvaluatorsEmptyState({}onBrowseLibrary={() => {capture("evaluators:empty_state_browse_library");+ capture("eval:onboarding_started", {+ entryPoint: "empty_state_browse_templates",+ hasExistingEvaluators: false,+ });onBrowseLibrary();}}/>diff --git a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/CreateRuleDialog.tsx b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/CreateRuleDialog.tsxindex cd83589264..5674d87e0d 100644--- a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/CreateRuleDialog.tsx+++ b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/CreateRuleDialog.tsx@@ -20,6 +20,7 @@ export function CreateRuleDialog({initialFilter,targetObject,successNotification,+ onCreated,}: {projectId: string;open: boolean;@@ -29,6 +30,7 @@ export function CreateRuleDialog({initialFilter?: FilterState;targetObject?: Extract<EvalTargetObject, "event" | "experiment">;successNotification: "toast" | "none";+ onCreated?: () => void;}) {const [evaluatorSearch, setEvaluatorSearch] = useState("");const [evaluatorSearchQuery, setEvaluatorSearchQuery] = useState("");@@ -98,6 +100,7 @@ export function CreateRuleDialog({targetObject={targetObject}evaluatorSearch={evaluatorSearch}successNotification={successNotification}+ onCreated={onCreated}onEvaluatorSearchChange={(value) => {setEvaluatorSearch(value);debouncedEvaluatorSearch(value);diff --git a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsxindex f89fd1ae83..1adaed79bf 100644--- a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx+++ b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx@@ -37,6 +37,7 @@ export function CreateRuleDialogContent({targetObject,evaluatorSearch,successNotification,+ onCreated,onEvaluatorSearchChange,}: {projectId: string;@@ -49,6 +50,7 @@ export function CreateRuleDialogContent({targetObject?: Extract<EvalTargetObject, "event" | "experiment">;evaluatorSearch: string;successNotification: "toast" | "none";+ onCreated?: () => void;onEvaluatorSearchChange: (search: string) => void;}) {const capture = usePostHogClientCapture();@@ -119,6 +121,7 @@ export function CreateRuleDialogContent({samplingPercent: Math.round(draft.sampling * 100),isEnabled: true,});+ onCreated?.();if (successNotification === "toast") {showSuccessToast({title: "Rule created",diff --git a/web/src/features/evals/v2/components/VariableMapping/components/EditableVariableMapping/EditableVariableMapping.tsx b/web/src/features/evals/v2/components/VariableMapping/components/EditableVariableMapping/EditableVariableMapping.tsxindex ec278fead7..06c22dd066 100644--- a/web/src/features/evals/v2/components/VariableMapping/components/EditableVariableMapping/EditableVariableMapping.tsx+++ b/web/src/features/evals/v2/components/VariableMapping/components/EditableVariableMapping/EditableVariableMapping.tsx@@ -33,6 +33,7 @@ import {experimentTargetEvalVariableColumns,} from "@langfuse/shared";import { usePostHogClientCapture } from "@/src/features/posthog-analytics";+import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";const TOOL_CALLS_COLUMN_ID = "toolCalls";@@ -293,6 +294,7 @@ function VariableMappingRow({onDelete?: () => void;}) {const capture = usePostHogClientCapture();+ const onboardingAnalytics = useEvalOnboardingAnalytics();const segments = useMemo(() =>fieldState.jsonSelector@@ -343,6 +345,10 @@ function VariableMappingRow({capture("evaluators:variable_mapping_configured", {method: "tree",});+ onboardingAnalytics?.completeStep({+ stepName: "variable_mapping_updated",+ method: "tree",+ });}onChange({selectedColumnId: columnId,@@ -357,6 +363,10 @@ function VariableMappingRow({capture("evaluators:variable_mapping_configured", {method: "json_path",});+ onboardingAnalytics?.completeStep({+ stepName: "variable_mapping_updated",+ method: "json_path",+ });}onChange({selectedColumnId: fieldState.selectedColumnId,diff --git a/web/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext.tsx b/web/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext.tsxnew file mode 100644index 0000000000..c2297db547--- /dev/null+++ b/web/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext.tsx@@ -0,0 +1,27 @@+import { createContext, useContext, type ReactNode } from "react";+import type { createEvalOnboardingAnalytics } from "@/src/features/evals/v2/fns/createEvalOnboardingAnalytics";++type EvalOnboardingAnalytics = ReturnType<typeof createEvalOnboardingAnalytics>;++// Null outside an evaluator creation flow (editing, rule editor), so shared+// editor components only report onboarding events while creating.+const EvalOnboardingAnalyticsContext =+ createContext<EvalOnboardingAnalytics | null>(null);++export function EvalOnboardingAnalyticsProvider({+ value,+ children,+}: {+ value: EvalOnboardingAnalytics | null;+ children: ReactNode;+}) {+ return (+ <EvalOnboardingAnalyticsContext.Provider value={value}>+ {children}+ </EvalOnboardingAnalyticsContext.Provider>+ );+}++export function useEvalOnboardingAnalytics() {+ return useContext(EvalOnboardingAnalyticsContext);+}diff --git a/web/src/features/evals/v2/fns/createEvalOnboardingAnalytics.clienttest.ts b/web/src/features/evals/v2/fns/createEvalOnboardingAnalytics.clienttest.tsnew file mode 100644index 0000000000..e3f0744aa2--- /dev/null+++ b/web/src/features/evals/v2/fns/createEvalOnboardingAnalytics.clienttest.ts@@ -0,0 +1,88 @@+import type { EvalTemplateType } from "@langfuse/shared";+import { describe, expect, it, vi } from "vitest";++import { createEvalOnboardingAnalytics } from "./createEvalOnboardingAnalytics";++function setup() {+ const capture = vi.fn();+ let evaluatorType: EvalTemplateType = "LLM_AS_JUDGE";+ const analytics = createEvalOnboardingAnalytics({+ capture,+ getEvaluatorType: () => evaluatorType,+ });+ return {+ analytics,+ capture,+ setEvaluatorType: (type: EvalTemplateType) => {+ evaluatorType = type;+ },+ };+}++describe("createEvalOnboardingAnalytics", () => {+ it("emits each step once, stamped with the evaluator type at the time", () => {+ const { analytics, capture, setEvaluatorType } = setup();++ analytics.completeStep({ stepName: "evaluator_name_updated" });+ setEvaluatorType("CODE");+ analytics.completeStep({ stepName: "evaluator_name_updated" });+ analytics.completeStep({ stepName: "evaluator_tested" });++ expect(capture.mock.calls).toEqual([+ [+ "eval:onboarding_step_completed",+ { stepName: "evaluator_name_updated", evaluatorType: "LLM_AS_JUDGE" },+ ],+ [+ "eval:onboarding_step_completed",+ { stepName: "evaluator_tested", evaluatorType: "CODE" },+ ],+ ]);+ });++ it("fires interaction events every time unless they share a once key", () => {+ const { analytics, capture } = setup();++ analytics.track("eval:onboarding_model_picker_opened", {});+ analytics.track("eval:onboarding_model_picker_opened", {});+ analytics.track(+ "eval:onboarding_sampling_changed",+ {},+ { onceKey: "sampling_changed" },+ );+ analytics.track(+ "eval:onboarding_sampling_changed",+ {},+ { onceKey: "sampling_changed" },+ );++ expect(capture.mock.calls.map(([event]) => event)).toEqual([+ "eval:onboarding_model_picker_opened",+ "eval:onboarding_model_picker_opened",+ "eval:onboarding_sampling_changed",+ ]);+ });++ it("completes onboarding once with the execute step", () => {+ const { analytics, capture } = setup();+ const execution = {+ executionPath: "test_filters" as const,+ hasBackfill: true,+ samplingPercent: 50,+ };++ analytics.completeOnboarding(execution);+ analytics.completeOnboarding(execution);++ expect(capture.mock.calls).toEqual([+ [+ "eval:onboarding_step_completed",+ { stepName: "execute", ...execution, evaluatorType: "LLM_AS_JUDGE" },+ ],+ [+ "eval:onboarding_completed",+ { ...execution, evaluatorType: "LLM_AS_JUDGE" },+ ],+ ]);+ });+});diff --git a/web/src/features/evals/v2/fns/createEvalOnboardingAnalytics.ts b/web/src/features/evals/v2/fns/createEvalOnboardingAnalytics.tsnew file mode 100644index 0000000000..f891ca96a7--- /dev/null+++ b/web/src/features/evals/v2/fns/createEvalOnboardingAnalytics.ts@@ -0,0 +1,75 @@+import type { EvalTemplateType } from "@langfuse/shared";+import type { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";+import type { EvalOnboardingEventMap } from "@/src/features/evals/v2/types/evalOnboardingAnalytics";++type Capture = ReturnType<typeof usePostHogClientCapture>;+type WithoutEvaluatorType<T> = T extends unknown+ ? Omit<T, "evaluatorType">+ : never;+type OnboardingStep = EvalOnboardingEventMap["eval:onboarding_step_completed"];+type ExecuteStep = Extract<OnboardingStep, { stepName: "execute" }>;+// Gallery events fire before the setup page exists and are captured directly.+type SetupEventName = Exclude<+ keyof EvalOnboardingEventMap,+ | "eval:onboarding_started"+ | "eval:onboarding_step_completed"+ | "eval:onboarding_completed"+ | "eval:onboarding_gallery_searched"+ | "eval:onboarding_gallery_section_selected"+>;++/**+ * Analytics for one evaluator setup attempt. Every event carries the evaluator+ * type at the moment of the action; each step is emitted at most once so the+ * funnel counts reaching a step, not how often it was repeated.+ */+export function createEvalOnboardingAnalytics({+ capture,+ getEvaluatorType,+}: {+ capture: Capture;+ getEvaluatorType: () => EvalTemplateType;+}) {+ const emittedKeys = new Set<string>();+ const claim = (key: string) => {+ if (emittedKeys.has(key)) return false;+ emittedKeys.add(key);+ return true;+ };++ const completeStep = (step: WithoutEvaluatorType<OnboardingStep>) => {+ if (!claim(`step:${step.stepName}`)) return;+ capture("eval:onboarding_step_completed", {+ ...step,+ evaluatorType: getEvaluatorType(),+ } as OnboardingStep);+ };++ return {+ track<E extends SetupEventName>(+ event: E,+ properties: WithoutEvaluatorType<EvalOnboardingEventMap[E]>,+ options?: { onceKey?: string },+ ) {+ if (options?.onceKey && !claim(options.onceKey)) return;+ // `properties` is checked against the event map by this method's+ // signature; TypeScript cannot relate the generic index back to the+ // registry's property map, so the hand-off is widened here.+ (capture as (event: E, properties: object) => void)(event, {+ ...properties,+ evaluatorType: getEvaluatorType(),+ });+ },+ completeStep,+ completeOnboarding(+ execution: Omit<ExecuteStep, "stepName" | "evaluatorType">,+ ) {+ if (!claim("completed")) return;+ completeStep({ stepName: "execute", ...execution });+ capture("eval:onboarding_completed", {+ ...execution,+ evaluatorType: getEvaluatorType(),+ });+ },+ };+}diff --git a/web/src/features/evals/v2/fns/templateGallery/getTemplateSelectionAnalyticsProperties.ts b/web/src/features/evals/v2/fns/templateGallery/getTemplateSelectionAnalyticsProperties.tsnew file mode 100644index 0000000000..8e34245312--- /dev/null+++ b/web/src/features/evals/v2/fns/templateGallery/getTemplateSelectionAnalyticsProperties.ts@@ -0,0 +1,15 @@+import type { GalleryTemplate } from "@/src/features/evals/v2/types/templateGallery";++// Managed template keys are Langfuse-owned; custom (project) templates are+// reported only as custom, never by id or name.+export function getTemplateSelectionAnalyticsProperties(+ template: GalleryTemplate,+) {+ return template.source === "managed"+ ? {+ evaluatorType: template.evaluator.type,+ managedTemplateKey: template.key,+ isCustomTemplate: false,+ }+ : { evaluatorType: template.type, isCustomTemplate: true };+}diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsxindex 2979bbfc0f..95e7ec2035 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx@@ -67,6 +67,8 @@ import {type EvaluatorCreationSource,} from "@/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties";import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";+import { createEvalOnboardingAnalytics } from "@/src/features/evals/v2/fns/createEvalOnboardingAnalytics";+import { EvalOnboardingAnalyticsProvider } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";type InitialEvaluator = {id: string;@@ -225,6 +227,14 @@ export function EvaluatorSetupPage(mode: props.mode,}),);+ const [onboardingAnalytics] = useState(() =>+ props.mode === "create"+ ? createEvalOnboardingAnalytics({+ capture,+ getEvaluatorType: () => evaluatorSetupStore.getState().type,+ })+ : null,+ );useEffect(() => {evaluatorSetupStore.getState()@@ -369,6 +379,9 @@ export function EvaluatorSetupPage(onSuccess: (result) => {setTestResult(result);if ("executionTraceId" in result) setHasCompletedTestCall(true);+ if ("success" in result && result.success) {+ onboardingAnalytics?.completeStep({ stepName: "evaluator_tested" });+ }setLastTestRunCostUsd("estimatedCostUsd" in result &&typeof result.estimatedCostUsd === "number"@@ -566,6 +579,10 @@ export function EvaluatorSetupPage(definition,});hasCreatedRef.current = true;+ onboardingAnalytics?.completeStep({+ stepName: "evaluator_saved",+ isBlocked: !shouldOfferRuleAttachment(evaluator),+ });capture("evaluators:create", {...getEvaluatorCreationAnalyticsProperties({evaluatorType: state.type,@@ -674,7 +691,13 @@ export function EvaluatorSetupPage(providerGroups={projectDefaultModel.providerGroups}providerAdapters={projectDefaultModel.providerAdapters}canSetProjectDefault={projectDefaultModel.canUpdate}- onConfigureProviders={projectDefaultModel.openProviderSettings}+ onConfigureProviders={() => {+ onboardingAnalytics?.track(+ "eval:onboarding_llm_connection_tab_opened",+ {},+ );+ projectDefaultModel.openProviderSettings();+ }}onSetProjectDefault={projectDefaultModel.update.requestUpdate}codeValidationResult={codeValidation.isPending ? null : codeValidation.validationResult@@ -687,8 +710,13 @@ export function EvaluatorSetupPage(? { state: "generating" }: {state: "idle",- onGenerate: () =>- requestNameSuggestion(true).catch(trpcErrorToast),+ onGenerate: () => {+ onboardingAnalytics?.track(+ "eval:onboarding_ai_generate_requested",+ { field: "name" },+ );+ requestNameSuggestion(true).catch(trpcErrorToast);+ },}}descriptionAIAssistance={@@ -700,6 +728,10 @@ export function EvaluatorSetupPage(state: "idle",// requestDescriptionSuggestion reports its own failures.onGenerate: () => {+ onboardingAnalytics?.track(+ "eval:onboarding_ai_generate_requested",+ { field: "description" },+ );requestDescriptionSuggestion();},}@@ -717,6 +749,10 @@ export function EvaluatorSetupPage(projectId={projectId}timeRange={absoluteTimeRange}onOpenTrace={(observation) => {+ onboardingAnalytics?.track(+ "eval:onboarding_sample_observation_previewed",+ {},+ );if (observation.traceId) {sampleTracePeekNavigation.openPeek(observation.traceId);}@@ -786,81 +822,84 @@ export function EvaluatorSetupPage() : undefined,}}>- <div className="flex min-h-0 flex-1 flex-col">- <TableHeaderControls- timeRange={timeRange}- setTimeRange={setTimeRange}- />- {initialEvaluator?.blockedAt ? (- <div className="mx-3 mt-3">- <EvaluatorBlockedBanner- projectId={projectId}- blockedAt={initialEvaluator.blockedAt}- blockReason={initialEvaluator.blockReason}- blockMessage={initialEvaluator.blockMessage}- canReactivate={canReactivate}- reactivationPending={reactivate.isPending}- onReactivate={() => {- capture("evaluators:reactivate", {- blockReason:- initialEvaluator.blockReason ?? "EVAL_MODEL_CONFIG_INVALID",- });- reactivate.mutate({- projectId,- evaluatorId: initialEvaluator.id,- });- }}- />- </div>- ) : null}- {isMobile ? (- <div className="min-h-0 flex-1 overflow-y-auto">- <div>{evaluatorEditor}</div>- <div className="border-t [&>aside]:h-auto">- {evaluatorTestPanel}+ <EvalOnboardingAnalyticsProvider value={onboardingAnalytics}>+ <div className="flex min-h-0 flex-1 flex-col">+ <TableHeaderControls+ timeRange={timeRange}+ setTimeRange={setTimeRange}+ />+ {initialEvaluator?.blockedAt ? (+ <div className="mx-3 mt-3">+ <EvaluatorBlockedBanner+ projectId={projectId}+ blockedAt={initialEvaluator.blockedAt}+ blockReason={initialEvaluator.blockReason}+ blockMessage={initialEvaluator.blockMessage}+ canReactivate={canReactivate}+ reactivationPending={reactivate.isPending}+ onReactivate={() => {+ capture("evaluators:reactivate", {+ blockReason:+ initialEvaluator.blockReason ??+ "EVAL_MODEL_CONFIG_INVALID",+ });+ reactivate.mutate({+ projectId,+ evaluatorId: initialEvaluator.id,+ });+ }}+ /></div>- </div>- ) : (- <ResizableSplitLayout- className="h-auto min-h-0 flex-1"- primaryContent={evaluatorEditor}- secondaryContent={evaluatorTestPanel}- open={testPanelOpen}- defaultPrimarySize={60}- defaultSecondarySize={40}- minPrimarySize={30}- minSecondarySize="360px"- collapsedSecondarySize="48px"- onOpenChange={- evaluatorSetupStore.getState().actions.setTestPanelOpen+ ) : null}+ {isMobile ? (+ <div className="min-h-0 flex-1 overflow-y-auto">+ <div>{evaluatorEditor}</div>+ <div className="border-t [&>aside]:h-auto">+ {evaluatorTestPanel}+ </div>+ </div>+ ) : (+ <ResizableSplitLayout+ className="h-auto min-h-0 flex-1"+ primaryContent={evaluatorEditor}+ secondaryContent={evaluatorTestPanel}+ open={testPanelOpen}+ defaultPrimarySize={60}+ defaultSecondarySize={40}+ minPrimarySize={30}+ minSecondarySize="360px"+ collapsedSecondarySize="48px"+ onOpenChange={+ evaluatorSetupStore.getState().actions.setTestPanelOpen+ }+ persistId="evaluator-test-panel"+ />+ )}+ <EvaluatorSetupFooter+ store={evaluatorSetupStore}+ initialSnapshot={initialSnapshot.current}+ isEditing={Boolean(initialEvaluator)}+ isSaving={+ saveInFlight ||+ create.isPending ||+ update.isPending ||+ suggestName.isPending ||+ suggestDescription.isPending+ }+ nameAIAssistanceAvailable={nameAIAssistanceAvailable}+ codeValidation={+ codeDraft.type === "CODE"+ ? {+ isValid: codeValidation.isValid,+ isPending: codeValidation.isPending,+ }+ : null}- persistId="evaluator-test-panel"+ onClose={requestClose}+ onSave={save}/>- )}- <EvaluatorSetupFooter- store={evaluatorSetupStore}- initialSnapshot={initialSnapshot.current}- isEditing={Boolean(initialEvaluator)}- isSaving={- saveInFlight ||- create.isPending ||- update.isPending ||- suggestName.isPending ||- suggestDescription.isPending- }- nameAIAssistanceAvailable={nameAIAssistanceAvailable}- codeValidation={- codeDraft.type === "CODE"- ? {- isValid: codeValidation.isValid,- isPending: codeValidation.isPending,- }- : null- }- onClose={requestClose}- onSave={save}- />- </div>+ </div>+ </EvalOnboardingAnalyticsProvider>{initialEvaluator ? (<EvaluatorVersionHistorySheetopen={historyOpen}@@ -934,6 +973,7 @@ export function EvaluatorSetupPage(<EvaluatorSavedDialogContainerprojectId={projectId}evaluator={savedEvaluator}+ onboardingAnalytics={onboardingAnalytics}onDismiss={async () => {await router.push(`/project/${projectId}/evals/${savedEvaluator.id}`,diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsxindex a1581da11b..3f9effe33f 100644--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx@@ -287,6 +287,13 @@ export default function EvaluatorsPage() {evaluators.data.totalItems === 0 &&!searchQuery &&filterState.length === 0;+ const openGalleryFromNewEvaluatorButton = () => {+ capture("eval:onboarding_started", {+ entryPoint: "new_evaluator_button",+ hasExistingEvaluators: evaluators.isSuccess ? !showOnboarding : undefined,+ });+ setGalleryOpen(true);+ };const hasExecutionReadAccess = useHasProjectAccess({projectId,scope: "evalJobExecution:read",@@ -527,11 +534,15 @@ export default function EvaluatorsPage() {onEdit={() =>router.push(`/project/${projectId}/evals/${row.original.id}`)}- onClone={() =>- router.push(+ onClone={() => {+ capture("eval:onboarding_started", {+ entryPoint: "clone_evaluator",+ hasExistingEvaluators: true,+ });+ return router.push(`/project/${projectId}/evals/new?evaluatorId=${encodeURIComponent(row.original.id)}`,- )- }+ );+ }}onDelete={() => setDeleteIds([row.original.id])}/></div>@@ -539,6 +550,7 @@ export default function EvaluatorsPage() {},],[+ capture,costs.data,costs.isPending,hasExecutionReadAccess,@@ -620,7 +632,7 @@ export default function EvaluatorsPage() {"Create reusable evaluator definitions and test them before activation.",},mobileActionButtons: showOnboarding ? (- <Button size="sm" onClick={() => setGalleryOpen(true)}>+ <Button size="sm" onClick={openGalleryFromNewEvaluatorButton}><Plus className="mr-2 h-4 w-4" />New evaluator</Button>@@ -678,7 +690,7 @@ export default function EvaluatorsPage() {{...evaluatorAlerts}/>)}- <Button onClick={() => setGalleryOpen(true)}>+ <Button onClick={openGalleryFromNewEvaluatorButton}><Plus className="mr-2 h-4 w-4" />New evaluator</Button>diff --git a/web/src/features/evals/v2/types/evalOnboardingAnalytics.ts b/web/src/features/evals/v2/types/evalOnboardingAnalytics.tsnew file mode 100644index 0000000000..9407ad55c5--- /dev/null+++ b/web/src/features/evals/v2/types/evalOnboardingAnalytics.ts@@ -0,0 +1,109 @@+import type { EvalTemplateType } from "@langfuse/shared";+import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";++// Funnel for creating an evaluator: gallery -> setup page -> saved dialog.+// Metadata only — enums, counts, lengths, booleans and Langfuse-owned template+// or section keys. Never search text, prompt content, evaluator/rule names,+// filter values or custom template ids.++type EvaluatorContext = { evaluatorType: EvalTemplateType };++type ExecutionPath = "test_filters" | "existing_rule" | "new_rule";++type Execution = {+ executionPath: ExecutionPath;+ hasBackfill: boolean;+ // Unknown for a rule created in the rule editor.+ samplingPercent?: number;+};++type TemplateSelection = EvaluatorContext & {+ managedTemplateKey?: string;+ isCustomTemplate: boolean;+};++type OnboardingStep =+ | (EvaluatorContext & {+ stepName: "new_from_scratch_selected";+ surface: "gallery";+ })+ | (TemplateSelection & {+ stepName: "suggestion_selected";+ surface: "gallery" | "empty_state";+ })+ | (TemplateSelection & {+ stepName: "template_selected";+ surface: "gallery";+ sectionKey: string;+ })+ | (EvaluatorContext & {+ stepName: "model_selected";+ modelMode: "default" | "custom";+ })+ | (EvaluatorContext & { stepName: "score_output_modified" })+ | (EvaluatorContext & {+ stepName: "variable_mapping_updated";+ method: "tree" | "json_path";+ })+ | (EvaluatorContext & { stepName: "evaluator_name_updated" })+ | (EvaluatorContext & {+ stepName: "evaluator_filter_changed";+ filterExperience: EvaluatorFilterExperience;+ filterCount: number;+ })+ | (EvaluatorContext & { stepName: "evaluator_tested" })+ | (EvaluatorContext & { stepName: "evaluator_saved"; isBlocked: boolean })+ | (EvaluatorContext & Execution & { stepName: "execute" });++export type EvalOnboardingEventMap = {+ "eval:onboarding_started": {+ entryPoint:+ | "new_evaluator_button"+ | "empty_state_starting_point"+ | "empty_state_browse_templates"+ | "clone_evaluator"+ | "batch_evaluation";+ hasExistingEvaluators?: boolean;+ };+ // Fires at most once per step and setup attempt; the interaction events+ // below fire on every action.+ "eval:onboarding_step_completed": OnboardingStep;+ "eval:onboarding_completed": EvaluatorContext & Execution;+ "eval:onboarding_gallery_searched": { queryLength: number };+ "eval:onboarding_gallery_section_selected": { sectionKey: string };+ "eval:onboarding_evaluator_type_changed": EvaluatorContext & {+ previousEvaluatorType: EvalTemplateType;+ };+ "eval:onboarding_preview_toggled": EvaluatorContext & { isEnabled: boolean };+ "eval:onboarding_sample_observation_previewed": EvaluatorContext;+ // `message_edited` (content or role, typed) fires once per setup attempt;+ // adding, removing and reordering messages fire on every action.+ "eval:onboarding_prompt_modified": EvaluatorContext & {+ modification:+ | "message_edited"+ | "message_added"+ | "message_removed"+ | "messages_reordered";+ };+ "eval:onboarding_llm_connection_tab_opened": EvaluatorContext;+ "eval:onboarding_model_picker_opened": EvaluatorContext;+ "eval:onboarding_model_changed": EvaluatorContext & {+ modelMode: "default" | "custom";+ };+ "eval:onboarding_ai_generate_requested": EvaluatorContext & {+ field: "name" | "description";+ };+ // Once per setup attempt: the slider reports every drag tick. The final+ // value is on `eval:onboarding_completed`.+ "eval:onboarding_sampling_changed": EvaluatorContext;+ "eval:onboarding_historic_eval_toggled": EvaluatorContext & {+ isEnabled: boolean;+ };+ "eval:onboarding_scope_changed": EvaluatorContext & {+ scope: "test_filters" | "different_scope";+ };+ "eval:onboarding_create_rule_opened": EvaluatorContext;+ "eval:onboarding_execution_skipped": EvaluatorContext & {+ method: "skip_button" | "dialog_dismissed" | "rule_editor_closed";+ };+};diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.tsindex 9510cfabf3..d617cf22a1 100644--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts@@ -2,6 +2,7 @@ import { type CaptureResult, type CaptureOptions } from "posthog-js";import { usePostHog } from "posthog-js/react";import { useCallback } from "react";import type { AnnotationEventMap } from "@/src/features/scores/lib/annotationAnalytics";+import type { EvalOnboardingEventMap } from "@/src/features/evals/v2/types/evalOnboardingAnalytics";export const V4_BETA_ENABLED_POSTHOG_PROPERTY = "v4BetaEnabled";@@ -219,6 +220,29 @@ const events = {"detach_evaluator","filter_reused",],+ // Evaluator creation funnel (gallery -> setup page -> saved dialog). Props+ // are typed in EvalOnboardingEventMap: metadata only, never search text,+ // prompt content, names or filter values.+ eval: [+ "onboarding_started",+ "onboarding_step_completed",+ "onboarding_completed",+ "onboarding_gallery_searched",+ "onboarding_gallery_section_selected",+ "onboarding_evaluator_type_changed",+ "onboarding_preview_toggled",+ "onboarding_sample_observation_previewed",+ "onboarding_prompt_modified",+ "onboarding_llm_connection_tab_opened",+ "onboarding_model_picker_opened",+ "onboarding_model_changed",+ "onboarding_ai_generate_requested",+ "onboarding_sampling_changed",+ "onboarding_historic_eval_toggled",+ "onboarding_scope_changed",+ "onboarding_create_rule_opened",+ "onboarding_execution_skipped",+ ],// One-shot batch evaluation from the events / experiments tables.// Counts and enums only — never mapping contents or observation payloads.batch_eval: ["run"],@@ -466,11 +490,10 @@ type EventName = {[Resource in keyof typeof events]: `${Resource}:${(typeof events)[Resource][number]}`;}[keyof typeof events];-type EventProperties = AnnotationEventMap & {- [E in Exclude<EventName, keyof AnnotationEventMap>]: Record<- string,- any- > | null;+type TypedEventMap = AnnotationEventMap & EvalOnboardingEventMap;++type EventProperties = TypedEventMap & {+ [E in Exclude<EventName, keyof TypedEventMap>]: Record<string, any> | null;};export const usePostHogClientCapture = () => {