🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): track filter analytics (#17066)
Tobias Wochinger70363626842b
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsxindex 70f4ad4c11..59cafeb9b7 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx@@ -24,6 +24,7 @@ import { api, type RouterOutputs } from "@/src/utils/api";import { trpcErrorToast } from "@/src/utils/trpcErrorToast";import { cn } from "@/src/utils/tailwind";import { classifySampleFiltersForRule } from "@/src/features/evals/v2/fns/rules/classifySampleFiltersForRule";+import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";import { EvaluatorSavedRuleFilterPreview } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedRuleFilterPreview";type Rule = RouterOutputs["evalsV2"]["rules"]["list"]["rules"][number];@@ -162,7 +163,7 @@ export function EvaluatorSavedDialogContainer({if (result.action === "created") {capture("evaluation_rules:create", {assignmentCount: 1,- filterCount: supportedRuleFilters.length,+ ...getFilterAnalyticsProperties(supportedRuleFilters),samplingPercent: Math.round(sampling * 100),isEnabled: true,source: "evaluator_create_test_filters",diff --git a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsxindex 76161ddce3..0d03058411 100644--- a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx+++ b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx@@ -22,6 +22,7 @@ import { trpcErrorToast } from "@/src/utils/trpcErrorToast";import { useLangfuseCloudRegion } from "@/src/features/organizations/hooks";import { useProject } from "@/src/features/projects/hooks";import { prepareNameForSave } from "@/src/features/evals/v2/fns/prepareNameForSave";+import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";import { resolveInitialRuleFilters } from "./resolveInitialRuleFilters";export function CreateRuleDialogContent({@@ -113,7 +114,7 @@ export function CreateRuleDialogContent({});capture("evaluation_rules:create", {assignmentCount: draft.assignments.length,- filterCount: draft.filter.length,+ ...getFilterAnalyticsProperties(draft.filter),samplingPercent: Math.round(draft.sampling * 100),isEnabled: true,});diff --git a/web/src/features/evals/v2/components/Rules/EditRuleDialog/components/EditRuleDialogContent/EditRuleDialogContent.tsx b/web/src/features/evals/v2/components/Rules/EditRuleDialog/components/EditRuleDialogContent/EditRuleDialogContent.tsxindex 0fdca6d761..5f8204f9f9 100644--- a/web/src/features/evals/v2/components/Rules/EditRuleDialog/components/EditRuleDialogContent/EditRuleDialogContent.tsx+++ b/web/src/features/evals/v2/components/Rules/EditRuleDialog/components/EditRuleDialogContent/EditRuleDialogContent.tsx@@ -14,6 +14,7 @@ import type { RuleEvaluatorOption } from "@/src/features/evals/v2/types/rules";import { usePostHogClientCapture } from "@/src/features/posthog-analytics";import { api, type RouterOutputs } from "@/src/utils/api";import { trpcErrorToast } from "@/src/utils/trpcErrorToast";+import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";type Rule = RouterOutputs["evalsV2"]["rules"]["get"];@@ -93,7 +94,7 @@ export function EditRuleDialogContent({});capture("evaluation_rules:update", {assignmentCount: draft.assignments.length,- filterCount: draft.filter.length,+ ...getFilterAnalyticsProperties(draft.filter),samplingPercent: Math.round(draft.sampling * 100),isEnabled: rule.enabled,});diff --git a/web/src/features/evals/v2/fns/getFilterAnalyticsProperties.clienttest.ts b/web/src/features/evals/v2/fns/getFilterAnalyticsProperties.clienttest.tsnew file mode 100644index 0000000000..3383030bbe--- /dev/null+++ b/web/src/features/evals/v2/fns/getFilterAnalyticsProperties.clienttest.ts@@ -0,0 +1,58 @@+import type { FilterState } from "@langfuse/shared";+import { describe, expect, it } from "vitest";++import { getFilterAnalyticsProperties } from "./getFilterAnalyticsProperties";++describe("getFilterAnalyticsProperties", () => {+ it("reports filter columns without filter values", () => {+ const filters = [+ {+ column: "type",+ type: "stringOptions",+ operator: "any of",+ value: ["GENERATION"],+ },+ {+ column: "type",+ type: "stringOptions",+ operator: "none of",+ value: ["SPAN"],+ },+ {+ column: "metadata",+ key: "customerId",+ type: "stringObject",+ operator: "=",+ value: "private-customer-id",+ },+ ] satisfies FilterState;++ expect(getFilterAnalyticsProperties(filters)).toEqual({+ filterCount: 3,+ filterColumns: ["type", "metadata"],+ usesExperimentFilter: false,+ });+ });++ it.each([+ "experimentId",+ "experimentName",+ "experimentDatasetId",+ "isExperimentItemRootSpan",+ ])("identifies the %s experiment filter", (column) => {+ const filters = [+ {+ column,+ type: "string",+ operator: "=",+ value: "value",+ },+ ] satisfies FilterState;++ expect(getFilterAnalyticsProperties(filters)).toEqual({+ filterCount: 1,+ filterColumns: [column],+ usesExperimentFilter: true,+ });+ });+});diff --git a/web/src/features/evals/v2/fns/getFilterAnalyticsProperties.ts b/web/src/features/evals/v2/fns/getFilterAnalyticsProperties.tsnew file mode 100644index 0000000000..8daa3e2d81--- /dev/null+++ b/web/src/features/evals/v2/fns/getFilterAnalyticsProperties.ts@@ -0,0 +1,20 @@+import type { FilterState } from "@langfuse/shared";++const EXPERIMENT_FILTER_COLUMNS = new Set([+ "experimentId",+ "experimentName",+ "experimentDatasetId",+ "isExperimentItemRootSpan",+]);++export function getFilterAnalyticsProperties(filters: FilterState) {+ const filterColumns = [...new Set(filters.map(({ column }) => column))];++ return {+ filterCount: filters.length,+ filterColumns,+ usesExperimentFilter: filterColumns.some((column) =>+ EXPERIMENT_FILTER_COLUMNS.has(column),+ ),+ };+}diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsxindex ec396fb03e..d76f622c9d 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx@@ -63,6 +63,7 @@ import {getJudgePromptAnalyticsProperties,type EvaluatorCreationSource,} from "@/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties";+import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";type InitialEvaluator = {id: string;@@ -503,6 +504,7 @@ export function EvaluatorSetupPage(capture("evaluators:update", {evaluatorType: state.type,filterExperience,+ ...getFilterAnalyticsProperties(state.sampleFilter),...(definition.type === "LLM_AS_JUDGE"? getJudgePromptAnalyticsProperties(definition.promptMessages): {}),@@ -550,6 +552,7 @@ export function EvaluatorSetupPage(: undefined,}),filterExperience,+ ...getFilterAnalyticsProperties(state.sampleFilter),});initialSnapshot.current = getCurrentSnapshot(state);await utils.evalsV2.filterOptions.invalidate({ projectId });