Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): track filter analytics (#17066)

Tobias Wochinger70363626842b
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
index 70f4ad4c11..59cafeb9b7 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
@@ -24,6 +24,7 @@ import { api, type RouterOutputs } from "@/src/utils/api";
import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
import { cn } from "@/src/utils/tailwind";
import { classifySampleFiltersForRule } from "@/src/features/evals/v2/fns/rules/classifySampleFiltersForRule";
+import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";
import { EvaluatorSavedRuleFilterPreview } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedRuleFilterPreview";
type Rule = RouterOutputs["evalsV2"]["rules"]["list"]["rules"][number];
@@ -162,7 +163,7 @@ export function EvaluatorSavedDialogContainer({
if (result.action === "created") {
capture("evaluation_rules:create", {
assignmentCount: 1,
- filterCount: supportedRuleFilters.length,
+ ...getFilterAnalyticsProperties(supportedRuleFilters),
samplingPercent: Math.round(sampling * 100),
isEnabled: true,
source: "evaluator_create_test_filters",
diff --git a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx
index 76161ddce3..0d03058411 100644
--- a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx
+++ b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx
@@ -22,6 +22,7 @@ import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
import { useLangfuseCloudRegion } from "@/src/features/organizations/hooks";
import { useProject } from "@/src/features/projects/hooks";
import { prepareNameForSave } from "@/src/features/evals/v2/fns/prepareNameForSave";
+import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";
import { resolveInitialRuleFilters } from "./resolveInitialRuleFilters";
export function CreateRuleDialogContent({
@@ -113,7 +114,7 @@ export function CreateRuleDialogContent({
});
capture("evaluation_rules:create", {
assignmentCount: draft.assignments.length,
- filterCount: draft.filter.length,
+ ...getFilterAnalyticsProperties(draft.filter),
samplingPercent: Math.round(draft.sampling * 100),
isEnabled: true,
});
diff --git a/web/src/features/evals/v2/components/Rules/EditRuleDialog/components/EditRuleDialogContent/EditRuleDialogContent.tsx b/web/src/features/evals/v2/components/Rules/EditRuleDialog/components/EditRuleDialogContent/EditRuleDialogContent.tsx
index 0fdca6d761..5f8204f9f9 100644
--- a/web/src/features/evals/v2/components/Rules/EditRuleDialog/components/EditRuleDialogContent/EditRuleDialogContent.tsx
+++ b/web/src/features/evals/v2/components/Rules/EditRuleDialog/components/EditRuleDialogContent/EditRuleDialogContent.tsx
@@ -14,6 +14,7 @@ import type { RuleEvaluatorOption } from "@/src/features/evals/v2/types/rules";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import { api, type RouterOutputs } from "@/src/utils/api";
import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
+import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";
type Rule = RouterOutputs["evalsV2"]["rules"]["get"];
@@ -93,7 +94,7 @@ export function EditRuleDialogContent({
});
capture("evaluation_rules:update", {
assignmentCount: draft.assignments.length,
- filterCount: draft.filter.length,
+ ...getFilterAnalyticsProperties(draft.filter),
samplingPercent: Math.round(draft.sampling * 100),
isEnabled: rule.enabled,
});
diff --git a/web/src/features/evals/v2/fns/getFilterAnalyticsProperties.clienttest.ts b/web/src/features/evals/v2/fns/getFilterAnalyticsProperties.clienttest.ts
new file mode 100644
index 0000000000..3383030bbe
--- /dev/null
+++ b/web/src/features/evals/v2/fns/getFilterAnalyticsProperties.clienttest.ts
@@ -0,0 +1,58 @@
+import type { FilterState } from "@langfuse/shared";
+import { describe, expect, it } from "vitest";
+
+import { getFilterAnalyticsProperties } from "./getFilterAnalyticsProperties";
+
+describe("getFilterAnalyticsProperties", () => {
+ it("reports filter columns without filter values", () => {
+ const filters = [
+ {
+ column: "type",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["GENERATION"],
+ },
+ {
+ column: "type",
+ type: "stringOptions",
+ operator: "none of",
+ value: ["SPAN"],
+ },
+ {
+ column: "metadata",
+ key: "customerId",
+ type: "stringObject",
+ operator: "=",
+ value: "private-customer-id",
+ },
+ ] satisfies FilterState;
+
+ expect(getFilterAnalyticsProperties(filters)).toEqual({
+ filterCount: 3,
+ filterColumns: ["type", "metadata"],
+ usesExperimentFilter: false,
+ });
+ });
+
+ it.each([
+ "experimentId",
+ "experimentName",
+ "experimentDatasetId",
+ "isExperimentItemRootSpan",
+ ])("identifies the %s experiment filter", (column) => {
+ const filters = [
+ {
+ column,
+ type: "string",
+ operator: "=",
+ value: "value",
+ },
+ ] satisfies FilterState;
+
+ expect(getFilterAnalyticsProperties(filters)).toEqual({
+ filterCount: 1,
+ filterColumns: [column],
+ usesExperimentFilter: true,
+ });
+ });
+});
diff --git a/web/src/features/evals/v2/fns/getFilterAnalyticsProperties.ts b/web/src/features/evals/v2/fns/getFilterAnalyticsProperties.ts
new file mode 100644
index 0000000000..8daa3e2d81
--- /dev/null
+++ b/web/src/features/evals/v2/fns/getFilterAnalyticsProperties.ts
@@ -0,0 +1,20 @@
+import type { FilterState } from "@langfuse/shared";
+
+const EXPERIMENT_FILTER_COLUMNS = new Set([
+ "experimentId",
+ "experimentName",
+ "experimentDatasetId",
+ "isExperimentItemRootSpan",
+]);
+
+export function getFilterAnalyticsProperties(filters: FilterState) {
+ const filterColumns = [...new Set(filters.map(({ column }) => column))];
+
+ return {
+ filterCount: filters.length,
+ filterColumns,
+ usesExperimentFilter: filterColumns.some((column) =>
+ EXPERIMENT_FILTER_COLUMNS.has(column),
+ ),
+ };
+}
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
index ec396fb03e..d76f622c9d 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
@@ -63,6 +63,7 @@ import {
getJudgePromptAnalyticsProperties,
type EvaluatorCreationSource,
} from "@/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties";
+import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";
type InitialEvaluator = {
id: string;
@@ -503,6 +504,7 @@ export function EvaluatorSetupPage(
capture("evaluators:update", {
evaluatorType: state.type,
filterExperience,
+ ...getFilterAnalyticsProperties(state.sampleFilter),
...(definition.type === "LLM_AS_JUDGE"
? getJudgePromptAnalyticsProperties(definition.promptMessages)
: {}),
@@ -550,6 +552,7 @@ export function EvaluatorSetupPage(
: undefined,
}),
filterExperience,
+ ...getFilterAnalyticsProperties(state.sampleFilter),
});
initialSnapshot.current = getCurrentSnapshot(state);
await utils.evalsV2.filterOptions.invalidate({ projectId });