🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix: translate event metadata null filters to physical columns (#17597)
Valery Meleshkin4a78fcf3ce47
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.tsindex 952c504f7f..81d14017be 100644--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts@@ -10,6 +10,8 @@ import {StringFilter,StringObjectFilter,} from "./clickhouse-filter";+import { createFilterFromFilterState } from "./factory";+import { experimentItemsTableNativeUiColumnDefinitions } from "../../tableMappings/mapExperimentItemsTable";// events_core_mv truncates metadata values to leftUTF8(v, 200), so the safety// boundary is 200 Unicode code points.@@ -198,6 +200,76 @@ describe("StringObjectFilter key presence/absence operators", () => {});});+describe("NullFilter metadata filters on events tables", () => {+ it.each([+ ["metadata", "metadata_names"],+ ["experiment_metadata", "experiment_metadata_names"],+ ["experiment_item_metadata", "experiment_item_metadata_names"],+ ] as const)(+ "compiles `%s` null checks against `%s`",+ (field, namesColumn) => {+ const isNullFilter = new NullFilter({+ clickhouseTable: "events_proto",+ field,+ operator: "is null",+ tablePrefix: "e",+ });+ const isNotNullFilter = new NullFilter({+ clickhouseTable: "events_proto",+ field,+ operator: "is not null",+ tablePrefix: "e",+ });++ expect(isNullFilter.apply()).toEqual({+ query: `empty(e.${namesColumn})`,+ params: {},+ });+ expect(isNotNullFilter.apply()).toEqual({+ query: `notEmpty(e.${namesColumn})`,+ params: {},+ });+ },+ );++ it("keeps null filters on legacy map tables unchanged", () => {+ const filter = new NullFilter({+ clickhouseTable: "traces",+ field: "metadata",+ operator: "is null",+ });++ expect(filter.apply()).toEqual({+ query: "metadata is null",+ params: {},+ });+ });+});++describe("NullFilter metadata aliases through table mappings", () => {+ it("compiles experiment item metadata null filters against its names array", () => {+ const [filter] = createFilterFromFilterState(+ [+ {+ column: "itemMetadata",+ operator: "is null",+ value: "",+ type: "null",+ },+ ],+ experimentItemsTableNativeUiColumnDefinitions,+ );++ expect(filter).toBeDefined();+ if (!filter) throw new Error("expected filter");++ expect(filter.apply()).toEqual({+ query: "empty(e.experiment_item_metadata_names)",+ params: {},+ });+ });+});+describe("StringObjectFilter empty-value rejection", () => {it.each(["contains", "starts with", "ends with"] as const)("rejects an empty value for the substring operator `%s`",diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.tsindex a1082f819e..697a68c93d 100644--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts@@ -9,6 +9,7 @@ import { clickhouseCompliantRandomCharacters } from "../../repositories";import { escapeSqlLikePattern } from "../../utils/sqlLike";import {assertValidFtsMatchFilter,+ bareFtsField,FTS_OPERATOR_DESCRIPTORS,isFtsEventsTable,isFtsMetadataField,@@ -52,6 +53,15 @@ const NON_EMPTY_VALUE_METADATA_OPERATORS = new Set<(typeof filterOperators)["stringObject"][number]>(["contains", "starts with", "ends with"]);+// Event tables expose these computed map aliases while storing each map as a+// pair of physical arrays. Null checks must target the corresponding names+// array because the qualified map aliases are not physical columns.+const EVENTS_METADATA_MAP_FIELDS = new Set([+ "metadata",+ "experiment_metadata",+ "experiment_item_metadata",+]);+export class StringFilter implements Filter {public clickhouseTable: string;public field: string;@@ -635,6 +645,22 @@ export class NullFilter implements Filter {apply(): ClickhouseFilter {const fieldWithPrefix = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;+ // Event metadata aliases are computed maps. Their physical names arrays+ // are non-nullable, so emptiness represents a null/absent map.+ if (+ isFtsEventsTable(this.clickhouseTable) &&+ EVENTS_METADATA_MAP_FIELDS.has(bareFtsField(this.field))+ ) {+ const metadataNames = `${fieldWithPrefix}_names`;+ return {+ query:+ this.operator === "is null"+ ? `empty(${metadataNames})`+ : `notEmpty(${metadataNames})`,+ params: {},+ };+ }+// '' ≡ NULL: treat empty string and NULL as the same valueif (this.emptyEqualsNull) {const isNull = this.operator === "is null";diff --git a/web/src/__tests__/server/event-query-builder.servertest.ts b/web/src/__tests__/server/event-query-builder.servertest.tsindex b202bf1ede..a5e55f2494 100644--- a/web/src/__tests__/server/event-query-builder.servertest.ts+++ b/web/src/__tests__/server/event-query-builder.servertest.ts@@ -8,6 +8,7 @@ import {EventsAggregationQueryBuilder,EventsQueryBuilder,eventsTableUiColumnDefinitions,+ experimentPreAggCols,ExperimentsAggregationQueryBuilder,} from "@langfuse/shared/src/server";import {@@ -237,6 +238,51 @@ describe("buildEventsFilterOptionsForColumnsQuery", () => {expect(built.params).not.toHaveProperty("optionReserved");});+ it("translates whole metadata null filters to the physical events columns", () => {+ const built = buildEventsFilterOptionsForColumnsQuery({+ projectId: "test-project",+ filter: [+ {+ column: "metadata",+ operator: "is null",+ value: "",+ type: "null",+ },+ ],+ columns: ["name"],+ limit: 10,+ });++ expect(built).not.toBeNull();+ if (!built) throw new Error("expected query");++ expect(built.query).toContain("empty(e.metadata_names)");+ expect(built.query).not.toContain("e.metadata is null");+ expect(built.query).toContain("FROM events_core e");+ });++ it("translates experiment metadata null filters through its table mapping", () => {+ const [filter] = createFilterFromFilterState(+ [+ {+ column: "metadata",+ operator: "is null",+ value: "",+ type: "null",+ },+ ],+ experimentPreAggCols,+ );++ expect(filter).toBeDefined();+ if (!filter) throw new Error("expected filter");++ expect(filter.apply()).toEqual({+ query: "empty(e.experiment_metadata_names)",+ params: {},+ });+ });+it("reuses row-selection score dependencies and full-table routing", () => {const built = buildEventsFilterOptionsForColumnsQuery({projectId: "test-project",