Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix: translate event metadata null filters to physical columns (#17597)

Valery Meleshkin4a78fcf3ce47
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts
index 952c504f7f..81d14017be 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts
@@ -10,6 +10,8 @@ import {
StringFilter,
StringObjectFilter,
} from "./clickhouse-filter";
+import { createFilterFromFilterState } from "./factory";
+import { experimentItemsTableNativeUiColumnDefinitions } from "../../tableMappings/mapExperimentItemsTable";
// events_core_mv truncates metadata values to leftUTF8(v, 200), so the safety
// boundary is 200 Unicode code points.
@@ -198,6 +200,76 @@ describe("StringObjectFilter key presence/absence operators", () => {
});
});
+describe("NullFilter metadata filters on events tables", () => {
+ it.each([
+ ["metadata", "metadata_names"],
+ ["experiment_metadata", "experiment_metadata_names"],
+ ["experiment_item_metadata", "experiment_item_metadata_names"],
+ ] as const)(
+ "compiles `%s` null checks against `%s`",
+ (field, namesColumn) => {
+ const isNullFilter = new NullFilter({
+ clickhouseTable: "events_proto",
+ field,
+ operator: "is null",
+ tablePrefix: "e",
+ });
+ const isNotNullFilter = new NullFilter({
+ clickhouseTable: "events_proto",
+ field,
+ operator: "is not null",
+ tablePrefix: "e",
+ });
+
+ expect(isNullFilter.apply()).toEqual({
+ query: `empty(e.${namesColumn})`,
+ params: {},
+ });
+ expect(isNotNullFilter.apply()).toEqual({
+ query: `notEmpty(e.${namesColumn})`,
+ params: {},
+ });
+ },
+ );
+
+ it("keeps null filters on legacy map tables unchanged", () => {
+ const filter = new NullFilter({
+ clickhouseTable: "traces",
+ field: "metadata",
+ operator: "is null",
+ });
+
+ expect(filter.apply()).toEqual({
+ query: "metadata is null",
+ params: {},
+ });
+ });
+});
+
+describe("NullFilter metadata aliases through table mappings", () => {
+ it("compiles experiment item metadata null filters against its names array", () => {
+ const [filter] = createFilterFromFilterState(
+ [
+ {
+ column: "itemMetadata",
+ operator: "is null",
+ value: "",
+ type: "null",
+ },
+ ],
+ experimentItemsTableNativeUiColumnDefinitions,
+ );
+
+ expect(filter).toBeDefined();
+ if (!filter) throw new Error("expected filter");
+
+ expect(filter.apply()).toEqual({
+ query: "empty(e.experiment_item_metadata_names)",
+ params: {},
+ });
+ });
+});
+
describe("StringObjectFilter empty-value rejection", () => {
it.each(["contains", "starts with", "ends with"] as const)(
"rejects an empty value for the substring operator `%s`",
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
index a1082f819e..697a68c93d 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
@@ -9,6 +9,7 @@ import { clickhouseCompliantRandomCharacters } from "../../repositories";
import { escapeSqlLikePattern } from "../../utils/sqlLike";
import {
assertValidFtsMatchFilter,
+ bareFtsField,
FTS_OPERATOR_DESCRIPTORS,
isFtsEventsTable,
isFtsMetadataField,
@@ -52,6 +53,15 @@ const NON_EMPTY_VALUE_METADATA_OPERATORS = new Set<
(typeof filterOperators)["stringObject"][number]
>(["contains", "starts with", "ends with"]);
+// Event tables expose these computed map aliases while storing each map as a
+// pair of physical arrays. Null checks must target the corresponding names
+// array because the qualified map aliases are not physical columns.
+const EVENTS_METADATA_MAP_FIELDS = new Set([
+ "metadata",
+ "experiment_metadata",
+ "experiment_item_metadata",
+]);
+
export class StringFilter implements Filter {
public clickhouseTable: string;
public field: string;
@@ -635,6 +645,22 @@ export class NullFilter implements Filter {
apply(): ClickhouseFilter {
const fieldWithPrefix = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
+ // Event metadata aliases are computed maps. Their physical names arrays
+ // are non-nullable, so emptiness represents a null/absent map.
+ if (
+ isFtsEventsTable(this.clickhouseTable) &&
+ EVENTS_METADATA_MAP_FIELDS.has(bareFtsField(this.field))
+ ) {
+ const metadataNames = `${fieldWithPrefix}_names`;
+ return {
+ query:
+ this.operator === "is null"
+ ? `empty(${metadataNames})`
+ : `notEmpty(${metadataNames})`,
+ params: {},
+ };
+ }
+
// '' ≡ NULL: treat empty string and NULL as the same value
if (this.emptyEqualsNull) {
const isNull = this.operator === "is null";
diff --git a/web/src/__tests__/server/event-query-builder.servertest.ts b/web/src/__tests__/server/event-query-builder.servertest.ts
index b202bf1ede..a5e55f2494 100644
--- a/web/src/__tests__/server/event-query-builder.servertest.ts
+++ b/web/src/__tests__/server/event-query-builder.servertest.ts
@@ -8,6 +8,7 @@ import {
EventsAggregationQueryBuilder,
EventsQueryBuilder,
eventsTableUiColumnDefinitions,
+ experimentPreAggCols,
ExperimentsAggregationQueryBuilder,
} from "@langfuse/shared/src/server";
import {
@@ -237,6 +238,51 @@ describe("buildEventsFilterOptionsForColumnsQuery", () => {
expect(built.params).not.toHaveProperty("optionReserved");
});
+ it("translates whole metadata null filters to the physical events columns", () => {
+ const built = buildEventsFilterOptionsForColumnsQuery({
+ projectId: "test-project",
+ filter: [
+ {
+ column: "metadata",
+ operator: "is null",
+ value: "",
+ type: "null",
+ },
+ ],
+ columns: ["name"],
+ limit: 10,
+ });
+
+ expect(built).not.toBeNull();
+ if (!built) throw new Error("expected query");
+
+ expect(built.query).toContain("empty(e.metadata_names)");
+ expect(built.query).not.toContain("e.metadata is null");
+ expect(built.query).toContain("FROM events_core e");
+ });
+
+ it("translates experiment metadata null filters through its table mapping", () => {
+ const [filter] = createFilterFromFilterState(
+ [
+ {
+ column: "metadata",
+ operator: "is null",
+ value: "",
+ type: "null",
+ },
+ ],
+ experimentPreAggCols,
+ );
+
+ expect(filter).toBeDefined();
+ if (!filter) throw new Error("expected filter");
+
+ expect(filter.apply()).toEqual({
+ query: "empty(e.experiment_metadata_names)",
+ params: {},
+ });
+ });
+
it("reuses row-selection score dependencies and full-table routing", () => {
const built = buildEventsFilterOptionsForColumnsQuery({
projectId: "test-project",