Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(shared): align in-memory null filters with column semantics (#17160)

Tobias Wochinger1bb49babf6ce
diff --git a/packages/shared/src/server/services/InMemoryFilterService.ts b/packages/shared/src/server/services/InMemoryFilterService.ts
index 53463e7b43..0c8dac0437 100644
--- a/packages/shared/src/server/services/InMemoryFilterService.ts
+++ b/packages/shared/src/server/services/InMemoryFilterService.ts
@@ -2,6 +2,10 @@ import { FilterCondition, FilterState } from "../../types";
import { logger } from "../logger";
import { encodeBooleanScoreEntry } from "../queries/clickhouse-sql/clickhouse-filter";
+export type InMemoryFilterOptions = {
+ emptyEqualsNullColumns?: ReadonlySet<string>;
+};
+
export class InMemoryFilterService {
/**
* Evaluates whether a data object matches the given filter conditions.
@@ -9,12 +13,14 @@ export class InMemoryFilterService {
* @param data - The data object to evaluate
* @param filter - The filter conditions to apply
* @param fieldMapper - Function to map filter column names to data object values
+ * @param options - Column-specific filter semantics
* @returns true if the data matches all filter conditions, false otherwise
*/
static evaluateFilter<T>(
data: T,
filter: FilterState,
fieldMapper: (data: T, column: string) => unknown,
+ options?: InMemoryFilterOptions,
): boolean {
try {
// If no filters, data matches
@@ -24,7 +30,9 @@ export class InMemoryFilterService {
// Evaluate each filter condition
for (const condition of filter) {
- if (!this.evaluateFilterCondition(data, condition, fieldMapper)) {
+ if (
+ !this.evaluateFilterCondition(data, condition, fieldMapper, options)
+ ) {
return false;
}
}
@@ -47,6 +55,7 @@ export class InMemoryFilterService {
data: T,
condition: FilterCondition,
fieldMapper: (data: T, column: string) => unknown,
+ options?: InMemoryFilterOptions,
): boolean {
const { column, type, operator } = condition;
@@ -111,7 +120,11 @@ export class InMemoryFilterService {
operator,
);
case "null":
- return this.evaluateNullFilter(fieldValue, operator);
+ return this.evaluateNullFilter(
+ fieldValue,
+ operator,
+ options?.emptyEqualsNullColumns?.has(column) ?? false,
+ );
case "positionInTrace":
// Position filters are applied after all other filters in DB queries.
// Ignore them in in-memory filtering.
@@ -434,14 +447,18 @@ export class InMemoryFilterService {
private static evaluateNullFilter(
fieldValue: unknown,
operator: string,
+ emptyEqualsNull: boolean,
): boolean {
+ const isNull =
+ fieldValue === null ||
+ fieldValue === undefined ||
+ (emptyEqualsNull && fieldValue === "");
+
switch (operator) {
case "is null":
- return (
- fieldValue === null || fieldValue === undefined || fieldValue === ""
- );
+ return isNull;
case "is not null":
- return fieldValue !== null && fieldValue !== undefined;
+ return !isNull;
default:
logger.error("Unsupported null filter operator", {
operator,
diff --git a/worker/src/__tests__/inMemoryFilterService.test.ts b/worker/src/__tests__/inMemoryFilterService.test.ts
index 00c4f9c486..d248159eca 100644
--- a/worker/src/__tests__/inMemoryFilterService.test.ts
+++ b/worker/src/__tests__/inMemoryFilterService.test.ts
@@ -1093,6 +1093,60 @@ describe("InMemoryFilterService", () => {
).toBe(true);
});
+ test("applies empty-string null semantics per column", () => {
+ const dataWithEmptyString = { ...mockData, release: "" };
+
+ expect(
+ InMemoryFilterService.evaluateFilter(
+ dataWithEmptyString,
+ [{ column: "release", type: "null", operator: "is null", value: "" }],
+ fieldMapper,
+ ),
+ ).toBe(false);
+
+ expect(
+ InMemoryFilterService.evaluateFilter(
+ dataWithEmptyString,
+ [
+ {
+ column: "release",
+ type: "null",
+ operator: "is not null",
+ value: "",
+ },
+ ],
+ fieldMapper,
+ ),
+ ).toBe(true);
+
+ const emptyEqualsNullColumns = new Set(["release"]);
+
+ expect(
+ InMemoryFilterService.evaluateFilter(
+ dataWithEmptyString,
+ [{ column: "release", type: "null", operator: "is null", value: "" }],
+ fieldMapper,
+ { emptyEqualsNullColumns },
+ ),
+ ).toBe(true);
+
+ expect(
+ InMemoryFilterService.evaluateFilter(
+ dataWithEmptyString,
+ [
+ {
+ column: "release",
+ type: "null",
+ operator: "is not null",
+ value: "",
+ },
+ ],
+ fieldMapper,
+ { emptyEqualsNullColumns },
+ ),
+ ).toBe(false);
+ });
+
test("evaluates multiple filters with AND logic", () => {
expect(
InMemoryFilterService.evaluateFilter(
diff --git a/worker/src/features/evaluation/observationEval/__tests__/filterEvaluation.test.ts b/worker/src/features/evaluation/observationEval/__tests__/filterEvaluation.test.ts
index d6c214b9c4..38627d2c15 100644
--- a/worker/src/features/evaluation/observationEval/__tests__/filterEvaluation.test.ts
+++ b/worker/src/features/evaluation/observationEval/__tests__/filterEvaluation.test.ts
@@ -1053,6 +1053,33 @@ describe("Filter Evaluation for Observation Evals", () => {
expect(matched).toBe(true);
});
+ it("should treat an empty parentObservationId as null", async () => {
+ const observation = createTestObservation({
+ project_id: projectId,
+ parent_span_id: "",
+ });
+
+ const isNullMatch = await testFilterMatch(observation, [
+ {
+ column: "parentObservationId",
+ type: "null",
+ operator: "is null",
+ value: "",
+ },
+ ]);
+ const isNotNullMatch = await testFilterMatch(observation, [
+ {
+ column: "parentObservationId",
+ type: "null",
+ operator: "is not null",
+ value: "",
+ },
+ ]);
+
+ expect(isNullMatch).toBe(true);
+ expect(isNotNullMatch).toBe(false);
+ });
+
it("should not match observations where parentObservationId is not null (child observations)", async () => {
const observation = createTestObservation({
project_id: projectId,
diff --git a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts
index 8ec12af32e..31f104cc6d 100644
--- a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts
+++ b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts
@@ -25,6 +25,10 @@ import {
import { createW3CTraceId } from "../../utils";
import { isInternalEvalEnvironment } from "../isEvalTargetEnvironmentAllowed";
+const OBSERVATION_FILTER_EMPTY_EQUALS_NULL_COLUMNS = new Set([
+ "parentObservationId",
+]);
+
interface ScheduleObservationEvalsParams {
observation: ObservationForEval;
configs: ObservationEvalRule[];
@@ -375,6 +379,9 @@ function evaluateFilter(
observation,
filterConditions,
fieldMapper,
+ {
+ emptyEqualsNullColumns: OBSERVATION_FILTER_EMPTY_EQUALS_NULL_COLUMNS,
+ },
);
return isFilterMatch;