🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(shared): align in-memory null filters with column semantics (#17160)
Tobias Wochinger1bb49babf6ce
diff --git a/packages/shared/src/server/services/InMemoryFilterService.ts b/packages/shared/src/server/services/InMemoryFilterService.tsindex 53463e7b43..0c8dac0437 100644--- a/packages/shared/src/server/services/InMemoryFilterService.ts+++ b/packages/shared/src/server/services/InMemoryFilterService.ts@@ -2,6 +2,10 @@ import { FilterCondition, FilterState } from "../../types";import { logger } from "../logger";import { encodeBooleanScoreEntry } from "../queries/clickhouse-sql/clickhouse-filter";+export type InMemoryFilterOptions = {+ emptyEqualsNullColumns?: ReadonlySet<string>;+};+export class InMemoryFilterService {/*** Evaluates whether a data object matches the given filter conditions.@@ -9,12 +13,14 @@ export class InMemoryFilterService {* @param data - The data object to evaluate* @param filter - The filter conditions to apply* @param fieldMapper - Function to map filter column names to data object values+ * @param options - Column-specific filter semantics* @returns true if the data matches all filter conditions, false otherwise*/static evaluateFilter<T>(data: T,filter: FilterState,fieldMapper: (data: T, column: string) => unknown,+ options?: InMemoryFilterOptions,): boolean {try {// If no filters, data matches@@ -24,7 +30,9 @@ export class InMemoryFilterService {// Evaluate each filter conditionfor (const condition of filter) {- if (!this.evaluateFilterCondition(data, condition, fieldMapper)) {+ if (+ !this.evaluateFilterCondition(data, condition, fieldMapper, options)+ ) {return false;}}@@ -47,6 +55,7 @@ export class InMemoryFilterService {data: T,condition: FilterCondition,fieldMapper: (data: T, column: string) => unknown,+ options?: InMemoryFilterOptions,): boolean {const { column, type, operator } = condition;@@ -111,7 +120,11 @@ export class InMemoryFilterService {operator,);case "null":- return this.evaluateNullFilter(fieldValue, operator);+ return this.evaluateNullFilter(+ fieldValue,+ operator,+ options?.emptyEqualsNullColumns?.has(column) ?? false,+ );case "positionInTrace":// Position filters are applied after all other filters in DB queries.// Ignore them in in-memory filtering.@@ -434,14 +447,18 @@ export class InMemoryFilterService {private static evaluateNullFilter(fieldValue: unknown,operator: string,+ emptyEqualsNull: boolean,): boolean {+ const isNull =+ fieldValue === null ||+ fieldValue === undefined ||+ (emptyEqualsNull && fieldValue === "");+switch (operator) {case "is null":- return (- fieldValue === null || fieldValue === undefined || fieldValue === ""- );+ return isNull;case "is not null":- return fieldValue !== null && fieldValue !== undefined;+ return !isNull;default:logger.error("Unsupported null filter operator", {operator,diff --git a/worker/src/__tests__/inMemoryFilterService.test.ts b/worker/src/__tests__/inMemoryFilterService.test.tsindex 00c4f9c486..d248159eca 100644--- a/worker/src/__tests__/inMemoryFilterService.test.ts+++ b/worker/src/__tests__/inMemoryFilterService.test.ts@@ -1093,6 +1093,60 @@ describe("InMemoryFilterService", () => {).toBe(true);});+ test("applies empty-string null semantics per column", () => {+ const dataWithEmptyString = { ...mockData, release: "" };++ expect(+ InMemoryFilterService.evaluateFilter(+ dataWithEmptyString,+ [{ column: "release", type: "null", operator: "is null", value: "" }],+ fieldMapper,+ ),+ ).toBe(false);++ expect(+ InMemoryFilterService.evaluateFilter(+ dataWithEmptyString,+ [+ {+ column: "release",+ type: "null",+ operator: "is not null",+ value: "",+ },+ ],+ fieldMapper,+ ),+ ).toBe(true);++ const emptyEqualsNullColumns = new Set(["release"]);++ expect(+ InMemoryFilterService.evaluateFilter(+ dataWithEmptyString,+ [{ column: "release", type: "null", operator: "is null", value: "" }],+ fieldMapper,+ { emptyEqualsNullColumns },+ ),+ ).toBe(true);++ expect(+ InMemoryFilterService.evaluateFilter(+ dataWithEmptyString,+ [+ {+ column: "release",+ type: "null",+ operator: "is not null",+ value: "",+ },+ ],+ fieldMapper,+ { emptyEqualsNullColumns },+ ),+ ).toBe(false);+ });+test("evaluates multiple filters with AND logic", () => {expect(InMemoryFilterService.evaluateFilter(diff --git a/worker/src/features/evaluation/observationEval/__tests__/filterEvaluation.test.ts b/worker/src/features/evaluation/observationEval/__tests__/filterEvaluation.test.tsindex d6c214b9c4..38627d2c15 100644--- a/worker/src/features/evaluation/observationEval/__tests__/filterEvaluation.test.ts+++ b/worker/src/features/evaluation/observationEval/__tests__/filterEvaluation.test.ts@@ -1053,6 +1053,33 @@ describe("Filter Evaluation for Observation Evals", () => {expect(matched).toBe(true);});+ it("should treat an empty parentObservationId as null", async () => {+ const observation = createTestObservation({+ project_id: projectId,+ parent_span_id: "",+ });++ const isNullMatch = await testFilterMatch(observation, [+ {+ column: "parentObservationId",+ type: "null",+ operator: "is null",+ value: "",+ },+ ]);+ const isNotNullMatch = await testFilterMatch(observation, [+ {+ column: "parentObservationId",+ type: "null",+ operator: "is not null",+ value: "",+ },+ ]);++ expect(isNullMatch).toBe(true);+ expect(isNotNullMatch).toBe(false);+ });+it("should not match observations where parentObservationId is not null (child observations)", async () => {const observation = createTestObservation({project_id: projectId,diff --git a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.tsindex 8ec12af32e..31f104cc6d 100644--- a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts+++ b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts@@ -25,6 +25,10 @@ import {import { createW3CTraceId } from "../../utils";import { isInternalEvalEnvironment } from "../isEvalTargetEnvironmentAllowed";+const OBSERVATION_FILTER_EMPTY_EQUALS_NULL_COLUMNS = new Set([+ "parentObservationId",+]);+interface ScheduleObservationEvalsParams {observation: ObservationForEval;configs: ObservationEvalRule[];@@ -375,6 +379,9 @@ function evaluateFilter(observation,filterConditions,fieldMapper,+ {+ emptyEqualsNullColumns: OBSERVATION_FILTER_EMPTY_EQUALS_NULL_COLUMNS,+ },);return isFilterMatch;