🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): align metadata contains-filter key-existence semantics (#16068)
Tobias Wochingerfc9164dcf094
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.tsindex a0a921cd44..73d95fa79c 100644--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts@@ -427,22 +427,29 @@ export class StringObjectFilter implements Filter {} else {// For observations/traces tables, use Map access: metadata[key]const column = `${prefix}${this.field}`;+ const valueAccessor = `${column}[{${varKeyName}: String}]`;+ // A missing key resolves the Map access to the empty-string default,+ // which would otherwise make `contains ""` (and every other operator's+ // empty-value comparison) incorrectly match rows that never had the+ // key. Require the key to exist first, mirroring the events-table fix+ // in PR #13369.+ const hasKey = `mapContains(${column}, {${varKeyName}: String})`;switch (this.operator) {case "=":- query = `${column}[{${varKeyName}: String}] = {${varValueName}: String}`;+ query = `${hasKey} AND (${valueAccessor} = {${varValueName}: String})`;break;case "contains":- query = `position(${column}[{${varKeyName}: String}], {${varValueName}: String}) > 0`;+ query = `${hasKey} AND (position(${valueAccessor}, {${varValueName}: String}) > 0)`;break;case "does not contain":- query = `position(${column}[{${varKeyName}: String}], {${varValueName}: String}) = 0`;+ query = `${hasKey} AND (position(${valueAccessor}, {${varValueName}: String}) = 0)`;break;case "starts with":- query = `startsWith(${column}[{${varKeyName}: String}], {${varValueName}: String})`;+ query = `${hasKey} AND (startsWith(${valueAccessor}, {${varValueName}: String}))`;break;case "ends with":- query = `endsWith(${column}[{${varKeyName}: String}], {${varValueName}: String})`;+ query = `${hasKey} AND (endsWith(${valueAccessor}, {${varValueName}: String}))`;break;default:throw new Error(`Unsupported operator: ${this.operator}`);diff --git a/packages/shared/src/server/services/InMemoryFilterService.ts b/packages/shared/src/server/services/InMemoryFilterService.tsindex 268969e214..cc9fdc892b 100644--- a/packages/shared/src/server/services/InMemoryFilterService.ts+++ b/packages/shared/src/server/services/InMemoryFilterService.ts@@ -327,9 +327,32 @@ export class InMemoryFilterService {return false;}- // Type assertion is safe here since we've checked typeof fieldValue === "object" above- const objectValue = (fieldValue as Record<string, unknown>)[key];- const stringValue = objectValue?.toString() || "";+ // Type assertion is safe here since we've checked typeof fieldValue === "object" above.+ // Use hasOwnProperty rather than bracket-access-is-undefined: a key that+ // collides with an Object.prototype name (e.g. "toString", "constructor")+ // would otherwise resolve the inherited property instead of undefined,+ // silently bypassing this guard.+ const record = fieldValue as Record<string, unknown>;+ if (!Object.prototype.hasOwnProperty.call(record, key)) {+ // The key does not exist on the object. Coalescing this to an empty+ // string below would make e.g. `contains ""` incorrectly match rows+ // that never had the key. Mirror the ClickHouse `mapContains` guard+ // (PR #13369) and require the key to exist for every operator.+ return false;+ }+ const objectValue = record[key];+ // Mirror the ClickHouse-side representation exactly: ingestion stores a+ // metadata value as `typeof value === "string" ? value : JSON.stringify(value)`+ // (see convertRecordValuesToString). `.toString()` diverges from that for+ // null ("" via `null?.toString()` vs. the stored "null"), arrays ("1,2"+ // vs. the stored "[1,2]"), and plain objects ("[object Object]" vs. the+ // stored '{"a":1}').+ const stringValue =+ typeof objectValue === "string"+ ? objectValue+ : objectValue === undefined+ ? ""+ : JSON.stringify(objectValue);return this.evaluateStringFilter(stringValue, filterValue, operator);}diff --git a/worker/src/__tests__/evalService.filtering.test.ts b/worker/src/__tests__/evalService.filtering.test.tsindex 82467e0153..f26ba722ed 100644--- a/worker/src/__tests__/evalService.filtering.test.ts+++ b/worker/src/__tests__/evalService.filtering.test.ts@@ -453,6 +453,47 @@ describe("test eval filtering", () => {expect(jobs[0].status.toString()).toBe("PENDING");}, 10_000);+ test("does not create eval job for a trace missing the metadata key when filtering on contains empty string", async ({+ expect,+ upsertTwoTraces,+ configureDefaultJobWithSingleFilter,+ createTwoEvalJobs,+ getJobs,+ traceId1,+ }) => {+ // trace1 has the "turn" metadata key, trace2 never had it set at all.+ await upsertTwoTraces([+ {+ metadata: { turn: "1" },+ },+ {+ metadata: {},+ },+ ]);++ // "contains ''" on a metadata key should behave as a key-existence+ // check, not match every trace regardless of whether the key was ever+ // set.+ await configureDefaultJobWithSingleFilter({+ type: "stringObject",+ key: "turn",+ value: "",+ column: "metadata",+ operator: "contains",+ });++ // No cachedTrace is passed here, so this exercises the database+ // fallback query (legacy traces table Map-column filter), not the+ // in-memory evaluator.+ await createTwoEvalJobs();++ const jobs = await getJobs();++ expect(jobs.length).toBe(1);+ expect(jobs[0].jobInputTraceId).toBe(traceId1);+ expect(jobs[0].status.toString()).toBe("PENDING");+ }, 10_000);+test("creates eval job only for matching version", async ({expect,upsertTwoTraces,diff --git a/worker/src/__tests__/inMemoryFilterService.test.ts b/worker/src/__tests__/inMemoryFilterService.test.tsindex fe2957c8b9..00c4f9c486 100644--- a/worker/src/__tests__/inMemoryFilterService.test.ts+++ b/worker/src/__tests__/inMemoryFilterService.test.ts@@ -842,6 +842,144 @@ describe("InMemoryFilterService", () => {fieldMapper,),).toBe(true);++ // A key that doesn't exist on the object must not match, even for an+ // empty-string comparison value. Otherwise `contains ""` would match+ // every row regardless of whether the key was ever set.+ expect(+ InMemoryFilterService.evaluateFilter(+ mockData,+ [+ {+ column: "metadata",+ type: "stringObject",+ key: "missingKey",+ operator: "contains",+ value: "",+ },+ ],+ fieldMapper,+ ),+ ).toBe(false);++ expect(+ InMemoryFilterService.evaluateFilter(+ mockData,+ [+ {+ column: "metadata",+ type: "stringObject",+ key: "missingKey",+ operator: "=",+ value: "",+ },+ ],+ fieldMapper,+ ),+ ).toBe(false);++ // A key present with an explicit JSON-null value is not a missing key —+ // ClickHouse stores it as the literal string "null" (ingestion runs+ // JSON.stringify on non-string metadata values). `= ""` must not match+ // it, the same way it wouldn't match the stored "null" string.+ const dataWithNullMetadataValue = {+ ...mockData,+ metadata: { ...mockData.metadata, nullField: null },+ };+ expect(+ InMemoryFilterService.evaluateFilter(+ dataWithNullMetadataValue,+ [+ {+ column: "metadata",+ type: "stringObject",+ key: "nullField",+ operator: "=",+ value: "",+ },+ ],+ fieldMapper,+ ),+ ).toBe(false);+ expect(+ InMemoryFilterService.evaluateFilter(+ dataWithNullMetadataValue,+ [+ {+ column: "metadata",+ type: "stringObject",+ key: "nullField",+ operator: "=",+ value: "null",+ },+ ],+ fieldMapper,+ ),+ ).toBe(true);++ // A filter key that collides with an inherited Object.prototype name+ // (e.g. "toString") must be treated as absent, not resolved to the+ // inherited property, which would silently bypass the key-existence+ // guard above.+ expect(+ InMemoryFilterService.evaluateFilter(+ mockData,+ [+ {+ column: "metadata",+ type: "stringObject",+ key: "toString",+ operator: "contains",+ value: "",+ },+ ],+ fieldMapper,+ ),+ ).toBe(false);++ // Array/object metadata values are stored in ClickHouse via+ // JSON.stringify, not the array/object's own `.toString()` (which+ // drops brackets for arrays and is useless for plain objects). The+ // in-memory representation must match that stored string exactly, the+ // same way it now does for null.+ const dataWithStructuredMetadataValues = {+ ...mockData,+ metadata: {+ ...mockData.metadata,+ arrayField: [1, 2],+ objectField: { a: 1 },+ },+ };+ expect(+ InMemoryFilterService.evaluateFilter(+ dataWithStructuredMetadataValues,+ [+ {+ column: "metadata",+ type: "stringObject",+ key: "arrayField",+ operator: "starts with",+ value: "[",+ },+ ],+ fieldMapper,+ ),+ ).toBe(true);+ expect(+ InMemoryFilterService.evaluateFilter(+ dataWithStructuredMetadataValues,+ [+ {+ column: "metadata",+ type: "stringObject",+ key: "objectField",+ operator: "contains",+ value: '{"a":1}',+ },+ ],+ fieldMapper,+ ),+ ).toBe(true);});test("evaluates numberObject filters correctly", () => {