🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): stop exporting customer-authored core data fields (#18701)
Max Deichmann22d83f887202
diff --git a/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts b/worker/src/queues/__tests__/coreDataS3ExportQueue.test.tsindex 3f49594d50..902c2ff001 100644--- a/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts+++ b/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts@@ -165,13 +165,52 @@ describe("coreDataTableExports", () => {"core/dashboardWidgets.jsonl",]),);++ const selectFor = (findMany: {+ mock: { calls: Array<[{ select?: Record<string, unknown> }?]> };+ }) => findMany.mock.calls[0]?.[0]?.select;++ const evaluatorVersionSelect = selectFor(+ vi.mocked(prisma.evaluatorVersion.findMany),+ );+ expect(evaluatorVersionSelect).toMatchObject({+ id: true,+ evaluatorId: true,+ version: true,+ vars: true,+ variableMapping: true,+ });++ expect(+ selectFor(vi.mocked(prisma.jobConfiguration.findMany)),+ ).not.toHaveProperty("filter");+ expect(+ selectFor(vi.mocked(prisma.jobConfiguration.findMany)),+ ).toMatchObject({ variableMapping: true });+ expect(+ selectFor(vi.mocked(prisma.evaluationRule.findMany)),+ ).not.toHaveProperty("filter");+ expect(+ selectFor(vi.mocked(prisma.evaluationRuleEvaluatorAssignment.findMany)),+ ).toMatchObject({ variableMapping: true });++ const widgetSelect = selectFor(vi.mocked(prisma.dashboardWidget.findMany));+ expect(widgetSelect).not.toHaveProperty("filters");+ expect(widgetSelect).toMatchObject({+ dimensions: true,+ metrics: true,+ chartType: true,+ chartConfig: true,+ });});});describe("mapDashboardWidgetToCoreDataRow", () => {- it("allowlists dimensions/metrics and replaces unknown strings with a sentinel", () => {+ it("allowlists dimensions, metrics, and chart type and drops filters", () => {const row = mapDashboardWidgetToCoreDataRow({id: "widget-1",+ projectId: "project-1",+ view: "OBSERVATIONS",dimensions: [{ field: "calledToolNames" },{ field: "customer entered free text" },@@ -180,14 +219,30 @@ describe("mapDashboardWidgetToCoreDataRow", () => {{ measure: "toolCalls", agg: "sum" },{ measure: "leaked customer string", agg: "not-an-agg" },],- filters: [],+ filters: [+ {+ column: "userId",+ operator: "any of",+ value: ["customer-user-42"],+ type: "stringOptions",+ },+ {+ column: "metadata",+ key: "customer-secret-key",+ operator: "=",+ value: "customer-value",+ type: "stringObject",+ },+ ],chartConfig: {type: "PIVOT_TABLE",row_limit: 20,+ bins: 8,defaultSort: { column: "customer column text", order: "DESC" },},});+ expect(row).not.toHaveProperty("filters");expect(row.dimensions).toStrictEqual([{ field: "calledToolNames" },{ field: "__invalid__" },@@ -199,50 +254,36 @@ describe("mapDashboardWidgetToCoreDataRow", () => {expect(row.chartConfig).toStrictEqual({type: "PIVOT_TABLE",row_limit: 20,- bins: null,+ bins: 8,});const serialized = JSON.stringify(row);expect(serialized).not.toContain("customer entered free text");expect(serialized).not.toContain("leaked customer string");expect(serialized).not.toContain("customer column text");+ expect(serialized).not.toContain("customer-user-42");+ expect(serialized).not.toContain("customer-secret-key");+ expect(serialized).not.toContain("customer-value");+ expect(serialized).not.toContain("userId");+ expect(serialized).not.toContain("stringOptions");});- it("keeps filter shape but strips customer-entered values and keys", () => {+ it("replaces an unknown chartConfig.type with the invalid sentinel", () => {const row = mapDashboardWidgetToCoreDataRow({- id: "widget-1",- projectId: "project-1",- view: "OBSERVATIONS",+ id: "widget-3",dimensions: [],- metrics: [{ measure: "toolCalls", agg: "sum" }],- chartConfig: { type: "LINE_TIME_SERIES" },- filters: [- {- column: "userId",- operator: "any of",- value: ["customer-user-42"],- type: "stringOptions",- },- {- column: "metadata",- key: "customer-secret-key",- operator: "=",- value: "customer-value",- type: "stringObject",- },- ],+ metrics: [],+ chartConfig: { type: "customer free text chart", row_limit: 5 },});- expect(row.filters).toStrictEqual([- { column: "userId", operator: "any of", type: "stringOptions" },- { column: "metadata", operator: "=", type: "stringObject" },- ]);- const serialized = JSON.stringify(row);- expect(serialized).not.toContain("customer-user-42");- expect(serialized).not.toContain("customer-secret-key");- expect(serialized).not.toContain("customer-value");+ expect(row.chartConfig).toStrictEqual({+ type: "__invalid__",+ row_limit: 5,+ bins: null,+ });+ expect(JSON.stringify(row)).not.toContain("customer free text chart");});- it("normalizes a non-array filters column to an empty list", () => {+ it("normalizes missing dimensions, metrics, and chart config without a filters field", () => {const row = mapDashboardWidgetToCoreDataRow({id: "widget-2",dimensions: null,@@ -251,9 +292,14 @@ describe("mapDashboardWidgetToCoreDataRow", () => {chartConfig: null,});- expect(row.filters).toStrictEqual([]);+ expect(row).not.toHaveProperty("filters");expect(row.dimensions).toStrictEqual([]);expect(row.metrics).toStrictEqual([]);+ expect(row.chartConfig).toStrictEqual({+ type: null,+ row_limit: null,+ bins: null,+ });});});diff --git a/worker/src/queues/coreDataS3ExportQueue.ts b/worker/src/queues/coreDataS3ExportQueue.tsindex 6ba3ce4963..ded0e292f1 100644--- a/worker/src/queues/coreDataS3ExportQueue.ts+++ b/worker/src/queues/coreDataS3ExportQueue.ts@@ -1,5 +1,6 @@import { Processor } from "bullmq";import { Readable } from "node:stream";+import { DashboardWidgetChartType } from "@prisma/client";import pLimit from "p-limit";import {logger,@@ -100,6 +101,9 @@ const KNOWN_WIDGET_DIMENSIONS = new Set(),);const KNOWN_WIDGET_AGGREGATIONS = new Set<string>(metricAggregations.options);+const KNOWN_WIDGET_CHART_TYPES = new Set<string>(+ Object.values(DashboardWidgetChartType),+);const INVALID_SENTINEL = "__invalid__";const allowlisted = (value: unknown, allowlist: Set<string>): string | null => {@@ -115,18 +119,16 @@ const allowlisted = (value: unknown, allowlist: Set<string>): string | null => {type DashboardWidgetCoreDataInput = {dimensions: unknown;metrics: unknown;- filters: unknown;chartConfig: unknown;+ /** Drops a supplied filters column so it cannot leak through the rest spread. */+ filters?: unknown;} & Record<string, unknown>;-// Widget filters carry customer-entered values (user ids, metadata values,-// tool names, ...). Analytics only needs which columns/operators are used, so-// the values (and metadata keys) are stripped before export; every other-// string field is allowlisted (see above).+/** Allowlists widget dimensions, metrics, and chart type, and keeps numeric chartConfig limits. */export const mapDashboardWidgetToCoreDataRow = ({dimensions,metrics,- filters,+ filters: _filters,chartConfig,...widget}: DashboardWidgetCoreDataInput) => {@@ -144,17 +146,8 @@ export const mapDashboardWidgetToCoreDataRow = ({agg: allowlisted(metric.agg, KNOWN_WIDGET_AGGREGATIONS),})): [],- filters: Array.isArray(filters)- ? filters.map((filter: Record<string, unknown>) => ({- column: filter.column ?? null,- operator: filter.operator ?? null,- type: filter.type ?? null,- }))- : [],- // Explicit safe scalars only; defaultSort.column is a free string and is- // deliberately not exported.chartConfig: {- type: typeof config.type === "string" ? config.type : null,+ type: allowlisted(config.type, KNOWN_WIDGET_CHART_TYPES),row_limit: typeof config.row_limit === "number" ? config.row_limit : null,bins: typeof config.bins === "number" ? config.bins : null,},@@ -603,8 +596,7 @@ export const coreDataTableExports: Array<uploadTableCoreDataJsonl({...args,tableName: "jobConfigurations",- // Keep this stable for downstream consumers while evaluator v2 exports- // are adopted independently.+ /** Omits the customer-authored job-configuration filter. */fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>prisma.jobConfiguration.findMany({take,@@ -619,7 +611,6 @@ export const coreDataTableExports: Array<blockReason: true,evalTemplateId: true,scoreName: true,- filter: true,targetObject: true,variableMapping: true,sampling: true,@@ -660,9 +651,7 @@ export const coreDataTableExports: Array<uploadTableCoreDataJsonl({...args,tableName: "evaluatorVersions",- // Evaluator definitions (prompt, code, model params, and output- // definitions) may contain customer data. Export metadata and mapping- // shape only, matching the legacy job-configuration export.+ /** Exports version metadata, vars, and variableMapping, and omits prompt, code, and model params. */fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>prisma.evaluatorVersion.findMany({take,@@ -687,6 +676,7 @@ export const coreDataTableExports: Array<uploadTableCoreDataJsonl({...args,tableName: "evaluationRules",+ /** Omits the customer-authored evaluation-rule filter. */fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>prisma.evaluationRule.findMany({take,@@ -699,7 +689,6 @@ export const coreDataTableExports: Array<name: true,status: true,targetObject: true,- filter: true,sampling: true,delay: true,timeScope: true,@@ -756,11 +745,7 @@ export const coreDataTableExports: Array<uploadTableCoreDataJsonl({...args,tableName: "dashboardWidgets",- // Customer-authored name/description are free text and intentionally- // excluded; filter values are stripped in the mapper. The remaining- // shape (view, metrics, dimensions, chart type) is what dashboard- // product decisions need (e.g. which aggregation users pair with a- // measure). projectId NULL marks Langfuse-owned template widgets.+ /** Allowlists widget dimensions, metrics, and chart type, and omits customer-authored filters. */fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>prisma.dashboardWidget.findMany({take,@@ -773,7 +758,6 @@ export const coreDataTableExports: Array<view: true,dimensions: true,metrics: true,- filters: true,chartType: true,chartConfig: true,minVersion: true,