Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): stop exporting customer-authored core data fields (#18701)

Max Deichmann22d83f887202
diff --git a/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts b/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts
index 3f49594d50..902c2ff001 100644
--- a/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts
+++ b/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts
@@ -165,13 +165,52 @@ describe("coreDataTableExports", () => {
"core/dashboardWidgets.jsonl",
]),
);
+
+ const selectFor = (findMany: {
+ mock: { calls: Array<[{ select?: Record<string, unknown> }?]> };
+ }) => findMany.mock.calls[0]?.[0]?.select;
+
+ const evaluatorVersionSelect = selectFor(
+ vi.mocked(prisma.evaluatorVersion.findMany),
+ );
+ expect(evaluatorVersionSelect).toMatchObject({
+ id: true,
+ evaluatorId: true,
+ version: true,
+ vars: true,
+ variableMapping: true,
+ });
+
+ expect(
+ selectFor(vi.mocked(prisma.jobConfiguration.findMany)),
+ ).not.toHaveProperty("filter");
+ expect(
+ selectFor(vi.mocked(prisma.jobConfiguration.findMany)),
+ ).toMatchObject({ variableMapping: true });
+ expect(
+ selectFor(vi.mocked(prisma.evaluationRule.findMany)),
+ ).not.toHaveProperty("filter");
+ expect(
+ selectFor(vi.mocked(prisma.evaluationRuleEvaluatorAssignment.findMany)),
+ ).toMatchObject({ variableMapping: true });
+
+ const widgetSelect = selectFor(vi.mocked(prisma.dashboardWidget.findMany));
+ expect(widgetSelect).not.toHaveProperty("filters");
+ expect(widgetSelect).toMatchObject({
+ dimensions: true,
+ metrics: true,
+ chartType: true,
+ chartConfig: true,
+ });
});
});
describe("mapDashboardWidgetToCoreDataRow", () => {
- it("allowlists dimensions/metrics and replaces unknown strings with a sentinel", () => {
+ it("allowlists dimensions, metrics, and chart type and drops filters", () => {
const row = mapDashboardWidgetToCoreDataRow({
id: "widget-1",
+ projectId: "project-1",
+ view: "OBSERVATIONS",
dimensions: [
{ field: "calledToolNames" },
{ field: "customer entered free text" },
@@ -180,14 +219,30 @@ describe("mapDashboardWidgetToCoreDataRow", () => {
{ measure: "toolCalls", agg: "sum" },
{ measure: "leaked customer string", agg: "not-an-agg" },
],
- filters: [],
+ filters: [
+ {
+ column: "userId",
+ operator: "any of",
+ value: ["customer-user-42"],
+ type: "stringOptions",
+ },
+ {
+ column: "metadata",
+ key: "customer-secret-key",
+ operator: "=",
+ value: "customer-value",
+ type: "stringObject",
+ },
+ ],
chartConfig: {
type: "PIVOT_TABLE",
row_limit: 20,
+ bins: 8,
defaultSort: { column: "customer column text", order: "DESC" },
},
});
+ expect(row).not.toHaveProperty("filters");
expect(row.dimensions).toStrictEqual([
{ field: "calledToolNames" },
{ field: "__invalid__" },
@@ -199,50 +254,36 @@ describe("mapDashboardWidgetToCoreDataRow", () => {
expect(row.chartConfig).toStrictEqual({
type: "PIVOT_TABLE",
row_limit: 20,
- bins: null,
+ bins: 8,
});
const serialized = JSON.stringify(row);
expect(serialized).not.toContain("customer entered free text");
expect(serialized).not.toContain("leaked customer string");
expect(serialized).not.toContain("customer column text");
+ expect(serialized).not.toContain("customer-user-42");
+ expect(serialized).not.toContain("customer-secret-key");
+ expect(serialized).not.toContain("customer-value");
+ expect(serialized).not.toContain("userId");
+ expect(serialized).not.toContain("stringOptions");
});
- it("keeps filter shape but strips customer-entered values and keys", () => {
+ it("replaces an unknown chartConfig.type with the invalid sentinel", () => {
const row = mapDashboardWidgetToCoreDataRow({
- id: "widget-1",
- projectId: "project-1",
- view: "OBSERVATIONS",
+ id: "widget-3",
dimensions: [],
- metrics: [{ measure: "toolCalls", agg: "sum" }],
- chartConfig: { type: "LINE_TIME_SERIES" },
- filters: [
- {
- column: "userId",
- operator: "any of",
- value: ["customer-user-42"],
- type: "stringOptions",
- },
- {
- column: "metadata",
- key: "customer-secret-key",
- operator: "=",
- value: "customer-value",
- type: "stringObject",
- },
- ],
+ metrics: [],
+ chartConfig: { type: "customer free text chart", row_limit: 5 },
});
- expect(row.filters).toStrictEqual([
- { column: "userId", operator: "any of", type: "stringOptions" },
- { column: "metadata", operator: "=", type: "stringObject" },
- ]);
- const serialized = JSON.stringify(row);
- expect(serialized).not.toContain("customer-user-42");
- expect(serialized).not.toContain("customer-secret-key");
- expect(serialized).not.toContain("customer-value");
+ expect(row.chartConfig).toStrictEqual({
+ type: "__invalid__",
+ row_limit: 5,
+ bins: null,
+ });
+ expect(JSON.stringify(row)).not.toContain("customer free text chart");
});
- it("normalizes a non-array filters column to an empty list", () => {
+ it("normalizes missing dimensions, metrics, and chart config without a filters field", () => {
const row = mapDashboardWidgetToCoreDataRow({
id: "widget-2",
dimensions: null,
@@ -251,9 +292,14 @@ describe("mapDashboardWidgetToCoreDataRow", () => {
chartConfig: null,
});
- expect(row.filters).toStrictEqual([]);
+ expect(row).not.toHaveProperty("filters");
expect(row.dimensions).toStrictEqual([]);
expect(row.metrics).toStrictEqual([]);
+ expect(row.chartConfig).toStrictEqual({
+ type: null,
+ row_limit: null,
+ bins: null,
+ });
});
});
diff --git a/worker/src/queues/coreDataS3ExportQueue.ts b/worker/src/queues/coreDataS3ExportQueue.ts
index 6ba3ce4963..ded0e292f1 100644
--- a/worker/src/queues/coreDataS3ExportQueue.ts
+++ b/worker/src/queues/coreDataS3ExportQueue.ts
@@ -1,5 +1,6 @@
import { Processor } from "bullmq";
import { Readable } from "node:stream";
+import { DashboardWidgetChartType } from "@prisma/client";
import pLimit from "p-limit";
import {
logger,
@@ -100,6 +101,9 @@ const KNOWN_WIDGET_DIMENSIONS = new Set(
),
);
const KNOWN_WIDGET_AGGREGATIONS = new Set<string>(metricAggregations.options);
+const KNOWN_WIDGET_CHART_TYPES = new Set<string>(
+ Object.values(DashboardWidgetChartType),
+);
const INVALID_SENTINEL = "__invalid__";
const allowlisted = (value: unknown, allowlist: Set<string>): string | null => {
@@ -115,18 +119,16 @@ const allowlisted = (value: unknown, allowlist: Set<string>): string | null => {
type DashboardWidgetCoreDataInput = {
dimensions: unknown;
metrics: unknown;
- filters: unknown;
chartConfig: unknown;
+ /** Drops a supplied filters column so it cannot leak through the rest spread. */
+ filters?: unknown;
} & Record<string, unknown>;
-// Widget filters carry customer-entered values (user ids, metadata values,
-// tool names, ...). Analytics only needs which columns/operators are used, so
-// the values (and metadata keys) are stripped before export; every other
-// string field is allowlisted (see above).
+/** Allowlists widget dimensions, metrics, and chart type, and keeps numeric chartConfig limits. */
export const mapDashboardWidgetToCoreDataRow = ({
dimensions,
metrics,
- filters,
+ filters: _filters,
chartConfig,
...widget
}: DashboardWidgetCoreDataInput) => {
@@ -144,17 +146,8 @@ export const mapDashboardWidgetToCoreDataRow = ({
agg: allowlisted(metric.agg, KNOWN_WIDGET_AGGREGATIONS),
}))
: [],
- filters: Array.isArray(filters)
- ? filters.map((filter: Record<string, unknown>) => ({
- column: filter.column ?? null,
- operator: filter.operator ?? null,
- type: filter.type ?? null,
- }))
- : [],
- // Explicit safe scalars only; defaultSort.column is a free string and is
- // deliberately not exported.
chartConfig: {
- type: typeof config.type === "string" ? config.type : null,
+ type: allowlisted(config.type, KNOWN_WIDGET_CHART_TYPES),
row_limit: typeof config.row_limit === "number" ? config.row_limit : null,
bins: typeof config.bins === "number" ? config.bins : null,
},
@@ -603,8 +596,7 @@ export const coreDataTableExports: Array<
uploadTableCoreDataJsonl({
...args,
tableName: "jobConfigurations",
- // Keep this stable for downstream consumers while evaluator v2 exports
- // are adopted independently.
+ /** Omits the customer-authored job-configuration filter. */
fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
prisma.jobConfiguration.findMany({
take,
@@ -619,7 +611,6 @@ export const coreDataTableExports: Array<
blockReason: true,
evalTemplateId: true,
scoreName: true,
- filter: true,
targetObject: true,
variableMapping: true,
sampling: true,
@@ -660,9 +651,7 @@ export const coreDataTableExports: Array<
uploadTableCoreDataJsonl({
...args,
tableName: "evaluatorVersions",
- // Evaluator definitions (prompt, code, model params, and output
- // definitions) may contain customer data. Export metadata and mapping
- // shape only, matching the legacy job-configuration export.
+ /** Exports version metadata, vars, and variableMapping, and omits prompt, code, and model params. */
fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
prisma.evaluatorVersion.findMany({
take,
@@ -687,6 +676,7 @@ export const coreDataTableExports: Array<
uploadTableCoreDataJsonl({
...args,
tableName: "evaluationRules",
+ /** Omits the customer-authored evaluation-rule filter. */
fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
prisma.evaluationRule.findMany({
take,
@@ -699,7 +689,6 @@ export const coreDataTableExports: Array<
name: true,
status: true,
targetObject: true,
- filter: true,
sampling: true,
delay: true,
timeScope: true,
@@ -756,11 +745,7 @@ export const coreDataTableExports: Array<
uploadTableCoreDataJsonl({
...args,
tableName: "dashboardWidgets",
- // Customer-authored name/description are free text and intentionally
- // excluded; filter values are stripped in the mapper. The remaining
- // shape (view, metrics, dimensions, chart type) is what dashboard
- // product decisions need (e.g. which aggregation users pair with a
- // measure). projectId NULL marks Langfuse-owned template widgets.
+ /** Allowlists widget dimensions, metrics, and chart type, and omits customer-authored filters. */
fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
prisma.dashboardWidget.findMany({
take,
@@ -773,7 +758,6 @@ export const coreDataTableExports: Array<
view: true,
dimensions: true,
metrics: true,
- filters: true,
chartType: true,
chartConfig: true,
minVersion: true,