Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(worker): export dashboards and widgets in the core data S3 export (#16621)

Trang Nguyend61909d76270
diff --git a/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts b/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts
index 51f29a081c..3f49594d50 100644
--- a/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts
+++ b/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts
@@ -4,6 +4,8 @@ import { type StorageService } from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import {
coreDataTableExports,
+ mapDashboardToCoreDataRow,
+ mapDashboardWidgetToCoreDataRow,
mapEvaluationRuleToCoreDataRow,
mapJobConfigurationToCoreDataRow,
mapUserToCoreDataRow,
@@ -129,6 +131,8 @@ describe("coreDataTableExports", () => {
prisma.evaluationRule,
prisma.evaluationRuleEvaluatorAssignment,
prisma.jobConfiguration,
+ prisma.dashboard,
+ prisma.dashboardWidget,
];
for (const delegate of delegates) {
vi.spyOn(delegate, "findMany").mockResolvedValue([]);
@@ -157,11 +161,139 @@ describe("coreDataTableExports", () => {
"core/evaluatorVersions.jsonl",
"core/evaluationRules.jsonl",
"core/evaluationRuleEvaluatorAssignments.jsonl",
+ "core/dashboards.jsonl",
+ "core/dashboardWidgets.jsonl",
]),
);
});
});
+describe("mapDashboardWidgetToCoreDataRow", () => {
+ it("allowlists dimensions/metrics and replaces unknown strings with a sentinel", () => {
+ const row = mapDashboardWidgetToCoreDataRow({
+ id: "widget-1",
+ dimensions: [
+ { field: "calledToolNames" },
+ { field: "customer entered free text" },
+ ],
+ metrics: [
+ { measure: "toolCalls", agg: "sum" },
+ { measure: "leaked customer string", agg: "not-an-agg" },
+ ],
+ filters: [],
+ chartConfig: {
+ type: "PIVOT_TABLE",
+ row_limit: 20,
+ defaultSort: { column: "customer column text", order: "DESC" },
+ },
+ });
+
+ expect(row.dimensions).toStrictEqual([
+ { field: "calledToolNames" },
+ { field: "__invalid__" },
+ ]);
+ expect(row.metrics).toStrictEqual([
+ { measure: "toolCalls", agg: "sum" },
+ { measure: "__invalid__", agg: "__invalid__" },
+ ]);
+ expect(row.chartConfig).toStrictEqual({
+ type: "PIVOT_TABLE",
+ row_limit: 20,
+ bins: null,
+ });
+ const serialized = JSON.stringify(row);
+ expect(serialized).not.toContain("customer entered free text");
+ expect(serialized).not.toContain("leaked customer string");
+ expect(serialized).not.toContain("customer column text");
+ });
+
+ it("keeps filter shape but strips customer-entered values and keys", () => {
+ const row = mapDashboardWidgetToCoreDataRow({
+ id: "widget-1",
+ projectId: "project-1",
+ view: "OBSERVATIONS",
+ dimensions: [],
+ metrics: [{ measure: "toolCalls", agg: "sum" }],
+ chartConfig: { type: "LINE_TIME_SERIES" },
+ filters: [
+ {
+ column: "userId",
+ operator: "any of",
+ value: ["customer-user-42"],
+ type: "stringOptions",
+ },
+ {
+ column: "metadata",
+ key: "customer-secret-key",
+ operator: "=",
+ value: "customer-value",
+ type: "stringObject",
+ },
+ ],
+ });
+
+ expect(row.filters).toStrictEqual([
+ { column: "userId", operator: "any of", type: "stringOptions" },
+ { column: "metadata", operator: "=", type: "stringObject" },
+ ]);
+ const serialized = JSON.stringify(row);
+ expect(serialized).not.toContain("customer-user-42");
+ expect(serialized).not.toContain("customer-secret-key");
+ expect(serialized).not.toContain("customer-value");
+ });
+
+ it("normalizes a non-array filters column to an empty list", () => {
+ const row = mapDashboardWidgetToCoreDataRow({
+ id: "widget-2",
+ dimensions: null,
+ metrics: null,
+ filters: null,
+ chartConfig: null,
+ });
+
+ expect(row.filters).toStrictEqual([]);
+ expect(row.dimensions).toStrictEqual([]);
+ expect(row.metrics).toStrictEqual([]);
+ });
+});
+
+describe("mapDashboardToCoreDataRow", () => {
+ it("exports only known placement fields from the definition", () => {
+ const row = mapDashboardToCoreDataRow({
+ id: "dashboard-1",
+ definition: {
+ widgets: [
+ {
+ type: "widget",
+ widgetId: "widget-1",
+ x: 0,
+ y: 0,
+ x_size: 6,
+ y_size: 4,
+ id: "placement-1",
+ unexpectedCustomerField: "customer text",
+ },
+ ],
+ unexpectedTopLevel: "more customer text",
+ },
+ });
+
+ expect(row.definition).toStrictEqual({
+ widgets: [
+ {
+ type: "widget",
+ widgetId: "widget-1",
+ x: 0,
+ y: 0,
+ x_size: 6,
+ y_size: 4,
+ },
+ ],
+ });
+ expect(JSON.stringify(row)).not.toContain("customer text");
+ });
+});
+
describe("uploadTableCoreDataJsonl", () => {
const createS3Client = (
uploadFileBuffered: ReturnType<typeof vi.fn>,
diff --git a/worker/src/queues/coreDataS3ExportQueue.ts b/worker/src/queues/coreDataS3ExportQueue.ts
index b0a2b76592..05769cf499 100644
--- a/worker/src/queues/coreDataS3ExportQueue.ts
+++ b/worker/src/queues/coreDataS3ExportQueue.ts
@@ -6,6 +6,7 @@ import {
StorageServiceFactory,
type StorageService,
} from "@langfuse/shared/src/server";
+import { metricAggregations, viewDeclarations } from "@langfuse/shared/query";
import { prisma } from "@langfuse/shared/src/db";
import { env } from "../env";
@@ -83,6 +84,108 @@ export const mapEvaluationRuleToCoreDataRow = ({
sampling: sampling.toNumber(),
});
+// Widget dimensions/metrics/chart config are persisted as free-form strings
+// (DimensionSchema/MetricSchema accept any z.string()), so an API client can
+// smuggle arbitrary text into them. The export therefore allowlists every
+// string against the query-model declarations and replaces unknown values
+// with a sentinel instead of forwarding them.
+const KNOWN_WIDGET_MEASURES = new Set(
+ Object.values(viewDeclarations).flatMap((versionViews) =>
+ Object.values(versionViews).flatMap((view) => Object.keys(view.measures)),
+ ),
+);
+const KNOWN_WIDGET_DIMENSIONS = new Set(
+ Object.values(viewDeclarations).flatMap((versionViews) =>
+ Object.values(versionViews).flatMap((view) => Object.keys(view.dimensions)),
+ ),
+);
+const KNOWN_WIDGET_AGGREGATIONS = new Set<string>(metricAggregations.options);
+const INVALID_SENTINEL = "__invalid__";
+
+const allowlisted = (value: unknown, allowlist: Set<string>): string | null =>
+ value == null
+ ? null
+ : typeof value === "string" && allowlist.has(value)
+ ? value
+ : INVALID_SENTINEL;
+
+type DashboardWidgetCoreDataInput = {
+ dimensions: unknown;
+ metrics: unknown;
+ filters: unknown;
+ chartConfig: unknown;
+} & Record<string, unknown>;
+
+// Widget filters carry customer-entered values (user ids, metadata values,
+// tool names, ...). Analytics only needs which columns/operators are used, so
+// the values (and metadata keys) are stripped before export; every other
+// string field is allowlisted (see above).
+export const mapDashboardWidgetToCoreDataRow = ({
+ dimensions,
+ metrics,
+ filters,
+ chartConfig,
+ ...widget
+}: DashboardWidgetCoreDataInput) => {
+ const config = (chartConfig ?? {}) as Record<string, unknown>;
+ return {
+ ...widget,
+ dimensions: Array.isArray(dimensions)
+ ? dimensions.map((dimension: Record<string, unknown>) => ({
+ field: allowlisted(dimension.field, KNOWN_WIDGET_DIMENSIONS),
+ }))
+ : [],
+ metrics: Array.isArray(metrics)
+ ? metrics.map((metric: Record<string, unknown>) => ({
+ measure: allowlisted(metric.measure, KNOWN_WIDGET_MEASURES),
+ agg: allowlisted(metric.agg, KNOWN_WIDGET_AGGREGATIONS),
+ }))
+ : [],
+ filters: Array.isArray(filters)
+ ? filters.map((filter: Record<string, unknown>) => ({
+ column: filter.column ?? null,
+ operator: filter.operator ?? null,
+ type: filter.type ?? null,
+ }))
+ : [],
+ // Explicit safe scalars only; defaultSort.column is a free string and is
+ // deliberately not exported.
+ chartConfig: {
+ type: typeof config.type === "string" ? config.type : null,
+ row_limit: typeof config.row_limit === "number" ? config.row_limit : null,
+ bins: typeof config.bins === "number" ? config.bins : null,
+ },
+ };
+};
+
+type DashboardCoreDataInput = {
+ definition: unknown;
+} & Record<string, unknown>;
+
+// Dashboard definitions hold widget placements; explicit field picks keep any
+// unexpected persisted keys out of the export.
+export const mapDashboardToCoreDataRow = ({
+ definition,
+ ...dashboard
+}: DashboardCoreDataInput) => {
+ const widgets = (definition as { widgets?: unknown })?.widgets;
+ return {
+ ...dashboard,
+ definition: {
+ widgets: Array.isArray(widgets)
+ ? widgets.map((placement: Record<string, unknown>) => ({
+ type: placement.type ?? null,
+ widgetId: placement.widgetId ?? null,
+ x: placement.x ?? null,
+ y: placement.y ?? null,
+ x_size: placement.x_size ?? null,
+ y_size: placement.y_size ?? null,
+ }))
+ : [],
+ },
+ };
+};
+
type TablePageArgs<TCursor> = {
lastRow: TCursor | null;
take: number;
@@ -623,6 +726,60 @@ export const coreDataTableExports: Array<
},
}),
}),
+ (args) =>
+ uploadTableCoreDataJsonl({
+ ...args,
+ tableName: "dashboards",
+ // Customer-authored name/description are free text and intentionally
+ // excluded; the definition JSON holds only widget placements (ids,
+ // positions, sizes). projectId NULL marks Langfuse-owned templates.
+ fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
+ prisma.dashboard.findMany({
+ take,
+ ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
+ orderBy: { id: "asc" },
+ select: {
+ id: true,
+ projectId: true,
+ createdBy: true,
+ definition: true,
+ createdAt: true,
+ updatedAt: true,
+ },
+ }),
+ mapRow: mapDashboardToCoreDataRow,
+ }),
+ (args) =>
+ uploadTableCoreDataJsonl({
+ ...args,
+ tableName: "dashboardWidgets",
+ // Customer-authored name/description are free text and intentionally
+ // excluded; filter values are stripped in the mapper. The remaining
+ // shape (view, metrics, dimensions, chart type) is what dashboard
+ // product decisions need (e.g. which aggregation users pair with a
+ // measure). projectId NULL marks Langfuse-owned template widgets.
+ fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>
+ prisma.dashboardWidget.findMany({
+ take,
+ ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),
+ orderBy: { id: "asc" },
+ select: {
+ id: true,
+ projectId: true,
+ createdBy: true,
+ view: true,
+ dimensions: true,
+ metrics: true,
+ filters: true,
+ chartType: true,
+ chartConfig: true,
+ minVersion: true,
+ createdAt: true,
+ updatedAt: true,
+ },
+ }),
+ mapRow: mapDashboardWidgetToCoreDataRow,
+ }),
];
export const coreDataS3ExportProcessor: Processor = async (): Promise<void> => {