🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(worker): export dashboards and widgets in the core data S3 export (#16621)
Trang Nguyend61909d76270
diff --git a/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts b/worker/src/queues/__tests__/coreDataS3ExportQueue.test.tsindex 51f29a081c..3f49594d50 100644--- a/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts+++ b/worker/src/queues/__tests__/coreDataS3ExportQueue.test.ts@@ -4,6 +4,8 @@ import { type StorageService } from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";import {coreDataTableExports,+ mapDashboardToCoreDataRow,+ mapDashboardWidgetToCoreDataRow,mapEvaluationRuleToCoreDataRow,mapJobConfigurationToCoreDataRow,mapUserToCoreDataRow,@@ -129,6 +131,8 @@ describe("coreDataTableExports", () => {prisma.evaluationRule,prisma.evaluationRuleEvaluatorAssignment,prisma.jobConfiguration,+ prisma.dashboard,+ prisma.dashboardWidget,];for (const delegate of delegates) {vi.spyOn(delegate, "findMany").mockResolvedValue([]);@@ -157,11 +161,139 @@ describe("coreDataTableExports", () => {"core/evaluatorVersions.jsonl","core/evaluationRules.jsonl","core/evaluationRuleEvaluatorAssignments.jsonl",+ "core/dashboards.jsonl",+ "core/dashboardWidgets.jsonl",]),);});});+describe("mapDashboardWidgetToCoreDataRow", () => {+ it("allowlists dimensions/metrics and replaces unknown strings with a sentinel", () => {+ const row = mapDashboardWidgetToCoreDataRow({+ id: "widget-1",+ dimensions: [+ { field: "calledToolNames" },+ { field: "customer entered free text" },+ ],+ metrics: [+ { measure: "toolCalls", agg: "sum" },+ { measure: "leaked customer string", agg: "not-an-agg" },+ ],+ filters: [],+ chartConfig: {+ type: "PIVOT_TABLE",+ row_limit: 20,+ defaultSort: { column: "customer column text", order: "DESC" },+ },+ });++ expect(row.dimensions).toStrictEqual([+ { field: "calledToolNames" },+ { field: "__invalid__" },+ ]);+ expect(row.metrics).toStrictEqual([+ { measure: "toolCalls", agg: "sum" },+ { measure: "__invalid__", agg: "__invalid__" },+ ]);+ expect(row.chartConfig).toStrictEqual({+ type: "PIVOT_TABLE",+ row_limit: 20,+ bins: null,+ });+ const serialized = JSON.stringify(row);+ expect(serialized).not.toContain("customer entered free text");+ expect(serialized).not.toContain("leaked customer string");+ expect(serialized).not.toContain("customer column text");+ });++ it("keeps filter shape but strips customer-entered values and keys", () => {+ const row = mapDashboardWidgetToCoreDataRow({+ id: "widget-1",+ projectId: "project-1",+ view: "OBSERVATIONS",+ dimensions: [],+ metrics: [{ measure: "toolCalls", agg: "sum" }],+ chartConfig: { type: "LINE_TIME_SERIES" },+ filters: [+ {+ column: "userId",+ operator: "any of",+ value: ["customer-user-42"],+ type: "stringOptions",+ },+ {+ column: "metadata",+ key: "customer-secret-key",+ operator: "=",+ value: "customer-value",+ type: "stringObject",+ },+ ],+ });++ expect(row.filters).toStrictEqual([+ { column: "userId", operator: "any of", type: "stringOptions" },+ { column: "metadata", operator: "=", type: "stringObject" },+ ]);+ const serialized = JSON.stringify(row);+ expect(serialized).not.toContain("customer-user-42");+ expect(serialized).not.toContain("customer-secret-key");+ expect(serialized).not.toContain("customer-value");+ });++ it("normalizes a non-array filters column to an empty list", () => {+ const row = mapDashboardWidgetToCoreDataRow({+ id: "widget-2",+ dimensions: null,+ metrics: null,+ filters: null,+ chartConfig: null,+ });++ expect(row.filters).toStrictEqual([]);+ expect(row.dimensions).toStrictEqual([]);+ expect(row.metrics).toStrictEqual([]);+ });+});++describe("mapDashboardToCoreDataRow", () => {+ it("exports only known placement fields from the definition", () => {+ const row = mapDashboardToCoreDataRow({+ id: "dashboard-1",+ definition: {+ widgets: [+ {+ type: "widget",+ widgetId: "widget-1",+ x: 0,+ y: 0,+ x_size: 6,+ y_size: 4,+ id: "placement-1",+ unexpectedCustomerField: "customer text",+ },+ ],+ unexpectedTopLevel: "more customer text",+ },+ });++ expect(row.definition).toStrictEqual({+ widgets: [+ {+ type: "widget",+ widgetId: "widget-1",+ x: 0,+ y: 0,+ x_size: 6,+ y_size: 4,+ },+ ],+ });+ expect(JSON.stringify(row)).not.toContain("customer text");+ });+});+describe("uploadTableCoreDataJsonl", () => {const createS3Client = (uploadFileBuffered: ReturnType<typeof vi.fn>,diff --git a/worker/src/queues/coreDataS3ExportQueue.ts b/worker/src/queues/coreDataS3ExportQueue.tsindex b0a2b76592..05769cf499 100644--- a/worker/src/queues/coreDataS3ExportQueue.ts+++ b/worker/src/queues/coreDataS3ExportQueue.ts@@ -6,6 +6,7 @@ import {StorageServiceFactory,type StorageService,} from "@langfuse/shared/src/server";+import { metricAggregations, viewDeclarations } from "@langfuse/shared/query";import { prisma } from "@langfuse/shared/src/db";import { env } from "../env";@@ -83,6 +84,108 @@ export const mapEvaluationRuleToCoreDataRow = ({sampling: sampling.toNumber(),});+// Widget dimensions/metrics/chart config are persisted as free-form strings+// (DimensionSchema/MetricSchema accept any z.string()), so an API client can+// smuggle arbitrary text into them. The export therefore allowlists every+// string against the query-model declarations and replaces unknown values+// with a sentinel instead of forwarding them.+const KNOWN_WIDGET_MEASURES = new Set(+ Object.values(viewDeclarations).flatMap((versionViews) =>+ Object.values(versionViews).flatMap((view) => Object.keys(view.measures)),+ ),+);+const KNOWN_WIDGET_DIMENSIONS = new Set(+ Object.values(viewDeclarations).flatMap((versionViews) =>+ Object.values(versionViews).flatMap((view) => Object.keys(view.dimensions)),+ ),+);+const KNOWN_WIDGET_AGGREGATIONS = new Set<string>(metricAggregations.options);+const INVALID_SENTINEL = "__invalid__";++const allowlisted = (value: unknown, allowlist: Set<string>): string | null =>+ value == null+ ? null+ : typeof value === "string" && allowlist.has(value)+ ? value+ : INVALID_SENTINEL;++type DashboardWidgetCoreDataInput = {+ dimensions: unknown;+ metrics: unknown;+ filters: unknown;+ chartConfig: unknown;+} & Record<string, unknown>;++// Widget filters carry customer-entered values (user ids, metadata values,+// tool names, ...). Analytics only needs which columns/operators are used, so+// the values (and metadata keys) are stripped before export; every other+// string field is allowlisted (see above).+export const mapDashboardWidgetToCoreDataRow = ({+ dimensions,+ metrics,+ filters,+ chartConfig,+ ...widget+}: DashboardWidgetCoreDataInput) => {+ const config = (chartConfig ?? {}) as Record<string, unknown>;+ return {+ ...widget,+ dimensions: Array.isArray(dimensions)+ ? dimensions.map((dimension: Record<string, unknown>) => ({+ field: allowlisted(dimension.field, KNOWN_WIDGET_DIMENSIONS),+ }))+ : [],+ metrics: Array.isArray(metrics)+ ? metrics.map((metric: Record<string, unknown>) => ({+ measure: allowlisted(metric.measure, KNOWN_WIDGET_MEASURES),+ agg: allowlisted(metric.agg, KNOWN_WIDGET_AGGREGATIONS),+ }))+ : [],+ filters: Array.isArray(filters)+ ? filters.map((filter: Record<string, unknown>) => ({+ column: filter.column ?? null,+ operator: filter.operator ?? null,+ type: filter.type ?? null,+ }))+ : [],+ // Explicit safe scalars only; defaultSort.column is a free string and is+ // deliberately not exported.+ chartConfig: {+ type: typeof config.type === "string" ? config.type : null,+ row_limit: typeof config.row_limit === "number" ? config.row_limit : null,+ bins: typeof config.bins === "number" ? config.bins : null,+ },+ };+};++type DashboardCoreDataInput = {+ definition: unknown;+} & Record<string, unknown>;++// Dashboard definitions hold widget placements; explicit field picks keep any+// unexpected persisted keys out of the export.+export const mapDashboardToCoreDataRow = ({+ definition,+ ...dashboard+}: DashboardCoreDataInput) => {+ const widgets = (definition as { widgets?: unknown })?.widgets;+ return {+ ...dashboard,+ definition: {+ widgets: Array.isArray(widgets)+ ? widgets.map((placement: Record<string, unknown>) => ({+ type: placement.type ?? null,+ widgetId: placement.widgetId ?? null,+ x: placement.x ?? null,+ y: placement.y ?? null,+ x_size: placement.x_size ?? null,+ y_size: placement.y_size ?? null,+ }))+ : [],+ },+ };+};+type TablePageArgs<TCursor> = {lastRow: TCursor | null;take: number;@@ -623,6 +726,60 @@ export const coreDataTableExports: Array<},}),}),+ (args) =>+ uploadTableCoreDataJsonl({+ ...args,+ tableName: "dashboards",+ // Customer-authored name/description are free text and intentionally+ // excluded; the definition JSON holds only widget placements (ids,+ // positions, sizes). projectId NULL marks Langfuse-owned templates.+ fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>+ prisma.dashboard.findMany({+ take,+ ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),+ orderBy: { id: "asc" },+ select: {+ id: true,+ projectId: true,+ createdBy: true,+ definition: true,+ createdAt: true,+ updatedAt: true,+ },+ }),+ mapRow: mapDashboardToCoreDataRow,+ }),+ (args) =>+ uploadTableCoreDataJsonl({+ ...args,+ tableName: "dashboardWidgets",+ // Customer-authored name/description are free text and intentionally+ // excluded; filter values are stripped in the mapper. The remaining+ // shape (view, metrics, dimensions, chart type) is what dashboard+ // product decisions need (e.g. which aggregation users pair with a+ // measure). projectId NULL marks Langfuse-owned template widgets.+ fetchPage: ({ lastRow, take }: TablePageArgs<{ id: string }>) =>+ prisma.dashboardWidget.findMany({+ take,+ ...(lastRow ? { cursor: { id: lastRow.id }, skip: 1 } : {}),+ orderBy: { id: "asc" },+ select: {+ id: true,+ projectId: true,+ createdBy: true,+ view: true,+ dimensions: true,+ metrics: true,+ filters: true,+ chartType: true,+ chartConfig: true,+ minVersion: true,+ createdAt: true,+ updatedAt: true,+ },+ }),+ mapRow: mapDashboardWidgetToCoreDataRow,+ }),];export const coreDataS3ExportProcessor: Processor = async (): Promise<void> => {