🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable no-nested-ternary */
// @vitest-environment node
import startCase from "lodash/startCase";
import { type z } from "zod";
import { type metricAggregations } from "@langfuse/shared/query";
import {
mapWidgetUiTableFilterToView,
normalizeStoredWidgetFiltersForEditor,
} from "@/src/features/dashboard/lib/dashboardUiTableToViewMapping";
import { isTimeSeriesChart } from "@/src/features/widgets/chart-library/utils";
import {
buildWidgetDescription,
buildWidgetName,
sanitizePivotTableDefaultSort,
type WidgetChartConfig,
} from "@/src/features/widgets/utils";
import {
applyChartTypeChange,
deriveEffectiveSort,
deriveWidgetBaseMinVersion,
deriveWidgetSuggestions,
resolveWidgetFormVersion,
toDefaultValues,
toSavePayload,
type WidgetFormValues,
type WidgetInitialValues,
type WidgetSavePayload,
} from "./widgetFormSchema";
/** The view version the app would seed with for a given widget (non-beta). */
const fixtureViewVersion = (iv: WidgetInitialValues) =>
resolveWidgetFormVersion({
view: iv.view,
baseMinVersion: deriveWidgetBaseMinVersion(iv),
activeVersion: "v1",
});
/**
* An INDEPENDENT reconstruction of the legacy `handleSaveWidget` object builder
* (WidgetForm.tsx :1430-1471), driven off the legacy single/pivot state seeding,
* the auto-name/description effects, and the pivot default-sort sanitize effect.
* It is deliberately written from the ORIGINAL component logic — not from the
* new adapters — so that a shape-mapping bug in `toDefaultValues`/`toSavePayload`
* is caught by the deep-equality assertion below.
*/
function legacyReconstruct(iv: WidgetInitialValues): WidgetSavePayload {
const isPivot = iv.chartType === "PIVOT_TABLE";
const selectedMeasure = iv.measure;
const selectedAggregation = iv.aggregation;
const selectedDimension = iv.dimension;
const selectedMetrics =
isPivot && iv.metrics?.length
? iv.metrics.map((m) => ({
id: `${m.agg}_${m.measure}`,
measure: m.measure,
aggregation: m.agg as z.infer<typeof metricAggregations>,
}))
: [
{
id: `${iv.aggregation}_${iv.measure}`,
measure: iv.measure,
aggregation: iv.aggregation,
},
];
const pivotDimensions =
isPivot && iv.dimensions?.length ? iv.dimensions.map((d) => d.field) : [];
const initialDefaultSort = isPivot
? sanitizePivotTableDefaultSort(iv.chartConfig?.defaultSort, {
dimensions: iv.dimensions ?? [],
metrics:
iv.metrics ??
(iv.measure && iv.aggregation
? [{ measure: iv.measure, agg: iv.aggregation }]
: []),
})
: undefined;
let defaultSortColumn = initialDefaultSort?.column ?? "none";
let defaultSortOrder: "ASC" | "DESC" = initialDefaultSort?.order ?? "DESC";
const rowLimit = iv.chartConfig?.row_limit ?? 100;
const histogramBins = iv.chartConfig?.bins ?? 10;
const userFilterState = normalizeStoredWidgetFiltersForEditor(
iv.view,
iv.filters ?? [],
).editorFilters;
// Legacy sort-sanitize effect (:541): drop a stale sort column.
if (isPivot) {
const sanitized = sanitizePivotTableDefaultSort(
defaultSortColumn !== "none"
? { column: defaultSortColumn, order: defaultSortOrder }
: undefined,
{
dimensions: pivotDimensions
.filter((field) => field && field !== "none")
.map((field) => ({ field })),
metrics: selectedMetrics
.filter((metric) => metric.measure && metric.measure !== "")
.map((metric) => ({
measure: metric.measure,
agg: metric.aggregation,
})),
},
);
if (defaultSortColumn !== "none" && !sanitized) {
defaultSortColumn = "none";
defaultSortOrder = "DESC";
}
}
// Legacy auto-name / auto-description (create: suggestion; edit: sticks).
const validMetricsForNaming = selectedMetrics.filter(
(m) => m.measure && m.measure !== "",
);
const dimensionForNaming =
isPivot && pivotDimensions.length > 0
? pivotDimensions.map(startCase).join(" and ")
: selectedDimension;
const metricNames =
isPivot && validMetricsForNaming.length > 0
? validMetricsForNaming.map((m) => m.id)
: undefined;
// buildWidgetName / buildWidgetDescription are shared with the adapter; the
// structural arg construction above is what the parity check exercises.
const suggestedName = buildWidgetName({
aggregation: isPivot ? "count" : selectedAggregation,
measure: isPivot ? "count" : selectedMeasure,
dimension: dimensionForNaming,
view: iv.view,
metrics: metricNames,
isMultiMetric: isPivot && validMetricsForNaming.length > 0,
});
const suggestedDescription = buildWidgetDescription({
aggregation: isPivot ? "count" : selectedAggregation,
measure: isPivot ? "count" : selectedMeasure,
dimension: dimensionForNaming,
view: iv.view,
filters: userFilterState,
metrics: metricNames,
isMultiMetric: isPivot && validMetricsForNaming.length > 0,
});
const widgetName = iv.name && iv.name.length > 0 ? iv.name : suggestedName;
const widgetDescription =
iv.description && iv.description.length > 0
? iv.description
: suggestedDescription;
const validMetrics = selectedMetrics.filter(
(m) => m.measure && m.measure !== "",
);
const saveDimensions = isPivot
? pivotDimensions.map((field) => ({ field }))
: selectedDimension !== "none"
? [{ field: selectedDimension }]
: [];
const saveMetrics = isPivot
? validMetrics.map((m) => ({ measure: m.measure, agg: m.aggregation }))
: [{ measure: selectedMeasure, agg: selectedAggregation }];
const normalizedUserFilters = mapWidgetUiTableFilterToView(
iv.view,
userFilterState,
);
const chartConfig: WidgetChartConfig = isTimeSeriesChart(iv.chartType)
? { type: iv.chartType }
: iv.chartType === "HISTOGRAM"
? { type: iv.chartType, bins: histogramBins }
: iv.chartType === "PIVOT_TABLE"
? {
type: iv.chartType,
row_limit: rowLimit,
defaultSort:
defaultSortColumn && defaultSortColumn !== "none"
? { column: defaultSortColumn, order: defaultSortOrder }
: undefined,
}
: { type: iv.chartType, row_limit: rowLimit };
return {
name: widgetName,
description: widgetDescription,
view: iv.view,
dimensions: saveDimensions,
metrics: saveMetrics,
filters: normalizedUserFilters,
chartType: iv.chartType,
chartConfig,
};
}
/** Runs the new adapter path: initialValues → form values → save payload. */
function adapterSavePayload(iv: WidgetInitialValues): WidgetSavePayload {
const values = toDefaultValues(iv, fixtureViewVersion(iv));
const suggestions = deriveWidgetSuggestions(values);
const effectiveSort = deriveEffectiveSort(values);
return toSavePayload(values, {
suggestedName: suggestions.name,
suggestedDescription: suggestions.description,
effectiveSort,
});
}
const fixtures: Record<string, WidgetInitialValues> = {
"regular line (create)": {
name: "",
description: "",
view: "observations",
measure: "latency",
aggregation: "avg",
dimension: "none",
filters: [],
chartType: "LINE_TIME_SERIES",
chartConfig: { type: "LINE_TIME_SERIES" },
},
"breakdown bar with dimension": {
name: "",
description: "",
view: "observations",
measure: "totalCost",
aggregation: "sum",
dimension: "environment",
filters: [],
chartType: "VERTICAL_BAR",
chartConfig: { type: "VERTICAL_BAR", row_limit: 25 },
},
"count measure big number": {
name: "",
description: "",
view: "observations",
measure: "count",
aggregation: "count",
dimension: "none",
filters: [],
chartType: "NUMBER",
chartConfig: { type: "NUMBER" },
},
histogram: {
name: "",
description: "",
view: "observations",
measure: "latency",
aggregation: "histogram",
dimension: "none",
filters: [],
chartType: "HISTOGRAM",
chartConfig: { type: "HISTOGRAM", bins: 25 },
},
"pivot with sort (edit-mode arrays)": {
name: "My Pivot Widget",
description: "A saved description",
view: "observations",
measure: "latency",
aggregation: "avg",
dimension: "environment",
filters: [],
chartType: "PIVOT_TABLE",
metrics: [
{ measure: "latency", agg: "avg" },
{ measure: "count", agg: "count" },
],
dimensions: [{ field: "environment" }, { field: "name" }],
chartConfig: {
type: "PIVOT_TABLE",
row_limit: 50,
defaultSort: { column: "avg_latency", order: "DESC" },
},
minVersion: 1,
},
"pivot with stale sort column dropped": {
name: "Stale Sort Pivot",
description: "desc",
view: "observations",
measure: "latency",
aggregation: "avg",
dimension: "environment",
filters: [],
chartType: "PIVOT_TABLE",
metrics: [{ measure: "latency", agg: "avg" }],
dimensions: [{ field: "environment" }],
// sort references a metric that is no longer present -> must be dropped
chartConfig: {
type: "PIVOT_TABLE",
row_limit: 100,
defaultSort: { column: "sum_totalCost", order: "ASC" },
},
minVersion: 1,
},
"edit non-pivot single metric (arrays ignored)": {
name: "Edited Widget",
description: "Edited description",
view: "observations",
measure: "totalCost",
aggregation: "sum",
dimension: "environment",
filters: [],
chartType: "LINE_TIME_SERIES",
metrics: [{ measure: "totalCost", agg: "sum" }],
dimensions: [{ field: "environment" }],
chartConfig: { type: "LINE_TIME_SERIES" },
minVersion: 1,
},
// Exercises the filter double-transform path: stored view-space filters go
// through normalizeStoredWidgetFiltersForEditor (on seed) and back through
// mapWidgetUiTableFilterToView (on save). Parity holds because BOTH the
// adapter and the legacy reconstruction apply the identical transforms.
"line with a stored environment filter": {
name: "",
description: "",
view: "observations",
measure: "count",
aggregation: "count",
dimension: "none",
filters: [
{
column: "environment",
type: "stringOptions",
operator: "any of",
value: ["production"],
},
],
chartType: "LINE_TIME_SERIES",
chartConfig: { type: "LINE_TIME_SERIES" },
},
// Historical widgets that stored count(toolCalls), the silent undercount.
// Opening them must not rewrite count to sum via defaultAggregation.
"saved toolCalls with count aggregation": {
name: "My tool calls widget",
description: "saved copy",
view: "observations",
measure: "toolCalls",
aggregation: "count",
dimension: "none",
filters: [],
chartType: "NUMBER",
chartConfig: { type: "NUMBER" },
minVersion: 1,
},
"saved toolCalls with sum aggregation": {
name: "Total tool calls",
description: "saved copy",
view: "observations",
measure: "toolCalls",
aggregation: "sum",
dimension: "none",
filters: [],
chartType: "NUMBER",
chartConfig: { type: "NUMBER" },
minVersion: 1,
},
"saved toolCallInvocations with count aggregation": {
name: "Invocations count",
description: "saved copy",
view: "observations",
measure: "toolCallInvocations",
aggregation: "count",
dimension: "calledToolNames",
filters: [],
chartType: "HORIZONTAL_BAR",
chartConfig: { type: "HORIZONTAL_BAR" },
minVersion: 1,
},
};
describe("widget form adapters round-trip parity", () => {
it.each(Object.entries(fixtures))(
"%s: toSavePayload(toDefaultValues(x)) matches the legacy save object",
(_name, iv) => {
expect(adapterSavePayload(iv)).toEqual(legacyReconstruct(iv));
},
);
it("regular line create produces the exact legacy save shape", () => {
expect(adapterSavePayload(fixtures["regular line (create)"])).toEqual({
name: "Avg Latency (Observations)",
description: "Shows the avg latency of Observations",
view: "observations",
dimensions: [],
metrics: [{ measure: "latency", agg: "avg" }],
filters: [],
chartType: "LINE_TIME_SERIES",
chartConfig: { type: "LINE_TIME_SERIES" },
});
});
it("pivot save carries row_limit + sanitized defaultSort and drops stale sort", () => {
const kept = adapterSavePayload(
fixtures["pivot with sort (edit-mode arrays)"],
);
expect(kept.chartConfig).toEqual({
type: "PIVOT_TABLE",
row_limit: 50,
defaultSort: { column: "avg_latency", order: "DESC" },
});
expect(kept.metrics).toEqual([
{ measure: "latency", agg: "avg" },
{ measure: "count", agg: "count" },
]);
expect(kept.dimensions).toEqual([
{ field: "environment" },
{ field: "name" },
]);
const dropped = adapterSavePayload(
fixtures["pivot with stale sort column dropped"],
);
expect(dropped.chartConfig).toEqual({
type: "PIVOT_TABLE",
row_limit: 100,
defaultSort: undefined,
});
});
it("count + NUMBER save carries row_limit and count metric", () => {
const payload = adapterSavePayload(fixtures["count measure big number"]);
expect(payload.chartType).toBe("NUMBER");
expect(payload.chartConfig).toEqual({ type: "NUMBER", row_limit: 100 });
expect(payload.metrics).toEqual([{ measure: "count", agg: "count" }]);
expect(payload.name).toBe("Count (Observations)");
});
});
describe("widget form view version", () => {
it("keeps a v1 base version for a v1-compatible create shape", () => {
expect(deriveWidgetBaseMinVersion(fixtures["regular line (create)"])).toBe(
1,
);
});
it("promotes a legacy widget when its current shape requires v2", () => {
expect(
resolveWidgetFormVersion({
view: "observations",
baseMinVersion: 1,
activeVersion: "v1",
shape: {
dimensions: [],
metrics: [{ measure: "count" }],
filters: [
{
column: "isRootObservation",
type: "boolean",
operator: "=",
value: true,
},
],
},
}),
).toBe("v2");
});
it("uses the events-backed v2 declaration for legacy trace widgets in v4", () => {
expect(
resolveWidgetFormVersion({
view: "traces",
baseMinVersion: 1,
activeVersion: "v2",
shape: {
dimensions: [],
metrics: [{ measure: "count" }],
filters: [],
},
}),
).toBe("v2");
});
});
describe("toDefaultValues normalizes malformed stored/imported widgets", () => {
it("heals a HISTOGRAM on the count measure to NUMBER (matches the old mount effect)", () => {
const iv: WidgetInitialValues = {
name: "Legacy Histogram",
description: "d",
view: "observations",
measure: "count",
aggregation: "histogram",
dimension: "none",
filters: [],
chartType: "HISTOGRAM",
chartConfig: { type: "HISTOGRAM", bins: 10 },
minVersion: 1,
};
const values = toDefaultValues(iv, fixtureViewVersion(iv));
expect(values.chart.type).toBe("NUMBER");
expect(values.metrics).toEqual([
{ measure: "count", aggregation: "count" },
]);
// And the save payload reflects the healed (valid) state.
const payload = adapterSavePayload(iv);
expect(payload.chartType).toBe("NUMBER");
expect(payload.metrics).toEqual([{ measure: "count", agg: "count" }]);
});
it("heals a HISTOGRAM with a stored non-histogram aggregation to the histogram aggregation", () => {
const iv: WidgetInitialValues = {
name: "",
description: "",
view: "observations",
measure: "latency",
aggregation: "avg",
dimension: "none",
filters: [],
chartType: "HISTOGRAM",
chartConfig: { type: "HISTOGRAM", bins: 10 },
};
const values = toDefaultValues(iv, fixtureViewVersion(iv));
expect(values.chart.type).toBe("HISTOGRAM");
expect(values.metrics[0].aggregation).toBe("histogram");
});
it("drops a stored breakdown dimension on a NUMBER chart (matches the old breakdown-wipe effect)", () => {
const iv: WidgetInitialValues = {
name: "Big number with a stray dimension",
description: "d",
view: "observations",
measure: "count",
aggregation: "count",
dimension: "environment",
filters: [],
chartType: "NUMBER",
chartConfig: { type: "NUMBER" },
minVersion: 1,
};
const values = toDefaultValues(iv, fixtureViewVersion(iv));
expect(values.dimensions).toEqual([]);
expect(adapterSavePayload(iv).dimensions).toEqual([]);
});
it("is a fixed point on a valid widget (normalizes to itself)", () => {
const iv = fixtures["breakdown bar with dimension"];
const once = toDefaultValues(iv, fixtureViewVersion(iv));
expect(once.chart.type).toBe("VERTICAL_BAR");
expect(once.dimensions).toEqual([{ field: "environment" }]);
expect(once.metrics).toEqual([
{ measure: "totalCost", aggregation: "sum" },
]);
});
});
describe("opening a saved widget does not apply defaultAggregation", () => {
it.each([
["saved toolCalls with count aggregation", "toolCalls", "count"],
["saved toolCalls with sum aggregation", "toolCalls", "sum"],
[
"saved toolCallInvocations with count aggregation",
"toolCallInvocations",
"count",
],
] as const)(
"%s keeps the stored %s + %s pair",
(fixtureName, measure, aggregation) => {
const iv = fixtures[fixtureName];
const values = toDefaultValues(iv, fixtureViewVersion(iv));
expect(values.metrics).toEqual([{ measure, aggregation }]);
const payload = adapterSavePayload(iv);
expect(payload.metrics).toEqual([{ measure, agg: aggregation }]);
expect(payload.name).toBe(iv.name);
expect(payload.description).toBe(iv.description);
},
);
it("keeps a pivot slot that stored count(toolCalls) next to avg(latency)", () => {
const iv: WidgetInitialValues = {
name: "Mixed pivot",
description: "saved",
view: "observations",
measure: "latency",
aggregation: "avg",
dimension: "environment",
filters: [],
chartType: "PIVOT_TABLE",
metrics: [
{ measure: "latency", agg: "avg" },
{ measure: "toolCalls", agg: "count" },
],
dimensions: [{ field: "environment" }],
chartConfig: { type: "PIVOT_TABLE", row_limit: 50 },
minVersion: 1,
};
expect(toDefaultValues(iv, fixtureViewVersion(iv)).metrics).toEqual([
{ measure: "latency", aggregation: "avg" },
{ measure: "toolCalls", aggregation: "count" },
]);
expect(adapterSavePayload(iv).metrics).toEqual([
{ measure: "latency", agg: "avg" },
{ measure: "toolCalls", agg: "count" },
]);
});
});
describe("applyChartTypeChange dimension boundary", () => {
const pivotWithDims: WidgetFormValues = {
name: null,
description: null,
view: "observations",
filters: [],
metrics: [
{ measure: "latency", aggregation: "avg" },
{ measure: "count", aggregation: "count" },
],
dimensions: [{ field: "environment" }, { field: "name" }],
chart: {
type: "PIVOT_TABLE",
bins: 10,
rowLimit: 100,
sort: null,
},
};
const barWithBreakdown: WidgetFormValues = {
name: null,
description: null,
view: "observations",
filters: [],
metrics: [{ measure: "latency", aggregation: "avg" }],
dimensions: [{ field: "environment" }],
chart: { type: "VERTICAL_BAR", bins: 10, rowLimit: 100, sort: null },
};
it("pivot([environment, name]) -> vertical bar clears the breakdown to none", () => {
const next = applyChartTypeChange(pivotWithDims, "VERTICAL_BAR", "v2");
expect(next.chart.type).toBe("VERTICAL_BAR");
expect(next.dimensions).toEqual([]);
// non-pivot trims to a single metric
expect(next.metrics).toEqual([{ measure: "latency", aggregation: "avg" }]);
});
it("vertical bar(breakdown=environment) -> pivot starts with empty row dimensions", () => {
const next = applyChartTypeChange(barWithBreakdown, "PIVOT_TABLE", "v2");
expect(next.chart.type).toBe("PIVOT_TABLE");
expect(next.dimensions).toEqual([]);
});
it("vertical bar -> line keeps the breakdown dimension (within non-pivot)", () => {
const next = applyChartTypeChange(
barWithBreakdown,
"LINE_TIME_SERIES",
"v2",
);
expect(next.chart.type).toBe("LINE_TIME_SERIES");
expect(next.dimensions).toEqual([{ field: "environment" }]);
});
});
describe("save seam: editor-space filters map to view space (double-map)", () => {
// The live path maps the SAME editor-space filters to view space twice, from
// two independent seams: the resolver (validation) and toSavePayload (save).
// The parity suite above bypasses the resolver, so exercise both mappings
// here and assert they agree AND that the mapping is idempotent (a filter
// already in view space maps to itself).
const stored: WidgetInitialValues =
fixtures["line with a stored environment filter"];
it("resolver mapping and toSavePayload agree, and the mapping is idempotent", () => {
const values = toDefaultValues(stored, fixtureViewVersion(stored));
// Resolver seam: maps the form's editor-space filters to view space.
const resolverMapped = mapWidgetUiTableFilterToView(
values.view,
values.filters,
);
// Save seam: toSavePayload maps the same editor-space filters independently.
const suggestions = deriveWidgetSuggestions(values);
const saved = toSavePayload(values, {
suggestedName: suggestions.name,
suggestedDescription: suggestions.description,
effectiveSort: deriveEffectiveSort(values),
});
expect(saved.filters).toEqual(resolverMapped);
expect(saved.filters.length).toBeGreaterThan(0);
// Idempotency: mapping the already-mapped (view-space) filters again is a
// no-op. This invariant is what lets the two seams double-map safely.
expect(mapWidgetUiTableFilterToView(values.view, saved.filters)).toEqual(
saved.filters,
);
});
});
// --- sum(toolCalls) reads as "number of tool calls" in suggested name/copy ---
describe("buildWidgetName / buildWidgetDescription for sum of tool calls", () => {
it("names the widget by meaning, not aggregation mechanics", () => {
expect(
buildWidgetName({
aggregation: "sum",
measure: "toolCalls",
dimension: "none",
view: "observations",
}),
).toBe("Number of Tool Calls (Observations)");
expect(
buildWidgetDescription({
aggregation: "sum",
measure: "toolCalls",
dimension: "none",
view: "observations",
filters: [],
}),
).toBe("Shows the number of tool calls of Observations");
});
it("keeps the generic template for other aggregations of toolCalls", () => {
expect(
buildWidgetName({
aggregation: "avg",
measure: "toolCalls",
dimension: "none",
view: "observations",
}),
).toBe("Avg Tool Calls (Observations)");
});
});