OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
chart: stop serializing a million "this field is undefined" markers
Carl-Gerhard Lindesvärde826a05abf35
diff --git a/packages/core/src/modules/chart/src/engine/format.ts b/packages/core/src/modules/chart/src/engine/format.tsindex 0159dd466..159e1af5d 100644--- a/packages/core/src/modules/chart/src/engine/format.ts+++ b/packages/core/src/modules/chart/src/engine/format.ts@@ -131,16 +131,24 @@ export function format(}: {}),},- data: cs.data.map((item, index) => ({- date: item.date,- count: item.count,- previous: previousSerie?.data[index]- ? getPreviousMetric(- item.count,- previousSerie.data[index]?.count ?? null- )- : undefined,- })),+ // The key is OMITTED rather than set to `undefined` when there is no+ // previous point. superjson records one metadata entry per `undefined`,+ // each carrying its full path string, so a 12-month three-breakdown+ // chart spent ~77 MB and several seconds encoding 1.69 M entries whose+ // entire content was "this field is undefined". Reading+ // `data[i].previous` still yields `undefined` either way, and superjson+ // already erased the key over the wire, so no consumer can tell.+ data: cs.data.map((item, index) => {+ const previousPoint = previousSerie?.data[index];+ if (!previousPoint) {+ return { date: item.date, count: item.count };+ }+ return {+ date: item.date,+ count: item.count,+ previous: getPreviousMetric(item.count, previousPoint.count ?? null),+ };+ }),};});diff --git a/packages/core/src/modules/report/report.constants.ts b/packages/core/src/modules/report/report.constants.tsindex 287f03457..df73da0c1 100644--- a/packages/core/src/modules/report/report.constants.ts+++ b/packages/core/src/modules/report/report.constants.ts@@ -1044,7 +1044,13 @@ export type IChartSerie = {data: {date: string;count: number;- previous: PreviousValue;+ /**+ * Absent, not `undefined`, when the request asked for no previous period.+ * superjson writes one metadata entry per explicit `undefined`, and a long+ * range with breakdowns produced millions of them; over the wire the two+ * are indistinguishable. Read it as optional.+ */+ previous?: PreviousValue;}[];};diff --git a/packages/core/src/shared/group-by-labels.ts b/packages/core/src/shared/group-by-labels.tsindex 3b084fca1..b31d65645 100644--- a/packages/core/src/shared/group-by-labels.ts+++ b/packages/core/src/shared/group-by-labels.ts@@ -60,16 +60,21 @@ export function groupByLabels(data: ISerieDataItem[]): GroupedResult[] {),}));+ // Every group is padded to the full set of dates so all series share an+ // x-axis. The lookup is a Map rather than a `.find()` per date: that scan+ // was O(groups x dates x points), and a 12-month chart with three+ // breakdowns reaches 4,640 groups over 365 dates. The OUTPUT is unchanged —+ // same dates, same order, same zero-filled points.+ const allDates = Array.from(timestamps);return result.filter((group) => group.name.length > 0).map((group) => {+ const byDate = new Map(group.data.map((dp) => [dp.date, dp]));return {...group,- // This will ensure that all dates are present in the data array- data: Array.from(timestamps).map((date) => {- const dataPoint = group.data.find((dp) => dp.date === date);- return dataPoint || { date, count: 0, total_count: 0 };- }),+ data: allDates.map(+ (date) => byDate.get(date) ?? { date, count: 0, total_count: 0 }+ ),};});}