Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

chart: stop serializing a million "this field is undefined" markers

Carl-Gerhard Lindesvärde826a05abf35
diff --git a/packages/core/src/modules/chart/src/engine/format.ts b/packages/core/src/modules/chart/src/engine/format.ts
index 0159dd466..159e1af5d 100644
--- a/packages/core/src/modules/chart/src/engine/format.ts
+++ b/packages/core/src/modules/chart/src/engine/format.ts
@@ -131,16 +131,24 @@ export function format(
}
: {}),
},
- data: cs.data.map((item, index) => ({
- date: item.date,
- count: item.count,
- previous: previousSerie?.data[index]
- ? getPreviousMetric(
- item.count,
- previousSerie.data[index]?.count ?? null
- )
- : undefined,
- })),
+ // The key is OMITTED rather than set to `undefined` when there is no
+ // previous point. superjson records one metadata entry per `undefined`,
+ // each carrying its full path string, so a 12-month three-breakdown
+ // chart spent ~77 MB and several seconds encoding 1.69 M entries whose
+ // entire content was "this field is undefined". Reading
+ // `data[i].previous` still yields `undefined` either way, and superjson
+ // already erased the key over the wire, so no consumer can tell.
+ data: cs.data.map((item, index) => {
+ const previousPoint = previousSerie?.data[index];
+ if (!previousPoint) {
+ return { date: item.date, count: item.count };
+ }
+ return {
+ date: item.date,
+ count: item.count,
+ previous: getPreviousMetric(item.count, previousPoint.count ?? null),
+ };
+ }),
};
});
diff --git a/packages/core/src/modules/report/report.constants.ts b/packages/core/src/modules/report/report.constants.ts
index 287f03457..df73da0c1 100644
--- a/packages/core/src/modules/report/report.constants.ts
+++ b/packages/core/src/modules/report/report.constants.ts
@@ -1044,7 +1044,13 @@ export type IChartSerie = {
data: {
date: string;
count: number;
- previous: PreviousValue;
+ /**
+ * Absent, not `undefined`, when the request asked for no previous period.
+ * superjson writes one metadata entry per explicit `undefined`, and a long
+ * range with breakdowns produced millions of them; over the wire the two
+ * are indistinguishable. Read it as optional.
+ */
+ previous?: PreviousValue;
}[];
};
diff --git a/packages/core/src/shared/group-by-labels.ts b/packages/core/src/shared/group-by-labels.ts
index 3b084fca1..b31d65645 100644
--- a/packages/core/src/shared/group-by-labels.ts
+++ b/packages/core/src/shared/group-by-labels.ts
@@ -60,16 +60,21 @@ export function groupByLabels(data: ISerieDataItem[]): GroupedResult[] {
),
}));
+ // Every group is padded to the full set of dates so all series share an
+ // x-axis. The lookup is a Map rather than a `.find()` per date: that scan
+ // was O(groups x dates x points), and a 12-month chart with three
+ // breakdowns reaches 4,640 groups over 365 dates. The OUTPUT is unchanged —
+ // same dates, same order, same zero-filled points.
+ const allDates = Array.from(timestamps);
return result
.filter((group) => group.name.length > 0)
.map((group) => {
+ const byDate = new Map(group.data.map((dp) => [dp.date, dp]));
return {
...group,
- // This will ensure that all dates are present in the data array
- data: Array.from(timestamps).map((date) => {
- const dataPoint = group.data.find((dp) => dp.date === date);
- return dataPoint || { date, count: 0, total_count: 0 };
- }),
+ data: allDates.map(
+ (date) => byDate.get(date) ?? { date, count: 0, total_count: 0 }
+ ),
};
});
}