Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix: better formula support

Carl-Gerhard Lindesvärdc1801adaa2f3
diff --git a/packages/trpc/package.json b/packages/trpc/package.json
index 6632ce43f..e5ade19c8 100644
--- a/packages/trpc/package.json
+++ b/packages/trpc/package.json
@@ -4,6 +4,7 @@
"type": "module",
"main": "index.ts",
"scripts": {
+ "test": "vitest",
"typecheck": "tsc --noEmit"
},
"dependencies": {
diff --git a/packages/trpc/src/routers/chart.helpers.test.ts b/packages/trpc/src/routers/chart.helpers.test.ts
new file mode 100644
index 000000000..31fcb7ff4
--- /dev/null
+++ b/packages/trpc/src/routers/chart.helpers.test.ts
@@ -0,0 +1,544 @@
+import type { IChartEvent, IChartInput } from '@openpanel/validation';
+import { describe, expect, it } from 'vitest';
+import { withFormula } from './chart.helpers';
+
+// Helper to create a test event
+function createEvent(
+ id: string,
+ name: string,
+ displayName?: string,
+): IChartEvent {
+ return {
+ id,
+ name,
+ displayName: displayName ?? '',
+ segment: 'event',
+ filters: [],
+ };
+}
+
+const createChartInput = (
+ rest: Pick<IChartInput, 'events' | 'formula'>,
+): IChartInput => {
+ return {
+ metric: 'sum',
+ chartType: 'linear',
+ interval: 'day',
+ breakdowns: [],
+ projectId: '1',
+ startDate: '2025-01-01',
+ endDate: '2025-01-01',
+ range: '30d',
+ previous: false,
+ formula: '',
+ ...rest,
+ };
+};
+
+// Helper to create a test series
+function createSeries(
+ name: string[],
+ event: IChartEvent,
+ data: Array<{ date: string; count: number }>,
+) {
+ return {
+ name,
+ event,
+ data: data.map((d) => ({ ...d, total_count: d.count })),
+ };
+}
+
+describe('withFormula', () => {
+ describe('edge cases', () => {
+ it('should return series unchanged when formula is empty', () => {
+ const events = [createEvent('evt1', 'event1')];
+ const series = [
+ createSeries(['event1'], events[0]!, [
+ { date: '2025-01-01', count: 10 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: '', events }),
+ series,
+ );
+
+ expect(result).toEqual(series);
+ });
+
+ it('should return series unchanged when series is empty', () => {
+ const events = [createEvent('evt1', 'event1')];
+ const result = withFormula(
+ createChartInput({ formula: 'A*100', events }),
+ [],
+ );
+
+ expect(result).toEqual([]);
+ });
+
+ it('should return series unchanged when series has no data', () => {
+ const events = [createEvent('evt1', 'event1')];
+ const series = [{ name: ['event1'], event: events[0]!, data: [] }];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A*100', events }),
+ series,
+ );
+
+ expect(result).toEqual(series);
+ });
+ });
+
+ describe('single event, no breakdown', () => {
+ it('should apply simple multiplication formula', () => {
+ const events = [createEvent('evt1', 'event1')];
+ const series = [
+ createSeries(['event1'], events[0]!, [
+ { date: '2025-01-01', count: 10 },
+ { date: '2025-01-02', count: 20 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A*100', events }),
+ series,
+ );
+
+ expect(result).toHaveLength(1);
+ expect(result[0]?.data).toEqual([
+ { date: '2025-01-01', count: 1000, total_count: 10 },
+ { date: '2025-01-02', count: 2000, total_count: 20 },
+ ]);
+ });
+
+ it('should apply addition formula', () => {
+ const events = [createEvent('evt1', 'event1')];
+ const series = [
+ createSeries(['event1'], events[0]!, [
+ { date: '2025-01-01', count: 5 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A+10', events }),
+ series,
+ );
+
+ expect(result[0]?.data[0]?.count).toBe(15);
+ });
+
+ it('should handle division formula', () => {
+ const events = [createEvent('evt1', 'event1')];
+ const series = [
+ createSeries(['event1'], events[0]!, [
+ { date: '2025-01-01', count: 100 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A/10', events }),
+ series,
+ );
+
+ expect(result[0]?.data[0]?.count).toBe(10);
+ });
+
+ it('should handle NaN and Infinity by returning 0', () => {
+ const events = [createEvent('evt1', 'event1')];
+ const series = [
+ createSeries(['event1'], events[0]!, [
+ { date: '2025-01-01', count: 0 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A/0', events }),
+ series,
+ );
+
+ expect(result[0]?.data[0]?.count).toBe(0);
+ });
+ });
+
+ describe('single event, with breakdown', () => {
+ it('should apply formula to each breakdown group', () => {
+ const events = [createEvent('evt1', 'screen_view')];
+ const series = [
+ createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 10 }]),
+ createSeries(['Android'], events[0]!, [
+ { date: '2025-01-01', count: 20 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A*100', events }),
+ series,
+ );
+
+ expect(result).toHaveLength(2);
+ expect(result[0]?.name).toEqual(['iOS']);
+ expect(result[0]?.data[0]?.count).toBe(1000);
+ expect(result[1]?.name).toEqual(['Android']);
+ expect(result[1]?.data[0]?.count).toBe(2000);
+ });
+
+ it('should handle multiple breakdown values', () => {
+ const events = [createEvent('evt1', 'screen_view')];
+ const series = [
+ createSeries(['iOS', 'US'], events[0]!, [
+ { date: '2025-01-01', count: 10 },
+ ]),
+ createSeries(['Android', 'US'], events[0]!, [
+ { date: '2025-01-01', count: 20 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A*2', events }),
+ series,
+ );
+
+ expect(result).toHaveLength(2);
+ expect(result[0]?.name).toEqual(['iOS', 'US']);
+ expect(result[0]?.data[0]?.count).toBe(20);
+ expect(result[1]?.name).toEqual(['Android', 'US']);
+ expect(result[1]?.data[0]?.count).toBe(40);
+ });
+ });
+
+ describe('multiple events, no breakdown', () => {
+ it('should combine two events with division formula', () => {
+ const events = [
+ createEvent('evt1', 'screen_view'),
+ createEvent('evt2', 'session_start'),
+ ];
+ const series = [
+ createSeries(['screen_view'], events[0]!, [
+ { date: '2025-01-01', count: 100 },
+ ]),
+ createSeries(['session_start'], events[1]!, [
+ { date: '2025-01-01', count: 50 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A/B', events }),
+ series,
+ );
+
+ expect(result).toHaveLength(1);
+ expect(result[0]?.data[0]?.count).toBe(2);
+ });
+
+ it('should combine two events with addition formula', () => {
+ const events = [
+ createEvent('evt1', 'event1'),
+ createEvent('evt2', 'event2'),
+ ];
+ const series = [
+ createSeries(['event1'], events[0]!, [
+ { date: '2025-01-01', count: 10 },
+ ]),
+ createSeries(['event2'], events[1]!, [
+ { date: '2025-01-01', count: 20 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A+B', events }),
+ series,
+ );
+
+ expect(result[0]?.data[0]?.count).toBe(30);
+ });
+
+ it('should handle three events', () => {
+ const events = [
+ createEvent('evt1', 'event1'),
+ createEvent('evt2', 'event2'),
+ createEvent('evt3', 'event3'),
+ ];
+ const series = [
+ createSeries(['event1'], events[0]!, [
+ { date: '2025-01-01', count: 10 },
+ ]),
+ createSeries(['event2'], events[1]!, [
+ { date: '2025-01-01', count: 20 },
+ ]),
+ createSeries(['event3'], events[2]!, [
+ { date: '2025-01-01', count: 30 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A+B+C', events }),
+ series,
+ );
+
+ expect(result[0]?.data[0]?.count).toBe(60);
+ });
+
+ it('should handle missing data points with 0', () => {
+ const events = [
+ createEvent('evt1', 'event1'),
+ createEvent('evt2', 'event2'),
+ ];
+ const series = [
+ createSeries(['event1'], events[0]!, [
+ { date: '2025-01-01', count: 10 },
+ { date: '2025-01-02', count: 20 },
+ ]),
+ createSeries(['event2'], events[1]!, [
+ { date: '2025-01-01', count: 5 },
+ // Missing 2025-01-02
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A+B', events }),
+ series,
+ );
+
+ expect(result[0]?.data[0]?.count).toBe(15); // 10 + 5
+ expect(result[0]?.data[1]?.count).toBe(20); // 20 + 0 (missing)
+ });
+ });
+
+ describe('multiple events, with breakdown', () => {
+ it('should match series by breakdown values and apply formula', () => {
+ const events = [
+ createEvent('evt1', 'screen_view'),
+ createEvent('evt2', 'session_start'),
+ ];
+ const series = [
+ // iOS breakdown
+ createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 100 }]),
+ createSeries(['iOS'], events[1]!, [{ date: '2025-01-01', count: 50 }]),
+ // Android breakdown
+ createSeries(['Android'], events[0]!, [
+ { date: '2025-01-01', count: 200 },
+ ]),
+ createSeries(['Android'], events[1]!, [
+ { date: '2025-01-01', count: 100 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A/B', events }),
+ series,
+ );
+
+ expect(result).toHaveLength(2);
+ // iOS: 100/50 = 2
+ expect(result[0]?.name).toEqual(['iOS']);
+ expect(result[0]?.data[0]?.count).toBe(2);
+ // Android: 200/100 = 2
+ expect(result[1]?.name).toEqual(['Android']);
+ expect(result[1]?.data[0]?.count).toBe(2);
+ });
+
+ it('should handle multiple breakdown values matching', () => {
+ const events = [
+ createEvent('evt1', 'screen_view'),
+ createEvent('evt2', 'session_start'),
+ ];
+ const series = [
+ createSeries(['iOS', 'US'], events[0]!, [
+ { date: '2025-01-01', count: 100 },
+ ]),
+ createSeries(['iOS', 'US'], events[1]!, [
+ { date: '2025-01-01', count: 50 },
+ ]),
+ createSeries(['Android', 'US'], events[0]!, [
+ { date: '2025-01-01', count: 200 },
+ ]),
+ createSeries(['Android', 'US'], events[1]!, [
+ { date: '2025-01-01', count: 100 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A/B', events }),
+ series,
+ );
+
+ expect(result).toHaveLength(2);
+ expect(result[0]?.name).toEqual(['iOS', 'US']);
+ expect(result[0]?.data[0]?.count).toBe(2);
+ expect(result[1]?.name).toEqual(['Android', 'US']);
+ expect(result[1]?.data[0]?.count).toBe(2);
+ });
+
+ it('should handle different date ranges across breakdown groups', () => {
+ const events = [
+ createEvent('evt1', 'screen_view'),
+ createEvent('evt2', 'session_start'),
+ ];
+ const series = [
+ createSeries(['iOS'], events[0]!, [
+ { date: '2025-01-01', count: 100 },
+ { date: '2025-01-02', count: 200 },
+ ]),
+ createSeries(['iOS'], events[1]!, [
+ { date: '2025-01-01', count: 50 },
+ { date: '2025-01-02', count: 100 },
+ ]),
+ createSeries(['Android'], events[0]!, [
+ { date: '2025-01-01', count: 300 },
+ // Missing 2025-01-02
+ ]),
+ createSeries(['Android'], events[1]!, [
+ { date: '2025-01-01', count: 150 },
+ { date: '2025-01-02', count: 200 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A/B', events }),
+ series,
+ );
+
+ expect(result).toHaveLength(2);
+ // iOS group
+ expect(result[0]?.name).toEqual(['iOS']);
+ expect(result[0]?.data[0]?.count).toBe(2); // 100/50
+ expect(result[0]?.data[1]?.count).toBe(2); // 200/100
+ // Android group
+ expect(result[1]?.name).toEqual(['Android']);
+ expect(result[1]?.data[0]?.count).toBe(2); // 300/150
+ expect(result[1]?.data[1]?.count).toBe(0); // 0/200 = 0 (missing A)
+ });
+ });
+
+ describe('complex formulas', () => {
+ it('should handle complex expressions', () => {
+ const events = [
+ createEvent('evt1', 'event1'),
+ createEvent('evt2', 'event2'),
+ createEvent('evt3', 'event3'),
+ ];
+ const series = [
+ createSeries(['event1'], events[0]!, [
+ { date: '2025-01-01', count: 10 },
+ ]),
+ createSeries(['event2'], events[1]!, [
+ { date: '2025-01-01', count: 20 },
+ ]),
+ createSeries(['event3'], events[2]!, [
+ { date: '2025-01-01', count: 30 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: '(A+B)*C', events }),
+ series,
+ );
+
+ // (10+20)*30 = 900
+ expect(result[0]?.data[0]?.count).toBe(900);
+ });
+
+ it('should handle percentage calculations', () => {
+ const events = [
+ createEvent('evt1', 'screen_view'),
+ createEvent('evt2', 'session_start'),
+ ];
+ const series = [
+ createSeries(['screen_view'], events[0]!, [
+ { date: '2025-01-01', count: 75 },
+ ]),
+ createSeries(['session_start'], events[1]!, [
+ { date: '2025-01-01', count: 100 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: '(A/B)*100', events }),
+ series,
+ );
+
+ // (75/100)*100 = 75
+ expect(result[0]?.data[0]?.count).toBe(75);
+ });
+ });
+
+ describe('error handling', () => {
+ it('should handle invalid formulas gracefully', () => {
+ const events = [createEvent('evt1', 'event1')];
+ const series = [
+ createSeries(['event1'], events[0]!, [
+ { date: '2025-01-01', count: 10 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'invalid formula', events }),
+ series,
+ );
+
+ // Should return 0 for invalid formulas
+ expect(result[0]?.data[0]?.count).toBe(0);
+ });
+
+ it('should handle division by zero', () => {
+ const events = [
+ createEvent('evt1', 'event1'),
+ createEvent('evt2', 'event2'),
+ ];
+ const series = [
+ createSeries(['event1'], events[0]!, [
+ { date: '2025-01-01', count: 10 },
+ ]),
+ createSeries(['event2'], events[1]!, [
+ { date: '2025-01-01', count: 0 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A/B', events }),
+ series,
+ );
+
+ // Division by zero should result in 0 (Infinity -> 0)
+ expect(result[0]?.data[0]?.count).toBe(0);
+ });
+ });
+
+ describe('real-world scenario: article hit ratio', () => {
+ it('should calculate hit ratio per article path', () => {
+ const events = [
+ createEvent('evt1', 'screen_view'),
+ createEvent('evt2', 'article_card_seen'),
+ ];
+ const series = [
+ // Article 1
+ createSeries(['/articles/1'], events[0]!, [
+ { date: '2025-01-01', count: 1000 },
+ ]),
+ createSeries(['/articles/1'], events[1]!, [
+ { date: '2025-01-01', count: 100 },
+ ]),
+ // Article 2
+ createSeries(['/articles/2'], events[0]!, [
+ { date: '2025-01-01', count: 500 },
+ ]),
+ createSeries(['/articles/2'], events[1]!, [
+ { date: '2025-01-01', count: 200 },
+ ]),
+ ];
+
+ const result = withFormula(
+ createChartInput({ formula: 'A/B', events }),
+ series,
+ );
+
+ expect(result).toHaveLength(2);
+ // Article 1: 1000/100 = 10
+ expect(result[0]?.name).toEqual(['/articles/1']);
+ expect(result[0]?.data[0]?.count).toBe(10);
+ // Article 2: 500/200 = 2.5
+ expect(result[1]?.name).toEqual(['/articles/2']);
+ expect(result[1]?.data[0]?.count).toBe(2.5);
+ });
+ });
+});
diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.ts
index 788f6e13d..40cf884f1 100644
--- a/packages/trpc/src/routers/chart.helpers.ts
+++ b/packages/trpc/src/routers/chart.helpers.ts
@@ -1,5 +1,5 @@
import * as mathjs from 'mathjs';
-import { last, pluck, reverse, uniq } from 'ramda';
+import { last, reverse } from 'ramda';
import sqlstring from 'sqlstring';
import type { ISerieDataItem } from '@openpanel/common';
@@ -31,7 +31,6 @@ import type {
IChartEvent,
IChartInput,
IChartInputWithDates,
- IChartRange,
IGetChartDataInput,
} from '@openpanel/validation';
@@ -43,74 +42,129 @@ export function withFormula(
return series;
}
- if (!series) {
+ if (!series || series.length === 0) {
return series;
}
- if (!series[0]) {
- return series;
- }
- if (!series[0].data) {
+ if (!series[0]?.data) {
return series;
}
- if (events.length === 1) {
- return series.map((serie) => {
- if (!serie.event.id) {
- return serie;
+ // Formulas always use alphabet IDs (A, B, C, etc.), not event IDs
+ // Group series by breakdown values (the name array)
+ // This allows us to match series from different events that have the same breakdown values
+
+ // Detect if we have breakdowns: when there are no breakdowns, name arrays contain event names
+ // When there are breakdowns, name arrays contain breakdown values (not event names)
+ const hasBreakdowns = series.some(
+ (serie) =>
+ serie.name.length > 0 &&
+ !events.some(
+ (event) =>
+ serie.name[0] === event.name || serie.name[0] === event.displayName,
+ ),
+ );
+
+ const seriesByBreakdown = new Map<string, typeof series>();
+
+ series.forEach((serie) => {
+ let breakdownKey: string;
+
+ if (hasBreakdowns) {
+ // With breakdowns: use the entire name array as the breakdown key
+ // The name array contains breakdown values (e.g., ["iOS"], ["Android"])
+ breakdownKey = serie.name.join(':::');
+ } else {
+ // Without breakdowns: group all series together regardless of event name
+ // This allows formulas to combine multiple events
+ breakdownKey = '';
+ }
+
+ if (!seriesByBreakdown.has(breakdownKey)) {
+ seriesByBreakdown.set(breakdownKey, []);
+ }
+ seriesByBreakdown.get(breakdownKey)!.push(serie);
+ });
+
+ // For each breakdown group, apply the formula
+ const result: typeof series = [];
+
+ for (const [breakdownKey, breakdownSeries] of seriesByBreakdown) {
+ // Group series by event to ensure we have one series per event
+ const seriesByEvent = new Map<string, (typeof series)[number]>();
+
+ breakdownSeries.forEach((serie) => {
+ const eventId = serie.event.id ?? serie.event.name;
+ // If we already have a series for this event in this breakdown group, skip it
+ // (shouldn't happen, but just in case)
+ if (!seriesByEvent.has(eventId)) {
+ seriesByEvent.set(eventId, serie);
+ }
+ });
+
+ // Get all unique dates across all series in this breakdown group
+ const allDates = new Set<string>();
+ breakdownSeries.forEach((serie) => {
+ serie.data.forEach((item) => {
+ allDates.add(item.date);
+ });
+ });
+
+ // Sort dates chronologically
+ const sortedDates = Array.from(allDates).sort(
+ (a, b) => new Date(a).getTime() - new Date(b).getTime(),
+ );
+
+ // Apply formula for each date, matching series by event index
+ const formulaData = sortedDates.map((date) => {
+ const scope: Record<string, number> = {};
+
+ // Build scope using alphabet IDs (A, B, C, etc.) for each event
+ // This matches how formulas are written (e.g., "A*100", "A/B", "A+B-C")
+ events.forEach((event, eventIndex) => {
+ const readableId = alphabetIds[eventIndex];
+ if (!readableId) {
+ throw new Error('no alphabet id for serie in withFormula');
+ }
+
+ // Find the series for this event in this breakdown group
+ const eventId = event.id ?? event.name;
+ const matchingSerie = seriesByEvent.get(eventId);
+
+ // Find the data point for this date
+ // If the series doesn't exist or the date is missing, use 0
+ const dataPoint = matchingSerie?.data.find((d) => d.date === date);
+ scope[readableId] = dataPoint?.count ?? 0;
+ });
+
+ // Evaluate the formula with the scope
+ let count: number;
+ try {
+ count = mathjs.parse(formula).compile().evaluate(scope) as number;
+ } catch (error) {
+ // If formula evaluation fails, return 0
+ count = 0;
}
return {
- ...serie,
- data: serie.data.map((item) => {
- serie.event.id;
- const scope = {
- [serie.event.id ?? '']: item?.count ?? 0,
- };
- const count = mathjs
- .parse(formula)
- .compile()
- .evaluate(scope) as number;
-
- return {
- ...item,
- count:
- Number.isNaN(count) || !Number.isFinite(count)
- ? null
- : round(count, 2),
- };
- }),
+ date,
+ count:
+ Number.isNaN(count) || !Number.isFinite(count) ? 0 : round(count, 2),
+ total_count: breakdownSeries[0]?.data.find((d) => d.date === date)
+ ?.total_count,
};
});
+
+ // Use the first series as a template, but replace its data with formula results
+ // Preserve the breakdown labels (name array) from the original series
+ const templateSerie = breakdownSeries[0]!;
+ result.push({
+ ...templateSerie,
+ data: formulaData,
+ });
}
- return [
- {
- ...series[0],
- data: series[0].data.map((item, dIndex) => {
- const scope = series.reduce((acc, item, index) => {
- const readableId = alphabetIds[index];
-
- if (!readableId) {
- throw new Error('no alphabet id for serie in withFormula');
- }
-
- return {
- ...acc,
- [readableId]: item.data[dIndex]?.count ?? 0,
- };
- }, {});
-
- const count = mathjs.parse(formula).compile().evaluate(scope) as number;
- return {
- ...item,
- count:
- Number.isNaN(count) || !Number.isFinite(count)
- ? null
- : round(count, 2),
- };
- }),
- },
- ];
+
+ return result;
}
function fillFunnel(funnel: { level: number; count: number }[], steps: number) {
@@ -314,9 +368,7 @@ export async function getChart(input: IChartInput) {
const previousSeries = result[1];
const limit = input.limit || 300;
const offset = input.offset || 0;
- const includeEventName =
- uniq(pluck('name', input.events)).length !==
- pluck('name', input.events).length && series.length > 1;
+ const includeEventAlphaId = input.events.length > 1;
const final: FinalChart = {
series: series.map((serie, index) => {
const eventIndex = input.events.findIndex(
@@ -343,7 +395,7 @@ export async function getChart(input: IChartInput) {
names:
input.breakdowns.length === 0 && serie.event.displayName
? [serie.event.displayName]
- : includeEventName
+ : includeEventAlphaId
? [`(${alphaId}) ${serie.name[0]}`, ...serie.name.slice(1)]
: serie.name,
event,
diff --git a/packages/trpc/vitest.config.ts b/packages/trpc/vitest.config.ts
new file mode 100644
index 000000000..f87a2039f
--- /dev/null
+++ b/packages/trpc/vitest.config.ts
@@ -0,0 +1,3 @@
+import { getSharedVitestConfig } from '../../vitest.shared';
+
+export default getSharedVitestConfig({ __dirname });