OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: better formula support
Carl-Gerhard Lindesvärdc1801adaa2f3
diff --git a/packages/trpc/package.json b/packages/trpc/package.jsonindex 6632ce43f..e5ade19c8 100644--- a/packages/trpc/package.json+++ b/packages/trpc/package.json@@ -4,6 +4,7 @@"type": "module","main": "index.ts","scripts": {+ "test": "vitest","typecheck": "tsc --noEmit"},"dependencies": {diff --git a/packages/trpc/src/routers/chart.helpers.test.ts b/packages/trpc/src/routers/chart.helpers.test.tsnew file mode 100644index 000000000..31fcb7ff4--- /dev/null+++ b/packages/trpc/src/routers/chart.helpers.test.ts@@ -0,0 +1,544 @@+import type { IChartEvent, IChartInput } from '@openpanel/validation';+import { describe, expect, it } from 'vitest';+import { withFormula } from './chart.helpers';++// Helper to create a test event+function createEvent(+ id: string,+ name: string,+ displayName?: string,+): IChartEvent {+ return {+ id,+ name,+ displayName: displayName ?? '',+ segment: 'event',+ filters: [],+ };+}++const createChartInput = (+ rest: Pick<IChartInput, 'events' | 'formula'>,+): IChartInput => {+ return {+ metric: 'sum',+ chartType: 'linear',+ interval: 'day',+ breakdowns: [],+ projectId: '1',+ startDate: '2025-01-01',+ endDate: '2025-01-01',+ range: '30d',+ previous: false,+ formula: '',+ ...rest,+ };+};++// Helper to create a test series+function createSeries(+ name: string[],+ event: IChartEvent,+ data: Array<{ date: string; count: number }>,+) {+ return {+ name,+ event,+ data: data.map((d) => ({ ...d, total_count: d.count })),+ };+}++describe('withFormula', () => {+ describe('edge cases', () => {+ it('should return series unchanged when formula is empty', () => {+ const events = [createEvent('evt1', 'event1')];+ const series = [+ createSeries(['event1'], events[0]!, [+ { date: '2025-01-01', count: 10 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: '', events }),+ series,+ );++ expect(result).toEqual(series);+ });++ it('should return series unchanged when series is empty', () => {+ const events = [createEvent('evt1', 'event1')];+ const result = withFormula(+ createChartInput({ formula: 'A*100', events }),+ [],+ );++ expect(result).toEqual([]);+ });++ it('should return series unchanged when series has no data', () => {+ const events = [createEvent('evt1', 'event1')];+ const series = [{ name: ['event1'], event: events[0]!, data: [] }];++ const result = withFormula(+ createChartInput({ formula: 'A*100', events }),+ series,+ );++ expect(result).toEqual(series);+ });+ });++ describe('single event, no breakdown', () => {+ it('should apply simple multiplication formula', () => {+ const events = [createEvent('evt1', 'event1')];+ const series = [+ createSeries(['event1'], events[0]!, [+ { date: '2025-01-01', count: 10 },+ { date: '2025-01-02', count: 20 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A*100', events }),+ series,+ );++ expect(result).toHaveLength(1);+ expect(result[0]?.data).toEqual([+ { date: '2025-01-01', count: 1000, total_count: 10 },+ { date: '2025-01-02', count: 2000, total_count: 20 },+ ]);+ });++ it('should apply addition formula', () => {+ const events = [createEvent('evt1', 'event1')];+ const series = [+ createSeries(['event1'], events[0]!, [+ { date: '2025-01-01', count: 5 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A+10', events }),+ series,+ );++ expect(result[0]?.data[0]?.count).toBe(15);+ });++ it('should handle division formula', () => {+ const events = [createEvent('evt1', 'event1')];+ const series = [+ createSeries(['event1'], events[0]!, [+ { date: '2025-01-01', count: 100 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A/10', events }),+ series,+ );++ expect(result[0]?.data[0]?.count).toBe(10);+ });++ it('should handle NaN and Infinity by returning 0', () => {+ const events = [createEvent('evt1', 'event1')];+ const series = [+ createSeries(['event1'], events[0]!, [+ { date: '2025-01-01', count: 0 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A/0', events }),+ series,+ );++ expect(result[0]?.data[0]?.count).toBe(0);+ });+ });++ describe('single event, with breakdown', () => {+ it('should apply formula to each breakdown group', () => {+ const events = [createEvent('evt1', 'screen_view')];+ const series = [+ createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 10 }]),+ createSeries(['Android'], events[0]!, [+ { date: '2025-01-01', count: 20 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A*100', events }),+ series,+ );++ expect(result).toHaveLength(2);+ expect(result[0]?.name).toEqual(['iOS']);+ expect(result[0]?.data[0]?.count).toBe(1000);+ expect(result[1]?.name).toEqual(['Android']);+ expect(result[1]?.data[0]?.count).toBe(2000);+ });++ it('should handle multiple breakdown values', () => {+ const events = [createEvent('evt1', 'screen_view')];+ const series = [+ createSeries(['iOS', 'US'], events[0]!, [+ { date: '2025-01-01', count: 10 },+ ]),+ createSeries(['Android', 'US'], events[0]!, [+ { date: '2025-01-01', count: 20 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A*2', events }),+ series,+ );++ expect(result).toHaveLength(2);+ expect(result[0]?.name).toEqual(['iOS', 'US']);+ expect(result[0]?.data[0]?.count).toBe(20);+ expect(result[1]?.name).toEqual(['Android', 'US']);+ expect(result[1]?.data[0]?.count).toBe(40);+ });+ });++ describe('multiple events, no breakdown', () => {+ it('should combine two events with division formula', () => {+ const events = [+ createEvent('evt1', 'screen_view'),+ createEvent('evt2', 'session_start'),+ ];+ const series = [+ createSeries(['screen_view'], events[0]!, [+ { date: '2025-01-01', count: 100 },+ ]),+ createSeries(['session_start'], events[1]!, [+ { date: '2025-01-01', count: 50 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A/B', events }),+ series,+ );++ expect(result).toHaveLength(1);+ expect(result[0]?.data[0]?.count).toBe(2);+ });++ it('should combine two events with addition formula', () => {+ const events = [+ createEvent('evt1', 'event1'),+ createEvent('evt2', 'event2'),+ ];+ const series = [+ createSeries(['event1'], events[0]!, [+ { date: '2025-01-01', count: 10 },+ ]),+ createSeries(['event2'], events[1]!, [+ { date: '2025-01-01', count: 20 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A+B', events }),+ series,+ );++ expect(result[0]?.data[0]?.count).toBe(30);+ });++ it('should handle three events', () => {+ const events = [+ createEvent('evt1', 'event1'),+ createEvent('evt2', 'event2'),+ createEvent('evt3', 'event3'),+ ];+ const series = [+ createSeries(['event1'], events[0]!, [+ { date: '2025-01-01', count: 10 },+ ]),+ createSeries(['event2'], events[1]!, [+ { date: '2025-01-01', count: 20 },+ ]),+ createSeries(['event3'], events[2]!, [+ { date: '2025-01-01', count: 30 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A+B+C', events }),+ series,+ );++ expect(result[0]?.data[0]?.count).toBe(60);+ });++ it('should handle missing data points with 0', () => {+ const events = [+ createEvent('evt1', 'event1'),+ createEvent('evt2', 'event2'),+ ];+ const series = [+ createSeries(['event1'], events[0]!, [+ { date: '2025-01-01', count: 10 },+ { date: '2025-01-02', count: 20 },+ ]),+ createSeries(['event2'], events[1]!, [+ { date: '2025-01-01', count: 5 },+ // Missing 2025-01-02+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A+B', events }),+ series,+ );++ expect(result[0]?.data[0]?.count).toBe(15); // 10 + 5+ expect(result[0]?.data[1]?.count).toBe(20); // 20 + 0 (missing)+ });+ });++ describe('multiple events, with breakdown', () => {+ it('should match series by breakdown values and apply formula', () => {+ const events = [+ createEvent('evt1', 'screen_view'),+ createEvent('evt2', 'session_start'),+ ];+ const series = [+ // iOS breakdown+ createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 100 }]),+ createSeries(['iOS'], events[1]!, [{ date: '2025-01-01', count: 50 }]),+ // Android breakdown+ createSeries(['Android'], events[0]!, [+ { date: '2025-01-01', count: 200 },+ ]),+ createSeries(['Android'], events[1]!, [+ { date: '2025-01-01', count: 100 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A/B', events }),+ series,+ );++ expect(result).toHaveLength(2);+ // iOS: 100/50 = 2+ expect(result[0]?.name).toEqual(['iOS']);+ expect(result[0]?.data[0]?.count).toBe(2);+ // Android: 200/100 = 2+ expect(result[1]?.name).toEqual(['Android']);+ expect(result[1]?.data[0]?.count).toBe(2);+ });++ it('should handle multiple breakdown values matching', () => {+ const events = [+ createEvent('evt1', 'screen_view'),+ createEvent('evt2', 'session_start'),+ ];+ const series = [+ createSeries(['iOS', 'US'], events[0]!, [+ { date: '2025-01-01', count: 100 },+ ]),+ createSeries(['iOS', 'US'], events[1]!, [+ { date: '2025-01-01', count: 50 },+ ]),+ createSeries(['Android', 'US'], events[0]!, [+ { date: '2025-01-01', count: 200 },+ ]),+ createSeries(['Android', 'US'], events[1]!, [+ { date: '2025-01-01', count: 100 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A/B', events }),+ series,+ );++ expect(result).toHaveLength(2);+ expect(result[0]?.name).toEqual(['iOS', 'US']);+ expect(result[0]?.data[0]?.count).toBe(2);+ expect(result[1]?.name).toEqual(['Android', 'US']);+ expect(result[1]?.data[0]?.count).toBe(2);+ });++ it('should handle different date ranges across breakdown groups', () => {+ const events = [+ createEvent('evt1', 'screen_view'),+ createEvent('evt2', 'session_start'),+ ];+ const series = [+ createSeries(['iOS'], events[0]!, [+ { date: '2025-01-01', count: 100 },+ { date: '2025-01-02', count: 200 },+ ]),+ createSeries(['iOS'], events[1]!, [+ { date: '2025-01-01', count: 50 },+ { date: '2025-01-02', count: 100 },+ ]),+ createSeries(['Android'], events[0]!, [+ { date: '2025-01-01', count: 300 },+ // Missing 2025-01-02+ ]),+ createSeries(['Android'], events[1]!, [+ { date: '2025-01-01', count: 150 },+ { date: '2025-01-02', count: 200 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A/B', events }),+ series,+ );++ expect(result).toHaveLength(2);+ // iOS group+ expect(result[0]?.name).toEqual(['iOS']);+ expect(result[0]?.data[0]?.count).toBe(2); // 100/50+ expect(result[0]?.data[1]?.count).toBe(2); // 200/100+ // Android group+ expect(result[1]?.name).toEqual(['Android']);+ expect(result[1]?.data[0]?.count).toBe(2); // 300/150+ expect(result[1]?.data[1]?.count).toBe(0); // 0/200 = 0 (missing A)+ });+ });++ describe('complex formulas', () => {+ it('should handle complex expressions', () => {+ const events = [+ createEvent('evt1', 'event1'),+ createEvent('evt2', 'event2'),+ createEvent('evt3', 'event3'),+ ];+ const series = [+ createSeries(['event1'], events[0]!, [+ { date: '2025-01-01', count: 10 },+ ]),+ createSeries(['event2'], events[1]!, [+ { date: '2025-01-01', count: 20 },+ ]),+ createSeries(['event3'], events[2]!, [+ { date: '2025-01-01', count: 30 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: '(A+B)*C', events }),+ series,+ );++ // (10+20)*30 = 900+ expect(result[0]?.data[0]?.count).toBe(900);+ });++ it('should handle percentage calculations', () => {+ const events = [+ createEvent('evt1', 'screen_view'),+ createEvent('evt2', 'session_start'),+ ];+ const series = [+ createSeries(['screen_view'], events[0]!, [+ { date: '2025-01-01', count: 75 },+ ]),+ createSeries(['session_start'], events[1]!, [+ { date: '2025-01-01', count: 100 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: '(A/B)*100', events }),+ series,+ );++ // (75/100)*100 = 75+ expect(result[0]?.data[0]?.count).toBe(75);+ });+ });++ describe('error handling', () => {+ it('should handle invalid formulas gracefully', () => {+ const events = [createEvent('evt1', 'event1')];+ const series = [+ createSeries(['event1'], events[0]!, [+ { date: '2025-01-01', count: 10 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'invalid formula', events }),+ series,+ );++ // Should return 0 for invalid formulas+ expect(result[0]?.data[0]?.count).toBe(0);+ });++ it('should handle division by zero', () => {+ const events = [+ createEvent('evt1', 'event1'),+ createEvent('evt2', 'event2'),+ ];+ const series = [+ createSeries(['event1'], events[0]!, [+ { date: '2025-01-01', count: 10 },+ ]),+ createSeries(['event2'], events[1]!, [+ { date: '2025-01-01', count: 0 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A/B', events }),+ series,+ );++ // Division by zero should result in 0 (Infinity -> 0)+ expect(result[0]?.data[0]?.count).toBe(0);+ });+ });++ describe('real-world scenario: article hit ratio', () => {+ it('should calculate hit ratio per article path', () => {+ const events = [+ createEvent('evt1', 'screen_view'),+ createEvent('evt2', 'article_card_seen'),+ ];+ const series = [+ // Article 1+ createSeries(['/articles/1'], events[0]!, [+ { date: '2025-01-01', count: 1000 },+ ]),+ createSeries(['/articles/1'], events[1]!, [+ { date: '2025-01-01', count: 100 },+ ]),+ // Article 2+ createSeries(['/articles/2'], events[0]!, [+ { date: '2025-01-01', count: 500 },+ ]),+ createSeries(['/articles/2'], events[1]!, [+ { date: '2025-01-01', count: 200 },+ ]),+ ];++ const result = withFormula(+ createChartInput({ formula: 'A/B', events }),+ series,+ );++ expect(result).toHaveLength(2);+ // Article 1: 1000/100 = 10+ expect(result[0]?.name).toEqual(['/articles/1']);+ expect(result[0]?.data[0]?.count).toBe(10);+ // Article 2: 500/200 = 2.5+ expect(result[1]?.name).toEqual(['/articles/2']);+ expect(result[1]?.data[0]?.count).toBe(2.5);+ });+ });+});diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.tsindex 788f6e13d..40cf884f1 100644--- a/packages/trpc/src/routers/chart.helpers.ts+++ b/packages/trpc/src/routers/chart.helpers.ts@@ -1,5 +1,5 @@import * as mathjs from 'mathjs';-import { last, pluck, reverse, uniq } from 'ramda';+import { last, reverse } from 'ramda';import sqlstring from 'sqlstring';import type { ISerieDataItem } from '@openpanel/common';@@ -31,7 +31,6 @@ import type {IChartEvent,IChartInput,IChartInputWithDates,- IChartRange,IGetChartDataInput,} from '@openpanel/validation';@@ -43,74 +42,129 @@ export function withFormula(return series;}- if (!series) {+ if (!series || series.length === 0) {return series;}- if (!series[0]) {- return series;- }- if (!series[0].data) {+ if (!series[0]?.data) {return series;}- if (events.length === 1) {- return series.map((serie) => {- if (!serie.event.id) {- return serie;+ // Formulas always use alphabet IDs (A, B, C, etc.), not event IDs+ // Group series by breakdown values (the name array)+ // This allows us to match series from different events that have the same breakdown values++ // Detect if we have breakdowns: when there are no breakdowns, name arrays contain event names+ // When there are breakdowns, name arrays contain breakdown values (not event names)+ const hasBreakdowns = series.some(+ (serie) =>+ serie.name.length > 0 &&+ !events.some(+ (event) =>+ serie.name[0] === event.name || serie.name[0] === event.displayName,+ ),+ );++ const seriesByBreakdown = new Map<string, typeof series>();++ series.forEach((serie) => {+ let breakdownKey: string;++ if (hasBreakdowns) {+ // With breakdowns: use the entire name array as the breakdown key+ // The name array contains breakdown values (e.g., ["iOS"], ["Android"])+ breakdownKey = serie.name.join(':::');+ } else {+ // Without breakdowns: group all series together regardless of event name+ // This allows formulas to combine multiple events+ breakdownKey = '';+ }++ if (!seriesByBreakdown.has(breakdownKey)) {+ seriesByBreakdown.set(breakdownKey, []);+ }+ seriesByBreakdown.get(breakdownKey)!.push(serie);+ });++ // For each breakdown group, apply the formula+ const result: typeof series = [];++ for (const [breakdownKey, breakdownSeries] of seriesByBreakdown) {+ // Group series by event to ensure we have one series per event+ const seriesByEvent = new Map<string, (typeof series)[number]>();++ breakdownSeries.forEach((serie) => {+ const eventId = serie.event.id ?? serie.event.name;+ // If we already have a series for this event in this breakdown group, skip it+ // (shouldn't happen, but just in case)+ if (!seriesByEvent.has(eventId)) {+ seriesByEvent.set(eventId, serie);+ }+ });++ // Get all unique dates across all series in this breakdown group+ const allDates = new Set<string>();+ breakdownSeries.forEach((serie) => {+ serie.data.forEach((item) => {+ allDates.add(item.date);+ });+ });++ // Sort dates chronologically+ const sortedDates = Array.from(allDates).sort(+ (a, b) => new Date(a).getTime() - new Date(b).getTime(),+ );++ // Apply formula for each date, matching series by event index+ const formulaData = sortedDates.map((date) => {+ const scope: Record<string, number> = {};++ // Build scope using alphabet IDs (A, B, C, etc.) for each event+ // This matches how formulas are written (e.g., "A*100", "A/B", "A+B-C")+ events.forEach((event, eventIndex) => {+ const readableId = alphabetIds[eventIndex];+ if (!readableId) {+ throw new Error('no alphabet id for serie in withFormula');+ }++ // Find the series for this event in this breakdown group+ const eventId = event.id ?? event.name;+ const matchingSerie = seriesByEvent.get(eventId);++ // Find the data point for this date+ // If the series doesn't exist or the date is missing, use 0+ const dataPoint = matchingSerie?.data.find((d) => d.date === date);+ scope[readableId] = dataPoint?.count ?? 0;+ });++ // Evaluate the formula with the scope+ let count: number;+ try {+ count = mathjs.parse(formula).compile().evaluate(scope) as number;+ } catch (error) {+ // If formula evaluation fails, return 0+ count = 0;}return {- ...serie,- data: serie.data.map((item) => {- serie.event.id;- const scope = {- [serie.event.id ?? '']: item?.count ?? 0,- };- const count = mathjs- .parse(formula)- .compile()- .evaluate(scope) as number;-- return {- ...item,- count:- Number.isNaN(count) || !Number.isFinite(count)- ? null- : round(count, 2),- };- }),+ date,+ count:+ Number.isNaN(count) || !Number.isFinite(count) ? 0 : round(count, 2),+ total_count: breakdownSeries[0]?.data.find((d) => d.date === date)+ ?.total_count,};});++ // Use the first series as a template, but replace its data with formula results+ // Preserve the breakdown labels (name array) from the original series+ const templateSerie = breakdownSeries[0]!;+ result.push({+ ...templateSerie,+ data: formulaData,+ });}- return [- {- ...series[0],- data: series[0].data.map((item, dIndex) => {- const scope = series.reduce((acc, item, index) => {- const readableId = alphabetIds[index];-- if (!readableId) {- throw new Error('no alphabet id for serie in withFormula');- }-- return {- ...acc,- [readableId]: item.data[dIndex]?.count ?? 0,- };- }, {});-- const count = mathjs.parse(formula).compile().evaluate(scope) as number;- return {- ...item,- count:- Number.isNaN(count) || !Number.isFinite(count)- ? null- : round(count, 2),- };- }),- },- ];++ return result;}function fillFunnel(funnel: { level: number; count: number }[], steps: number) {@@ -314,9 +368,7 @@ export async function getChart(input: IChartInput) {const previousSeries = result[1];const limit = input.limit || 300;const offset = input.offset || 0;- const includeEventName =- uniq(pluck('name', input.events)).length !==- pluck('name', input.events).length && series.length > 1;+ const includeEventAlphaId = input.events.length > 1;const final: FinalChart = {series: series.map((serie, index) => {const eventIndex = input.events.findIndex(@@ -343,7 +395,7 @@ export async function getChart(input: IChartInput) {names:input.breakdowns.length === 0 && serie.event.displayName? [serie.event.displayName]- : includeEventName+ : includeEventAlphaId? [`(${alphaId}) ${serie.name[0]}`, ...serie.name.slice(1)]: serie.name,event,diff --git a/packages/trpc/vitest.config.ts b/packages/trpc/vitest.config.tsnew file mode 100644index 000000000..f87a2039f--- /dev/null+++ b/packages/trpc/vitest.config.ts@@ -0,0 +1,3 @@+import { getSharedVitestConfig } from '../../vitest.shared';++export default getSharedVitestConfig({ __dirname });