OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
add charts to export api
Carl-Gerhard Lindesvärdf20cca6e15c6
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 9ef23fa95..c8612c7b8 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -22,6 +22,7 @@"@openpanel/queue": "workspace:*","@openpanel/redis": "workspace:*","@openpanel/trpc": "workspace:*",+ "@openpanel/validation": "workspace:*","@trpc/server": "^10.45.1","fastify": "^4.25.2","fastify-metrics": "^11.0.0",@@ -35,7 +36,8 @@"sqlstring": "^2.3.3","superjson": "^1.13.3","ua-parser-js": "^1.0.37",- "url-metadata": "^4.1.0"+ "url-metadata": "^4.1.0",+ "zod": "^3.22.4"},"devDependencies": {"@openpanel/eslint-config": "workspace:*",@@ -61,4 +63,4 @@]},"prettier": "@openpanel/prettier-config"-}+}\ No newline at end of filediff --git a/apps/api/src/controllers/export.controller.ts b/apps/api/src/controllers/export.controller.tsindex 0e442b890..ccb9f6476 100644--- a/apps/api/src/controllers/export.controller.ts+++ b/apps/api/src/controllers/export.controller.ts@@ -1,42 +1,39 @@+import { parseQueryString } from '@/utils/parse-zod-query-string';import type { FastifyReply, FastifyRequest } from 'fastify';+import { z } from 'zod';import type { GetEventListOptions } from '@openpanel/db';import { ClientType, db, getEventList, getEventsCount } from '@openpanel/db';+import { getChart } from '@openpanel/trpc/src/routers/chart';+import { zChartInput } from '@openpanel/validation';-type EventsQuery = {- project_id?: string;- event?: string | string[];- start?: string;- end?: string;- page?: string;- limit?: string;-};-export async function events(+async function getProjectId(request: FastifyRequest<{- Querystring: EventsQuery;+ Querystring: {+ project_id?: string;+ projectId?: string;+ };}>,reply: FastifyReply) {- const query = request.query;- const limit = parseInt(query.limit || '50', 10);- const page = parseInt(query.page || '1', 10);+ let projectId = request.query.projectId || request.query.project_id;- if (query.project_id) {+ if (projectId) {if (request.client?.type === ClientType.read &&- request.client?.projectId !== query.project_id+ request.client?.projectId !== projectId) {reply.status(403).send({error: 'Forbidden',message: 'You do not have access to this project',});- return;+ return '';}const project = await db.project.findUnique({where: {organizationSlug: request.client?.organizationSlug,- id: query.project_id,+ id: projectId,},});@@ -45,29 +42,64 @@ export async function events(error: 'Not Found',message: 'Project not found',});- return;+ return '';}}- const projectId = query.project_id ?? request.client?.projectId;+ if (!projectId && request.client?.projectId) {+ projectId = request.client?.projectId;+ }if (!projectId) {reply.status(400).send({error: 'Bad Request',message: 'project_id is required',});- return;+ return '';+ }++ return projectId;+}++const eventsScheme = z.object({+ project_id: z.string().optional(),+ projectId: z.string().optional(),+ event: z.union([z.string(), z.array(z.string())]).optional(),+ start: z.coerce.string().optional(),+ end: z.coerce.string().optional(),+ page: z.coerce.number().optional().default(1),+ limit: z.coerce.number().optional().default(50),+});++export async function events(+ request: FastifyRequest<{+ Querystring: z.infer<typeof eventsScheme>;+ }>,+ reply: FastifyReply+) {+ const query = eventsScheme.safeParse(request.query);++ if (query.success === false) {+ return reply.status(400).send({+ error: 'Bad Request',+ message: 'Invalid query parameters',+ details: query.error.errors,+ });}+ const projectId = await getProjectId(request, reply);+ const limit = query.data.limit;+ const page = Math.max(query.data.page, 1);const take = Math.max(Math.min(limit, 50), 1);const cursor = page - 1;const options: GetEventListOptions = {projectId,- events: (Array.isArray(query.event) ? query.event : [query.event]).filter(- (s): s is string => typeof s === 'string'- ),- startDate: query.start ? new Date(query.start) : undefined,- endDate: query.end ? new Date(query.end) : undefined,+ events: (Array.isArray(query.data.event)+ ? query.data.event+ : [query.data.event]+ ).filter((s): s is string => typeof s === 'string'),+ startDate: query.data.start ? new Date(query.data.start) : undefined,+ endDate: query.data.end ? new Date(query.data.end) : undefined,cursor,take,meta: false,@@ -89,3 +121,39 @@ export async function events(data,});}++const chartSchemeFull = zChartInput.pick({+ events: true,+ breakdowns: true,+ projectId: true,+ interval: true,+ range: true,+ previous: true,+ startDate: true,+ endDate: true,+});++export async function charts(+ request: FastifyRequest<{+ Querystring: Record<string, string>;+ }>,+ reply: FastifyReply+) {+ const query = chartSchemeFull.safeParse(parseQueryString(request.query));++ if (query.success === false) {+ return reply.status(400).send({+ error: 'Bad Request',+ message: 'Invalid query parameters',+ details: query.error.errors,+ });+ }++ return getChart({+ ...query.data,+ name: 'export-api',+ metric: 'sum',+ lineType: 'monotone',+ chartType: 'linear',+ });+}diff --git a/apps/api/src/routes/export.router.ts b/apps/api/src/routes/export.router.tsindex 03ed838bd..f1f61d72d 100644--- a/apps/api/src/routes/export.router.ts+++ b/apps/api/src/routes/export.router.ts@@ -31,6 +31,12 @@ const eventRouter: FastifyPluginCallback = (fastify, opts, done) => {url: '/events',handler: controller.events,});++ fastify.route({+ method: 'GET',+ url: '/charts',+ handler: controller.charts,+ });done();};diff --git a/apps/api/src/utils/parse-zod-query-string.ts b/apps/api/src/utils/parse-zod-query-string.tsnew file mode 100644index 000000000..e492e8acf--- /dev/null+++ b/apps/api/src/utils/parse-zod-query-string.ts@@ -0,0 +1,19 @@+import { getSafeJson } from '@openpanel/common';++export const parseQueryString = (obj: Record<string, any>): any => {+ return Object.fromEntries(+ Object.entries(obj).map(([k, v]) => {+ if (typeof v === 'object') return [k, parseQueryString(v)];+ if (!isNaN(parseFloat(v))) return [k, parseFloat(v)];+ if (v === 'true') return [k, true];+ if (v === 'false') return [k, false];+ if (typeof v === 'string') {+ if (getSafeJson(v) !== null) {+ return [k, getSafeJson(v)];+ }+ return [k, v];+ }+ return [k, null];+ })+ );+};diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.tsindex 240150f30..43bb0b621 100644--- a/packages/trpc/src/routers/chart.helpers.ts+++ b/packages/trpc/src/routers/chart.helpers.ts@@ -1,10 +1,8 @@import {differenceInMilliseconds,- endOfDay,endOfMonth,endOfYear,formatISO,- startOfDay,startOfMonth,startOfYear,subDays,@@ -14,14 +12,13 @@ import {subYears,} from 'date-fns';import * as mathjs from 'mathjs';-import { repeat, reverse, sort } from 'ramda';+import { repeat, reverse } from 'ramda';import { escape } from 'sqlstring';import { completeTimeline, round } from '@openpanel/common';import { alphabetIds, NOT_SET_VALUE } from '@openpanel/constants';import {chQuery,- convertClickhouseDateToJs,createSqlBuilder,formatClickhouseDate,getChartSql,@@ -44,7 +41,7 @@ export interface ResultItem {}function getEventLegend(event: IChartEvent) {- return event.displayName ?? `${event.name} (${event.id})`;+ return event.displayName ?? event.name;}export function withFormula(@@ -68,12 +65,16 @@ export function withFormula(if (events.length === 1) {return series.map((serie) => {+ if (!serie.event.id) {+ return serie;+ }+return {...serie,data: serie.data.map((item) => {serie.event.id;const scope = {- [serie.event.id]: item?.count ?? 0,+ [serie.event.id ?? '']: item?.count ?? 0,};const count = mathjs.parse(formula)@@ -97,6 +98,10 @@ export function withFormula(...series[0],data: series[0].data.map((item, dIndex) => {const scope = series.reduce((acc, item) => {+ if (!item.event.id) {+ return acc;+ }+return {...acc,[item.event.id]: item.data[dIndex]?.count ?? 0,diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.tsindex 214e17a2d..b950bdad4 100644--- a/packages/trpc/src/routers/chart.ts+++ b/packages/trpc/src/routers/chart.ts@@ -205,153 +205,155 @@ export const chartRouter = createTRPCRouter({}}- const currentPeriod = getChartStartEndDate(input);- const previousPeriod = getChartPrevStartEndDate({- range: input.range,- ...currentPeriod,- });+ return getChart(input);+ }),+});- const promises = [getSeriesFromEvents({ ...input, ...currentPeriod })];+export async function getChart(input: IChartInput) {+ const currentPeriod = getChartStartEndDate(input);+ const previousPeriod = getChartPrevStartEndDate({+ range: input.range,+ ...currentPeriod,+ });- if (input.previous) {- promises.push(- getSeriesFromEvents({- ...input,- ...previousPeriod,- })- );- }+ const promises = [getSeriesFromEvents({ ...input, ...currentPeriod })];- const result = await Promise.all(promises);- const series = result[0]!;- const previousSeries = result[1];-- const final: FinalChart = {- events: input.events,- series: series.map((serie, index) => {- const previousSerie = previousSeries?.find(- (item) => item.name === serie.name- );- const metrics = {- sum: sum(serie.data.map((item) => item.count)),- average: round(average(serie.data.map((item) => item.count)), 2),- min: min(serie.data.map((item) => item.count)),- max: max(serie.data.map((item) => item.count)),- };+ if (input.previous) {+ promises.push(+ getSeriesFromEvents({+ ...input,+ ...previousPeriod,+ })+ );+ }- return {- id: slug(serie.name), // TODO: Remove this (temporary fix for the frontend- name: serie.name,- event: {- ...serie.event,- displayName: serie.event.displayName ?? serie.event.name,- },- metrics: {- ...metrics,- previous: {- sum: getPreviousMetric(- metrics.sum,- previousSerie- ? sum(previousSerie?.data.map((item) => item.count))- : null- ),- average: getPreviousMetric(- metrics.average,- previousSerie- ? round(- average(previousSerie?.data.map((item) => item.count)),- 2- )- : null- ),- min: getPreviousMetric(- metrics.sum,- previousSerie- ? min(previousSerie?.data.map((item) => item.count))- : null- ),- max: getPreviousMetric(- metrics.sum,- previousSerie- ? max(previousSerie?.data.map((item) => item.count))- : null- ),- },+ const result = await Promise.all(promises);+ const series = result[0]!;+ const previousSeries = result[1];++ const final: FinalChart = {+ events: input.events,+ series: series.map((serie) => {+ const previousSerie = previousSeries?.find(+ (item) => item.name === serie.name+ );+ const metrics = {+ sum: sum(serie.data.map((item) => item.count)),+ average: round(average(serie.data.map((item) => item.count)), 2),+ min: min(serie.data.map((item) => item.count)),+ max: max(serie.data.map((item) => item.count)),+ };++ return {+ id: slug(serie.name), // TODO: Remove this (temporary fix for the frontend+ name: serie.name,+ event: {+ ...serie.event,+ displayName: serie.event.displayName ?? serie.event.name,+ },+ metrics: {+ ...metrics,+ previous: {+ sum: getPreviousMetric(+ metrics.sum,+ previousSerie+ ? sum(previousSerie?.data.map((item) => item.count))+ : null+ ),+ average: getPreviousMetric(+ metrics.average,+ previousSerie+ ? round(+ average(previousSerie?.data.map((item) => item.count)),+ 2+ )+ : null+ ),+ min: getPreviousMetric(+ metrics.sum,+ previousSerie+ ? min(previousSerie?.data.map((item) => item.count))+ : null+ ),+ max: getPreviousMetric(+ metrics.sum,+ previousSerie+ ? max(previousSerie?.data.map((item) => item.count))+ : null+ ),},- data: serie.data.map((item, index) => ({- date: item.date,- count: item.count ?? 0,- label: item.label,- previous: previousSerie?.data[index]- ? getPreviousMetric(- item.count ?? 0,- previousSerie?.data[index]?.count ?? null- )- : null,- })),- };- }),- metrics: {- sum: 0,- average: 0,- min: 0,- max: 0,- previous: {- sum: null,- average: null,- min: null,- max: null,},+ data: serie.data.map((item, index) => ({+ date: item.date,+ count: item.count ?? 0,+ label: item.label,+ previous: previousSerie?.data[index]+ ? getPreviousMetric(+ item.count ?? 0,+ previousSerie?.data[index]?.count ?? null+ )+ : null,+ })),+ };+ }),+ metrics: {+ sum: 0,+ average: 0,+ min: 0,+ max: 0,+ previous: {+ sum: null,+ average: null,+ min: null,+ max: null,},- };+ },+ };- final.metrics.sum = sum(final.series.map((item) => item.metrics.sum));- final.metrics.average = round(- average(final.series.map((item) => item.metrics.average)),- 2- );- final.metrics.min = min(final.series.map((item) => item.metrics.min));- final.metrics.max = max(final.series.map((item) => item.metrics.max));- final.metrics.previous = {- sum: getPreviousMetric(- final.metrics.sum,- sum(final.series.map((item) => item.metrics.previous.sum?.value ?? 0))- ),- average: getPreviousMetric(- final.metrics.average,- round(- average(- final.series.map(- (item) => item.metrics.previous.average?.value ?? 0- )- ),- 2- )- ),- min: getPreviousMetric(- final.metrics.min,- min(final.series.map((item) => item.metrics.previous.min?.value ?? 0))- ),- max: getPreviousMetric(- final.metrics.max,- max(final.series.map((item) => item.metrics.previous.max?.value ?? 0))- ),- };+ final.metrics.sum = sum(final.series.map((item) => item.metrics.sum));+ final.metrics.average = round(+ average(final.series.map((item) => item.metrics.average)),+ 2+ );+ final.metrics.min = min(final.series.map((item) => item.metrics.min));+ final.metrics.max = max(final.series.map((item) => item.metrics.max));+ final.metrics.previous = {+ sum: getPreviousMetric(+ final.metrics.sum,+ sum(final.series.map((item) => item.metrics.previous.sum?.value ?? 0))+ ),+ average: getPreviousMetric(+ final.metrics.average,+ round(+ average(+ final.series.map((item) => item.metrics.previous.average?.value ?? 0)+ ),+ 2+ )+ ),+ min: getPreviousMetric(+ final.metrics.min,+ min(final.series.map((item) => item.metrics.previous.min?.value ?? 0))+ ),+ max: getPreviousMetric(+ final.metrics.max,+ max(final.series.map((item) => item.metrics.previous.max?.value ?? 0))+ ),+ };- // Sort by sum- final.series = final.series.sort((a, b) => {- if (input.chartType === 'linear') {- const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0);- const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0);- return sumB - sumA;- } else {- return b.metrics[input.metric] - a.metrics[input.metric];- }- });+ // Sort by sum+ final.series = final.series.sort((a, b) => {+ if (input.chartType === 'linear') {+ const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0);+ const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0);+ return sumB - sumA;+ } else {+ return b.metrics[input.metric] - a.metrics[input.metric];+ }+ });- return final;- }),-});+ return final;+}export function getPreviousMetric(current: number,diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.tsindex 029023c41..ba1584a00 100644--- a/packages/validation/src/index.ts+++ b/packages/validation/src/index.ts@@ -19,7 +19,7 @@ export function objectToZodEnums<K extends string>(export const mapKeys = objectToZodEnums;export const zChartEvent = z.object({- id: z.string(),+ id: z.string().optional(),name: z.string(),displayName: z.string().optional(),property: z.string().optional(),@@ -34,7 +34,7 @@ export const zChartEvent = z.object({]),filters: z.array(z.object({- id: z.string(),+ id: z.string().optional(),name: z.string(),operator: z.enum(objectToZodEnums(operators)),value: z.array(z.string().or(z.number()).or(z.boolean()).or(z.null())),@@ -42,7 +42,7 @@ export const zChartEvent = z.object({),});export const zChartBreakdown = z.object({- id: z.string(),+ id: z.string().optional(),name: z.string(),});diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 77272f35b..601f6e68a 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -59,6 +59,9 @@ importers:'@openpanel/trpc':specifier: workspace:*version: link:../../packages/trpc+ '@openpanel/validation':+ specifier: workspace:*+ version: link:../../packages/validation'@trpc/server':specifier: ^10.45.1version: 10.45.1@@ -101,6 +104,9 @@ importers:url-metadata:specifier: ^4.1.0version: 4.1.0+ zod:+ specifier: ^3.22.4+ version: 3.22.4devDependencies:'@openpanel/eslint-config':specifier: workspace:*