Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

add charts to export api

Carl-Gerhard Lindesvärdf20cca6e15c6
diff --git a/apps/api/package.json b/apps/api/package.json
index 9ef23fa95..c8612c7b8 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -22,6 +22,7 @@
"@openpanel/queue": "workspace:*",
"@openpanel/redis": "workspace:*",
"@openpanel/trpc": "workspace:*",
+ "@openpanel/validation": "workspace:*",
"@trpc/server": "^10.45.1",
"fastify": "^4.25.2",
"fastify-metrics": "^11.0.0",
@@ -35,7 +36,8 @@
"sqlstring": "^2.3.3",
"superjson": "^1.13.3",
"ua-parser-js": "^1.0.37",
- "url-metadata": "^4.1.0"
+ "url-metadata": "^4.1.0",
+ "zod": "^3.22.4"
},
"devDependencies": {
"@openpanel/eslint-config": "workspace:*",
@@ -61,4 +63,4 @@
]
},
"prettier": "@openpanel/prettier-config"
-}
+}
\ No newline at end of file
diff --git a/apps/api/src/controllers/export.controller.ts b/apps/api/src/controllers/export.controller.ts
index 0e442b890..ccb9f6476 100644
--- a/apps/api/src/controllers/export.controller.ts
+++ b/apps/api/src/controllers/export.controller.ts
@@ -1,42 +1,39 @@
+import { parseQueryString } from '@/utils/parse-zod-query-string';
import type { FastifyReply, FastifyRequest } from 'fastify';
+import { z } from 'zod';
import type { GetEventListOptions } from '@openpanel/db';
import { ClientType, db, getEventList, getEventsCount } from '@openpanel/db';
+import { getChart } from '@openpanel/trpc/src/routers/chart';
+import { zChartInput } from '@openpanel/validation';
-type EventsQuery = {
- project_id?: string;
- event?: string | string[];
- start?: string;
- end?: string;
- page?: string;
- limit?: string;
-};
-export async function events(
+async function getProjectId(
request: FastifyRequest<{
- Querystring: EventsQuery;
+ Querystring: {
+ project_id?: string;
+ projectId?: string;
+ };
}>,
reply: FastifyReply
) {
- const query = request.query;
- const limit = parseInt(query.limit || '50', 10);
- const page = parseInt(query.page || '1', 10);
+ let projectId = request.query.projectId || request.query.project_id;
- if (query.project_id) {
+ if (projectId) {
if (
request.client?.type === ClientType.read &&
- request.client?.projectId !== query.project_id
+ request.client?.projectId !== projectId
) {
reply.status(403).send({
error: 'Forbidden',
message: 'You do not have access to this project',
});
- return;
+ return '';
}
const project = await db.project.findUnique({
where: {
organizationSlug: request.client?.organizationSlug,
- id: query.project_id,
+ id: projectId,
},
});
@@ -45,29 +42,64 @@ export async function events(
error: 'Not Found',
message: 'Project not found',
});
- return;
+ return '';
}
}
- const projectId = query.project_id ?? request.client?.projectId;
+ if (!projectId && request.client?.projectId) {
+ projectId = request.client?.projectId;
+ }
if (!projectId) {
reply.status(400).send({
error: 'Bad Request',
message: 'project_id is required',
});
- return;
+ return '';
+ }
+
+ return projectId;
+}
+
+const eventsScheme = z.object({
+ project_id: z.string().optional(),
+ projectId: z.string().optional(),
+ event: z.union([z.string(), z.array(z.string())]).optional(),
+ start: z.coerce.string().optional(),
+ end: z.coerce.string().optional(),
+ page: z.coerce.number().optional().default(1),
+ limit: z.coerce.number().optional().default(50),
+});
+
+export async function events(
+ request: FastifyRequest<{
+ Querystring: z.infer<typeof eventsScheme>;
+ }>,
+ reply: FastifyReply
+) {
+ const query = eventsScheme.safeParse(request.query);
+
+ if (query.success === false) {
+ return reply.status(400).send({
+ error: 'Bad Request',
+ message: 'Invalid query parameters',
+ details: query.error.errors,
+ });
}
+ const projectId = await getProjectId(request, reply);
+ const limit = query.data.limit;
+ const page = Math.max(query.data.page, 1);
const take = Math.max(Math.min(limit, 50), 1);
const cursor = page - 1;
const options: GetEventListOptions = {
projectId,
- events: (Array.isArray(query.event) ? query.event : [query.event]).filter(
- (s): s is string => typeof s === 'string'
- ),
- startDate: query.start ? new Date(query.start) : undefined,
- endDate: query.end ? new Date(query.end) : undefined,
+ events: (Array.isArray(query.data.event)
+ ? query.data.event
+ : [query.data.event]
+ ).filter((s): s is string => typeof s === 'string'),
+ startDate: query.data.start ? new Date(query.data.start) : undefined,
+ endDate: query.data.end ? new Date(query.data.end) : undefined,
cursor,
take,
meta: false,
@@ -89,3 +121,39 @@ export async function events(
data,
});
}
+
+const chartSchemeFull = zChartInput.pick({
+ events: true,
+ breakdowns: true,
+ projectId: true,
+ interval: true,
+ range: true,
+ previous: true,
+ startDate: true,
+ endDate: true,
+});
+
+export async function charts(
+ request: FastifyRequest<{
+ Querystring: Record<string, string>;
+ }>,
+ reply: FastifyReply
+) {
+ const query = chartSchemeFull.safeParse(parseQueryString(request.query));
+
+ if (query.success === false) {
+ return reply.status(400).send({
+ error: 'Bad Request',
+ message: 'Invalid query parameters',
+ details: query.error.errors,
+ });
+ }
+
+ return getChart({
+ ...query.data,
+ name: 'export-api',
+ metric: 'sum',
+ lineType: 'monotone',
+ chartType: 'linear',
+ });
+}
diff --git a/apps/api/src/routes/export.router.ts b/apps/api/src/routes/export.router.ts
index 03ed838bd..f1f61d72d 100644
--- a/apps/api/src/routes/export.router.ts
+++ b/apps/api/src/routes/export.router.ts
@@ -31,6 +31,12 @@ const eventRouter: FastifyPluginCallback = (fastify, opts, done) => {
url: '/events',
handler: controller.events,
});
+
+ fastify.route({
+ method: 'GET',
+ url: '/charts',
+ handler: controller.charts,
+ });
done();
};
diff --git a/apps/api/src/utils/parse-zod-query-string.ts b/apps/api/src/utils/parse-zod-query-string.ts
new file mode 100644
index 000000000..e492e8acf
--- /dev/null
+++ b/apps/api/src/utils/parse-zod-query-string.ts
@@ -0,0 +1,19 @@
+import { getSafeJson } from '@openpanel/common';
+
+export const parseQueryString = (obj: Record<string, any>): any => {
+ return Object.fromEntries(
+ Object.entries(obj).map(([k, v]) => {
+ if (typeof v === 'object') return [k, parseQueryString(v)];
+ if (!isNaN(parseFloat(v))) return [k, parseFloat(v)];
+ if (v === 'true') return [k, true];
+ if (v === 'false') return [k, false];
+ if (typeof v === 'string') {
+ if (getSafeJson(v) !== null) {
+ return [k, getSafeJson(v)];
+ }
+ return [k, v];
+ }
+ return [k, null];
+ })
+ );
+};
diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.ts
index 240150f30..43bb0b621 100644
--- a/packages/trpc/src/routers/chart.helpers.ts
+++ b/packages/trpc/src/routers/chart.helpers.ts
@@ -1,10 +1,8 @@
import {
differenceInMilliseconds,
- endOfDay,
endOfMonth,
endOfYear,
formatISO,
- startOfDay,
startOfMonth,
startOfYear,
subDays,
@@ -14,14 +12,13 @@ import {
subYears,
} from 'date-fns';
import * as mathjs from 'mathjs';
-import { repeat, reverse, sort } from 'ramda';
+import { repeat, reverse } from 'ramda';
import { escape } from 'sqlstring';
import { completeTimeline, round } from '@openpanel/common';
import { alphabetIds, NOT_SET_VALUE } from '@openpanel/constants';
import {
chQuery,
- convertClickhouseDateToJs,
createSqlBuilder,
formatClickhouseDate,
getChartSql,
@@ -44,7 +41,7 @@ export interface ResultItem {
}
function getEventLegend(event: IChartEvent) {
- return event.displayName ?? `${event.name} (${event.id})`;
+ return event.displayName ?? event.name;
}
export function withFormula(
@@ -68,12 +65,16 @@ export function withFormula(
if (events.length === 1) {
return series.map((serie) => {
+ if (!serie.event.id) {
+ return serie;
+ }
+
return {
...serie,
data: serie.data.map((item) => {
serie.event.id;
const scope = {
- [serie.event.id]: item?.count ?? 0,
+ [serie.event.id ?? '']: item?.count ?? 0,
};
const count = mathjs
.parse(formula)
@@ -97,6 +98,10 @@ export function withFormula(
...series[0],
data: series[0].data.map((item, dIndex) => {
const scope = series.reduce((acc, item) => {
+ if (!item.event.id) {
+ return acc;
+ }
+
return {
...acc,
[item.event.id]: item.data[dIndex]?.count ?? 0,
diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts
index 214e17a2d..b950bdad4 100644
--- a/packages/trpc/src/routers/chart.ts
+++ b/packages/trpc/src/routers/chart.ts
@@ -205,153 +205,155 @@ export const chartRouter = createTRPCRouter({
}
}
- const currentPeriod = getChartStartEndDate(input);
- const previousPeriod = getChartPrevStartEndDate({
- range: input.range,
- ...currentPeriod,
- });
+ return getChart(input);
+ }),
+});
- const promises = [getSeriesFromEvents({ ...input, ...currentPeriod })];
+export async function getChart(input: IChartInput) {
+ const currentPeriod = getChartStartEndDate(input);
+ const previousPeriod = getChartPrevStartEndDate({
+ range: input.range,
+ ...currentPeriod,
+ });
- if (input.previous) {
- promises.push(
- getSeriesFromEvents({
- ...input,
- ...previousPeriod,
- })
- );
- }
+ const promises = [getSeriesFromEvents({ ...input, ...currentPeriod })];
- const result = await Promise.all(promises);
- const series = result[0]!;
- const previousSeries = result[1];
-
- const final: FinalChart = {
- events: input.events,
- series: series.map((serie, index) => {
- const previousSerie = previousSeries?.find(
- (item) => item.name === serie.name
- );
- const metrics = {
- sum: sum(serie.data.map((item) => item.count)),
- average: round(average(serie.data.map((item) => item.count)), 2),
- min: min(serie.data.map((item) => item.count)),
- max: max(serie.data.map((item) => item.count)),
- };
+ if (input.previous) {
+ promises.push(
+ getSeriesFromEvents({
+ ...input,
+ ...previousPeriod,
+ })
+ );
+ }
- return {
- id: slug(serie.name), // TODO: Remove this (temporary fix for the frontend
- name: serie.name,
- event: {
- ...serie.event,
- displayName: serie.event.displayName ?? serie.event.name,
- },
- metrics: {
- ...metrics,
- previous: {
- sum: getPreviousMetric(
- metrics.sum,
- previousSerie
- ? sum(previousSerie?.data.map((item) => item.count))
- : null
- ),
- average: getPreviousMetric(
- metrics.average,
- previousSerie
- ? round(
- average(previousSerie?.data.map((item) => item.count)),
- 2
- )
- : null
- ),
- min: getPreviousMetric(
- metrics.sum,
- previousSerie
- ? min(previousSerie?.data.map((item) => item.count))
- : null
- ),
- max: getPreviousMetric(
- metrics.sum,
- previousSerie
- ? max(previousSerie?.data.map((item) => item.count))
- : null
- ),
- },
+ const result = await Promise.all(promises);
+ const series = result[0]!;
+ const previousSeries = result[1];
+
+ const final: FinalChart = {
+ events: input.events,
+ series: series.map((serie) => {
+ const previousSerie = previousSeries?.find(
+ (item) => item.name === serie.name
+ );
+ const metrics = {
+ sum: sum(serie.data.map((item) => item.count)),
+ average: round(average(serie.data.map((item) => item.count)), 2),
+ min: min(serie.data.map((item) => item.count)),
+ max: max(serie.data.map((item) => item.count)),
+ };
+
+ return {
+ id: slug(serie.name), // TODO: Remove this (temporary fix for the frontend
+ name: serie.name,
+ event: {
+ ...serie.event,
+ displayName: serie.event.displayName ?? serie.event.name,
+ },
+ metrics: {
+ ...metrics,
+ previous: {
+ sum: getPreviousMetric(
+ metrics.sum,
+ previousSerie
+ ? sum(previousSerie?.data.map((item) => item.count))
+ : null
+ ),
+ average: getPreviousMetric(
+ metrics.average,
+ previousSerie
+ ? round(
+ average(previousSerie?.data.map((item) => item.count)),
+ 2
+ )
+ : null
+ ),
+ min: getPreviousMetric(
+ metrics.sum,
+ previousSerie
+ ? min(previousSerie?.data.map((item) => item.count))
+ : null
+ ),
+ max: getPreviousMetric(
+ metrics.sum,
+ previousSerie
+ ? max(previousSerie?.data.map((item) => item.count))
+ : null
+ ),
},
- data: serie.data.map((item, index) => ({
- date: item.date,
- count: item.count ?? 0,
- label: item.label,
- previous: previousSerie?.data[index]
- ? getPreviousMetric(
- item.count ?? 0,
- previousSerie?.data[index]?.count ?? null
- )
- : null,
- })),
- };
- }),
- metrics: {
- sum: 0,
- average: 0,
- min: 0,
- max: 0,
- previous: {
- sum: null,
- average: null,
- min: null,
- max: null,
},
+ data: serie.data.map((item, index) => ({
+ date: item.date,
+ count: item.count ?? 0,
+ label: item.label,
+ previous: previousSerie?.data[index]
+ ? getPreviousMetric(
+ item.count ?? 0,
+ previousSerie?.data[index]?.count ?? null
+ )
+ : null,
+ })),
+ };
+ }),
+ metrics: {
+ sum: 0,
+ average: 0,
+ min: 0,
+ max: 0,
+ previous: {
+ sum: null,
+ average: null,
+ min: null,
+ max: null,
},
- };
+ },
+ };
- final.metrics.sum = sum(final.series.map((item) => item.metrics.sum));
- final.metrics.average = round(
- average(final.series.map((item) => item.metrics.average)),
- 2
- );
- final.metrics.min = min(final.series.map((item) => item.metrics.min));
- final.metrics.max = max(final.series.map((item) => item.metrics.max));
- final.metrics.previous = {
- sum: getPreviousMetric(
- final.metrics.sum,
- sum(final.series.map((item) => item.metrics.previous.sum?.value ?? 0))
- ),
- average: getPreviousMetric(
- final.metrics.average,
- round(
- average(
- final.series.map(
- (item) => item.metrics.previous.average?.value ?? 0
- )
- ),
- 2
- )
- ),
- min: getPreviousMetric(
- final.metrics.min,
- min(final.series.map((item) => item.metrics.previous.min?.value ?? 0))
- ),
- max: getPreviousMetric(
- final.metrics.max,
- max(final.series.map((item) => item.metrics.previous.max?.value ?? 0))
- ),
- };
+ final.metrics.sum = sum(final.series.map((item) => item.metrics.sum));
+ final.metrics.average = round(
+ average(final.series.map((item) => item.metrics.average)),
+ 2
+ );
+ final.metrics.min = min(final.series.map((item) => item.metrics.min));
+ final.metrics.max = max(final.series.map((item) => item.metrics.max));
+ final.metrics.previous = {
+ sum: getPreviousMetric(
+ final.metrics.sum,
+ sum(final.series.map((item) => item.metrics.previous.sum?.value ?? 0))
+ ),
+ average: getPreviousMetric(
+ final.metrics.average,
+ round(
+ average(
+ final.series.map((item) => item.metrics.previous.average?.value ?? 0)
+ ),
+ 2
+ )
+ ),
+ min: getPreviousMetric(
+ final.metrics.min,
+ min(final.series.map((item) => item.metrics.previous.min?.value ?? 0))
+ ),
+ max: getPreviousMetric(
+ final.metrics.max,
+ max(final.series.map((item) => item.metrics.previous.max?.value ?? 0))
+ ),
+ };
- // Sort by sum
- final.series = final.series.sort((a, b) => {
- if (input.chartType === 'linear') {
- const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
- const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
- return sumB - sumA;
- } else {
- return b.metrics[input.metric] - a.metrics[input.metric];
- }
- });
+ // Sort by sum
+ final.series = final.series.sort((a, b) => {
+ if (input.chartType === 'linear') {
+ const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
+ const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
+ return sumB - sumA;
+ } else {
+ return b.metrics[input.metric] - a.metrics[input.metric];
+ }
+ });
- return final;
- }),
-});
+ return final;
+}
export function getPreviousMetric(
current: number,
diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts
index 029023c41..ba1584a00 100644
--- a/packages/validation/src/index.ts
+++ b/packages/validation/src/index.ts
@@ -19,7 +19,7 @@ export function objectToZodEnums<K extends string>(
export const mapKeys = objectToZodEnums;
export const zChartEvent = z.object({
- id: z.string(),
+ id: z.string().optional(),
name: z.string(),
displayName: z.string().optional(),
property: z.string().optional(),
@@ -34,7 +34,7 @@ export const zChartEvent = z.object({
]),
filters: z.array(
z.object({
- id: z.string(),
+ id: z.string().optional(),
name: z.string(),
operator: z.enum(objectToZodEnums(operators)),
value: z.array(z.string().or(z.number()).or(z.boolean()).or(z.null())),
@@ -42,7 +42,7 @@ export const zChartEvent = z.object({
),
});
export const zChartBreakdown = z.object({
- id: z.string(),
+ id: z.string().optional(),
name: z.string(),
});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 77272f35b..601f6e68a 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -59,6 +59,9 @@ importers:
'@openpanel/trpc':
specifier: workspace:*
version: link:../../packages/trpc
+ '@openpanel/validation':
+ specifier: workspace:*
+ version: link:../../packages/validation
'@trpc/server':
specifier: ^10.45.1
version: 10.45.1
@@ -101,6 +104,9 @@ importers:
url-metadata:
specifier: ^4.1.0
version: 4.1.0
+ zod:
+ specifier: ^3.22.4
+ version: 3.22.4
devDependencies:
'@openpanel/eslint-config':
specifier: workspace:*