Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

perf(profile): compute profile metrics in one events scan (#441)

Ritik Jain7993bb4a7c08
diff --git a/packages/db/src/services/profile-metrics-sql.test.ts b/packages/db/src/services/profile-metrics-sql.test.ts
new file mode 100644
index 000000000..8ae256503
--- /dev/null
+++ b/packages/db/src/services/profile-metrics-sql.test.ts
@@ -0,0 +1,82 @@
+/**
+ * SQL-shape tests for the profile metrics query.
+ *
+ * Same strategy as chart-sql.test.ts / funnel-sql.test.ts: string assertions
+ * always run; `EXPLAIN` validation runs against a locally reachable
+ * ClickHouse (`pnpm dock:up`) and skips otherwise.
+ */
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+
+import { ch } from '../clickhouse/client';
+import { buildProfileMetricsSql } from './profile.service';
+
+const PROJECT_ID = 'test-sql-validation';
+const PROFILE_ID = 'profile-1';
+
+let chReachable = false;
+
+beforeAll(async () => {
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ try {
+ await ch.command({ query: 'SELECT 1' });
+ chReachable = true;
+ } catch {
+ chReachable = false;
+ }
+});
+
+afterAll(() => {
+ vi.restoreAllMocks();
+});
+
+const itCH = (name: string, fn: () => Promise<void>) =>
+ it(name, async (ctx) => {
+ if (!chReachable) {
+ ctx.skip('ClickHouse not reachable at CLICKHOUSE_URL');
+ }
+ await fn();
+ });
+
+describe('buildProfileMetricsSql', () => {
+ // Every metric is a plain aggregate over the same
+ // `profile_id = X AND project_id = Y` slice, so the query must read the
+ // events table exactly once — the per-metric-CTE shape scanned it eight
+ // times per profile view.
+ it('scans the events table exactly once', () => {
+ const sql = buildProfileMetricsSql(PROFILE_ID, PROJECT_ID);
+ expect(sql.match(/FROM events\b/g)).toHaveLength(1);
+ expect(sql.match(/FROM profiles\b/g)).toHaveLength(1);
+ });
+
+ it('keeps every metric of the old per-CTE shape', () => {
+ const sql = buildProfileMetricsSql(PROFILE_ID, PROJECT_ID);
+ for (const metric of [
+ 'lastSeen',
+ 'firstSeen',
+ 'screenViews',
+ 'sessions',
+ 'durationAvg',
+ 'durationP90',
+ 'totalEvents',
+ 'uniqueDaysActive',
+ 'bounceRate',
+ 'avgEventsPerSession',
+ 'conversionEvents',
+ 'avgTimeBetweenSessions',
+ 'revenue',
+ ]) {
+ expect(sql).toContain(`as ${metric}`);
+ }
+ });
+
+ it('escapes the identifiers', () => {
+ const sql = buildProfileMetricsSql("p'--", "x'--");
+ expect(sql).not.toContain("p'--");
+ expect(sql).toContain("'p\\'--'");
+ });
+
+ itCH('parses and resolves against ClickHouse', async () => {
+ const sql = buildProfileMetricsSql(PROFILE_ID, PROJECT_ID);
+ await ch.command({ query: `EXPLAIN ${sql}` });
+ });
+});
diff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.ts
index d32f38abb..5eddf7ae1 100644
--- a/packages/db/src/services/profile.service.ts
+++ b/packages/db/src/services/profile.service.ts
@@ -33,72 +33,74 @@ export interface IProfileMetrics {
avgTimeBetweenSessions: number;
revenue: number;
}
-export function getProfileMetrics(profileId: string, projectId: string) {
- return chQuery<
- Omit<IProfileMetrics, 'lastSeen' | 'firstSeen'> & {
- lastSeen: string;
- firstSeen: string;
- }
- >(`
+/**
+ * SQL for the profile metrics panel: one point-read of the profile row
+ * (first/last seen) plus ONE conditional-aggregate scan of the profile's
+ * events.
+ *
+ * Previously every metric lived in its own CTE, so the same
+ * `profile_id = X AND project_id = Y` slice of the events table was scanned
+ * eight times per profile view. All of those metrics are plain aggregates
+ * over the same rows, so they collapse into a single pass with
+ * countIf/avgIf/sumIf — identical values, one scan.
+ *
+ * Exported for SQL-shape tests.
+ */
+export function buildProfileMetricsSql(
+ profileId: string,
+ projectId: string
+): string {
+ const pid = sqlstring.escape(projectId);
+ const uid = sqlstring.escape(profileId);
+ return `
WITH profileSeen AS (
SELECT created_at as firstSeen, last_seen_at as lastSeen
FROM ${TABLE_NAMES.profiles} FINAL
- WHERE id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
+ WHERE id = ${uid} AND project_id = ${pid}
LIMIT 1
),
- screenViews AS (
- SELECT count(*) as screenViews FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
- ),
- sessions AS (
- SELECT count(*) as sessions FROM ${TABLE_NAMES.events} WHERE name = 'session_start' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
- ),
- duration AS (
- SELECT
- round(avg(duration) / 1000 / 60, 2) as durationAvg,
- round(quantilesExactInclusive(0.9)(duration)[1] / 1000 / 60, 2) as durationP90
- FROM ${TABLE_NAMES.events}
- WHERE name = 'session_end' AND duration != 0 AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
- ),
- totalEvents AS (
- SELECT count(*) as totalEvents FROM ${TABLE_NAMES.events} WHERE profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
- ),
- uniqueDaysActive AS (
- SELECT count(DISTINCT toDate(created_at)) as uniqueDaysActive FROM ${TABLE_NAMES.events} WHERE profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
- ),
- bounceRate AS (
- SELECT round(avg(properties['__bounce'] = '1') * 100, 4) as bounceRate FROM ${TABLE_NAMES.events} WHERE name = 'session_end' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
- ),
- avgEventsPerSession AS (
- SELECT round((SELECT totalEvents FROM totalEvents) / nullIf((SELECT sessions FROM sessions), 0), 2) as avgEventsPerSession
- ),
- conversionEvents AS (
- SELECT count(*) as conversionEvents FROM ${TABLE_NAMES.events} WHERE name NOT IN ('screen_view', 'session_start', 'session_end') AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
- ),
- avgTimeBetweenSessions AS (
- SELECT
- CASE
- WHEN (SELECT sessions FROM sessions) <= 1 THEN 0
- ELSE round(dateDiff('second', (SELECT firstSeen FROM profileSeen), (SELECT lastSeen FROM profileSeen)) / nullIf((SELECT sessions FROM sessions) - 1, 0), 1)
- END as avgTimeBetweenSessions
- ),
- revenue AS (
- SELECT sum(revenue) as revenue FROM ${TABLE_NAMES.events} WHERE name = 'revenue' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
+ eventStats AS (
+ SELECT
+ countIf(name = 'screen_view') as screenViews,
+ countIf(name = 'session_start') as sessions,
+ round(avgIf(duration, name = 'session_end' AND duration != 0) / 1000 / 60, 2) as durationAvg,
+ round(quantilesExactInclusiveIf(0.9)(duration, name = 'session_end' AND duration != 0)[1] / 1000 / 60, 2) as durationP90,
+ count(*) as totalEvents,
+ count(DISTINCT toDate(created_at)) as uniqueDaysActive,
+ round(avgIf(properties['__bounce'] = '1', name = 'session_end') * 100, 4) as bounceRate,
+ countIf(name NOT IN ('screen_view', 'session_start', 'session_end')) as conversionEvents,
+ sumIf(revenue, name = 'revenue') as revenue
+ FROM ${TABLE_NAMES.events}
+ WHERE profile_id = ${uid} AND project_id = ${pid}
)
- SELECT
+ SELECT
(SELECT lastSeen FROM profileSeen) as lastSeen,
(SELECT firstSeen FROM profileSeen) as firstSeen,
- (SELECT screenViews FROM screenViews) as screenViews,
- (SELECT sessions FROM sessions) as sessions,
- (SELECT durationAvg FROM duration) as durationAvg,
- (SELECT durationP90 FROM duration) as durationP90,
- (SELECT totalEvents FROM totalEvents) as totalEvents,
- (SELECT uniqueDaysActive FROM uniqueDaysActive) as uniqueDaysActive,
- (SELECT bounceRate FROM bounceRate) as bounceRate,
- (SELECT avgEventsPerSession FROM avgEventsPerSession) as avgEventsPerSession,
- (SELECT conversionEvents FROM conversionEvents) as conversionEvents,
- (SELECT avgTimeBetweenSessions FROM avgTimeBetweenSessions) as avgTimeBetweenSessions,
- (SELECT revenue FROM revenue) as revenue
- `)
+ screenViews,
+ sessions,
+ durationAvg,
+ durationP90,
+ totalEvents,
+ uniqueDaysActive,
+ bounceRate,
+ round(totalEvents / nullIf(sessions, 0), 2) as avgEventsPerSession,
+ conversionEvents,
+ CASE
+ WHEN sessions <= 1 THEN 0
+ ELSE round(dateDiff('second', (SELECT firstSeen FROM profileSeen), (SELECT lastSeen FROM profileSeen)) / nullIf(sessions - 1, 0), 1)
+ END as avgTimeBetweenSessions,
+ revenue
+ FROM eventStats
+ `;
+}
+
+export function getProfileMetrics(profileId: string, projectId: string) {
+ return chQuery<
+ Omit<IProfileMetrics, 'lastSeen' | 'firstSeen'> & {
+ lastSeen: string;
+ firstSeen: string;
+ }
+ >(buildProfileMetricsSql(profileId, projectId))
.then((data) => data[0]!)
.then((data) => {
return {