Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

perf(project): read project event counts from distinct_event_names_mv (#439)

Ritik Jaind2e8a2663877
diff --git a/packages/db/src/services/project.service.ts b/packages/db/src/services/project.service.ts
index df3f0b111..1639ba1c4 100644
--- a/packages/db/src/services/project.service.ts
+++ b/packages/db/src/services/project.service.ts
@@ -92,9 +92,29 @@ export async function getProjects({
return projects;
}
+/**
+ * Fast approximate count of a project's events (excluding session_start /
+ * session_end), read from distinct_event_names_mv instead of the raw events
+ * table.
+ *
+ * Why not count raw events: `name NOT IN (...)` is a negation, so it can't
+ * prune with idx_name and scans the project's whole events slice — and this
+ * runs from the sessions job on every batch. The MV already accumulates
+ * `count() AS event_count` per (project_id, name) insert block, so summing
+ * it reads a few thousand pre-aggregated rows instead of billions of raw
+ * ones.
+ *
+ * Why approximate: event_count is a plain UInt64, not an aggregate state,
+ * so MV counter rows whose (project_id, name, created_at) sort key collides
+ * collapse on merge keeping only one block's count. Live ingestion rarely
+ * ties on the ms timestamp; bulk imports with coarse timestamps are where
+ * collisions come from. The error is strictly downward (measured 0.0011%
+ * low on a 1.46B-event project) — acceptable for this display/onboarding
+ * counter.
+ */
export const getProjectEventsCount = async (projectId: string) => {
const res = await chQuery<{ count: number }>(
- `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`
+ `SELECT sum(event_count) as count FROM ${TABLE_NAMES.event_names_mv} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`
);
return res[0]?.count;
};