🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Clickhouse opts (#1086)
Goldflag01b5884efcda
diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.ymlindex ddd306ba0..e49fe6d29 100644--- a/docker-compose.cloud.yml+++ b/docker-compose.cloud.yml@@ -244,9 +244,10 @@ configs:<profiles><default><enable_json_type>1</enable_json_type>- <!-- 26.3+ enables async_insert by default; the backend already- batches app-side, so keep inserts synchronous -->- <async_insert>0</async_insert>+ <!-- Combine concurrent inserts into larger blocks inside ClickHouse.+ Wait for the flush so the backend still receives insert errors. -->+ <async_insert>1</async_insert>+ <wait_for_async_insert>1</wait_for_async_insert><log_queries>1</log_queries><log_query_threads>0</log_query_threads><log_processors_profiles>0</log_processors_profiles>diff --git a/docker-compose.yml b/docker-compose.ymlindex 470ca771d..c59898ec2 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -224,9 +224,10 @@ configs:<profiles><default><enable_json_type>1</enable_json_type>- <!-- 26.3+ enables async_insert by default; the backend already- batches app-side, so keep inserts synchronous -->- <async_insert>0</async_insert>+ <!-- Combine concurrent inserts into larger blocks inside ClickHouse.+ Wait for the flush so the backend still receives insert errors. -->+ <async_insert>1</async_insert>+ <wait_for_async_insert>1</wait_for_async_insert><log_queries>0</log_queries><log_query_threads>0</log_query_threads><log_processors_profiles>0</log_processors_profiles>diff --git a/server/src/api/sites/getSitesFromOrg.ts b/server/src/api/sites/getSitesFromOrg.tsindex 4edd16cd7..555a377a7 100644--- a/server/src/api/sites/getSitesFromOrg.ts+++ b/server/src/api/sites/getSitesFromOrg.ts@@ -3,11 +3,12 @@ import { FastifyRequest, FastifyReply } from "fastify";import { clickhouse } from "../../db/clickhouse/clickhouse.js";import { db } from "../../db/postgres/postgres.js";import { sites, member, organization, team, teamSiteAccess } from "../../db/postgres/schema.js";-import { IS_CLOUD, DEFAULT_EVENT_LIMIT } from "../../lib/const.js";+import { DEFAULT_EVENT_LIMIT, IS_CLOUD, LITE_DASHBOARD } from "../../lib/const.js";import { getUserIdFromRequest } from "../../lib/auth-utils.js";import { filterSitesByMemberAccess } from "../../lib/siteAccess.js";import { processResults } from "../analytics/utils/utils.js";import { getSubscriptionInner } from "../stripe/getSubscription.js";+import { buildSiteSessionCountsQuery } from "./siteSessionCountsQuery.js";export async function getSitesFromOrg(req: FastifyRequest<{@@ -57,15 +58,8 @@ export async function getSitesFromOrg(const siteIds = sitesData.map(site => site.siteId);const sessionCountsResult = await clickhouse.query({- query: `- SELECT- site_id,- uniqExact(session_id) AS total_sessions- FROM events- WHERE timestamp >= now() - INTERVAL 1 DAY- AND site_id IN (${siteIds.join(",")})- GROUP BY site_id- `,+ query: buildSiteSessionCountsQuery(LITE_DASHBOARD),+ query_params: { siteIds },format: "JSONEachRow",});const sessionCounts = await processResults(sessionCountsResult);diff --git a/server/src/api/sites/siteSessionCountsQuery.test.ts b/server/src/api/sites/siteSessionCountsQuery.test.tsnew file mode 100644index 000000000..b9f9a867d--- /dev/null+++ b/server/src/api/sites/siteSessionCountsQuery.test.ts@@ -0,0 +1,24 @@+import { describe, expect, it } from "vitest";+import { buildSiteSessionCountsQuery } from "./siteSessionCountsQuery.js";++describe("buildSiteSessionCountsQuery", () => {+ it("uses the hourly overview rollup when lite dashboards are enabled", () => {+ const query = buildSiteSessionCountsQuery(true);++ expect(query).toContain("uniqMerge(sessions) AS total_sessions");+ expect(query).toContain("FROM overview_hourly_mv_target");+ expect(query).toContain("event_hour >= now() - INTERVAL 1 DAY");+ expect(query).toContain("site_id IN {siteIds:Array(UInt16)}");+ expect(query).not.toContain("FROM events");+ expect(query).not.toContain("uniqExact");+ });++ it("keeps the raw-events fallback when lite dashboards are disabled", () => {+ const query = buildSiteSessionCountsQuery(false);++ expect(query).toContain("uniqExact(session_id) AS total_sessions");+ expect(query).toContain("FROM events");+ expect(query).toContain("timestamp >= now() - INTERVAL 1 DAY");+ expect(query).toContain("site_id IN {siteIds:Array(UInt16)}");+ });+});diff --git a/server/src/api/sites/siteSessionCountsQuery.ts b/server/src/api/sites/siteSessionCountsQuery.tsnew file mode 100644index 000000000..b137274d2--- /dev/null+++ b/server/src/api/sites/siteSessionCountsQuery.ts@@ -0,0 +1,23 @@+export function buildSiteSessionCountsQuery(useLiteDashboard: boolean) {+ if (useLiteDashboard) {+ return `+ SELECT+ site_id,+ uniqMerge(sessions) AS total_sessions+ FROM overview_hourly_mv_target+ WHERE event_hour >= now() - INTERVAL 1 DAY+ AND site_id IN {siteIds:Array(UInt16)}+ GROUP BY site_id+ `;+ }++ return `+ SELECT+ site_id,+ uniqExact(session_id) AS total_sessions+ FROM events+ WHERE timestamp >= now() - INTERVAL 1 DAY+ AND site_id IN {siteIds:Array(UInt16)}+ GROUP BY site_id+ `;+}diff --git a/server/src/db/clickhouse/clickhouse.test.ts b/server/src/db/clickhouse/clickhouse.test.tsnew file mode 100644index 000000000..23b04c5f3--- /dev/null+++ b/server/src/db/clickhouse/clickhouse.test.ts@@ -0,0 +1,106 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const state = vi.hoisted(() => ({+ existingSessionHourlyView: undefined as string | undefined,+}));++const mocks = vi.hoisted(() => ({+ exec: vi.fn(),+ query: vi.fn(),+ logger: {+ debug: vi.fn(),+ error: vi.fn(),+ info: vi.fn(),+ },+}));++vi.mock("@clickhouse/client", () => ({+ createClient: () => ({+ exec: mocks.exec,+ query: mocks.query,+ }),+}));++vi.mock("../../lib/const.js", () => ({+ IS_CLOUD: false,+ LITE_DASHBOARD: true,+}));++vi.mock("../../lib/logger/logger.js", () => ({+ createServiceLogger: () => mocks.logger,+}));++import { initializeClickhouse } from "./clickhouse.js";++function executedQueries() {+ return mocks.exec.mock.calls.map(([args]) => args.query as string);+}++function findSessionHourlyCreateQuery() {+ return executedQueries().find(query => /CREATE MATERIALIZED VIEW session_hourly_mv\b/.test(query));+}++describe("session hourly materialized view initialization", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ state.existingSessionHourlyView = undefined;+ mocks.exec.mockResolvedValue(undefined);+ mocks.query.mockImplementation(async ({ query }: { query: string }) => {+ if (query.includes("FROM system.columns")) {+ return { json: async () => [] };+ }++ if (query.includes("FROM system.tables")) {+ return {+ json: async () =>+ state.existingSessionHourlyView ? [{ create_table_query: state.existingSessionHourlyView }] : [],+ };+ }++ throw new Error(`Unexpected ClickHouse query: ${query}`);+ });+ });++ it("creates the hourly rollup from finalized sessions instead of raw events", async () => {+ await initializeClickhouse();++ const createQuery = findSessionHourlyCreateQuery();+ expect(createQuery).toBeDefined();+ expect(createQuery).toContain("REFRESH EVERY 1 HOUR");+ expect(createQuery).toContain("FROM sessions_mv_target FINAL");+ expect(createQuery).not.toMatch(/FROM events\b/);+ expect(executedQueries()).not.toContainEqual(expect.stringMatching(/DROP VIEW/));+ });++ it("replaces the old five-minute raw-events definition", async () => {+ state.existingSessionHourlyView = `+ CREATE MATERIALIZED VIEW analytics.session_hourly_mv+ REFRESH EVERY 5 MINUTE+ TO analytics.session_hourly_mv_target+ AS SELECT * FROM analytics.events+ `;++ await initializeClickhouse();++ const queries = executedQueries();+ const dropIndex = queries.findIndex(query => query.includes("DROP VIEW IF EXISTS session_hourly_mv SYNC"));+ const createIndex = queries.findIndex(query => /CREATE MATERIALIZED VIEW session_hourly_mv\b/.test(query));++ expect(dropIndex).toBeGreaterThanOrEqual(0);+ expect(createIndex).toBeGreaterThan(dropIndex);+ });++ it("leaves an already-correct definition unchanged", async () => {+ state.existingSessionHourlyView = `+ CREATE MATERIALIZED VIEW analytics.session_hourly_mv+ REFRESH EVERY 1 HOUR+ TO analytics.session_hourly_mv_target+ AS SELECT * FROM analytics.sessions_mv_target FINAL+ `;++ await initializeClickhouse();++ expect(findSessionHourlyCreateQuery()).toBeUndefined();+ expect(executedQueries()).not.toContainEqual(expect.stringMatching(/DROP VIEW/));+ });+});diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.tsindex c1b4b56b2..0ef30612e 100644--- a/server/src/db/clickhouse/clickhouse.ts+++ b/server/src/db/clickhouse/clickhouse.ts@@ -92,6 +92,75 @@ async function ensureEventsColumns() {);}+const SESSION_HOURLY_MV_NAME = "session_hourly_mv";+const SESSION_HOURLY_MV_REFRESH_INTERVAL = "REFRESH EVERY 1 HOUR";+const SESSION_HOURLY_MV_SOURCE = "sessions_mv_target FINAL";++const SESSION_HOURLY_MV_CREATE_QUERY = `+ CREATE MATERIALIZED VIEW ${SESSION_HOURLY_MV_NAME}+ ${SESSION_HOURLY_MV_REFRESH_INTERVAL}+ TO session_hourly_mv_target+ AS+ SELECT+ site_id,+ toStartOfHour(start_time) AS session_hour,+ count() AS sessions,+ sum(session_pageviews) AS pageviews,+ uniqState(user_id) AS users,+ sum(toUInt64(end_time - start_time)) AS total_session_duration_seconds,+ countIf(session_pageviews = 1) AS bounced_sessions+ FROM (+ SELECT+ site_id,+ user_id,+ start_time,+ end_time,+ pageviews AS session_pageviews+ FROM ${SESSION_HOURLY_MV_SOURCE}+ ) AS sessions+ GROUP BY site_id, session_hour+`;++async function getTableCreateQuery(table: string) {+ const result = await clickhouse.query({+ query: `+ SELECT create_table_query+ FROM system.tables+ WHERE database = currentDatabase()+ AND name = {table:String}+ LIMIT 1+ `,+ query_params: { table },+ format: "JSONEachRow",+ });++ const rows = await result.json<{ create_table_query: string }>();+ return rows[0]?.create_table_query;+}++async function ensureSessionHourlyMaterializedView() {+ const existingCreateQuery = await getTableCreateQuery(SESSION_HOURLY_MV_NAME);+ const isCurrent =+ existingCreateQuery?.includes(SESSION_HOURLY_MV_REFRESH_INTERVAL) &&+ existingCreateQuery.includes(SESSION_HOURLY_MV_SOURCE);++ if (isCurrent) {+ logger.debug("Session hourly materialized view is up to date");+ return;+ }++ if (existingCreateQuery) {+ logger.info("Replacing outdated session hourly materialized view");+ await execClickhouseInitStep(+ "drop outdated session hourly materialized view",+ `DROP VIEW IF EXISTS ${SESSION_HOURLY_MV_NAME} SYNC`,+ { lockAcquireTimeoutSeconds: 15 }+ );+ }++ await execClickhouseInitStep("create session hourly materialized view", SESSION_HOURLY_MV_CREATE_QUERY);+}+export const initializeClickhouse = async () => {// Create events tableawait execClickhouseInitStep(@@ -562,9 +631,8 @@ async function initializeLiteDashboardMVs() {// Session-keyed hourly rollup, populated by a REFRESHABLE materialized view.// Streaming MVs can't compute bounce_rate or session_duration because they// see one event at a time, never the full per-session state. The refreshable- // MV re-runs its SELECT every 5 minutes against the fully-merged sessions,- // which is cheap (~720 rows/site/month) and unlocks all 6 overview metrics- // from a single table read.+ // MV compacts the already-populated per-session rollup hourly instead of+ // repeatedly rebuilding sessions from every raw event.await clickhouse.exec({query: `CREATE TABLE IF NOT EXISTS session_hourly_mv_target (@@ -582,32 +650,8 @@ async function initializeLiteDashboardMVs() {`,});- await clickhouse.exec({- query: `- CREATE MATERIALIZED VIEW IF NOT EXISTS session_hourly_mv- REFRESH EVERY 5 MINUTE- TO session_hourly_mv_target- AS- SELECT- site_id,- toStartOfHour(session_start) AS session_hour,- count() AS sessions,- sum(session_pageviews) AS pageviews,- uniqState(user_id) AS users,- sum(toUInt64(session_end - session_start)) AS total_session_duration_seconds,- countIf(session_pageviews = 1) AS bounced_sessions- FROM (- SELECT- site_id,- session_id,- any(user_id) AS user_id,- countIf(type = 'pageview') AS session_pageviews,- min(timestamp) AS session_start,- max(timestamp) AS session_end- FROM events- GROUP BY site_id, session_id- ) AS s- GROUP BY site_id, session_hour- `,- });+ // CREATE IF NOT EXISTS cannot update an already-installed view. Inspect the+ // stored definition so deployments replace the old five-minute raw-events+ // refresh once, while subsequent startups remain no-ops.+ await ensureSessionHourlyMaterializedView();}