Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Clickhouse opts (#1086)

Goldflag01b5884efcda
diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.yml
index ddd306ba0..e49fe6d29 100644
--- a/docker-compose.cloud.yml
+++ b/docker-compose.cloud.yml
@@ -244,9 +244,10 @@ configs:
<profiles>
<default>
<enable_json_type>1</enable_json_type>
- <!-- 26.3+ enables async_insert by default; the backend already
- batches app-side, so keep inserts synchronous -->
- <async_insert>0</async_insert>
+ <!-- Combine concurrent inserts into larger blocks inside ClickHouse.
+ Wait for the flush so the backend still receives insert errors. -->
+ <async_insert>1</async_insert>
+ <wait_for_async_insert>1</wait_for_async_insert>
<log_queries>1</log_queries>
<log_query_threads>0</log_query_threads>
<log_processors_profiles>0</log_processors_profiles>
diff --git a/docker-compose.yml b/docker-compose.yml
index 470ca771d..c59898ec2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -224,9 +224,10 @@ configs:
<profiles>
<default>
<enable_json_type>1</enable_json_type>
- <!-- 26.3+ enables async_insert by default; the backend already
- batches app-side, so keep inserts synchronous -->
- <async_insert>0</async_insert>
+ <!-- Combine concurrent inserts into larger blocks inside ClickHouse.
+ Wait for the flush so the backend still receives insert errors. -->
+ <async_insert>1</async_insert>
+ <wait_for_async_insert>1</wait_for_async_insert>
<log_queries>0</log_queries>
<log_query_threads>0</log_query_threads>
<log_processors_profiles>0</log_processors_profiles>
diff --git a/server/src/api/sites/getSitesFromOrg.ts b/server/src/api/sites/getSitesFromOrg.ts
index 4edd16cd7..555a377a7 100644
--- a/server/src/api/sites/getSitesFromOrg.ts
+++ b/server/src/api/sites/getSitesFromOrg.ts
@@ -3,11 +3,12 @@ import { FastifyRequest, FastifyReply } from "fastify";
import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { db } from "../../db/postgres/postgres.js";
import { sites, member, organization, team, teamSiteAccess } from "../../db/postgres/schema.js";
-import { IS_CLOUD, DEFAULT_EVENT_LIMIT } from "../../lib/const.js";
+import { DEFAULT_EVENT_LIMIT, IS_CLOUD, LITE_DASHBOARD } from "../../lib/const.js";
import { getUserIdFromRequest } from "../../lib/auth-utils.js";
import { filterSitesByMemberAccess } from "../../lib/siteAccess.js";
import { processResults } from "../analytics/utils/utils.js";
import { getSubscriptionInner } from "../stripe/getSubscription.js";
+import { buildSiteSessionCountsQuery } from "./siteSessionCountsQuery.js";
export async function getSitesFromOrg(
req: FastifyRequest<{
@@ -57,15 +58,8 @@ export async function getSitesFromOrg(
const siteIds = sitesData.map(site => site.siteId);
const sessionCountsResult = await clickhouse.query({
- query: `
- SELECT
- site_id,
- uniqExact(session_id) AS total_sessions
- FROM events
- WHERE timestamp >= now() - INTERVAL 1 DAY
- AND site_id IN (${siteIds.join(",")})
- GROUP BY site_id
- `,
+ query: buildSiteSessionCountsQuery(LITE_DASHBOARD),
+ query_params: { siteIds },
format: "JSONEachRow",
});
const sessionCounts = await processResults(sessionCountsResult);
diff --git a/server/src/api/sites/siteSessionCountsQuery.test.ts b/server/src/api/sites/siteSessionCountsQuery.test.ts
new file mode 100644
index 000000000..b9f9a867d
--- /dev/null
+++ b/server/src/api/sites/siteSessionCountsQuery.test.ts
@@ -0,0 +1,24 @@
+import { describe, expect, it } from "vitest";
+import { buildSiteSessionCountsQuery } from "./siteSessionCountsQuery.js";
+
+describe("buildSiteSessionCountsQuery", () => {
+ it("uses the hourly overview rollup when lite dashboards are enabled", () => {
+ const query = buildSiteSessionCountsQuery(true);
+
+ expect(query).toContain("uniqMerge(sessions) AS total_sessions");
+ expect(query).toContain("FROM overview_hourly_mv_target");
+ expect(query).toContain("event_hour >= now() - INTERVAL 1 DAY");
+ expect(query).toContain("site_id IN {siteIds:Array(UInt16)}");
+ expect(query).not.toContain("FROM events");
+ expect(query).not.toContain("uniqExact");
+ });
+
+ it("keeps the raw-events fallback when lite dashboards are disabled", () => {
+ const query = buildSiteSessionCountsQuery(false);
+
+ expect(query).toContain("uniqExact(session_id) AS total_sessions");
+ expect(query).toContain("FROM events");
+ expect(query).toContain("timestamp >= now() - INTERVAL 1 DAY");
+ expect(query).toContain("site_id IN {siteIds:Array(UInt16)}");
+ });
+});
diff --git a/server/src/api/sites/siteSessionCountsQuery.ts b/server/src/api/sites/siteSessionCountsQuery.ts
new file mode 100644
index 000000000..b137274d2
--- /dev/null
+++ b/server/src/api/sites/siteSessionCountsQuery.ts
@@ -0,0 +1,23 @@
+export function buildSiteSessionCountsQuery(useLiteDashboard: boolean) {
+ if (useLiteDashboard) {
+ return `
+ SELECT
+ site_id,
+ uniqMerge(sessions) AS total_sessions
+ FROM overview_hourly_mv_target
+ WHERE event_hour >= now() - INTERVAL 1 DAY
+ AND site_id IN {siteIds:Array(UInt16)}
+ GROUP BY site_id
+ `;
+ }
+
+ return `
+ SELECT
+ site_id,
+ uniqExact(session_id) AS total_sessions
+ FROM events
+ WHERE timestamp >= now() - INTERVAL 1 DAY
+ AND site_id IN {siteIds:Array(UInt16)}
+ GROUP BY site_id
+ `;
+}
diff --git a/server/src/db/clickhouse/clickhouse.test.ts b/server/src/db/clickhouse/clickhouse.test.ts
new file mode 100644
index 000000000..23b04c5f3
--- /dev/null
+++ b/server/src/db/clickhouse/clickhouse.test.ts
@@ -0,0 +1,106 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const state = vi.hoisted(() => ({
+ existingSessionHourlyView: undefined as string | undefined,
+}));
+
+const mocks = vi.hoisted(() => ({
+ exec: vi.fn(),
+ query: vi.fn(),
+ logger: {
+ debug: vi.fn(),
+ error: vi.fn(),
+ info: vi.fn(),
+ },
+}));
+
+vi.mock("@clickhouse/client", () => ({
+ createClient: () => ({
+ exec: mocks.exec,
+ query: mocks.query,
+ }),
+}));
+
+vi.mock("../../lib/const.js", () => ({
+ IS_CLOUD: false,
+ LITE_DASHBOARD: true,
+}));
+
+vi.mock("../../lib/logger/logger.js", () => ({
+ createServiceLogger: () => mocks.logger,
+}));
+
+import { initializeClickhouse } from "./clickhouse.js";
+
+function executedQueries() {
+ return mocks.exec.mock.calls.map(([args]) => args.query as string);
+}
+
+function findSessionHourlyCreateQuery() {
+ return executedQueries().find(query => /CREATE MATERIALIZED VIEW session_hourly_mv\b/.test(query));
+}
+
+describe("session hourly materialized view initialization", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ state.existingSessionHourlyView = undefined;
+ mocks.exec.mockResolvedValue(undefined);
+ mocks.query.mockImplementation(async ({ query }: { query: string }) => {
+ if (query.includes("FROM system.columns")) {
+ return { json: async () => [] };
+ }
+
+ if (query.includes("FROM system.tables")) {
+ return {
+ json: async () =>
+ state.existingSessionHourlyView ? [{ create_table_query: state.existingSessionHourlyView }] : [],
+ };
+ }
+
+ throw new Error(`Unexpected ClickHouse query: ${query}`);
+ });
+ });
+
+ it("creates the hourly rollup from finalized sessions instead of raw events", async () => {
+ await initializeClickhouse();
+
+ const createQuery = findSessionHourlyCreateQuery();
+ expect(createQuery).toBeDefined();
+ expect(createQuery).toContain("REFRESH EVERY 1 HOUR");
+ expect(createQuery).toContain("FROM sessions_mv_target FINAL");
+ expect(createQuery).not.toMatch(/FROM events\b/);
+ expect(executedQueries()).not.toContainEqual(expect.stringMatching(/DROP VIEW/));
+ });
+
+ it("replaces the old five-minute raw-events definition", async () => {
+ state.existingSessionHourlyView = `
+ CREATE MATERIALIZED VIEW analytics.session_hourly_mv
+ REFRESH EVERY 5 MINUTE
+ TO analytics.session_hourly_mv_target
+ AS SELECT * FROM analytics.events
+ `;
+
+ await initializeClickhouse();
+
+ const queries = executedQueries();
+ const dropIndex = queries.findIndex(query => query.includes("DROP VIEW IF EXISTS session_hourly_mv SYNC"));
+ const createIndex = queries.findIndex(query => /CREATE MATERIALIZED VIEW session_hourly_mv\b/.test(query));
+
+ expect(dropIndex).toBeGreaterThanOrEqual(0);
+ expect(createIndex).toBeGreaterThan(dropIndex);
+ });
+
+ it("leaves an already-correct definition unchanged", async () => {
+ state.existingSessionHourlyView = `
+ CREATE MATERIALIZED VIEW analytics.session_hourly_mv
+ REFRESH EVERY 1 HOUR
+ TO analytics.session_hourly_mv_target
+ AS SELECT * FROM analytics.sessions_mv_target FINAL
+ `;
+
+ await initializeClickhouse();
+
+ expect(findSessionHourlyCreateQuery()).toBeUndefined();
+ expect(executedQueries()).not.toContainEqual(expect.stringMatching(/DROP VIEW/));
+ });
+});
diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.ts
index c1b4b56b2..0ef30612e 100644
--- a/server/src/db/clickhouse/clickhouse.ts
+++ b/server/src/db/clickhouse/clickhouse.ts
@@ -92,6 +92,75 @@ async function ensureEventsColumns() {
);
}
+const SESSION_HOURLY_MV_NAME = "session_hourly_mv";
+const SESSION_HOURLY_MV_REFRESH_INTERVAL = "REFRESH EVERY 1 HOUR";
+const SESSION_HOURLY_MV_SOURCE = "sessions_mv_target FINAL";
+
+const SESSION_HOURLY_MV_CREATE_QUERY = `
+ CREATE MATERIALIZED VIEW ${SESSION_HOURLY_MV_NAME}
+ ${SESSION_HOURLY_MV_REFRESH_INTERVAL}
+ TO session_hourly_mv_target
+ AS
+ SELECT
+ site_id,
+ toStartOfHour(start_time) AS session_hour,
+ count() AS sessions,
+ sum(session_pageviews) AS pageviews,
+ uniqState(user_id) AS users,
+ sum(toUInt64(end_time - start_time)) AS total_session_duration_seconds,
+ countIf(session_pageviews = 1) AS bounced_sessions
+ FROM (
+ SELECT
+ site_id,
+ user_id,
+ start_time,
+ end_time,
+ pageviews AS session_pageviews
+ FROM ${SESSION_HOURLY_MV_SOURCE}
+ ) AS sessions
+ GROUP BY site_id, session_hour
+`;
+
+async function getTableCreateQuery(table: string) {
+ const result = await clickhouse.query({
+ query: `
+ SELECT create_table_query
+ FROM system.tables
+ WHERE database = currentDatabase()
+ AND name = {table:String}
+ LIMIT 1
+ `,
+ query_params: { table },
+ format: "JSONEachRow",
+ });
+
+ const rows = await result.json<{ create_table_query: string }>();
+ return rows[0]?.create_table_query;
+}
+
+async function ensureSessionHourlyMaterializedView() {
+ const existingCreateQuery = await getTableCreateQuery(SESSION_HOURLY_MV_NAME);
+ const isCurrent =
+ existingCreateQuery?.includes(SESSION_HOURLY_MV_REFRESH_INTERVAL) &&
+ existingCreateQuery.includes(SESSION_HOURLY_MV_SOURCE);
+
+ if (isCurrent) {
+ logger.debug("Session hourly materialized view is up to date");
+ return;
+ }
+
+ if (existingCreateQuery) {
+ logger.info("Replacing outdated session hourly materialized view");
+ await execClickhouseInitStep(
+ "drop outdated session hourly materialized view",
+ `DROP VIEW IF EXISTS ${SESSION_HOURLY_MV_NAME} SYNC`,
+ { lockAcquireTimeoutSeconds: 15 }
+ );
+ }
+
+ await execClickhouseInitStep("create session hourly materialized view", SESSION_HOURLY_MV_CREATE_QUERY);
+}
+
export const initializeClickhouse = async () => {
// Create events table
await execClickhouseInitStep(
@@ -562,9 +631,8 @@ async function initializeLiteDashboardMVs() {
// Session-keyed hourly rollup, populated by a REFRESHABLE materialized view.
// Streaming MVs can't compute bounce_rate or session_duration because they
// see one event at a time, never the full per-session state. The refreshable
- // MV re-runs its SELECT every 5 minutes against the fully-merged sessions,
- // which is cheap (~720 rows/site/month) and unlocks all 6 overview metrics
- // from a single table read.
+ // MV compacts the already-populated per-session rollup hourly instead of
+ // repeatedly rebuilding sessions from every raw event.
await clickhouse.exec({
query: `
CREATE TABLE IF NOT EXISTS session_hourly_mv_target (
@@ -582,32 +650,8 @@ async function initializeLiteDashboardMVs() {
`,
});
- await clickhouse.exec({
- query: `
- CREATE MATERIALIZED VIEW IF NOT EXISTS session_hourly_mv
- REFRESH EVERY 5 MINUTE
- TO session_hourly_mv_target
- AS
- SELECT
- site_id,
- toStartOfHour(session_start) AS session_hour,
- count() AS sessions,
- sum(session_pageviews) AS pageviews,
- uniqState(user_id) AS users,
- sum(toUInt64(session_end - session_start)) AS total_session_duration_seconds,
- countIf(session_pageviews = 1) AS bounced_sessions
- FROM (
- SELECT
- site_id,
- session_id,
- any(user_id) AS user_id,
- countIf(type = 'pageview') AS session_pageviews,
- min(timestamp) AS session_start,
- max(timestamp) AS session_end
- FROM events
- GROUP BY site_id, session_id
- ) AS s
- GROUP BY site_id, session_hour
- `,
- });
+ // CREATE IF NOT EXISTS cannot update an already-installed view. Inspect the
+ // stored definition so deployments replace the old five-minute raw-events
+ // refresh once, while subsequent startups remain no-ops.
+ await ensureSessionHourlyMaterializedView();
}