Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor ClickHouse initialization and schema management

Bill Yang2fa9766977fa
diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.ts
index 0ef30612e..507029824 100644
--- a/server/src/db/clickhouse/clickhouse.ts
+++ b/server/src/db/clickhouse/clickhouse.ts
@@ -1,657 +1,18 @@
-import { createClient } from "@clickhouse/client";
import { IS_CLOUD, LITE_DASHBOARD } from "../../lib/const.js";
-import { createServiceLogger } from "../../lib/logger/logger.js";
+import { initializeCloudTables } from "./schema/cloud.js";
+import { initializeCoreTables } from "./schema/core.js";
+import { initializeLiteDashboardMVs } from "./schema/liteDashboard.js";
-const CLICKHOUSE_REQUEST_TIMEOUT_MS = 300_000;
-
-export const clickhouse = createClient({
- url: process.env.CLICKHOUSE_HOST,
- database: process.env.CLICKHOUSE_DB,
- password: process.env.CLICKHOUSE_PASSWORD,
- request_timeout: CLICKHOUSE_REQUEST_TIMEOUT_MS,
-});
-
-const logger = createServiceLogger("clickhouse");
-
-async function execClickhouseInitStep(
- step: string,
- query: string,
- options?: { optional?: boolean; lockAcquireTimeoutSeconds?: number }
-) {
- try {
- await clickhouse.exec({
- query,
- clickhouse_settings: options?.lockAcquireTimeoutSeconds
- ? { lock_acquire_timeout: options.lockAcquireTimeoutSeconds }
- : undefined,
- });
- } catch (error) {
- logger.error(
- { err: error, step, requestTimeoutMs: CLICKHOUSE_REQUEST_TIMEOUT_MS },
- "ClickHouse initialization step failed"
- );
- if (!options?.optional) {
- throw error;
- }
- }
-}
-
-type ColumnDefinition = {
- name: string;
- definition: string;
-};
-
-const EVENTS_COLUMNS_TO_ENSURE: ColumnDefinition[] = [
- { name: "lcp", definition: "lcp Nullable(Float64)" },
- { name: "cls", definition: "cls Nullable(Float64)" },
- { name: "inp", definition: "inp Nullable(Float64)" },
- { name: "fcp", definition: "fcp Nullable(Float64)" },
- { name: "ttfb", definition: "ttfb Nullable(Float64)" },
- { name: "ip", definition: "ip Nullable(String)" },
- { name: "timezone", definition: "timezone LowCardinality(String) DEFAULT ''" },
- { name: "identified_user_id", definition: "identified_user_id String DEFAULT ''" },
- { name: "import_id", definition: "import_id Nullable(UUID)" },
- { name: "tag", definition: "tag LowCardinality(String) DEFAULT ''" },
- { name: "feature_flags", definition: "feature_flags Map(String, String) DEFAULT map()" },
-];
-
-async function getTableColumns(table: string) {
- const result = await clickhouse.query({
- query: `
- SELECT name
- FROM system.columns
- WHERE database = currentDatabase()
- AND table = {table:String}
- `,
- query_params: { table },
- format: "JSONEachRow",
- });
-
- const rows = await result.json<{ name: string }>();
- return new Set(rows.map(row => row.name));
-}
-
-async function ensureEventsColumns() {
- const existingColumns = await getTableColumns("events");
- const missingColumns = EVENTS_COLUMNS_TO_ENSURE.filter(column => !existingColumns.has(column.name));
-
- if (missingColumns.length === 0) {
- logger.debug("Events table columns are up to date");
- return;
- }
-
- logger.info({ missingColumns: missingColumns.map(column => column.name) }, "Adding missing events table columns");
-
- await execClickhouseInitStep(
- "add missing events columns",
- `
- ALTER TABLE events
- ${missingColumns.map(column => `ADD COLUMN IF NOT EXISTS ${column.definition}`).join(",\n ")}
- `,
- { lockAcquireTimeoutSeconds: 15 }
- );
-}
-
-const SESSION_HOURLY_MV_NAME = "session_hourly_mv";
-const SESSION_HOURLY_MV_REFRESH_INTERVAL = "REFRESH EVERY 1 HOUR";
-const SESSION_HOURLY_MV_SOURCE = "sessions_mv_target FINAL";
-
-const SESSION_HOURLY_MV_CREATE_QUERY = `
- CREATE MATERIALIZED VIEW ${SESSION_HOURLY_MV_NAME}
- ${SESSION_HOURLY_MV_REFRESH_INTERVAL}
- TO session_hourly_mv_target
- AS
- SELECT
- site_id,
- toStartOfHour(start_time) AS session_hour,
- count() AS sessions,
- sum(session_pageviews) AS pageviews,
- uniqState(user_id) AS users,
- sum(toUInt64(end_time - start_time)) AS total_session_duration_seconds,
- countIf(session_pageviews = 1) AS bounced_sessions
- FROM (
- SELECT
- site_id,
- user_id,
- start_time,
- end_time,
- pageviews AS session_pageviews
- FROM ${SESSION_HOURLY_MV_SOURCE}
- ) AS sessions
- GROUP BY site_id, session_hour
-`;
-
-async function getTableCreateQuery(table: string) {
- const result = await clickhouse.query({
- query: `
- SELECT create_table_query
- FROM system.tables
- WHERE database = currentDatabase()
- AND name = {table:String}
- LIMIT 1
- `,
- query_params: { table },
- format: "JSONEachRow",
- });
-
- const rows = await result.json<{ create_table_query: string }>();
- return rows[0]?.create_table_query;
-}
-
-async function ensureSessionHourlyMaterializedView() {
- const existingCreateQuery = await getTableCreateQuery(SESSION_HOURLY_MV_NAME);
- const isCurrent =
- existingCreateQuery?.includes(SESSION_HOURLY_MV_REFRESH_INTERVAL) &&
- existingCreateQuery.includes(SESSION_HOURLY_MV_SOURCE);
-
- if (isCurrent) {
- logger.debug("Session hourly materialized view is up to date");
- return;
- }
-
- if (existingCreateQuery) {
- logger.info("Replacing outdated session hourly materialized view");
- await execClickhouseInitStep(
- "drop outdated session hourly materialized view",
- `DROP VIEW IF EXISTS ${SESSION_HOURLY_MV_NAME} SYNC`,
- { lockAcquireTimeoutSeconds: 15 }
- );
- }
-
- await execClickhouseInitStep("create session hourly materialized view", SESSION_HOURLY_MV_CREATE_QUERY);
-}
+export { clickhouse } from "./client.js";
export const initializeClickhouse = async () => {
- // Create events table
- await execClickhouseInitStep(
- "create events table",
- `
- CREATE TABLE IF NOT EXISTS events (
- site_id UInt16,
- timestamp DateTime,
- session_id String,
- user_id String,
- hostname String,
- pathname String,
- querystring String, /* URL parameters stored in raw format */
- url_parameters Map(String, String), /* Structured storage for all URL parameters */
- page_title String,
- referrer String,
- channel String,
- browser LowCardinality(String),
- browser_version LowCardinality(String),
- operating_system LowCardinality(String),
- operating_system_version LowCardinality(String),
- language LowCardinality(String),
- country LowCardinality(FixedString(2)),
- region LowCardinality(String),
- city String,
- lat Float64,
- lon Float64,
- screen_width UInt16,
- screen_height UInt16,
- device_type LowCardinality(String),
- type LowCardinality(String) DEFAULT 'pageview',
- event_name String,
- props JSON,
- lcp Nullable(Float64),
- cls Nullable(Float64),
- inp Nullable(Float64),
- fcp Nullable(Float64),
- ttfb Nullable(Float64),
- ip Nullable(String),
- timezone LowCardinality(String) DEFAULT '',
- identified_user_id String DEFAULT '',
- import_id Nullable(UUID),
- tag LowCardinality(String) DEFAULT '',
- feature_flags Map(String, String) DEFAULT map()
- )
- ENGINE = MergeTree()
- PARTITION BY toYYYYMM(timestamp)
- ORDER BY (site_id, timestamp)
- `
- );
-
- // Heal tables created by older versions (or by the window where the CREATE
- // TABLE above was missing these columns). Only columns that are actually
- // missing are altered in β€” even a no-op ALTER needs the events table ALTER lock.
- await ensureEventsColumns();
-
- await execClickhouseInitStep(
- "create bot events table",
- `
- CREATE TABLE IF NOT EXISTS bot_events (
- site_id UInt16,
- timestamp DateTime,
- session_id String,
- user_id String,
- hostname String,
- pathname String,
- querystring String,
- referrer String,
- browser LowCardinality(String),
- browser_version LowCardinality(String),
- operating_system LowCardinality(String),
- operating_system_version LowCardinality(String),
- country LowCardinality(FixedString(2)),
- region LowCardinality(String),
- city String,
- lat Float64,
- lon Float64,
- screen_width UInt16,
- screen_height UInt16,
- device_type LowCardinality(String),
- type LowCardinality(String) DEFAULT 'pageview',
- asn Nullable(UInt32),
- asn_org String DEFAULT '',
- detected_ua_pattern Bool DEFAULT false,
- detected_header_heuristics Bool DEFAULT false,
- detected_client_signals Bool DEFAULT false,
- detected_bot_asn Bool DEFAULT false,
- detected_rate_anomaly Bool DEFAULT false,
- matched_ua_pattern String DEFAULT '',
- bot_category LowCardinality(String) DEFAULT ''
- )
- ENGINE = MergeTree()
- PARTITION BY toYYYYMM(timestamp)
- ORDER BY (site_id, timestamp)
- TTL timestamp + INTERVAL 3 MONTH
- `
- );
-
- // Create session replay tables
- await clickhouse.exec({
- query: `
- CREATE TABLE IF NOT EXISTS session_replay_events (
- site_id UInt16,
- session_id String,
- user_id String,
- timestamp DateTime64(3),
- event_type LowCardinality(String),
- event_data String,
- event_data_key Nullable(String), -- R2 storage key for cloud deployments
- batch_index Nullable(UInt16), -- Index within the R2 batch
- sequence_number UInt32,
- event_size_bytes UInt32,
- viewport_width Nullable(UInt16),
- viewport_height Nullable(UInt16),
- is_complete UInt8 DEFAULT 0
- )
- ENGINE = MergeTree()
- PARTITION BY toYYYYMM(timestamp)
- ORDER BY (site_id, session_id, sequence_number)
- TTL toDateTime(timestamp) + INTERVAL 30 DAY
- `,
- });
-
- await clickhouse.exec({
- query: `
- ALTER TABLE session_replay_events
- ADD COLUMN IF NOT EXISTS event_data_key Nullable(String), -- R2 storage key for cloud deployments
- ADD COLUMN IF NOT EXISTS batch_index Nullable(UInt16), -- Index within the R2 batch
- ADD COLUMN IF NOT EXISTS identified_user_id String DEFAULT ''
- `,
- });
-
- await clickhouse.exec({
- query: `
- CREATE TABLE IF NOT EXISTS session_replay_metadata (
- site_id UInt16,
- session_id String,
- user_id String,
- start_time DateTime,
- end_time Nullable(DateTime),
- duration_ms Nullable(UInt32),
- event_count UInt32,
- compressed_size_bytes UInt32,
- page_url String,
- country LowCardinality(FixedString(2)),
- region LowCardinality(String),
- city String,
- lat Float64,
- lon Float64,
- browser LowCardinality(String),
- browser_version LowCardinality(String),
- operating_system LowCardinality(String),
- operating_system_version LowCardinality(String),
- language LowCardinality(String),
- screen_width UInt16,
- screen_height UInt16,
- device_type LowCardinality(String),
- channel String,
- hostname String,
- referrer String,
- has_replay_data UInt8 DEFAULT 1,
- created_at DateTime DEFAULT now()
- )
- ENGINE = ReplacingMergeTree(created_at)
- PARTITION BY toYYYYMM(start_time)
- ORDER BY (site_id, session_id)
- TTL start_time + INTERVAL 30 DAY
- `,
- });
-
- await clickhouse.exec({
- query: `
- ALTER TABLE session_replay_metadata
- ADD COLUMN IF NOT EXISTS identified_user_id String DEFAULT ''
- `,
- });
-
- // Create uptime monitor events table
- await clickhouse.exec({
- query: `
- CREATE TABLE IF NOT EXISTS monitor_events (
- monitor_id UInt32,
- organization_id String,
- timestamp DateTime,
-
- -- Monitor metadata
- monitor_type LowCardinality(String), -- 'http', 'tcp'
- monitor_url String,
- monitor_name String,
- region LowCardinality(String) DEFAULT 'local',
-
- -- Response data
- status LowCardinality(String), -- 'success', 'failure', 'timeout'
- status_code Nullable(UInt16), -- HTTP status code
- response_time_ms UInt32,
-
- -- HTTP timing breakdown (all in milliseconds)
- dns_time_ms Nullable(UInt32),
- tcp_time_ms Nullable(UInt32),
- tls_time_ms Nullable(UInt32),
- ttfb_ms Nullable(UInt32), -- Time to first byte
- transfer_time_ms Nullable(UInt32),
-
- -- Validation results
- validation_errors Array(String), -- Array of failed validation rules
-
- -- Response metadata (for HTTP)
- response_headers Map(String, String),
- response_size_bytes Nullable(UInt32),
-
- -- TCP specific
- port Nullable(UInt16),
-
- -- Error information
- error_message Nullable(String),
- error_type Nullable(String) -- 'dns_failure', 'connection_timeout', 'ssl_error', etc.
- )
- ENGINE = MergeTree()
- PARTITION BY toYYYYMM(timestamp)
- ORDER BY (organization_id, monitor_id, timestamp)
- SETTINGS ttl_only_drop_parts = 1
- `,
- });
+ await initializeCoreTables();
if (IS_CLOUD) {
- await clickhouse.exec({
- query: `
- CREATE TABLE IF NOT EXISTS hourly_events_by_site_mv_target (
- event_hour DateTime, -- The specific hour
- site_id UInt16,
- event_count UInt64 -- The count of events for that site in that hour
- )
- ENGINE = SummingMergeTree() -- Sums 'event_count' for rows with the same sorting key
- PARTITION BY toYYYYMM(event_hour)
- ORDER BY (event_hour, site_id)
- TTL event_hour + INTERVAL 60 DAY
- `,
- });
-
- // 2. Create the Materialized View
- // This MV will populate the 'hourly_events_by_site_mv_target' table.
- await clickhouse.exec({
- query: `
- CREATE MATERIALIZED VIEW IF NOT EXISTS hourly_events_by_site_mv
- TO hourly_events_by_site_mv_target -- Name of the target table
- AS SELECT
- toStartOfHour(timestamp) AS event_hour,
- site_id,
- count() AS event_count
- FROM events
- GROUP BY event_hour, site_id
- `,
- });
+ await initializeCloudTables();
}
if (LITE_DASHBOARD) {
await initializeLiteDashboardMVs();
}
};
-
-// Materialized views that back the simplified high-traffic dashboard.
-// All views are hourly-bucketed and feed the lite endpoints; raw `events`
-// is still queried for filters that aren't keyed in these MVs.
-async function initializeLiteDashboardMVs() {
- // Per-session rollup. Replaces the AllSessionPageviews / FilteredSessions
- // CTEs that getOverview and getOverviewBucketed run over raw events.
- //
- // This is a streaming MV: it fires once per inserted block and writes a
- // PARTIAL session row for whatever events were in that block. AggregatingMergeTree
- // then composes the SimpleAggregateFunction columns (min/max/sum) across parts
- // sharing (site_id, session_id), and the read queries additionally re-aggregate
- // by session_id at query time β€” so pageviews/start_time/end_time are always
- // correct even before merges complete.
- //
- // The plain metadata columns below (country, region, device_type, browser,
- // operating_system, hostname) are session-INVARIANT, so every partial row of a
- // session carries the same value and `any()` is well-defined. Entry/acquisition
- // attributes (entry_page, referrer, channel) are deliberately NOT stored here:
- // they are first-event values that differ across a session's blocks, can't be
- // composed correctly per-partial-row, and so are served by the raw-events
- // fallback instead (see LITE_SESSION_FILTER_COLUMNS in lite/utils.ts).
- await clickhouse.exec({
- query: `
- CREATE TABLE IF NOT EXISTS sessions_mv_target (
- site_id UInt16,
- session_id String,
- user_id String,
- start_time SimpleAggregateFunction(min, DateTime),
- end_time SimpleAggregateFunction(max, DateTime),
- pageviews SimpleAggregateFunction(sum, UInt64),
- events SimpleAggregateFunction(sum, UInt64),
- country LowCardinality(FixedString(2)),
- region LowCardinality(String),
- device_type LowCardinality(String),
- browser LowCardinality(String),
- operating_system LowCardinality(String),
- hostname String,
- last_seen SimpleAggregateFunction(max, DateTime)
- )
- ENGINE = AggregatingMergeTree()
- PARTITION BY toYYYYMM(start_time)
- ORDER BY (site_id, session_id)
- `,
- });
-
- await clickhouse.exec({
- query: `
- CREATE MATERIALIZED VIEW IF NOT EXISTS sessions_mv
- TO sessions_mv_target
- AS SELECT
- site_id,
- session_id,
- any(user_id) AS user_id,
- min(timestamp) AS start_time,
- max(timestamp) AS end_time,
- countIf(type = 'pageview') AS pageviews,
- count() AS events,
- any(country) AS country,
- any(region) AS region,
- any(device_type) AS device_type,
- any(browser) AS browser,
- any(operating_system) AS operating_system,
- any(hostname) AS hostname,
- max(timestamp) AS last_seen
- FROM events
- GROUP BY site_id, session_id
- `,
- });
-
- // Hourly overview rollup. Drives the bucketed chart and overview cards.
- await clickhouse.exec({
- query: `
- CREATE TABLE IF NOT EXISTS overview_hourly_mv_target (
- site_id UInt16,
- event_hour DateTime,
- pageviews SimpleAggregateFunction(sum, UInt64),
- events SimpleAggregateFunction(sum, UInt64),
- users AggregateFunction(uniq, String),
- sessions AggregateFunction(uniq, String)
- )
- ENGINE = AggregatingMergeTree()
- PARTITION BY toYYYYMM(event_hour)
- ORDER BY (site_id, event_hour)
- `,
- });
-
- await clickhouse.exec({
- query: `
- CREATE MATERIALIZED VIEW IF NOT EXISTS overview_hourly_mv
- TO overview_hourly_mv_target
- AS SELECT
- site_id,
- toStartOfHour(timestamp) AS event_hour,
- countIf(type = 'pageview') AS pageviews,
- count() AS events,
- uniqState(user_id) AS users,
- uniqState(session_id) AS sessions
- FROM events
- GROUP BY site_id, event_hour
- `,
- });
-
- // Top-pathname rollup. Cardinality is bounded by hour, so high-URL sites
- // still get smaller-than-raw storage. Filtered queries fall back to events.
- await clickhouse.exec({
- query: `
- CREATE TABLE IF NOT EXISTS pathname_hourly_mv_target (
- site_id UInt16,
- event_hour DateTime,
- pathname String,
- hostname String,
- pageviews SimpleAggregateFunction(sum, UInt64),
- users AggregateFunction(uniq, String),
- sessions AggregateFunction(uniq, String)
- )
- ENGINE = AggregatingMergeTree()
- PARTITION BY toYYYYMM(event_hour)
- ORDER BY (site_id, event_hour, pathname)
- `,
- });
-
- await clickhouse.exec({
- query: `
- CREATE MATERIALIZED VIEW IF NOT EXISTS pathname_hourly_mv
- TO pathname_hourly_mv_target
- AS SELECT
- site_id,
- toStartOfHour(timestamp) AS event_hour,
- pathname,
- any(hostname) AS hostname,
- countIf(type = 'pageview') AS pageviews,
- uniqState(user_id) AS users,
- uniqState(session_id) AS sessions
- FROM events
- WHERE type = 'pageview'
- GROUP BY site_id, event_hour, pathname
- `,
- });
-
- // Country/region rollup. Cardinality is naturally bounded.
- await clickhouse.exec({
- query: `
- CREATE TABLE IF NOT EXISTS country_hourly_mv_target (
- site_id UInt16,
- event_hour DateTime,
- country LowCardinality(FixedString(2)),
- region LowCardinality(String),
- pageviews SimpleAggregateFunction(sum, UInt64),
- users AggregateFunction(uniq, String),
- sessions AggregateFunction(uniq, String)
- )
- ENGINE = AggregatingMergeTree()
- PARTITION BY toYYYYMM(event_hour)
- ORDER BY (site_id, event_hour, country, region)
- `,
- });
-
- await clickhouse.exec({
- query: `
- CREATE MATERIALIZED VIEW IF NOT EXISTS country_hourly_mv
- TO country_hourly_mv_target
- AS SELECT
- site_id,
- toStartOfHour(timestamp) AS event_hour,
- country,
- region,
- countIf(type = 'pageview') AS pageviews,
- uniqState(user_id) AS users,
- uniqState(session_id) AS sessions
- FROM events
- GROUP BY site_id, event_hour, country, region
- `,
- });
-
- // Device-type rollup.
- await clickhouse.exec({
- query: `
- CREATE TABLE IF NOT EXISTS device_type_hourly_mv_target (
- site_id UInt16,
- event_hour DateTime,
- device_type LowCardinality(String),
- pageviews SimpleAggregateFunction(sum, UInt64),
- users AggregateFunction(uniq, String),
- sessions AggregateFunction(uniq, String)
- )
- ENGINE = AggregatingMergeTree()
- PARTITION BY toYYYYMM(event_hour)
- ORDER BY (site_id, event_hour, device_type)
- `,
- });
-
- await clickhouse.exec({
- query: `
- CREATE MATERIALIZED VIEW IF NOT EXISTS device_type_hourly_mv
- TO device_type_hourly_mv_target
- AS SELECT
- site_id,
- toStartOfHour(timestamp) AS event_hour,
- device_type,
- countIf(type = 'pageview') AS pageviews,
- uniqState(user_id) AS users,
- uniqState(session_id) AS sessions
- FROM events
- GROUP BY site_id, event_hour, device_type
- `,
- });
-
- // Session-keyed hourly rollup, populated by a REFRESHABLE materialized view.
- // Streaming MVs can't compute bounce_rate or session_duration because they
- // see one event at a time, never the full per-session state. The refreshable
- // MV compacts the already-populated per-session rollup hourly instead of
- // repeatedly rebuilding sessions from every raw event.
- await clickhouse.exec({
- query: `
- CREATE TABLE IF NOT EXISTS session_hourly_mv_target (
- site_id UInt16,
- session_hour DateTime,
- sessions UInt64,
- pageviews UInt64,
- users AggregateFunction(uniq, String),
- total_session_duration_seconds UInt64,
- bounced_sessions UInt64
- )
- ENGINE = MergeTree()
- PARTITION BY toYYYYMM(session_hour)
- ORDER BY (site_id, session_hour)
- `,
- });
-
- // CREATE IF NOT EXISTS cannot update an already-installed view. Inspect the
- // stored definition so deployments replace the old five-minute raw-events
- // refresh once, while subsequent startups remain no-ops.
- await ensureSessionHourlyMaterializedView();
-}
diff --git a/server/src/db/clickhouse/client.ts b/server/src/db/clickhouse/client.ts
new file mode 100644
index 000000000..ead68a145
--- /dev/null
+++ b/server/src/db/clickhouse/client.ts
@@ -0,0 +1,10 @@
+import { createClient } from "@clickhouse/client";
+
+export const CLICKHOUSE_REQUEST_TIMEOUT_MS = 300_000;
+
+export const clickhouse = createClient({
+ url: process.env.CLICKHOUSE_HOST,
+ database: process.env.CLICKHOUSE_DB,
+ password: process.env.CLICKHOUSE_PASSWORD,
+ request_timeout: CLICKHOUSE_REQUEST_TIMEOUT_MS,
+});
diff --git a/server/src/db/clickhouse/initUtils.ts b/server/src/db/clickhouse/initUtils.ts
new file mode 100644
index 000000000..06e3c6015
--- /dev/null
+++ b/server/src/db/clickhouse/initUtils.ts
@@ -0,0 +1,60 @@
+import { createServiceLogger } from "../../lib/logger/logger.js";
+import { CLICKHOUSE_REQUEST_TIMEOUT_MS, clickhouse } from "./client.js";
+
+export const clickhouseInitLogger = createServiceLogger("clickhouse");
+
+export async function execClickhouseInitStep(
+ step: string,
+ query: string,
+ options?: { optional?: boolean; lockAcquireTimeoutSeconds?: number }
+) {
+ try {
+ await clickhouse.exec({
+ query,
+ clickhouse_settings: options?.lockAcquireTimeoutSeconds
+ ? { lock_acquire_timeout: options.lockAcquireTimeoutSeconds }
+ : undefined,
+ });
+ } catch (error) {
+ clickhouseInitLogger.error(
+ { err: error, step, requestTimeoutMs: CLICKHOUSE_REQUEST_TIMEOUT_MS },
+ "ClickHouse initialization step failed"
+ );
+ if (!options?.optional) {
+ throw error;
+ }
+ }
+}
+
+export async function getTableColumns(table: string) {
+ const result = await clickhouse.query({
+ query: `
+ SELECT name
+ FROM system.columns
+ WHERE database = currentDatabase()
+ AND table = {table:String}
+ `,
+ query_params: { table },
+ format: "JSONEachRow",
+ });
+
+ const rows = await result.json<{ name: string }>();
+ return new Set(rows.map(row => row.name));
+}
+
+export async function getTableCreateQuery(table: string) {
+ const result = await clickhouse.query({
+ query: `
+ SELECT create_table_query
+ FROM system.tables
+ WHERE database = currentDatabase()
+ AND name = {table:String}
+ LIMIT 1
+ `,
+ query_params: { table },
+ format: "JSONEachRow",
+ });
+
+ const rows = await result.json<{ create_table_query: string }>();
+ return rows[0]?.create_table_query;
+}
diff --git a/server/src/db/clickhouse/schema/cloud.ts b/server/src/db/clickhouse/schema/cloud.ts
new file mode 100644
index 000000000..3208c9d80
--- /dev/null
+++ b/server/src/db/clickhouse/schema/cloud.ts
@@ -0,0 +1,33 @@
+import { execClickhouseInitStep } from "../initUtils.js";
+
+// Hourly per-site event counts, used by cloud usage tracking / billing.
+export async function initializeCloudTables() {
+ await execClickhouseInitStep(
+ "create hourly events by site target table",
+ `
+ CREATE TABLE IF NOT EXISTS hourly_events_by_site_mv_target (
+ event_hour DateTime, -- The specific hour
+ site_id UInt16,
+ event_count UInt64 -- The count of events for that site in that hour
+ )
+ ENGINE = SummingMergeTree() -- Sums 'event_count' for rows with the same sorting key
+ PARTITION BY toYYYYMM(event_hour)
+ ORDER BY (event_hour, site_id)
+ TTL event_hour + INTERVAL 60 DAY
+ `
+ );
+
+ await execClickhouseInitStep(
+ "create hourly events by site materialized view",
+ `
+ CREATE MATERIALIZED VIEW IF NOT EXISTS hourly_events_by_site_mv
+ TO hourly_events_by_site_mv_target -- Name of the target table
+ AS SELECT
+ toStartOfHour(timestamp) AS event_hour,
+ site_id,
+ count() AS event_count
+ FROM events
+ GROUP BY event_hour, site_id
+ `
+ );
+}
diff --git a/server/src/db/clickhouse/schema/core.ts b/server/src/db/clickhouse/schema/core.ts
new file mode 100644
index 000000000..a05b75db1
--- /dev/null
+++ b/server/src/db/clickhouse/schema/core.ts
@@ -0,0 +1,268 @@
+import { clickhouseInitLogger as logger, execClickhouseInitStep, getTableColumns } from "../initUtils.js";
+
+type ColumnDefinition = {
+ name: string;
+ definition: string;
+};
+
+const EVENTS_COLUMNS_TO_ENSURE: ColumnDefinition[] = [
+ { name: "lcp", definition: "lcp Nullable(Float64)" },
+ { name: "cls", definition: "cls Nullable(Float64)" },
+ { name: "inp", definition: "inp Nullable(Float64)" },
+ { name: "fcp", definition: "fcp Nullable(Float64)" },
+ { name: "ttfb", definition: "ttfb Nullable(Float64)" },
+ { name: "ip", definition: "ip Nullable(String)" },
+ { name: "timezone", definition: "timezone LowCardinality(String) DEFAULT ''" },
+ { name: "identified_user_id", definition: "identified_user_id String DEFAULT ''" },
+ { name: "import_id", definition: "import_id Nullable(UUID)" },
+ { name: "tag", definition: "tag LowCardinality(String) DEFAULT ''" },
+ { name: "feature_flags", definition: "feature_flags Map(String, String) DEFAULT map()" },
+];
+
+async function ensureEventsColumns() {
+ const existingColumns = await getTableColumns("events");
+ const missingColumns = EVENTS_COLUMNS_TO_ENSURE.filter(column => !existingColumns.has(column.name));
+
+ if (missingColumns.length === 0) {
+ logger.debug("Events table columns are up to date");
+ return;
+ }
+
+ logger.info({ missingColumns: missingColumns.map(column => column.name) }, "Adding missing events table columns");
+
+ await execClickhouseInitStep(
+ "add missing events columns",
+ `
+ ALTER TABLE events
+ ${missingColumns.map(column => `ADD COLUMN IF NOT EXISTS ${column.definition}`).join(",\n ")}
+ `,
+ { lockAcquireTimeoutSeconds: 15 }
+ );
+}
+
+export async function initializeCoreTables() {
+ await execClickhouseInitStep(
+ "create events table",
+ `
+ CREATE TABLE IF NOT EXISTS events (
+ site_id UInt16,
+ timestamp DateTime,
+ session_id String,
+ user_id String,
+ hostname String,
+ pathname String,
+ querystring String, /* URL parameters stored in raw format */
+ url_parameters Map(String, String), /* Structured storage for all URL parameters */
+ page_title String,
+ referrer String,
+ channel String,
+ browser LowCardinality(String),
+ browser_version LowCardinality(String),
+ operating_system LowCardinality(String),
+ operating_system_version LowCardinality(String),
+ language LowCardinality(String),
+ country LowCardinality(FixedString(2)),
+ region LowCardinality(String),
+ city String,
+ lat Float64,
+ lon Float64,
+ screen_width UInt16,
+ screen_height UInt16,
+ device_type LowCardinality(String),
+ type LowCardinality(String) DEFAULT 'pageview',
+ event_name String,
+ props JSON,
+ lcp Nullable(Float64),
+ cls Nullable(Float64),
+ inp Nullable(Float64),
+ fcp Nullable(Float64),
+ ttfb Nullable(Float64),
+ ip Nullable(String),
+ timezone LowCardinality(String) DEFAULT '',
+ identified_user_id String DEFAULT '',
+ import_id Nullable(UUID),
+ tag LowCardinality(String) DEFAULT '',
+ feature_flags Map(String, String) DEFAULT map()
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(timestamp)
+ ORDER BY (site_id, timestamp)
+ `
+ );
+
+ // Heal tables created by older versions (or by the window where the CREATE
+ // TABLE above was missing these columns). Only columns that are actually
+ // missing are altered in β€” even a no-op ALTER needs the events table ALTER lock.
+ await ensureEventsColumns();
+
+ await execClickhouseInitStep(
+ "create bot events table",
+ `
+ CREATE TABLE IF NOT EXISTS bot_events (
+ site_id UInt16,
+ timestamp DateTime,
+ session_id String,
+ user_id String,
+ hostname String,
+ pathname String,
+ querystring String,
+ referrer String,
+ browser LowCardinality(String),
+ browser_version LowCardinality(String),
+ operating_system LowCardinality(String),
+ operating_system_version LowCardinality(String),
+ country LowCardinality(FixedString(2)),
+ region LowCardinality(String),
+ city String,
+ lat Float64,
+ lon Float64,
+ screen_width UInt16,
+ screen_height UInt16,
+ device_type LowCardinality(String),
+ type LowCardinality(String) DEFAULT 'pageview',
+ asn Nullable(UInt32),
+ asn_org String DEFAULT '',
+ detected_ua_pattern Bool DEFAULT false,
+ detected_header_heuristics Bool DEFAULT false,
+ detected_client_signals Bool DEFAULT false,
+ detected_bot_asn Bool DEFAULT false,
+ detected_rate_anomaly Bool DEFAULT false,
+ matched_ua_pattern String DEFAULT '',
+ bot_category LowCardinality(String) DEFAULT ''
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(timestamp)
+ ORDER BY (site_id, timestamp)
+ TTL timestamp + INTERVAL 3 MONTH
+ `
+ );
+
+ await execClickhouseInitStep(
+ "create session replay events table",
+ `
+ CREATE TABLE IF NOT EXISTS session_replay_events (
+ site_id UInt16,
+ session_id String,
+ user_id String,
+ timestamp DateTime64(3),
+ event_type LowCardinality(String),
+ event_data String,
+ event_data_key Nullable(String), -- R2 storage key for cloud deployments
+ batch_index Nullable(UInt16), -- Index within the R2 batch
+ sequence_number UInt32,
+ event_size_bytes UInt32,
+ viewport_width Nullable(UInt16),
+ viewport_height Nullable(UInt16),
+ is_complete UInt8 DEFAULT 0
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(timestamp)
+ ORDER BY (site_id, session_id, sequence_number)
+ TTL toDateTime(timestamp) + INTERVAL 30 DAY
+ `
+ );
+
+ await execClickhouseInitStep(
+ "add session replay events columns",
+ `
+ ALTER TABLE session_replay_events
+ ADD COLUMN IF NOT EXISTS event_data_key Nullable(String), -- R2 storage key for cloud deployments
+ ADD COLUMN IF NOT EXISTS batch_index Nullable(UInt16), -- Index within the R2 batch
+ ADD COLUMN IF NOT EXISTS identified_user_id String DEFAULT ''
+ `
+ );
+
+ await execClickhouseInitStep(
+ "create session replay metadata table",
+ `
+ CREATE TABLE IF NOT EXISTS session_replay_metadata (
+ site_id UInt16,
+ session_id String,
+ user_id String,
+ start_time DateTime,
+ end_time Nullable(DateTime),
+ duration_ms Nullable(UInt32),
+ event_count UInt32,
+ compressed_size_bytes UInt32,
+ page_url String,
+ country LowCardinality(FixedString(2)),
+ region LowCardinality(String),
+ city String,
+ lat Float64,
+ lon Float64,
+ browser LowCardinality(String),
+ browser_version LowCardinality(String),
+ operating_system LowCardinality(String),
+ operating_system_version LowCardinality(String),
+ language LowCardinality(String),
+ screen_width UInt16,
+ screen_height UInt16,
+ device_type LowCardinality(String),
+ channel String,
+ hostname String,
+ referrer String,
+ has_replay_data UInt8 DEFAULT 1,
+ created_at DateTime DEFAULT now()
+ )
+ ENGINE = ReplacingMergeTree(created_at)
+ PARTITION BY toYYYYMM(start_time)
+ ORDER BY (site_id, session_id)
+ TTL start_time + INTERVAL 30 DAY
+ `
+ );
+
+ await execClickhouseInitStep(
+ "add session replay metadata columns",
+ `
+ ALTER TABLE session_replay_metadata
+ ADD COLUMN IF NOT EXISTS identified_user_id String DEFAULT ''
+ `
+ );
+
+ await execClickhouseInitStep(
+ "create monitor events table",
+ `
+ CREATE TABLE IF NOT EXISTS monitor_events (
+ monitor_id UInt32,
+ organization_id String,
+ timestamp DateTime,
+
+ -- Monitor metadata
+ monitor_type LowCardinality(String), -- 'http', 'tcp'
+ monitor_url String,
+ monitor_name String,
+ region LowCardinality(String) DEFAULT 'local',
+
+ -- Response data
+ status LowCardinality(String), -- 'success', 'failure', 'timeout'
+ status_code Nullable(UInt16), -- HTTP status code
+ response_time_ms UInt32,
+
+ -- HTTP timing breakdown (all in milliseconds)
+ dns_time_ms Nullable(UInt32),
+ tcp_time_ms Nullable(UInt32),
+ tls_time_ms Nullable(UInt32),
+ ttfb_ms Nullable(UInt32), -- Time to first byte
+ transfer_time_ms Nullable(UInt32),
+
+ -- Validation results
+ validation_errors Array(String), -- Array of failed validation rules
+
+ -- Response metadata (for HTTP)
+ response_headers Map(String, String),
+ response_size_bytes Nullable(UInt32),
+
+ -- TCP specific
+ port Nullable(UInt16),
+
+ -- Error information
+ error_message Nullable(String),
+ error_type Nullable(String) -- 'dns_failure', 'connection_timeout', 'ssl_error', etc.
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(timestamp)
+ ORDER BY (organization_id, monitor_id, timestamp)
+ SETTINGS ttl_only_drop_parts = 1
+ `
+ );
+}
diff --git a/server/src/db/clickhouse/schema/liteDashboard.ts b/server/src/db/clickhouse/schema/liteDashboard.ts
new file mode 100644
index 000000000..ef86f74bb
--- /dev/null
+++ b/server/src/db/clickhouse/schema/liteDashboard.ts
@@ -0,0 +1,299 @@
+import { clickhouseInitLogger as logger, execClickhouseInitStep, getTableCreateQuery } from "../initUtils.js";
+
+const SESSION_HOURLY_MV_NAME = "session_hourly_mv";
+const SESSION_HOURLY_MV_REFRESH_INTERVAL = "REFRESH EVERY 1 HOUR";
+const SESSION_HOURLY_MV_SOURCE = "sessions_mv_target FINAL";
+
+const SESSION_HOURLY_MV_CREATE_QUERY = `
+ CREATE MATERIALIZED VIEW ${SESSION_HOURLY_MV_NAME}
+ ${SESSION_HOURLY_MV_REFRESH_INTERVAL}
+ TO session_hourly_mv_target
+ AS
+ SELECT
+ site_id,
+ toStartOfHour(start_time) AS session_hour,
+ count() AS sessions,
+ sum(session_pageviews) AS pageviews,
+ uniqState(user_id) AS users,
+ sum(toUInt64(end_time - start_time)) AS total_session_duration_seconds,
+ countIf(session_pageviews = 1) AS bounced_sessions
+ FROM (
+ SELECT
+ site_id,
+ user_id,
+ start_time,
+ end_time,
+ pageviews AS session_pageviews
+ FROM ${SESSION_HOURLY_MV_SOURCE}
+ ) AS sessions
+ GROUP BY site_id, session_hour
+`;
+
+async function ensureSessionHourlyMaterializedView() {
+ const existingCreateQuery = await getTableCreateQuery(SESSION_HOURLY_MV_NAME);
+ const isCurrent =
+ existingCreateQuery?.includes(SESSION_HOURLY_MV_REFRESH_INTERVAL) &&
+ existingCreateQuery.includes(SESSION_HOURLY_MV_SOURCE);
+
+ if (isCurrent) {
+ logger.debug("Session hourly materialized view is up to date");
+ return;
+ }
+
+ if (existingCreateQuery) {
+ logger.info("Replacing outdated session hourly materialized view");
+ await execClickhouseInitStep(
+ "drop outdated session hourly materialized view",
+ `DROP VIEW IF EXISTS ${SESSION_HOURLY_MV_NAME} SYNC`,
+ { lockAcquireTimeoutSeconds: 15 }
+ );
+ }
+
+ await execClickhouseInitStep("create session hourly materialized view", SESSION_HOURLY_MV_CREATE_QUERY);
+}
+
+// Materialized views that back the simplified high-traffic dashboard.
+// All views are hourly-bucketed and feed the lite endpoints; raw `events`
+// is still queried for filters that aren't keyed in these MVs.
+export async function initializeLiteDashboardMVs() {
+ // Per-session rollup. Replaces the AllSessionPageviews / FilteredSessions
+ // CTEs that getOverview and getOverviewBucketed run over raw events.
+ //
+ // This is a streaming MV: it fires once per inserted block and writes a
+ // PARTIAL session row for whatever events were in that block. AggregatingMergeTree
+ // then composes the SimpleAggregateFunction columns (min/max/sum) across parts
+ // sharing (site_id, session_id), and the read queries additionally re-aggregate
+ // by session_id at query time β€” so pageviews/start_time/end_time are always
+ // correct even before merges complete.
+ //
+ // The plain metadata columns below (country, region, device_type, browser,
+ // operating_system, hostname) are session-INVARIANT, so every partial row of a
+ // session carries the same value and `any()` is well-defined. Entry/acquisition
+ // attributes (entry_page, referrer, channel) are deliberately NOT stored here:
+ // they are first-event values that differ across a session's blocks, can't be
+ // composed correctly per-partial-row, and so are served by the raw-events
+ // fallback instead (see LITE_SESSION_FILTER_COLUMNS in lite/utils.ts).
+ await execClickhouseInitStep(
+ "create sessions rollup target table",
+ `
+ CREATE TABLE IF NOT EXISTS sessions_mv_target (
+ site_id UInt16,
+ session_id String,
+ user_id String,
+ start_time SimpleAggregateFunction(min, DateTime),
+ end_time SimpleAggregateFunction(max, DateTime),
+ pageviews SimpleAggregateFunction(sum, UInt64),
+ events SimpleAggregateFunction(sum, UInt64),
+ country LowCardinality(FixedString(2)),
+ region LowCardinality(String),
+ device_type LowCardinality(String),
+ browser LowCardinality(String),
+ operating_system LowCardinality(String),
+ hostname String,
+ last_seen SimpleAggregateFunction(max, DateTime)
+ )
+ ENGINE = AggregatingMergeTree()
+ PARTITION BY toYYYYMM(start_time)
+ ORDER BY (site_id, session_id)
+ `
+ );
+
+ await execClickhouseInitStep(
+ "create sessions rollup materialized view",
+ `
+ CREATE MATERIALIZED VIEW IF NOT EXISTS sessions_mv
+ TO sessions_mv_target
+ AS SELECT
+ site_id,
+ session_id,
+ any(user_id) AS user_id,
+ min(timestamp) AS start_time,
+ max(timestamp) AS end_time,
+ countIf(type = 'pageview') AS pageviews,
+ count() AS events,
+ any(country) AS country,
+ any(region) AS region,
+ any(device_type) AS device_type,
+ any(browser) AS browser,
+ any(operating_system) AS operating_system,
+ any(hostname) AS hostname,
+ max(timestamp) AS last_seen
+ FROM events
+ GROUP BY site_id, session_id
+ `
+ );
+
+ // Hourly overview rollup. Drives the bucketed chart and overview cards.
+ await execClickhouseInitStep(
+ "create hourly overview rollup target table",
+ `
+ CREATE TABLE IF NOT EXISTS overview_hourly_mv_target (
+ site_id UInt16,
+ event_hour DateTime,
+ pageviews SimpleAggregateFunction(sum, UInt64),
+ events SimpleAggregateFunction(sum, UInt64),
+ users AggregateFunction(uniq, String),
+ sessions AggregateFunction(uniq, String)
+ )
+ ENGINE = AggregatingMergeTree()
+ PARTITION BY toYYYYMM(event_hour)
+ ORDER BY (site_id, event_hour)
+ `
+ );
+
+ await execClickhouseInitStep(
+ "create hourly overview rollup materialized view",
+ `
+ CREATE MATERIALIZED VIEW IF NOT EXISTS overview_hourly_mv
+ TO overview_hourly_mv_target
+ AS SELECT
+ site_id,
+ toStartOfHour(timestamp) AS event_hour,
+ countIf(type = 'pageview') AS pageviews,
+ count() AS events,
+ uniqState(user_id) AS users,
+ uniqState(session_id) AS sessions
+ FROM events
+ GROUP BY site_id, event_hour
+ `
+ );
+
+ // Top-pathname rollup. Cardinality is bounded by hour, so high-URL sites
+ // still get smaller-than-raw storage. Filtered queries fall back to events.
+ await execClickhouseInitStep(
+ "create hourly pathname rollup target table",
+ `
+ CREATE TABLE IF NOT EXISTS pathname_hourly_mv_target (
+ site_id UInt16,
+ event_hour DateTime,
+ pathname String,
+ hostname String,
+ pageviews SimpleAggregateFunction(sum, UInt64),
+ users AggregateFunction(uniq, String),
+ sessions AggregateFunction(uniq, String)
+ )
+ ENGINE = AggregatingMergeTree()
+ PARTITION BY toYYYYMM(event_hour)
+ ORDER BY (site_id, event_hour, pathname)
+ `
+ );
+
+ await execClickhouseInitStep(
+ "create hourly pathname rollup materialized view",
+ `
+ CREATE MATERIALIZED VIEW IF NOT EXISTS pathname_hourly_mv
+ TO pathname_hourly_mv_target
+ AS SELECT
+ site_id,
+ toStartOfHour(timestamp) AS event_hour,
+ pathname,
+ any(hostname) AS hostname,
+ countIf(type = 'pageview') AS pageviews,
+ uniqState(user_id) AS users,
+ uniqState(session_id) AS sessions
+ FROM events
+ WHERE type = 'pageview'
+ GROUP BY site_id, event_hour, pathname
+ `
+ );
+
+ // Country/region rollup. Cardinality is naturally bounded.
+ await execClickhouseInitStep(
+ "create hourly country rollup target table",
+ `
+ CREATE TABLE IF NOT EXISTS country_hourly_mv_target (
+ site_id UInt16,
+ event_hour DateTime,
+ country LowCardinality(FixedString(2)),
+ region LowCardinality(String),
+ pageviews SimpleAggregateFunction(sum, UInt64),
+ users AggregateFunction(uniq, String),
+ sessions AggregateFunction(uniq, String)
+ )
+ ENGINE = AggregatingMergeTree()
+ PARTITION BY toYYYYMM(event_hour)
+ ORDER BY (site_id, event_hour, country, region)
+ `
+ );
+
+ await execClickhouseInitStep(
+ "create hourly country rollup materialized view",
+ `
+ CREATE MATERIALIZED VIEW IF NOT EXISTS country_hourly_mv
+ TO country_hourly_mv_target
+ AS SELECT
+ site_id,
+ toStartOfHour(timestamp) AS event_hour,
+ country,
+ region,
+ countIf(type = 'pageview') AS pageviews,
+ uniqState(user_id) AS users,
+ uniqState(session_id) AS sessions
+ FROM events
+ GROUP BY site_id, event_hour, country, region
+ `
+ );
+
+ // Device-type rollup.
+ await execClickhouseInitStep(
+ "create hourly device type rollup target table",
+ `
+ CREATE TABLE IF NOT EXISTS device_type_hourly_mv_target (
+ site_id UInt16,
+ event_hour DateTime,
+ device_type LowCardinality(String),
+ pageviews SimpleAggregateFunction(sum, UInt64),
+ users AggregateFunction(uniq, String),
+ sessions AggregateFunction(uniq, String)
+ )
+ ENGINE = AggregatingMergeTree()
+ PARTITION BY toYYYYMM(event_hour)
+ ORDER BY (site_id, event_hour, device_type)
+ `
+ );
+
+ await execClickhouseInitStep(
+ "create hourly device type rollup materialized view",
+ `
+ CREATE MATERIALIZED VIEW IF NOT EXISTS device_type_hourly_mv
+ TO device_type_hourly_mv_target
+ AS SELECT
+ site_id,
+ toStartOfHour(timestamp) AS event_hour,
+ device_type,
+ countIf(type = 'pageview') AS pageviews,
+ uniqState(user_id) AS users,
+ uniqState(session_id) AS sessions
+ FROM events
+ GROUP BY site_id, event_hour, device_type
+ `
+ );
+
+ // Session-keyed hourly rollup, populated by a REFRESHABLE materialized view.
+ // Streaming MVs can't compute bounce_rate or session_duration because they
+ // see one event at a time, never the full per-session state. The refreshable
+ // MV compacts the already-populated per-session rollup hourly instead of
+ // repeatedly rebuilding sessions from every raw event.
+ await execClickhouseInitStep(
+ "create session hourly rollup target table",
+ `
+ CREATE TABLE IF NOT EXISTS session_hourly_mv_target (
+ site_id UInt16,
+ session_hour DateTime,
+ sessions UInt64,
+ pageviews UInt64,
+ users AggregateFunction(uniq, String),
+ total_session_duration_seconds UInt64,
+ bounced_sessions UInt64
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(session_hour)
+ ORDER BY (site_id, session_hour)
+ `
+ );
+
+ // CREATE IF NOT EXISTS cannot update an already-installed view. Inspect the
+ // stored definition so deployments replace the old five-minute raw-events
+ // refresh once, while subsequent startups remain no-ops.
+ await ensureSessionHourlyMaterializedView();
+}