Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Clickhouse compression

Blue Mouse5cd3b6e01e5f
diff --git a/backend/migrations/clickhouse/2025_12_14_data_compression.js b/backend/migrations/clickhouse/2025_12_14_data_compression.js
new file mode 100644
index 000000000..cbe953b8a
--- /dev/null
+++ b/backend/migrations/clickhouse/2025_12_14_data_compression.js
@@ -0,0 +1,96 @@
+// Migration to add compression codecs to existing tables for improved storage and query performance
+// ZSTD(3) for String columns, Delta(4)+LZ4 for DateTime columns
+const { queriesRunner, dbName } = require('./setup')
+
+const COMPRESSION_QUERIES = [
+ // Analytics table - String columns with ZSTD(3)
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN host Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN pg Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN ref Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN so Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN me Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN ca Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN te Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN co Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN ct Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN brv Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN osv Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN profileId Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN \`meta.key\` Array(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN \`meta.value\` Array(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.analytics MODIFY COLUMN created DateTime('UTC') CODEC(Delta(4), LZ4)`,
+
+ // CustomEV table
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN host Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN pg Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN ev String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN ref Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN so Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN me Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN ca Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN te Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN co Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN ct Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN brv Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN osv Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN profileId Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN \`meta.key\` Array(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN \`meta.value\` Array(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.customEV MODIFY COLUMN created DateTime('UTC') CODEC(Delta(4), LZ4)`,
+
+ // Performance table
+ `ALTER TABLE ${dbName}.performance MODIFY COLUMN host Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.performance MODIFY COLUMN pg Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.performance MODIFY COLUMN ct Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.performance MODIFY COLUMN brv Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.performance MODIFY COLUMN created DateTime('UTC') CODEC(Delta(4), LZ4)`,
+
+ // Errors table - stackTrace especially benefits from compression
+ `ALTER TABLE ${dbName}.errors MODIFY COLUMN host Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.errors MODIFY COLUMN pg Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.errors MODIFY COLUMN ct Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.errors MODIFY COLUMN brv Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.errors MODIFY COLUMN osv Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.errors MODIFY COLUMN profileId Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.errors MODIFY COLUMN name String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.errors MODIFY COLUMN message Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.errors MODIFY COLUMN stackTrace Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.errors MODIFY COLUMN filename Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.errors MODIFY COLUMN \`meta.key\` Array(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.errors MODIFY COLUMN \`meta.value\` Array(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.errors MODIFY COLUMN created DateTime('UTC') CODEC(Delta(4), LZ4)`,
+
+ // Sessions table
+ `ALTER TABLE ${dbName}.sessions MODIFY COLUMN profileId Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.sessions MODIFY COLUMN firstSeen DateTime('UTC') CODEC(Delta(4), LZ4)`,
+ `ALTER TABLE ${dbName}.sessions MODIFY COLUMN lastSeen DateTime('UTC') CODEC(Delta(4), LZ4)`,
+
+ // Feature flag evaluations
+ `ALTER TABLE ${dbName}.feature_flag_evaluations MODIFY COLUMN flagId String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.feature_flag_evaluations MODIFY COLUMN flagKey String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.feature_flag_evaluations MODIFY COLUMN profileId String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.feature_flag_evaluations MODIFY COLUMN created DateTime('UTC') CODEC(Delta(4), LZ4)`,
+
+ // Experiment exposures
+ `ALTER TABLE ${dbName}.experiment_exposures MODIFY COLUMN experimentId String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.experiment_exposures MODIFY COLUMN variantKey String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.experiment_exposures MODIFY COLUMN profileId String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.experiment_exposures MODIFY COLUMN created DateTime('UTC') CODEC(Delta(4), LZ4)`,
+
+ // Captcha table
+ `ALTER TABLE ${dbName}.captcha MODIFY COLUMN os Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.captcha MODIFY COLUMN created DateTime('UTC') CODEC(Delta(4), LZ4)`,
+]
+
+const runCompression = async () => {
+ try {
+ await queriesRunner(COMPRESSION_QUERIES)
+ console.log('[SUCCESS] Compression codecs applied to all tables')
+ } catch (reason) {
+ console.error(`[ERROR] Error applying compression: ${reason}`)
+ }
+}
+
+runCompression()
+
+module.exports = { runCompression }
diff --git a/backend/migrations/clickhouse/initialise_database.js b/backend/migrations/clickhouse/initialise_database.js
index 795311b9e..fe8e4ebc3 100644
--- a/backend/migrations/clickhouse/initialise_database.js
+++ b/backend/migrations/clickhouse/initialise_database.js
@@ -8,32 +8,32 @@ const CLICKHOUSE_INIT_QUERIES = [
`CREATE TABLE IF NOT EXISTS ${dbName}.analytics
(
psid Nullable(UInt64),
- profileId Nullable(String),
+ profileId Nullable(String) CODEC(ZSTD(3)),
pid FixedString(12),
- host Nullable(String),
- pg Nullable(String),
+ host Nullable(String) CODEC(ZSTD(3)),
+ pg Nullable(String) CODEC(ZSTD(3)),
dv LowCardinality(Nullable(String)),
br LowCardinality(Nullable(String)),
- brv Nullable(String),
+ brv Nullable(String) CODEC(ZSTD(3)),
os LowCardinality(Nullable(String)),
- osv Nullable(String),
+ osv Nullable(String) CODEC(ZSTD(3)),
lc LowCardinality(Nullable(String)),
- ref Nullable(String),
- so Nullable(String),
- me Nullable(String),
- ca Nullable(String),
- te Nullable(String),
- co Nullable(String),
+ ref Nullable(String) CODEC(ZSTD(3)),
+ so Nullable(String) CODEC(ZSTD(3)),
+ me Nullable(String) CODEC(ZSTD(3)),
+ ca Nullable(String) CODEC(ZSTD(3)),
+ te Nullable(String) CODEC(ZSTD(3)),
+ co Nullable(String) CODEC(ZSTD(3)),
cc Nullable(FixedString(2)),
rg LowCardinality(Nullable(String)),
rgc LowCardinality(Nullable(String)),
- ct Nullable(String),
+ ct Nullable(String) CODEC(ZSTD(3)),
meta Nested
(
- key String,
- value String
+ key String CODEC(ZSTD(3)),
+ value String CODEC(ZSTD(3))
),
- created DateTime('UTC')
+ created DateTime('UTC') CODEC(Delta(4), LZ4)
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
@@ -43,33 +43,33 @@ const CLICKHOUSE_INIT_QUERIES = [
`CREATE TABLE IF NOT EXISTS ${dbName}.customEV
(
psid Nullable(UInt64),
- profileId Nullable(String),
+ profileId Nullable(String) CODEC(ZSTD(3)),
pid FixedString(12),
- host Nullable(String),
- ev String,
- pg Nullable(String),
+ host Nullable(String) CODEC(ZSTD(3)),
+ ev String CODEC(ZSTD(3)),
+ pg Nullable(String) CODEC(ZSTD(3)),
dv LowCardinality(Nullable(String)),
br LowCardinality(Nullable(String)),
- brv Nullable(String),
+ brv Nullable(String) CODEC(ZSTD(3)),
os LowCardinality(Nullable(String)),
- osv Nullable(String),
+ osv Nullable(String) CODEC(ZSTD(3)),
lc LowCardinality(Nullable(String)),
- ref Nullable(String),
- so Nullable(String),
- me Nullable(String),
- ca Nullable(String),
- te Nullable(String),
- co Nullable(String),
+ ref Nullable(String) CODEC(ZSTD(3)),
+ so Nullable(String) CODEC(ZSTD(3)),
+ me Nullable(String) CODEC(ZSTD(3)),
+ ca Nullable(String) CODEC(ZSTD(3)),
+ te Nullable(String) CODEC(ZSTD(3)),
+ co Nullable(String) CODEC(ZSTD(3)),
cc Nullable(FixedString(2)),
rg LowCardinality(Nullable(String)),
rgc LowCardinality(Nullable(String)),
- ct Nullable(String),
+ ct Nullable(String) CODEC(ZSTD(3)),
meta Nested
(
- key String,
- value String
+ key String CODEC(ZSTD(3)),
+ value String CODEC(ZSTD(3))
),
- created DateTime('UTC')
+ created DateTime('UTC') CODEC(Delta(4), LZ4)
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
@@ -79,15 +79,15 @@ const CLICKHOUSE_INIT_QUERIES = [
`CREATE TABLE IF NOT EXISTS ${dbName}.performance
(
pid FixedString(12),
- host Nullable(String),
- pg Nullable(String),
+ host Nullable(String) CODEC(ZSTD(3)),
+ pg Nullable(String) CODEC(ZSTD(3)),
dv LowCardinality(Nullable(String)),
br LowCardinality(Nullable(String)),
- brv Nullable(String),
+ brv Nullable(String) CODEC(ZSTD(3)),
cc Nullable(FixedString(2)),
rg LowCardinality(Nullable(String)),
rgc LowCardinality(Nullable(String)),
- ct Nullable(String),
+ ct Nullable(String) CODEC(ZSTD(3)),
dns Nullable(UInt32),
tls Nullable(UInt32),
conn Nullable(UInt32),
@@ -96,7 +96,7 @@ const CLICKHOUSE_INIT_QUERIES = [
domLoad Nullable(UInt32),
pageLoad Nullable(UInt32),
ttfb Nullable(UInt32),
- created DateTime('UTC')
+ created DateTime('UTC') CODEC(Delta(4), LZ4)
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
@@ -106,33 +106,33 @@ const CLICKHOUSE_INIT_QUERIES = [
`CREATE TABLE IF NOT EXISTS ${dbName}.errors
(
psid Nullable(UInt64),
- profileId Nullable(String),
+ profileId Nullable(String) CODEC(ZSTD(3)),
eid FixedString(32),
pid FixedString(12),
- host Nullable(String),
- pg Nullable(String),
+ host Nullable(String) CODEC(ZSTD(3)),
+ pg Nullable(String) CODEC(ZSTD(3)),
dv LowCardinality(Nullable(String)),
br LowCardinality(Nullable(String)),
- brv Nullable(String),
+ brv Nullable(String) CODEC(ZSTD(3)),
os LowCardinality(Nullable(String)),
- osv Nullable(String),
+ osv Nullable(String) CODEC(ZSTD(3)),
lc LowCardinality(Nullable(String)),
cc LowCardinality(Nullable(FixedString(2))),
rg LowCardinality(Nullable(String)),
rgc LowCardinality(Nullable(String)),
- ct Nullable(String),
- name String,
- message Nullable(String),
- stackTrace Nullable(String),
+ ct Nullable(String) CODEC(ZSTD(3)),
+ name String CODEC(ZSTD(3)),
+ message Nullable(String) CODEC(ZSTD(3)),
+ stackTrace Nullable(String) CODEC(ZSTD(3)),
meta Nested
(
- key String,
- value String
+ key String CODEC(ZSTD(3)),
+ value String CODEC(ZSTD(3))
),
lineno Nullable(UInt32),
colno Nullable(UInt32),
- filename Nullable(String),
- created DateTime('UTC')
+ filename Nullable(String) CODEC(ZSTD(3)),
+ created DateTime('UTC') CODEC(Delta(4), LZ4)
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
@@ -153,9 +153,9 @@ const CLICKHOUSE_INIT_QUERIES = [
(
psid UInt64,
pid FixedString(12),
- profileId Nullable(String),
- firstSeen DateTime('UTC'),
- lastSeen DateTime('UTC'),
+ profileId Nullable(String) CODEC(ZSTD(3)),
+ firstSeen DateTime('UTC') CODEC(Delta(4), LZ4),
+ lastSeen DateTime('UTC') CODEC(Delta(4), LZ4),
pageviews UInt32 DEFAULT 1,
events UInt32 DEFAULT 0
)
@@ -163,28 +163,15 @@ const CLICKHOUSE_INIT_QUERIES = [
ORDER BY (pid, psid)
PARTITION BY toYYYYMM(firstSeen);`,
- // Session duration table
- `CREATE TABLE IF NOT EXISTS ${dbName}.sessions
- (
- psid UInt64,
- pid FixedString(12),
- profileId Nullable(String),
- firstSeen DateTime('UTC'),
- lastSeen DateTime('UTC')
- )
- ENGINE = ReplacingMergeTree(lastSeen)
- ORDER BY (pid, psid)
- PARTITION BY toYYYYMM(firstSeen);`,
-
// Feature flag evaluations table
`CREATE TABLE IF NOT EXISTS ${dbName}.feature_flag_evaluations
(
pid FixedString(12),
- flagId String,
- flagKey String,
+ flagId String CODEC(ZSTD(3)),
+ flagKey String CODEC(ZSTD(3)),
result UInt8,
- profileId String,
- created DateTime('UTC')
+ profileId String CODEC(ZSTD(3)),
+ created DateTime('UTC') CODEC(Delta(4), LZ4)
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
@@ -195,10 +182,10 @@ const CLICKHOUSE_INIT_QUERIES = [
`CREATE TABLE IF NOT EXISTS ${dbName}.experiment_exposures
(
pid FixedString(12),
- experimentId String,
- variantKey String,
- profileId String,
- created DateTime('UTC')
+ experimentId String CODEC(ZSTD(3)),
+ variantKey String CODEC(ZSTD(3)),
+ profileId String CODEC(ZSTD(3)),
+ created DateTime('UTC') CODEC(Delta(4), LZ4)
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
@@ -211,10 +198,10 @@ const CLICKHOUSE_INIT_QUERIES = [
pid FixedString(12),
dv LowCardinality(Nullable(String)),
br LowCardinality(Nullable(String)),
- os Nullable(String),
+ os Nullable(String) CODEC(ZSTD(3)),
cc Nullable(FixedString(2)),
manuallyPassed UInt8,
- created DateTime('UTC')
+ created DateTime('UTC') CODEC(Delta(4), LZ4)
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
diff --git a/backend/migrations/clickhouse/initialise_selfhosted.js b/backend/migrations/clickhouse/initialise_selfhosted.js
index f53922bb1..eb4f050b9 100644
--- a/backend/migrations/clickhouse/initialise_selfhosted.js
+++ b/backend/migrations/clickhouse/initialise_selfhosted.js
@@ -1,4 +1,4 @@
-// This script initialises the Swetrix cloud edition database and tables if they're absent
+// This script initialises the Swetrix selfhosted edition database and tables if they're absent
const { queriesRunner, dbName } = require('./setup')
const { initialiseDatabase } = require('./initialise_database')
@@ -6,14 +6,14 @@ const CLICKHOUSE_INIT_QUERIES = [
`CREATE TABLE IF NOT EXISTS ${dbName}.user
(
id FixedString(36),
- email String,
- password String,
- timezone String,
- timeFormat String,
+ email String CODEC(ZSTD(3)),
+ password String CODEC(ZSTD(3)),
+ timezone String CODEC(ZSTD(3)),
+ timeFormat String CODEC(ZSTD(3)),
showLiveVisitorsInTitle Int8,
- onboardingStep String,
+ onboardingStep String CODEC(ZSTD(3)),
hasCompletedOnboarding Int8,
- apiKey Nullable(String)
+ apiKey Nullable(String) CODEC(ZSTD(3))
)
ENGINE = MergeTree()
PRIMARY KEY id;`,
@@ -21,17 +21,17 @@ const CLICKHOUSE_INIT_QUERIES = [
`CREATE TABLE IF NOT EXISTS ${dbName}.project
(
id FixedString(12),
- name String,
- origins Nullable(String),
- ipBlacklist Nullable(String),
- countryBlacklist Nullable(String),
+ name String CODEC(ZSTD(3)),
+ origins Nullable(String) CODEC(ZSTD(3)),
+ ipBlacklist Nullable(String) CODEC(ZSTD(3)),
+ countryBlacklist Nullable(String) CODEC(ZSTD(3)),
active Int8,
public Int8,
isPasswordProtected Int8,
- botsProtectionLevel String DEFAULT 'basic',
- passwordHash Nullable(String),
- adminId Nullable(String),
- created DateTime
+ botsProtectionLevel String CODEC(ZSTD(3)) DEFAULT 'basic',
+ passwordHash Nullable(String) CODEC(ZSTD(3)),
+ adminId Nullable(String) CODEC(ZSTD(3)),
+ created DateTime CODEC(Delta(4), LZ4)
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
@@ -39,19 +39,19 @@ const CLICKHOUSE_INIT_QUERIES = [
`CREATE TABLE IF NOT EXISTS ${dbName}.refresh_token
(
- userId String,
- refreshToken String
+ userId String CODEC(ZSTD(3)),
+ refreshToken String CODEC(ZSTD(3))
)
ENGINE = MergeTree()
ORDER BY userId;`,
`CREATE TABLE IF NOT EXISTS ${dbName}.funnel
(
- id String,
- name String,
- steps String,
+ id String CODEC(ZSTD(3)),
+ name String CODEC(ZSTD(3)),
+ steps String CODEC(ZSTD(3)),
projectId FixedString(12),
- created DateTime
+ created DateTime CODEC(Delta(4), LZ4)
)
ENGINE = MergeTree()
PRIMARY KEY id;`,
@@ -60,11 +60,11 @@ const CLICKHOUSE_INIT_QUERIES = [
(
id FixedString(36),
projectId FixedString(12),
- name String,
- type String,
- filters Nullable(String),
- createdAt DateTime,
- updatedAt DateTime
+ name String CODEC(ZSTD(3)),
+ type String CODEC(ZSTD(3)),
+ filters Nullable(String) CODEC(ZSTD(3)),
+ createdAt DateTime CODEC(Delta(4), LZ4),
+ updatedAt DateTime CODEC(Delta(4), LZ4)
)
ENGINE = MergeTree()
PRIMARY KEY id;`,
@@ -73,11 +73,11 @@ const CLICKHOUSE_INIT_QUERIES = [
(
id FixedString(36),
viewId FixedString(36),
- customEventName String,
- metaKey Nullable(String),
- metaValue Nullable(String),
- metricKey String,
- metaValueType String
+ customEventName String CODEC(ZSTD(3)),
+ metaKey Nullable(String) CODEC(ZSTD(3)),
+ metaValue Nullable(String) CODEC(ZSTD(3)),
+ metricKey String CODEC(ZSTD(3)),
+ metaValueType String CODEC(ZSTD(3))
)
ENGINE = MergeTree()
PRIMARY KEY id;`,
@@ -87,10 +87,10 @@ const CLICKHOUSE_INIT_QUERIES = [
id FixedString(36),
userId FixedString(36),
projectId FixedString(12),
- role String,
+ role String CODEC(ZSTD(3)),
confirmed Int8,
- created DateTime,
- updated DateTime
+ created DateTime CODEC(Delta(4), LZ4),
+ updated DateTime CODEC(Delta(4), LZ4)
)
ENGINE = MergeTree()
PRIMARY KEY id;`,
@@ -100,8 +100,8 @@ const CLICKHOUSE_INIT_QUERIES = [
id FixedString(36),
projectId FixedString(12),
date Date,
- text String,
- created DateTime
+ text String CODEC(ZSTD(3)),
+ created DateTime CODEC(Delta(4), LZ4)
)
ENGINE = MergeTree()
PRIMARY KEY id;`,
diff --git a/backend/migrations/clickhouse/selfhosted_2025_12_14_data_compression.js b/backend/migrations/clickhouse/selfhosted_2025_12_14_data_compression.js
new file mode 100644
index 000000000..291a3ad7f
--- /dev/null
+++ b/backend/migrations/clickhouse/selfhosted_2025_12_14_data_compression.js
@@ -0,0 +1,69 @@
+// Migration to add compression codecs to selfhosted-specific tables
+// ZSTD(3) for String columns, Delta(4)+LZ4 for DateTime columns
+const { queriesRunner, dbName } = require('./setup')
+
+const COMPRESSION_QUERIES = [
+ // User table
+ `ALTER TABLE ${dbName}.user MODIFY COLUMN email String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.user MODIFY COLUMN password String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.user MODIFY COLUMN timezone String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.user MODIFY COLUMN timeFormat String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.user MODIFY COLUMN onboardingStep String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.user MODIFY COLUMN apiKey Nullable(String) CODEC(ZSTD(3))`,
+
+ // Project table
+ `ALTER TABLE ${dbName}.project MODIFY COLUMN name String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.project MODIFY COLUMN origins Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.project MODIFY COLUMN ipBlacklist Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.project MODIFY COLUMN countryBlacklist Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.project MODIFY COLUMN botsProtectionLevel String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.project MODIFY COLUMN passwordHash Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.project MODIFY COLUMN adminId Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.project MODIFY COLUMN created DateTime CODEC(Delta(4), LZ4)`,
+
+ // Refresh token table
+ `ALTER TABLE ${dbName}.refresh_token MODIFY COLUMN userId String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.refresh_token MODIFY COLUMN refreshToken String CODEC(ZSTD(3))`,
+
+ // Funnel table
+ `ALTER TABLE ${dbName}.funnel MODIFY COLUMN id String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.funnel MODIFY COLUMN name String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.funnel MODIFY COLUMN steps String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.funnel MODIFY COLUMN created DateTime CODEC(Delta(4), LZ4)`,
+
+ // Project views table
+ `ALTER TABLE ${dbName}.project_views MODIFY COLUMN name String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.project_views MODIFY COLUMN type String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.project_views MODIFY COLUMN filters Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.project_views MODIFY COLUMN createdAt DateTime CODEC(Delta(4), LZ4)`,
+ `ALTER TABLE ${dbName}.project_views MODIFY COLUMN updatedAt DateTime CODEC(Delta(4), LZ4)`,
+
+ // Projects views custom events table
+ `ALTER TABLE ${dbName}.projects_views_custom_events MODIFY COLUMN customEventName String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.projects_views_custom_events MODIFY COLUMN metaKey Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.projects_views_custom_events MODIFY COLUMN metaValue Nullable(String) CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.projects_views_custom_events MODIFY COLUMN metricKey String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.projects_views_custom_events MODIFY COLUMN metaValueType String CODEC(ZSTD(3))`,
+
+ // Project share table
+ `ALTER TABLE ${dbName}.project_share MODIFY COLUMN role String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.project_share MODIFY COLUMN created DateTime CODEC(Delta(4), LZ4)`,
+ `ALTER TABLE ${dbName}.project_share MODIFY COLUMN updated DateTime CODEC(Delta(4), LZ4)`,
+
+ // Annotation table
+ `ALTER TABLE ${dbName}.annotation MODIFY COLUMN text String CODEC(ZSTD(3))`,
+ `ALTER TABLE ${dbName}.annotation MODIFY COLUMN created DateTime CODEC(Delta(4), LZ4)`,
+]
+
+const runCompression = async () => {
+ try {
+ await queriesRunner(COMPRESSION_QUERIES)
+ console.log('[SUCCESS] Compression codecs applied to selfhosted tables')
+ } catch (reason) {
+ console.error(`[ERROR] Error applying compression: ${reason}`)
+ }
+}
+
+runCompression()
+
+module.exports = { runCompression }