Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Ch migration (#667)

Bill Yang4d844d3b48c6
diff --git a/clickhouse/MIGRATION.md b/clickhouse/MIGRATION.md
new file mode 100644
index 000000000..e78bfd13d
--- /dev/null
+++ b/clickhouse/MIGRATION.md
@@ -0,0 +1,72 @@
+Migration Steps
+
+On Source Server (your current production):
+
+# 1. Create a backup of the ClickHouse data volume
+
+docker run --rm \
+ -v rybbit_clickhouse-data:/source \
+ -v /tmp:/backup \
+ alpine tar czf /backup/clickhouse-data.tar.gz -C /source .
+
+# 2. Copy to your local machine or transfer directly to new server
+
+scp /tmp/clickhouse-data.tar.gz user@new-server:/tmp/
+
+On New Server:
+
+# 1. Upload the docker-compose file and create .env
+
+# Create .env file with your settings:
+
+cat > .env << EOF
+CLICKHOUSE_DB=analytics
+CLICKHOUSE_USER=default
+CLICKHOUSE_PASSWORD=frog
+EOF
+
+# 2. Create the volume and restore data (BEFORE starting ClickHouse)
+
+docker volume create clickhouse-data
+
+docker run --rm \
+ -v clickhouse-data:/target \
+ -v /tmp:/backup \
+ alpine tar xzf /backup/clickhouse-data.tar.gz -C /target
+
+# 3. Start ClickHouse
+
+docker-compose -f docker-compose.clickhouse.yml up -d
+
+# 4. Verify data
+
+docker exec clickhouse clickhouse-client --password=frog --query="SHOW DATABASES"
+docker exec clickhouse clickhouse-client --password=frog --query="SELECT count() FROM analytics.events"
+
+✅ Yes, Volume Copy Works Because:
+
+1. ClickHouse stores everything in /var/lib/clickhouse
+
+
+ - Table definitions
+ - Data files (compressed)
+ - Metadata
+
+2. Same ClickHouse version (25.4.2) - compatible format
+3. Same configurations - copied from your cloud docker-compose
+
+Quick Test Locally First:
+
+# On your Mac (if you want to test before remote server)
+
+cd /Users/bill/Desktop/rybbit/server/src/db/clickhouse
+
+# Start the standalone ClickHouse
+
+docker-compose -f docker-compose.clickhouse.yml up -d
+
+# Test connection
+
+docker exec clickhouse clickhouse-client --password=frog --query="SELECT 1"
+
+Time estimate: ~10-30 minutes depending on your data size and network speed between servers.
diff --git a/clickhouse/docker-compose.clickhouse.yml b/clickhouse/docker-compose.clickhouse.yml
new file mode 100644
index 000000000..7028dc67d
--- /dev/null
+++ b/clickhouse/docker-compose.clickhouse.yml
@@ -0,0 +1,99 @@
+# Standalone ClickHouse server for migration testing
+# Usage: docker compose -f docker-compose.clickhouse.yml up -d
+
+services:
+ clickhouse:
+ container_name: clickhouse
+ image: clickhouse/clickhouse-server:25.4.2
+ volumes:
+ - clickhouse-data:/var/lib/clickhouse
+ environment:
+ - CLICKHOUSE_DB=${CLICKHOUSE_DB:-analytics}
+ - CLICKHOUSE_USER=${CLICKHOUSE_USER:-default}
+ - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-frog}
+ healthcheck:
+ test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8123/ping"]
+ interval: 3s
+ timeout: 5s
+ retries: 5
+ start_period: 10s
+ restart: unless-stopped
+ ports:
+ - "8123:8123" # HTTP interface
+ - "9000:9000" # Native protocol
+ mem_limit: 12g
+ mem_reservation: 6g
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "50m"
+ max-file: "3"
+ configs:
+ - source: clickhouse_network
+ target: /etc/clickhouse-server/config.d/network.xml
+ - source: clickhouse_json
+ target: /etc/clickhouse-server/config.d/enable_json.xml
+ - source: clickhouse_logging
+ target: /etc/clickhouse-server/config.d/logging_rules.xml
+ - source: clickhouse_user_logging
+ target: /etc/clickhouse-server/config.d/user_logging.xml
+
+volumes:
+ clickhouse-data:
+ external: true
+
+configs:
+ clickhouse_network:
+ content: |
+ <clickhouse>
+ <listen_host>0.0.0.0</listen_host>
+ </clickhouse>
+
+ clickhouse_json:
+ content: |
+ <clickhouse>
+ <settings>
+ <enable_json_type>1</enable_json_type>
+ <max_concurrent_queries>20</max_concurrent_queries>
+ <background_pool_size>4</background_pool_size>
+ <background_merges_mutations_concurrency_ratio>1</background_merges_mutations_concurrency_ratio>
+ <mark_cache_size>524288000</mark_cache_size>
+ </settings>
+ </clickhouse>
+
+ clickhouse_logging:
+ content: |
+ <clickhouse>
+ <logger>
+ <level>warning</level>
+ <console>true</console>
+ </logger>
+ <query_thread_log remove="remove"/>
+ <query_log remove="remove"/>
+ <text_log remove="remove"/>
+ <trace_log remove="remove"/>
+ <metric_log remove="remove"/>
+ <asynchronous_metric_log remove="remove"/>
+ <session_log remove="remove"/>
+ <part_log remove="remove"/>
+ <latency_log remove="remove"/>
+ <processors_profile_log remove="remove"/>
+ </clickhouse>
+
+ clickhouse_user_logging:
+ content: |
+ <clickhouse>
+ <profiles>
+ <default>
+ <log_queries>0</log_queries>
+ <log_query_threads>0</log_query_threads>
+ <log_processors_profiles>0</log_processors_profiles>
+ <max_memory_usage>8000000000</max_memory_usage>
+ <max_bytes_before_external_group_by>4000000000</max_bytes_before_external_group_by>
+ <max_bytes_before_external_sort>4000000000</max_bytes_before_external_sort>
+ <max_threads>4</max_threads>
+ <max_block_size>8192</max_block_size>
+ <input_format_parallel_parsing>0</input_format_parallel_parsing>
+ </default>
+ </profiles>
+ </clickhouse>
diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.yml
index 5cae27622..2ace992ca 100644
--- a/docker-compose.cloud.yml
+++ b/docker-compose.cloud.yml
@@ -87,7 +87,7 @@ services:
dockerfile: ./server/Dockerfile
environment:
- NODE_ENV=production
- - CLICKHOUSE_HOST=http://clickhouse:8123
+ - CLICKHOUSE_HOST=${CLICKHOUSE_HOST:-http://clickhouse:8123}
- CLICKHOUSE_DB=${CLICKHOUSE_DB:-analytics}
- CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-frog}
- POSTGRES_HOST=postgres