Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix(db): merging profiles

Carl-Gerhard Lindesvärdcf949646eb9f
diff --git a/packages/db/migrations/20240917184138_fix_profile_merges.sql b/packages/db/migrations/20240917184138_fix_profile_merges.sql
new file mode 100644
index 000000000..ef419637b
--- /dev/null
+++ b/packages/db/migrations/20240917184138_fix_profile_merges.sql
@@ -0,0 +1,55 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE profiles_fixed
+(
+ `id` String,
+ `is_external` Bool,
+ `first_name` String,
+ `last_name` String,
+ `email` String,
+ `avatar` String,
+ `properties` Map(String, String),
+ `project_id` String,
+ `created_at` DateTime,
+ INDEX idx_first_name first_name TYPE bloom_filter GRANULARITY 1,
+ INDEX idx_last_name last_name TYPE bloom_filter GRANULARITY 1,
+ INDEX idx_email email TYPE bloom_filter GRANULARITY 1
+)
+ENGINE = ReplacingMergeTree(created_at)
+PARTITION BY toYYYYMM(created_at)
+ORDER BY (project_id, id)
+SETTINGS index_granularity = 8192;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+INSERT INTO profiles_fixed SELECT
+ id,
+ is_external,
+ first_name,
+ last_name,
+ email,
+ avatar,
+ properties,
+ project_id,
+ created_at
+FROM profiles;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+OPTIMIZE TABLE profiles_fixed FINAL;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+RENAME TABLE profiles TO profiles_old, profiles_fixed TO profiles;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+DROP TABLE profiles_old;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+-- This is a destructive migration, so the down migration is not provided.
+-- If needed, you should restore from a backup.
+SELECT 'down migration not implemented';
+-- +goose StatementEnd
diff --git a/packages/db/migrations/goose b/packages/db/migrations/goose
index ede0d3395..5138f2f2d 100755
--- a/packages/db/migrations/goose
+++ b/packages/db/migrations/goose
@@ -8,4 +8,20 @@ fi
export GOOSE_DBSTRING=$CLICKHOUSE_URL
+echo "Clickhouse migration script"
+echo ""
+echo "================="
+echo "Selected database: $GOOSE_DBSTRING"
+echo "================="
+echo ""
+if [ "$1" != "create" ] && [ -z "$CI" ]; then
+ read -p "Are you sure you want to run migrations on this database? (y/n) " -n 1 -r
+ echo
+ if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+ echo "Migration cancelled."
+ exit 0
+ fi
+fi
+
+
goose clickhouse --dir ./migrations $@
\ No newline at end of file
diff --git a/self-hosting/docker-compose.template.yml b/self-hosting/docker-compose.template.yml
index 736a80eaf..f9e573256 100644
--- a/self-hosting/docker-compose.template.yml
+++ b/self-hosting/docker-compose.template.yml
@@ -21,7 +21,7 @@ services:
volumes:
- op-db-data:/var/lib/postgresql/data
healthcheck:
- test: ['CMD-SHELL', 'pg_isready -U postgres']
+ test: [ 'CMD-SHELL', 'pg_isready -U postgres' ]
interval: 10s
timeout: 5s
retries: 5
@@ -37,7 +37,7 @@ services:
restart: always
volumes:
- op-kv-data:/data
- command: ['redis-server', '--maxmemory-policy', 'noeviction']
+ command: [ 'redis-server', '--maxmemory-policy', 'noeviction' ]
# Uncomment to expose ports
# ports:
# - 6379:6379
@@ -56,7 +56,7 @@ services:
- ./clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/op-user-config.xml:ro
- ./clickhouse/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
healthcheck:
- test: ['CMD-SHELL', 'clickhouse-client --query "SELECT 1"']
+ test: [ 'CMD-SHELL', 'clickhouse-client --query "SELECT 1"' ]
interval: 10s
timeout: 5s
retries: 5
@@ -88,7 +88,7 @@ services:
echo 'ClickHouse is ready'
echo 'Running migrations...'
- pnpm -r run migrate:deploy
+ CI=true pnpm -r run migrate:deploy
pnpm start
"