:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
#!/bin/bash
set -e # Exit on any command failure
set -o pipefail # Exit if any command in a pipeline fails
SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
# --scope flag: run specific migration(s) only. Without --scope, runs all.
# Usage: bin/migrate [--scope=postgres] [--scope=clickhouse] [--scope=async] ...
SCOPES=()
for arg in "$@"; do
[[ "$arg" == --scope=* ]] && SCOPES+=("${arg#*=}")
done
run_scope() {
[[ ${#SCOPES[@]} -eq 0 ]] && return 0
for s in "${SCOPES[@]}"; do [[ "$s" == "$1" ]] && return 0; done
return 1
}
# timestamp function with nanosecond precision if supported, otherwise fallback to seconds with .000000000 suffix for uniform formatting
if [ "$(date +%N)" != "N" ]; then
timestamp() { date +%s.%N; }
else
timestamp() { echo "$(date +%s).000000000"; }
fi
# Report the runner result without loading Django.
FAILED_PHASE=unknown
CLICKHOUSE_STATUS=
CLICKHOUSE_PID=
publish_outcome() {
if [ -n "${MIGRATION_JOB_NAME:-}" ] && [ -n "${MIGRATION_NAMESPACE:-}" ]; then
python3 "$SCRIPT_DIR/report-migration-outcome" "$1" || true
fi
}
report_outcome() {
local status=$?
if [ "$status" -eq 0 ]; then
FAILED_PHASE=none
fi
echo "[migrate] outcome: phase=$FAILED_PHASE status=$status"
publish_outcome "$FAILED_PHASE"
if [ -n "$CLICKHOUSE_STATUS" ]; then
rm -f "$CLICKHOUSE_STATUS"
fi
}
trap report_outcome EXIT
# NOTE when running in docker, rust might not exist so we need to check for it
if [ -d "$SCRIPT_DIR/../rust/bin" ] && [ "${DEPLOYMENT:-}" != "hobby" ]; then
if run_scope "cyclotron"; then
bash $SCRIPT_DIR/../rust/bin/migrate-cyclotron-node
if [ $? -ne 0 ]; then
echo "Error in rust/bin/migrate-cyclotron-node, exiting."
exit 1
fi
fi
if run_scope "behavioral-cohorts"; then
echo "Running behavioral cohorts migrations via external non-django migrator..."
bash $SCRIPT_DIR/../rust/bin/migrate-behavioral-cohorts
if [ $? -ne 0 ]; then
echo "Error in rust/bin/migrate-behavioral-cohorts, exiting."
exit 1
fi
fi
if run_scope "flags-read-store"; then
echo "Running flags read store migrations via external non-django migrator..."
bash $SCRIPT_DIR/../rust/bin/migrate-flags-read-store
if [ $? -ne 0 ]; then
echo "Error in rust/bin/migrate-flags-read-store, exiting."
exit 1
fi
fi
fi
if run_scope "clickhouse"; then RUN_CLICKHOUSE=1; else RUN_CLICKHOUSE=0; fi
if run_scope "postgres"; then RUN_POSTGRES=1; else RUN_POSTGRES=0; fi
# Run ClickHouse in background only when both CH and Postgres run (original parallel behavior)
if [ "$RUN_CLICKHOUSE" = "1" ] && [ "$RUN_POSTGRES" = "1" ]; then
CLICKHOUSE_STATUS=$(mktemp)
echo "0" > "$CLICKHOUSE_STATUS"
(
if ! python manage.py migrate_clickhouse; then
echo "Error in migrate_clickhouse, setting error status."
echo "1" > "$CLICKHOUSE_STATUS"
exit 1
fi
if ! python manage.py sync_replicated_schema; then
echo "Error in sync_replicated_schema, setting error status."
echo "1" > "$CLICKHOUSE_STATUS"
exit 1
fi
) &
CLICKHOUSE_PID=$!
elif [ "$RUN_CLICKHOUSE" = "1" ]; then
# ClickHouse only - run directly
python manage.py migrate_clickhouse || { FAILED_PHASE=clickhouse; echo "Error in migrate_clickhouse, exiting."; exit 1; }
python manage.py sync_replicated_schema || { FAILED_PHASE=clickhouse; echo "Error in sync_replicated_schema, exiting."; exit 1; }
fi
if [ "$RUN_POSTGRES" = "1" ]; then
# Run postgres migrations with retry logic
# If POSTHOG_POSTGRES_DIRECT_HOST is set, use direct connection (bypasses PgBouncer)
# which allows lock_timeout to be set in Django database OPTIONS
MIGRATE_MAX_RETRIES=${MIGRATE_MAX_RETRIES:-10}
MIGRATE_RETRY_DELAY=${MIGRATE_RETRY_DELAY:-3}
MIGRATE_BACKOFF=${MIGRATE_BACKOFF:-2}
# Must match UNSUPPORTED_DATABASE_EXIT_CODE in posthog/management/commands/migrate.py
MIGRATE_UNSUPPORTED_DATABASE=78
MIGRATE_ATTEMPT=1
MIGRATE_SUCCESS=false
MIGRATE_START_TIME=$(timestamp)
# Build migrate command
# --production skips local-dev features (orphan check, migration caching)
# Use direct connection if available (bypasses PgBouncer for lock_timeout support)
if [ -n "$POSTHOG_POSTGRES_DIRECT_HOST" ]; then
MIGRATE_CMD="python manage.py migrate --noinput --database=default_direct"
echo "[migrate] Using direct database connection (lock_timeout enabled)"
else
MIGRATE_CMD="python manage.py migrate --noinput"
fi
if [ -n "${DEPLOYMENT:-}" ]; then
MIGRATE_CMD="$MIGRATE_CMD --production"
fi
while [ $MIGRATE_ATTEMPT -le $MIGRATE_MAX_RETRIES ]; do
echo "[migrate] Django migration attempt $MIGRATE_ATTEMPT/$MIGRATE_MAX_RETRIES"
# Product databases (products/db_routing.yaml) lose lock races the same way, so they
# share the retry. A retry re-runs the main migrate too, which is a no-op by then.
if $MIGRATE_CMD; then
if python manage.py migrate_product_databases; then
MIGRATE_SUCCESS=true
break
else
MIGRATE_STATUS=$?
FAILED_PHASE=productdb
fi
else
MIGRATE_STATUS=$?
FAILED_PHASE=postgres
fi
echo "[migrate] Django migration failed"
if [ "$MIGRATE_STATUS" = "$MIGRATE_UNSUPPORTED_DATABASE" ]; then
echo "[migrate] the database version is not supported, so retrying cannot help"
break
fi
if [ $MIGRATE_ATTEMPT -ge $MIGRATE_MAX_RETRIES ]; then
echo "[migrate] max retries reached, aborting"
break
fi
echo "[migrate] waiting ${MIGRATE_RETRY_DELAY}s before retry..."
sleep $MIGRATE_RETRY_DELAY
MIGRATE_RETRY_DELAY=$((MIGRATE_RETRY_DELAY * MIGRATE_BACKOFF))
MIGRATE_ATTEMPT=$((MIGRATE_ATTEMPT + 1))
done
MIGRATE_END_TIME=$(timestamp)
MIGRATE_DURATION=$(python3 -c "print($MIGRATE_END_TIME - $MIGRATE_START_TIME)")
# Report migration metrics to pushgateway (if configured)
if [ "$MIGRATE_SUCCESS" = "true" ]; then
FAILED_PHASE=unknown
python manage.py report_migration_metric --attempts=$MIGRATE_ATTEMPT --duration=$MIGRATE_DURATION --success || true
else
publish_outcome "$FAILED_PHASE"
python manage.py report_migration_metric --attempts=$MIGRATE_ATTEMPT --duration=$MIGRATE_DURATION || true
echo "Error in $FAILED_PHASE migrations, killing background process and exiting."
[ -n "$CLICKHOUSE_PID" ] && kill "$CLICKHOUSE_PID" 2>/dev/null || true
exit 1
fi
fi
# Persons migrations via Django management command (connects to PERSONS_DB_WRITER_URL).
# Hobby: skips partitioning migrations (persons live in the main DB).
# Local dev (DEBUG=1): creates the persons database if needed and applies all migrations.
# Production: skipped — persons migrations are handled by the sqlx-migrate K8s job.
if run_scope "persons"; then
if [ "${DEPLOYMENT:-}" = "hobby" ]; then
echo "Running persons migrations via Django management command (hobby)..."
python manage.py apply_persons_migrations --hobby
if [ $? -ne 0 ]; then
echo "Error in apply_persons_migrations, exiting."
exit 1
fi
elif [ "${DEBUG:-0}" = "1" ]; then
echo "Running persons migrations via Django management command..."
python manage.py apply_persons_migrations --ensure-database
if [ $? -ne 0 ]; then
echo "Error in apply_persons_migrations, exiting."
exit 1
fi
fi
fi
if run_scope "async"; then
# NOTE: we do not apply any non-noop migrations here. Rather these are run
# manually within the UI. See https://posthog.com/docs/runbook/async-migrations
# for details.
python manage.py run_async_migrations --complete-noop-migrations
if [ $? -ne 0 ]; then
echo "Error in run_async_migrations, killing background process and exiting."
[ -n "$CLICKHOUSE_PID" ] && kill $CLICKHOUSE_PID 2>/dev/null || true
[ -n "$CLICKHOUSE_STATUS" ] && rm -f $CLICKHOUSE_STATUS
exit 1
fi
# NOTE: this check should not fail if a migration isn't complete but within the
# given async migration posthog version range, thus this should not block e.g.
# k8s pod deployments.
python manage.py run_async_migrations --check
if [ $? -ne 0 ]; then
echo "Error in run_async_migrations --check, killing background process and exiting."
[ -n "$CLICKHOUSE_PID" ] && kill $CLICKHOUSE_PID 2>/dev/null || true
[ -n "$CLICKHOUSE_STATUS" ] && rm -f $CLICKHOUSE_STATUS
exit 1
fi
fi
# Wait for background ClickHouse migrations if running in parallel mode
if [ -n "$CLICKHOUSE_PID" ]; then
CLICKHOUSE_WAIT_STATUS=0
wait "$CLICKHOUSE_PID" || CLICKHOUSE_WAIT_STATUS=$?
CH_STATUS=$(cat "$CLICKHOUSE_STATUS")
if [ "$CH_STATUS" != "0" ] || [ "$CLICKHOUSE_WAIT_STATUS" -ne 0 ]; then
FAILED_PHASE=clickhouse
fi
if [ "$FAILED_PHASE" = "clickhouse" ]; then
echo "Error in ClickHouse migrations, exiting."
exit 1
fi
fi
# OAuth applications task sandboxes mint their agent tokens under. Idempotent, and a no-op
# in the production regions, so it is safe to run on every deploy.
if run_scope "tasks-oauth"; then
if [ "${DEPLOYMENT:-}" != "hobby" ]; then
echo "Setting up task sandbox OAuth applications..."
python manage.py setup_tasks_oauth || { echo "Error in setup_tasks_oauth, exiting."; exit 1; }
fi
fi
# Temporal worker schedules. Skipped on hobby deploys (no Temporal cluster) and
# on local dev (DEBUG=1, matching the persons-migration gate above).
# Non-fatal: scheduling is reconciled idempotently on every deploy, so a
# transient Temporal outage should not block the migrate job.
if run_scope "temporal-schedules"; then
if [ "${DEPLOYMENT:-}" != "hobby" ] && [ "${DEBUG:-0}" != "1" ]; then
echo "Scheduling Temporal worker workflows..."
# timeout guards against an unreachable Temporal frontend: the client has
# no connect/RPC deadline of its own, so a network black-hole would hang
# the whole migrate job otherwise. 90s leaves room for Django startup on a
# cold container; reconciliation is idempotent so a killed run resumes next deploy.
if ! timeout 90 python manage.py schedule_temporal_workflows; then
echo "Warning: schedule_temporal_workflows failed, continuing."
fi
fi
fi
echo "All migrations completed successfully."