Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

10.7 KB
Raw Blame
#!/bin/bash
set -e              # Exit on any command failure
set -o pipefail     # Exit if any command in a pipeline fails

SCRIPT_DIR=$(dirname "$(readlink -f "$0")")

# --scope flag: run specific migration(s) only. Without --scope, runs all.
# Usage: bin/migrate [--scope=postgres] [--scope=clickhouse] [--scope=async] ...
SCOPES=()
for arg in "$@"; do
    [[ "$arg" == --scope=* ]] && SCOPES+=("${arg#*=}")
done
run_scope() {
    [[ ${#SCOPES[@]} -eq 0 ]] && return 0
    for s in "${SCOPES[@]}"; do [[ "$s" == "$1" ]] && return 0; done
    return 1
}

# timestamp function with nanosecond precision if supported, otherwise fallback to seconds with .000000000 suffix for uniform formatting
if [ "$(date +%N)" != "N" ]; then
    timestamp() { date +%s.%N; }
else
    timestamp() { echo "$(date +%s).000000000"; }
fi

# Report the runner result without loading Django.
FAILED_PHASE=unknown
CLICKHOUSE_STATUS=
CLICKHOUSE_PID=
publish_outcome() {
    if [ -n "${MIGRATION_JOB_NAME:-}" ] && [ -n "${MIGRATION_NAMESPACE:-}" ]; then
        python3 "$SCRIPT_DIR/report-migration-outcome" "$1" || true
    fi
}
report_outcome() {
    local status=$?
    if [ "$status" -eq 0 ]; then
        FAILED_PHASE=none
    fi
    echo "[migrate] outcome: phase=$FAILED_PHASE status=$status"
    publish_outcome "$FAILED_PHASE"
    if [ -n "$CLICKHOUSE_STATUS" ]; then
        rm -f "$CLICKHOUSE_STATUS"
    fi
}
trap report_outcome EXIT

# NOTE when running in docker, rust might not exist so we need to check for it
if [ -d "$SCRIPT_DIR/../rust/bin" ] && [ "${DEPLOYMENT:-}" != "hobby" ]; then
    if run_scope "cyclotron"; then
        bash $SCRIPT_DIR/../rust/bin/migrate-cyclotron-node
        if [ $? -ne 0 ]; then
            echo "Error in rust/bin/migrate-cyclotron-node, exiting."
            exit 1
        fi
    fi
    
    if run_scope "behavioral-cohorts"; then
        echo "Running behavioral cohorts migrations via external non-django migrator..."
        bash $SCRIPT_DIR/../rust/bin/migrate-behavioral-cohorts
        if [ $? -ne 0 ]; then
            echo "Error in rust/bin/migrate-behavioral-cohorts, exiting."
            exit 1
        fi
    fi

    if run_scope "flags-read-store"; then
        echo "Running flags read store migrations via external non-django migrator..."
        bash $SCRIPT_DIR/../rust/bin/migrate-flags-read-store
        if [ $? -ne 0 ]; then
            echo "Error in rust/bin/migrate-flags-read-store, exiting."
            exit 1
        fi
    fi
fi

if run_scope "clickhouse"; then RUN_CLICKHOUSE=1; else RUN_CLICKHOUSE=0; fi
if run_scope "postgres"; then RUN_POSTGRES=1; else RUN_POSTGRES=0; fi

# Run ClickHouse in background only when both CH and Postgres run (original parallel behavior)
if [ "$RUN_CLICKHOUSE" = "1" ] && [ "$RUN_POSTGRES" = "1" ]; then
    CLICKHOUSE_STATUS=$(mktemp)
    echo "0" > "$CLICKHOUSE_STATUS"
    
    (
        if ! python manage.py migrate_clickhouse; then
            echo "Error in migrate_clickhouse, setting error status."
            echo "1" > "$CLICKHOUSE_STATUS"
            exit 1
        fi

        if ! python manage.py sync_replicated_schema; then
            echo "Error in sync_replicated_schema, setting error status."
            echo "1" > "$CLICKHOUSE_STATUS"
            exit 1
        fi
    ) &
    CLICKHOUSE_PID=$!
    elif [ "$RUN_CLICKHOUSE" = "1" ]; then
    # ClickHouse only - run directly
    python manage.py migrate_clickhouse || { FAILED_PHASE=clickhouse; echo "Error in migrate_clickhouse, exiting."; exit 1; }
    python manage.py sync_replicated_schema || { FAILED_PHASE=clickhouse; echo "Error in sync_replicated_schema, exiting."; exit 1; }
fi

if [ "$RUN_POSTGRES" = "1" ]; then
    # Run postgres migrations with retry logic
    # If POSTHOG_POSTGRES_DIRECT_HOST is set, use direct connection (bypasses PgBouncer)
    # which allows lock_timeout to be set in Django database OPTIONS
    MIGRATE_MAX_RETRIES=${MIGRATE_MAX_RETRIES:-10}
    MIGRATE_RETRY_DELAY=${MIGRATE_RETRY_DELAY:-3}
    MIGRATE_BACKOFF=${MIGRATE_BACKOFF:-2}
    # Must match UNSUPPORTED_DATABASE_EXIT_CODE in posthog/management/commands/migrate.py
    MIGRATE_UNSUPPORTED_DATABASE=78
    
    MIGRATE_ATTEMPT=1
    MIGRATE_SUCCESS=false
    MIGRATE_START_TIME=$(timestamp)
    
    # Build migrate command
    # --production skips local-dev features (orphan check, migration caching)
    # Use direct connection if available (bypasses PgBouncer for lock_timeout support)
    if [ -n "$POSTHOG_POSTGRES_DIRECT_HOST" ]; then
        MIGRATE_CMD="python manage.py migrate --noinput --database=default_direct"
        echo "[migrate] Using direct database connection (lock_timeout enabled)"
    else
        MIGRATE_CMD="python manage.py migrate --noinput"
    fi

    if [ -n "${DEPLOYMENT:-}" ]; then
        MIGRATE_CMD="$MIGRATE_CMD --production"
    fi
    
    while [ $MIGRATE_ATTEMPT -le $MIGRATE_MAX_RETRIES ]; do
        echo "[migrate] Django migration attempt $MIGRATE_ATTEMPT/$MIGRATE_MAX_RETRIES"

        # Product databases (products/db_routing.yaml) lose lock races the same way, so they
        # share the retry. A retry re-runs the main migrate too, which is a no-op by then.
        if $MIGRATE_CMD; then
            if python manage.py migrate_product_databases; then
                MIGRATE_SUCCESS=true
                break
            else
                MIGRATE_STATUS=$?
                FAILED_PHASE=productdb
            fi
        else
            MIGRATE_STATUS=$?
            FAILED_PHASE=postgres
        fi
        
        echo "[migrate] Django migration failed"
        
        if [ "$MIGRATE_STATUS" = "$MIGRATE_UNSUPPORTED_DATABASE" ]; then
            echo "[migrate] the database version is not supported, so retrying cannot help"
            break
        fi
        
        if [ $MIGRATE_ATTEMPT -ge $MIGRATE_MAX_RETRIES ]; then
            echo "[migrate] max retries reached, aborting"
            break
        fi
        
        echo "[migrate] waiting ${MIGRATE_RETRY_DELAY}s before retry..."
        sleep $MIGRATE_RETRY_DELAY
        
        MIGRATE_RETRY_DELAY=$((MIGRATE_RETRY_DELAY * MIGRATE_BACKOFF))
        MIGRATE_ATTEMPT=$((MIGRATE_ATTEMPT + 1))
    done
    
    MIGRATE_END_TIME=$(timestamp)
    MIGRATE_DURATION=$(python3 -c "print($MIGRATE_END_TIME - $MIGRATE_START_TIME)")
    
    # Report migration metrics to pushgateway (if configured)
    if [ "$MIGRATE_SUCCESS" = "true" ]; then
        FAILED_PHASE=unknown
        python manage.py report_migration_metric --attempts=$MIGRATE_ATTEMPT --duration=$MIGRATE_DURATION --success || true
    else
        publish_outcome "$FAILED_PHASE"
        python manage.py report_migration_metric --attempts=$MIGRATE_ATTEMPT --duration=$MIGRATE_DURATION || true
        echo "Error in $FAILED_PHASE migrations, killing background process and exiting."
        [ -n "$CLICKHOUSE_PID" ] && kill "$CLICKHOUSE_PID" 2>/dev/null || true
        exit 1
    fi

fi

# Persons migrations via Django management command (connects to PERSONS_DB_WRITER_URL).
# Hobby: skips partitioning migrations (persons live in the main DB).
# Local dev (DEBUG=1): creates the persons database if needed and applies all migrations.
# Production: skipped — persons migrations are handled by the sqlx-migrate K8s job.
if run_scope "persons"; then
    if [ "${DEPLOYMENT:-}" = "hobby" ]; then
        echo "Running persons migrations via Django management command (hobby)..."
        python manage.py apply_persons_migrations --hobby
        if [ $? -ne 0 ]; then
            echo "Error in apply_persons_migrations, exiting."
            exit 1
        fi
    elif [ "${DEBUG:-0}" = "1" ]; then
        echo "Running persons migrations via Django management command..."
        python manage.py apply_persons_migrations --ensure-database
        if [ $? -ne 0 ]; then
            echo "Error in apply_persons_migrations, exiting."
            exit 1
        fi
    fi
fi

if run_scope "async"; then
    # NOTE: we do not apply any non-noop migrations here. Rather these are run
    # manually within the UI. See https://posthog.com/docs/runbook/async-migrations
    # for details.
    python manage.py run_async_migrations --complete-noop-migrations
    if [ $? -ne 0 ]; then
        echo "Error in run_async_migrations, killing background process and exiting."
        [ -n "$CLICKHOUSE_PID" ] && kill $CLICKHOUSE_PID 2>/dev/null || true
        [ -n "$CLICKHOUSE_STATUS" ] && rm -f $CLICKHOUSE_STATUS
        exit 1
    fi
    
    # NOTE: this check should not fail if a migration isn't complete but within the
    # given async migration posthog version range, thus this should not block e.g.
    # k8s pod deployments.
    python manage.py run_async_migrations --check
    if [ $? -ne 0 ]; then
        echo "Error in run_async_migrations --check, killing background process and exiting."
        [ -n "$CLICKHOUSE_PID" ] && kill $CLICKHOUSE_PID 2>/dev/null || true
        [ -n "$CLICKHOUSE_STATUS" ] && rm -f $CLICKHOUSE_STATUS
        exit 1
    fi
fi

# Wait for background ClickHouse migrations if running in parallel mode
if [ -n "$CLICKHOUSE_PID" ]; then
    CLICKHOUSE_WAIT_STATUS=0
    wait "$CLICKHOUSE_PID" || CLICKHOUSE_WAIT_STATUS=$?
    CH_STATUS=$(cat "$CLICKHOUSE_STATUS")
    if [ "$CH_STATUS" != "0" ] || [ "$CLICKHOUSE_WAIT_STATUS" -ne 0 ]; then
        FAILED_PHASE=clickhouse
    fi

    if [ "$FAILED_PHASE" = "clickhouse" ]; then
        echo "Error in ClickHouse migrations, exiting."
        exit 1
    fi
fi

# OAuth applications task sandboxes mint their agent tokens under. Idempotent, and a no-op
# in the production regions, so it is safe to run on every deploy.
if run_scope "tasks-oauth"; then
    if [ "${DEPLOYMENT:-}" != "hobby" ]; then
        echo "Setting up task sandbox OAuth applications..."
        python manage.py setup_tasks_oauth || { echo "Error in setup_tasks_oauth, exiting."; exit 1; }
    fi
fi

# Temporal worker schedules. Skipped on hobby deploys (no Temporal cluster) and
# on local dev (DEBUG=1, matching the persons-migration gate above).
# Non-fatal: scheduling is reconciled idempotently on every deploy, so a
# transient Temporal outage should not block the migrate job.
if run_scope "temporal-schedules"; then
    if [ "${DEPLOYMENT:-}" != "hobby" ] && [ "${DEBUG:-0}" != "1" ]; then
        echo "Scheduling Temporal worker workflows..."
        # timeout guards against an unreachable Temporal frontend: the client has
        # no connect/RPC deadline of its own, so a network black-hole would hang
        # the whole migrate job otherwise. 90s leaves room for Django startup on a
        # cold container; reconciliation is idempotent so a killed run resumes next deploy.
        if ! timeout 90 python manage.py schedule_temporal_workflows; then
            echo "Warning: schedule_temporal_workflows failed, continuing."
        fi
    fi
fi

echo "All migrations completed successfully."