Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.3 KB
Raw Blame
#!/usr/bin/env bash

set -e
set -o pipefail

# Copies objects out of the legacy MinIO volume into the SeaweedFS services that replaced
# it. MinIO is no longer a service in docker-compose.hobby.yml, so the source is a
# temporary container mounted on the orphaned volume. Each destination is the store the
# stack actually reads that data from.

MINIO_IMAGE="quay.io/minio/minio:RELEASE.2025-04-22T22-12-26Z"
SOURCE_CONTAINER="posthog-legacy-minio"
S3_KEY="object_storage_root_user"
S3_SECRET="object_storage_root_password"

OBJECTSTORAGE_ENDPOINT="http://objectstorage:19000"
SEAWEEDFS_ENDPOINT="http://seaweedfs:8333"

# `service|destination`, in run order. general-storage covers the whole posthog bucket
# apart from session recordings and the query cache, so a prefix no one listed here still
# comes across. source-maps repeats a prefix general-storage already copied, because
# cymbal reads symbol sets from seaweedfs while Django reads them from objectstorage.
ALL_SERVICES=(
    "general-storage|$OBJECTSTORAGE_ENDPOINT"
    "ai-blobs|$OBJECTSTORAGE_ENDPOINT"
    "source-maps|$SEAWEEDFS_ENDPOINT"
    "session-recordings|$SEAWEEDFS_ENDPOINT"
    "session-recordings-lts|$SEAWEEDFS_ENDPOINT"
)

# Destinations for a single-service run. A named prefix goes to the store the app reads
# it from, so `source-maps` on its own lands where Django looks.
SINGLE_SERVICES=(
    "general-storage|$OBJECTSTORAGE_ENDPOINT"
    "ai-blobs|$OBJECTSTORAGE_ENDPOINT"
    "exports|$OBJECTSTORAGE_ENDPOINT"
    "media-uploads|$OBJECTSTORAGE_ENDPOINT"
    "source-maps|$OBJECTSTORAGE_ENDPOINT"
    "query-cache|$OBJECTSTORAGE_ENDPOINT"
    "session-recordings|$SEAWEEDFS_ENDPOINT"
    "session-recordings-lts|$SEAWEEDFS_ENDPOINT"
)

if command -v docker-compose >/dev/null 2>&1; then
    COMPOSE=(docker-compose)
else
    COMPOSE=(docker compose)
fi

show_help() {
    cat <<'EOF'
Copy hobby object storage out of MinIO into SeaweedFS

PostHog replaced MinIO with SeaweedFS. SeaweedFS starts on a new volume and cannot read
the MinIO format, so your old objects sit in the `objectstorage` volume until this script
copies them across. Run it from the directory that holds your docker-compose.yml, with
the stack up.

A full run copies the whole `posthog` bucket into objectstorage, the `ai-blobs` bucket
into objectstorage, and session recordings into seaweedfs. Symbol sets are copied to both
stores, because cymbal reads them from seaweedfs and Django reads them from objectstorage.
The query cache is skipped, because PostHog rebuilds it.

Usage:
  ./posthog/bin/migrate-storage-hobby                 Copy everything
  ./posthog/bin/migrate-storage-hobby exports         Copy one service
  ./posthog/bin/migrate-storage-hobby --dry-run       Show what would be copied
  ./posthog/bin/migrate-storage-hobby --volume <id>   Read a specific MinIO volume

Services:
  general-storage, ai-blobs, exports, media-uploads, source-maps, query-cache,
  session-recordings, session-recordings-lts

Options:
  --dry-run        List the objects instead of copying them
  --force          Overwrite objects that already exist in the destination
  --workers <n>    Number of concurrent workers (default: 5)
  --volume <id>    MinIO volume to read, if the script picks the wrong one
  --list           List the MinIO volumes on this host
EOF
}

# A MinIO data volume always has a .minio.sys directory at its root.
is_minio_volume() {
    docker run --rm -v "$1:/data:ro" alpine test -d /data/.minio.sys 2>/dev/null
}

volume_size() {
    docker run --rm -v "$1:/data:ro" alpine du -sh /data 2>/dev/null | awk '{print $1}' || echo "unknown"
}

minio_volumes() {
    local vol
    while read -r vol; do
        if is_minio_volume "$vol"; then
            echo "$vol"
        fi
    done < <(docker volume ls -q)
}

list_minio_volumes() {
    local vol
    echo "Scanning for MinIO volumes..."
    echo ""
    while read -r vol; do
        [ -z "$vol" ] && continue
        echo "  $vol ($(volume_size "$vol"))"
    done < <(minio_volumes)
}

# The hobby stack named its MinIO volume `objectstorage`, which compose prefixes with the
# project name. Anything else on the host — another install, an unrelated MinIO — is not
# this stack's data, and copying it would import another tenant's objects.
detect_minio_volume() {
    local vol candidates=()
    while read -r vol; do
        [ -z "$vol" ] && continue
        if [ "$vol" = "objectstorage" ] || [[ "$vol" == *_objectstorage ]]; then
            candidates+=("$vol")
        fi
    done < <(minio_volumes)

    if [ ${#candidates[@]} -eq 1 ]; then
        echo "${candidates[0]}"
        return 0
    fi

    if [ ${#candidates[@]} -gt 1 ]; then
        echo "❌ This host has more than one legacy objectstorage volume:" >&2
        printf '   %s\n' "${candidates[@]}" >&2
        echo "Pick the one for this install with --volume <id>." >&2
        return 1
    fi

    return 0
}

# The copy runs inside the plugins container, so both endpoints must be reachable on the
# stack's network. Read the network off a running service rather than guessing the compose
# project name.
compose_network() {
    local container
    container=$("${COMPOSE[@]}" ps -q objectstorage 2>/dev/null | head -1)
    if [ -z "$container" ]; then
        return 1
    fi
    docker inspect -f '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$container" | head -1
}

stop_source() {
    docker rm -f "$SOURCE_CONTAINER" >/dev/null 2>&1 || true
}

start_source() {
    local volume="$1" network="$2"

    stop_source
    trap stop_source EXIT

    docker run -d \
        --name "$SOURCE_CONTAINER" \
        --network "$network" \
        -v "$volume:/data" \
        -e MINIO_ROOT_USER="$S3_KEY" \
        -e MINIO_ROOT_PASSWORD="$S3_SECRET" \
        "$MINIO_IMAGE" \
        server --address ":19000" /data >/dev/null

    for _ in $(seq 1 30); do
        if docker logs "$SOURCE_CONTAINER" 2>&1 | grep -q "API:"; then
            break
        fi
        sleep 1
    done
    sleep 2
}

# NODE_ENV=development clears the migration script's own guard against running outside a
# development environment. The plugins image ships NODE_ENV=production, and both endpoints
# here are inside the stack's network. The migration script exits non-zero if any object
# failed, and set -e stops the run there rather than reporting a clean copy.
run_service() {
    local service="$1" endpoint="$2"

    echo ""
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    echo "🔄 $service → $endpoint"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    "${COMPOSE[@]}" run --rm --no-deps \
        -e NODE_ENV=development \
        -e OBJECT_STORAGE_ENDPOINT="http://$SOURCE_CONTAINER:19000" \
        -e OBJECT_STORAGE_ACCESS_KEY_ID="$S3_KEY" \
        -e OBJECT_STORAGE_SECRET_ACCESS_KEY="$S3_SECRET" \
        -e SESSION_RECORDING_V2_S3_ENDPOINT="$endpoint" \
        -e SESSION_RECORDING_V2_S3_ACCESS_KEY_ID="$S3_KEY" \
        -e SESSION_RECORDING_V2_S3_SECRET_ACCESS_KEY="$S3_SECRET" \
        plugins \
        node nodejs/bin/migrate-minio-to-seaweedfs.js --service "$service" --mode migrate "${EXTRA_ARGS[@]}"
}

destination_for() {
    local service="$1" entry
    for entry in "${SINGLE_SERVICES[@]}"; do
        if [ "${entry%%|*}" = "$service" ]; then
            echo "${entry##*|}"
            return 0
        fi
    done
    return 1
}

SERVICE=""
VOLUME=""
DRY_RUN=false
EXTRA_ARGS=()

while [[ $# -gt 0 ]]; do
    case "$1" in
        --help|-h)
            show_help; exit 0 ;;
        --list)
            list_minio_volumes; exit 0 ;;
        --volume)
            VOLUME="$2"; shift 2 ;;
        --dry-run|-n)
            DRY_RUN=true; EXTRA_ARGS+=("--dry-run"); shift ;;
        --force)
            EXTRA_ARGS+=("$1"); shift ;;
        --workers)
            EXTRA_ARGS+=("$1" "$2"); shift 2 ;;
        *)
            if destination_for "$1" >/dev/null; then
                SERVICE="$1"; shift
            else
                echo "Unknown argument: $1"
                echo "Run with --help to see the services and options."
                exit 1
            fi ;;
    esac
done

if [ -z "$VOLUME" ]; then
    VOLUME=$(detect_minio_volume)
fi

if [ -z "$VOLUME" ]; then
    echo "No legacy objectstorage volume found on this host, so there is nothing to copy."
    echo "If you moved the data, pass the volume with --volume <id>."
    exit 0
fi

if ! is_minio_volume "$VOLUME"; then
    echo "❌ Volume $VOLUME holds no MinIO data (no .minio.sys directory)."
    exit 1
fi

NETWORK=$(compose_network || true)
if [ -z "$NETWORK" ]; then
    echo "❌ The objectstorage service is not running."
    echo "Start the stack with 'docker-compose up -d', then run this again."
    exit 1
fi

echo ""
echo "MinIO volume: $VOLUME ($(volume_size "$VOLUME"))"
echo "Stack network: $NETWORK"
if [ "$DRY_RUN" = true ]; then
    echo "Dry run: nothing is copied."
fi
echo ""
read -r -p "Continue? [y/N] " confirm
if [[ ! "$confirm" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
    echo "Cancelled."
    exit 0
fi

echo ""
echo "Starting a temporary MinIO container to read the old volume..."
start_source "$VOLUME" "$NETWORK"

if [ -n "$SERVICE" ]; then
    run_service "$SERVICE" "$(destination_for "$SERVICE")"
else
    for entry in "${ALL_SERVICES[@]}"; do
        run_service "${entry%%|*}" "${entry##*|}"
    done
fi

stop_source
trap - EXIT

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

# Only a complete run that copied something can say the volume is safe to drop. A dry run
# copies nothing, and a single service leaves the rest of the volume behind.
if [ "$DRY_RUN" = true ]; then
    echo "🔍 Dry run finished. Nothing was copied."
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    echo ""
    echo "Run the same command without --dry-run to copy the objects."
    echo ""
elif [ -n "$SERVICE" ]; then
    echo "✅ Copied $SERVICE"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    echo ""
    echo "The rest of the volume is still only in MinIO. Keep volume $VOLUME until you"
    echo "have run this script without a service argument."
    echo ""
else
    echo "✅ Copy finished"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    echo ""
    echo "The query cache was skipped, because PostHog rebuilds it."
    echo ""
    echo "Check your exports, media, symbol sets and recordings in PostHog. Once you are"
    echo "sure everything you need is there, free the disk space with:"
    echo ""
    echo "  docker volume rm $VOLUME"
    echo ""
    echo "Until you run that, the old objects are still on disk, so there is no rush."
    echo ""
fi