Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

59.9 KB
Raw Blame
"""The preview *stack* — provider-agnostic.

``PostHogPreviewStack`` knows how to bring PostHog up inside any
``PreviewBackend``: build from the checkout, migrate, seed demo data, wait for
health. It talks to the box only through ``backend.exec`` / ``write_file`` /
``run_long`` / ``wait_http_ok`` — it never knows or cares whether the box is a
hogbox or a droplet. Swapping the layer changes the backend, not this file.

Recipe (mount-over-image — the default, ~minutes per PR):
  - Run the ready-made published image (``ghcr.io/posthog/posthog:master``) and
    bind-mount the PR's backend source (``posthog``/``ee``/``products``) over the
    image's ``/code``. The image is the prod Dockerfile: it runs from
    ``WORKDIR /code`` via ``./bin/docker-server`` with the frontend baked at
    ``/code/frontend/dist``. Mounting the source swaps the BACKEND code live — no
    per-PR build. (Frontend stays at the image's version; frontend hot-mount is a
    later iteration.) DEBUG=0 is required: the prod image lacks DEBUG-only apps.
  - DB coherence: the restored golden's DB was migrated + seeded against the same
    image tag, so a restore only needs the PR's *delta* migrations on top
    (``migrate`` + ``migrate_clickhouse``). Reseeding is skipped — the golden is
    already seeded (CI passes ``--no-seed``).
  - ``build`` ESCAPE HATCH: pass ``image=None`` to build ``web`` from the
    checkout instead (``build: .`` — cold ~20 min; used when baking a golden, or
    to pin nothing). ``reset_db`` wipes pg+clickhouse so they migrate fresh
    against the built code. ``compose run`` has no ``--no-build`` flag (that's
    ``up``-only); what prevents a build is the image being present, so always
    pull/pin the image before any ``run``.
"""

from __future__ import annotations

import sys
import secrets

from . import timing
from .backend import PreviewBackend

# The tool's compose override MUST stay in sync with the golden bake script (hogland
# scripts/posthog-preview-setup.sh): both write docker-compose.preview.yml, and
# regenerating it here CLOBBERS the golden's baked copy — so anything the bake
# adds (env, services) must be mirrored here or a per-PR override drops it. That
# drift is exactly what broke previews fleet-wide from 2026-07-06 to 2026-07-10
# (missing PERSONHOG_ADDR -> "personhog client not configured" 500s); see
# write_override for the full post-mortem. JS_URL="" makes the SPA load assets
# relative to the request origin; the wildcard CSRF origin trusts every box's
# edge host for the login POST. One recipe then serves any box at its own edge
# URL.
# Built as a single comma-separated string (Django's get_list() splits on commas)
# — a join, not a tuple, so there's no ambiguity about what reaches the env.
_CSRF_TRUSTED_ORIGINS = ",".join(f"https://*.boxes.hogland.{env}.posthog.dev" for env in ("dev", "prod-us", "prod-eu"))
# The seeded demo login. These are the defaults of posthog's
# `manage.py generate_demo_data` (--email / --password, see
# posthog/management/commands/generate_demo_data.py), which generate_demo_data()
# runs unchanged. Not importable here — the tool runs in an SDK-only env with no
# posthog Django on the path — so they're duplicated with that source named. If
# the seed defaults ever change, change these too.
_STATIC_PROXY_CADDYFILE = """\
:8000 {
	@capture path /e /e/* /i/v0 /i/v0/* /i/v1/analytics/events /batch /batch/* /capture /capture/*
	handle @capture {
		reverse_proxy capture:3000
	}
	@flags path /flags /flags/* /api/feature_flag/local_evaluation /api/feature_flag/local_evaluation/*
	handle @flags {
		reverse_proxy feature-flags:3001
	}
	@otlp path /i/v1/logs /i/v1/logs/* /i/v1/traces /i/v1/traces/* /i/v1/metrics /i/v1/metrics/*
	handle @otlp {
		reverse_proxy capture-logs:4318
	}
	@static {
		path /static/*
		file {
			root /srv
		}
	}
	handle @static {
		root * /srv
		header Cache-Control "public, max-age=3600"
		file_server {
			precompressed br gzip
		}
	}
	handle {
		reverse_proxy web:8000
	}
}
"""
_TELEMETRY_PIPELINE = ("otel-collector", "capture-logs", "ingestion-logs", "ingestion-traces", "ingestion-metrics")

_OTEL_COLLECTOR_CONFIG = """\
extensions:
    docker_observer:
    health_check:

receivers:
    receiver_creator:
        watch_observers: [docker_observer]
        receivers:
            filelog:
                rule: type == "container" and labels["com.docker.compose.project"] == "${{env:COMPOSE_PROJECT}}" and not (name matches "{pipeline}")
                config:
                    include:
                        - '`"/var/lib/docker/containers/" + container_id + "/*.log"`'
                    operators:
                        - type: regex_replace
                          regex_name: ansi_control_sequences
                          field: body
                        - type: json_parser
                          timestamp:
                              parse_from: attributes.time
                              layout: '%Y-%m-%dT%H:%M:%S.%LZ'
                          on_error: drop
                        - type: move
                          from: attributes["log"]
                          to: body
                        - type: add
                          field: resource["service.name"]
                          value: '`labels["com.docker.compose.service"]`'
                        - type: add
                          field: resource["container.name"]
                          value: '`name`'
                        - type: json_parser
                          parse_from: body
                          on_error: send_quiet
                          if: "hasPrefix(body, '{{')"
    otlp:
        protocols:
            grpc:
                endpoint: 0.0.0.0:4317
            http:
                endpoint: 0.0.0.0:4318
    prometheus:
        config:
            scrape_configs:
                - job_name: plugins
                  scrape_interval: 15s
                  metrics_path: /_metrics
                  static_configs:
                      - targets: ['plugins:6738']
                - job_name: ingestion-general
                  scrape_interval: 15s
                  metrics_path: /_metrics
                  static_configs:
                      - targets: ['ingestion-general:6738']

exporters:
    otlphttp:
        endpoint: http://capture-logs:4318
        compression: none
        tls:
            insecure: true
        headers:
            authorization: Bearer {token}
        retry_on_failure:
            max_elapsed_time: 30s

processors:
    memory_limiter:
        check_interval: 1s
        limit_mib: 512
        spike_limit_mib: 128
    batch:

service:
    pipelines:
        traces:
            receivers: [otlp]
            processors: [memory_limiter, batch]
            exporters: [otlphttp]
        logs:
            receivers: [otlp, receiver_creator]
            processors: [memory_limiter, batch]
            exporters: [otlphttp]
        metrics:
            receivers: [otlp, prometheus]
            processors: [memory_limiter, batch]
            exporters: [otlphttp]
    extensions: [docker_observer, health_check]
"""

_OTEL_ENV = [
    "      - OTEL_SDK_DISABLED=false",
    "      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317",
    "      - OTEL_TRACES_SAMPLER=parentbased_traceidratio",
    "      - OTEL_TRACES_SAMPLER_ARG=1",
    "      - OTEL_METRICS_EXPORT_URL=http://otel-collector:4318/v1/metrics",
    "      - OTEL_METRICS_EXPORT_TOKEN=collector-adds-the-project-token",
]

_DEMO_EMAIL = "[email protected]"
_DEMO_PASSWORD = "12345678"

# PostHog's prod settings refuse to boot on the default SECRET_KEY, so the
# override must supply one (the migrate `run --rm web` one-off needs it too).
#
# It MUST be random per preview, NOT a shared constant. Previews are served on
# PUBLIC URLs (pen-<id>.boxes.hogland.<env>.posthog.dev) — the old
# "tailnet-only" assumption was stale — so a globally derivable SECRET_KEY lets
# anyone forge session cookies against every preview. We mint one random key per
# provision (see __init__) and pin it in the compose override so ALL of a
# single preview's processes share it, while different previews never do. A
# re-provision of the same PR rotates the key; that's fine — preview logins are
# throwaway demo sessions, so dropping them on re-provision is acceptable.


class PostHogPreviewStack:
    # Default: run the ready-made published image and mount PR source over it.
    # Pass image=None to fall back to the build-from-checkout escape hatch.
    IMAGE = "ghcr.io/posthog/posthog:master"
    CDP_IMAGE = "ghcr.io/posthog/posthog-node:master"
    SQLX_MIGRATE_IMAGE = "ghcr.io/posthog/posthog/sqlx-migrate:master"
    CYCLOTRON_NODE_DATABASE_URL = "postgres://posthog:posthog@db:5432/cyclotron_node"
    STATIC_PROXY_IMAGE = "caddy:2-alpine"
    STATIC_PROXY_CADDYFILE = "preview-static.Caddyfile"
    OTEL_COLLECTOR_CONFIG = "preview-otel-collector.yaml"
    COMPOSE_PROJECT = "posthog"
    SELF_CAPTURE_SERVICES = ["capture", "feature-flags", "property-defs-rs", "ingestion-general"]
    TELEMETRY_SERVICES = ["capture-logs", "ingestion-logs", "ingestion-traces", "ingestion-metrics", "otel-collector"]
    CELERY_SERVICES = ["worker"]
    CELERY_CONCURRENCY = 2
    FLAGS_REDIS_URL = "redis://redis7:6379/1"
    REPO_DIR = "/home/hog/posthog"
    COMPOSE = "docker-compose.dev-full.yml"
    OVERRIDE = "docker-compose.preview.yml"
    # Dependency services (published images, pulled by `up`).
    # personhog-replica/router are defined in the OVERRIDE (not dev-full.yml) —
    # compose merges both files, so starting them here works. They must be in
    # this list: nothing else starts them (up_web brings up only `web`, no
    # depends_on edges), so a cold/reset box would otherwise boot web with
    # PERSONHOG_ADDR pointing at a service that was never started. On the warm
    # golden they're already running and `up` is a no-op.
    #
    # temporal is here too: the server has to be listening before web schedules
    # anything, and auto-setup creates its schemas in the same postgres, so it
    # wants the same "start early, wait for pg" slot as the rest. The WORKER is
    # deliberately NOT in this list — it imports Django and touches the schema,
    # so it can only start after migrate(), and it has to be (re)created from the
    # override to pick up the PR's bind mounts. up_web handles it.
    DEPS = [
        "db",
        "redis7",
        "valkey",
        "clickhouse",
        "zookeeper",
        "kafka",
        "objectstorage",
        "personhog-replica",
        "personhog-router",
        "temporal",
    ]
    # App services built from the checkout (build escape hatch only). They share
    # a build context, so the second one is a pure cache hit — but compose tags a
    # build-without-image service as <project>-<service>, i.e. per service. So
    # both must be listed or `up --no-build temporal-django-worker` finds no
    # image on the build path.
    BUILD_SERVICES = ["web", "temporal-django-worker"]
    # PR backend source bind-mounted over the image's /code (backend hot-mount).
    # The frontend stays baked in the image; mounting these swaps backend live.
    # The personhog gRPC stubs are installed into the image's site-packages, not copied under
    # /code. /code comes first on sys.path, so the PR's copy mounted there shadows the image's
    # and a PR that changes a proto runs against its own stubs.
    MOUNTS = [
        ("posthog", "/code/posthog"),
        ("ee", "/code/ee"),
        ("products", "/code/products"),
        ("packages/personhog-proto/personhog", "/code/personhog"),
        ("rust/persons_migrations", "/code/rust/persons_migrations"),
        ("rust/persons_migrations_hobby", "/code/rust/persons_migrations_hobby"),
        ("frontend/src/lib/constants.tsx", "/code/frontend/src/lib/constants.tsx"),
    ]

    def __init__(
        self,
        backend: PreviewBackend,
        *,
        branch: str | None = None,
        image: str | None = None,
        repo_dir: str | None = None,
        seed_demo_data: bool = True,
        reset_db: bool = False,
        mount: bool = True,
        frontend_dist_tar: str | None = None,
    ):
        self.backend = backend
        # One random Django SECRET_KEY per stack (i.e. per provisioned box). Pinned
        # into the compose override so every process of THIS preview shares it, and
        # never shared across previews — see the module-level note above.
        self.secret_key = secrets.token_hex(32)
        self.oidc_private_key = ""  # see _ensure_oidc_private_key
        self.branch = branch
        # Default (None) -> the ready-made image; "" -> build-from-checkout escape
        # hatch; any tag -> run that published image.
        self.image = self.IMAGE if image is None else image
        # Bind-mount PR source over the image's /code (only meaningful with a
        # pinned image — the build path bakes the code in, so no mount needed).
        self.mount = mount
        self.repo_dir = repo_dir or self.REPO_DIR
        self.seed_demo_data = seed_demo_data
        # reset_db wipes pg + clickhouse before migrating, so the DB is migrated
        # fresh against the built code rather than inheriting a snapshot's
        # drifted DB. Set when baking a golden; a restore-and-delta leaves False.
        self.reset_db = reset_db
        # Optional gzipped tar of a prebuilt frontend/dist (built in CI with the
        # Turbo cache). When set, swap_frontend serves the PR's own frontend
        # instead of the golden image's :master SPA. None => keep :master.
        self.frontend_dist_tar = frontend_dist_tar

    # --- public API ----------------------------------------------------------
    def bring_up(self) -> str:
        """Provision the box and bring PostHog up; return the public URL."""
        self.backend.provision()
        self.start_runtime()
        url = self.backend.web_url
        if self.branch:
            self.checkout_branch(self.branch)
        self._ensure_oidc_private_key()
        self.write_override()
        if self.image:
            self.pull_image()  # escape hatch: run a published image, skip build
        else:
            self.build_app()  # native path: build the checkout (PR code)
        if self.reset_db:
            self.reset_database()
        # Prepare the database and service-backed templates BEFORE web serves:
        # web is not restarted to pick up a PR's delta migrations, so all setup
        # must finish before it boots.
        self.up_deps()
        self.migrate()
        self.start_cdp_service()
        self.sync_hog_function_templates()
        try:
            self.sync_feature_flags()
        except Exception as e:  # noqa: BLE001
            sys.stderr.write(f"[hogbox-preview] feature flag sync skipped (preview still usable): {e}\n")
        if self.seed_demo_data:
            # Best-effort: a transient build/model issue shouldn't sink an
            # otherwise-good preview — it just opens empty.
            try:
                self.generate_demo_data()
            except Exception as e:  # noqa: BLE001
                sys.stderr.write(f"[hogbox-preview] demo-data seeding skipped (preview still usable): {e}\n")
        try:
            self.warm_flag_caches()
        except Exception as e:  # noqa: BLE001
            sys.stderr.write(f"[hogbox-preview] flag cache warm skipped (preview still usable): {e}\n")
        if self.frontend_dist_tar:
            # Serve the PR's frontend (else it's the golden's :master SPA). Must
            # run before up_web so the fresh web container reads the new index
            # template + collected statics.
            self.swap_frontend()
        if self.image:
            self.export_image_files(static=not self.frontend_dist_tar)
        self.up_web()
        self.wait_for_health()
        self.deep_health()
        self.report_pipeline_health()
        return url

    def swap_frontend_only(self) -> str:
        """Lay the PR's frontend onto an ALREADY-up preview box; return its URL.

        The second half of the parallel CI flow: bring_up() first brings the box
        healthy on the golden's :master SPA with NO dist (so it can run the
        moment the box is restored, in parallel with the runner building the PR
        frontend), then — once that build finishes — this swaps the freshly-built
        dist in. It reuses the exact override + collectstatic + web-recreate the
        single-pass bring_up runs after seeding, so there's no second recipe to
        keep in sync. Attaches to the live box by pen/name rather than restoring
        a new one.
        """
        if not self.frontend_dist_tar:
            raise RuntimeError("swap_frontend_only requires a frontend dist (pass --frontend-dist)")
        self.backend.attach()
        # Keep the box's existing SECRET_KEY: recreating web in up_web must NOT
        # rotate the key bring_up already migrated + seeded under, or anything it
        # wrote encrypted becomes undecryptable (and any live session drops).
        self._reuse_existing_secret_key()
        self._ensure_oidc_private_key()
        # Rewrite the override so it now carries the frontend/dist + staticfiles
        # mounts (write_override only adds them when a dist is set), lay the dist
        # in + re-run collectstatic into the mounted staticfiles/, then recreate
        # web so the fresh container reads the PR's index + statics.
        self.write_override()
        self.swap_frontend()
        self.up_web()
        self.wait_for_health()
        return self.backend.web_url

    def _override_value(self, name: str) -> str:
        """Read one environment value out of the override the box already runs
        with. Empty when the box has no override yet, or no such entry."""
        return self.backend.exec(
            f"sed -n 's/.*{name}=//p' {self.repo_dir}/{self.OVERRIDE} 2>/dev/null | head -n1",
            timeout=60,
        ).stdout.strip()

    def _reuse_existing_secret_key(self) -> None:
        """Adopt the SECRET_KEY the box already runs with so a deferred swap
        doesn't rotate it. Falls back to the freshly-minted key when the override
        can't be read — shouldn't happen post-bring_up, but a random key is a
        safe default either way."""
        self.secret_key = self._override_value("SECRET_KEY") or self.secret_key

    def _ensure_oidc_private_key(self) -> None:
        """Give the box an RSA key for OAuth token signing, once per box.

        PostHog signs OAuth tokens with RS256 and refuses to save an OAuth
        application without OIDC_RSA_PRIVATE_KEY, so a preview cannot host an
        OAuth client (PostHog Desktop, for one) until this is set. Adopted from
        the override when the box already has one, because rotating it would
        invalidate every token the preview already issued. Left empty when the
        box cannot mint one, which serves everything except OAuth.
        """
        existing = self._override_value("OIDC_RSA_PRIVATE_KEY")
        if existing:
            self.oidc_private_key = existing
            return
        # One line, because it lives in a compose environment entry. The Django
        # setting turns the escapes back into newlines.
        r = self.backend.exec(
            r"""openssl genrsa 2048 2>/dev/null | openssl pkcs8 -topk8 -nocrypt -outform PEM """
            r"""| awk 'NF {sub(/\r/, ""); printf "%s\\n", $0}'""",
            timeout=120,
        )
        key = r.stdout.strip()
        if not key.startswith("-----BEGIN PRIVATE KEY-----"):
            sys.stderr.write(
                "[hogbox-preview] could not mint an RSA key; OAuth applications will not save on this preview\n"
            )
            return
        self.oidc_private_key = key

    # --- steps (each usable standalone, mirroring bin/hobby-ci.py) -----------
    def start_runtime(self) -> None:
        # Ensure docker is up before any compose command. The current golden is
        # baked WARM (stack already running — see tools/hogbox-preview/STARTUP.md),
        # so this is a no-op there; it stays as a safety net for a cold/stopped
        # golden or a box whose daemon didn't auto-start. Idempotent either way —
        # returns once `docker info` answers.
        self.backend.run_long(
            "systemctl start docker; "
            "for _ in $(seq 1 30); do docker info >/dev/null 2>&1 && break; sleep 2; done; "
            "docker info >/dev/null 2>&1",
            name="start-docker",
            timeout=180,
        )

    def checkout_branch(self, branch: str) -> None:
        # The box's exec API runs as root while the repo is hog-owned, so git
        # flags "dubious ownership" — scope a safe.directory exception per
        # command. (The rest of the stack runs as root too, matching how the
        # golden's setup script built it.)
        timing.stage(f"in-box PR checkout ({branch})")
        safe = f"git -c safe.directory={self.repo_dir}"
        self.backend.run_long(
            f"cd {self.repo_dir} && {safe} fetch --depth 1 origin {branch} && {safe} checkout --force FETCH_HEAD",
            name="checkout",
            timeout=600,
        )

    def write_override(self) -> None:
        # This override MUST stay in sync with hogland's
        # scripts/posthog-preview-setup.sh (the golden bake script). Both write
        # the same docker-compose.preview.yml; the bake bakes it into the golden,
        # this regenerates it per PR — and regenerating it CLOBBERS the baked one.
        # When the two drifted, previews broke fleet-wide: the bake added
        # PERSONHOG_ADDR + the personhog services on 2026-07-06 (master's Django
        # hard-requires the personhog service for group-type lookups since #65968
        # — require_personhog_client() raises "personhog client not configured"
        # without it), this file didn't, and every preview 500'd on
        # /api/projects/@current/ (and environment/@current, team create, some
        # HogQL paths) from 2026-07-06 to 2026-07-10 until this was fixed.
        #
        # Services this file must keep in sync with the bake: web,
        # personhog-replica, personhog-router, temporal, temporal-django-worker.
        # The temporal pair fails EVEN QUIETER than personhog did — web still
        # accepts the schedule, nothing services the queue, and a materialized
        # view just sits there forever with no error on any surface.
        #
        # A web recreate still happens in up_web — but only to bind-mount the PR's
        # backend source over the image's /code (you can't add a mount to a
        # running container). On the warm golden that's a warm single-worker import
        # rather than a cold rebuild; #315's win
        # is making that recreate warm and serving the frontend relative
        # (JS_URL=""), not removing it. Keeping the env constant means web only
        # ever recreates for the mount, never for config drift.
        timing.stage("write compose override (backend mount)")
        self.backend.write_file(f"{self.repo_dir}/{self.STATIC_PROXY_CADDYFILE}", _STATIC_PROXY_CADDYFILE)
        lines = ["# Generated by tools/hogbox-preview.", "services:", "  web:"]
        if self.image:
            lines.append(f"    image: {self.image}")
        lines.append("    ports: !reset []")
        if self.image and self.mount:
            # Bind-mount PR backend source over the image's /code — no rebuild.
            # Compose concatenates volume lists across files, so dev-full's
            # .:/app/posthog stays and these are appended.
            lines.append("    volumes:")
            mounts = list(self.MOUNTS)
            if self.frontend_dist_tar:
                # Serve the PR's frontend too: its built dist (the SPA index
                # template) + the statics collectstatic writes into staticfiles/.
                # Both are mounted so swap_frontend's `run` container and the
                # serving web container share them — static is served by
                # WhiteNoise from staticfiles/, NOT frontend/dist, hence both.
                mounts += [("frontend/dist", "/code/frontend/dist"), ("staticfiles", "/code/staticfiles")]
            lines += [f"      - ./{src}:{dst}" for src, dst in mounts]
        lines += [
            "    environment:",
            # The OAuth metadata documents (RFC 8414, RFC 9728) build their
            # issuer and endpoints from SITE_URL, so a placeholder would send a
            # discovery client to its own machine. Serving still needs no per-box
            # env: JS_URL="" keeps assets relative, and the CSRF origin is a
            # wildcard.
            f"      - SITE_URL={self.backend.web_url}",
            "      - JS_URL=",
            f"      - EXTRA_CSRF_TRUSTED_ORIGINS={_CSRF_TRUSTED_ORIGINS}",
            "      - DISABLE_SECURE_SSL_REDIRECT=1",
            "      - DEBUG=0",
            # Random per-preview key (see self.secret_key) — PostHog's prod
            # settings refuse to boot on the default, and the migrate one-off
            # (compose run --rm web) needs it too. Not shared across previews, so
            # a public preview URL can't be used to forge sessions on another.
            f"      - SECRET_KEY={self.secret_key}",
            f"      - OIDC_RSA_PRIVATE_KEY={self.oidc_private_key}",
            # Each worker costs a full Django import at boot, so a preview runs one.
            # Under ASGI one worker serves one sync request at a time, and a page
            # load fires dozens in parallel. WSGI serves them from a thread pool.
            "      - GRANIAN_WORKERS=1",
            "      - GRANIAN_INTERFACE=wsgi",
            "      - GRANIAN_BLOCKING_THREADS=16",
            # master's Django hard-requires the personhog service for group-type
            # lookups (require_personhog_client() raises "personhog client not
            # configured" without it — #65968). Same addr the dev/hobby composes
            # use; the router service is defined below. See write_override's note.
            "      - PERSONHOG_ADDR=personhog-router:50052",
            # A preview runs DEBUG=0, and posthog otherwise derives
            # USE_LOCAL_SETUP from TEST/DEBUG — so every data-warehouse path that
            # branches on it (delta write creds, HogQL S3 read creds, the
            # saved-query url_pattern) reaches for real AWS creds instead of the
            # stack's own MinIO at objectstorage:19000. Forcing it on is what
            # makes materialization work in a self-hosted-shaped stack. Needs the
            # companion settings change that reads this from the env; it's an
            # inert no-op on an image that predates it.
            "      - USE_LOCAL_SETUP=1",
            # ee/urls.py registers /admin/* only when ADMIN_PORTAL_ENABLED is true,
            # and ee/settings.py defaults it to DEMO or DEBUG — both false here. So
            # without this, Django has no /admin route at all: the request falls
            # through to the SPA, which prefixes any path it doesn't know with the
            # project id, and /admin lands on /project/1/admin. A box reaches admin
            # only over the tailnet (hogland's box-front is an internal NLB behind
            # split-DNS), and the seeded demo user is staff, so /admin is as exposed
            # as the rest of the preview and no more.
            "      - ADMIN_PORTAL_ENABLED=1",
            "      - SELF_CAPTURE=1",
            "      - SELF_CAPTURE_HOST=http://static-proxy:8000",
            f"      - FLAGS_REDIS_URL={self.FLAGS_REDIS_URL}",
            "      - OTEL_SERVICE_NAME=posthog-web",
            *_OTEL_ENV,
        ]
        lines += [
            "  static-proxy:",
            f"    image: {self.STATIC_PROXY_IMAGE}",
            "    restart: always",
            "    ports:",
            f"      - {self.backend.web_port}:8000",
            "    volumes:",
            f"      - ./{self.STATIC_PROXY_CADDYFILE}:/etc/caddy/Caddyfile:ro",
            "      - ./staticfiles:/srv/static:ro",
            "    depends_on:",
            "      - web",
        ]
        lines += [
            "  plugins:",
            f"    image: {self.CDP_IMAGE}",
            # The checkout mount from dev-full masks the compiled code in the published image.
            "    volumes: !override []",
            "    environment:",
            f"      - CYCLOTRON_NODE_DATABASE_URL={self.CYCLOTRON_NODE_DATABASE_URL}",
            "      - CDP_REDIS_HOST=redis7",
            "      - CDP_VALKEY_HOST=valkey",
            "      - CDP_VALKEY_PORT=6379",
            "      - ENCRYPTION_SALT_KEYS=00beef0000beef0000beef0000beef00",
            "      - PERSONHOG_ADDR=personhog-router:50052",
            "      - PERSONHOG_ENABLED=true",
            *_OTEL_ENV,
            "  cyclotron-node-migrate:",
            f"    image: {self.SQLX_MIGRATE_IMAGE}",
            "    profiles:",
            "      - migrate",
            "    command: cyclotron-node",
            "    environment:",
            f"      - CYCLOTRON_NODE_DATABASE_URL={self.CYCLOTRON_NODE_DATABASE_URL}",
            "    volumes:",
            "      - ./rust/cyclotron-node-migrations:/migrations/cyclotron-node-migrations:ro",
        ]
        lines += self._self_capture_services() + self._telemetry_services() + self._celery_worker_service()
        # Mirror the bake script's personhog service definitions (hogland
        # scripts/posthog-preview-setup.sh): dev-full.yml carries NO personhog
        # services (dev runs them via hogli), so define them here the way HOBBY
        # does — extend the base definitions and point the replica at the MAIN
        # posthog DB (persons live there for single-node deployments). The
        # golden's warm containers keep running regardless, but a cold/reset
        # path that `up`s the full project would lose them if they weren't here.
        # Tag matches the bake: the part of self.image after the last ':'.
        personhog_tag = self.image.rsplit(":", 1)[-1] if self.image else "master"
        lines += [
            "  personhog-replica:",
            "    extends:",
            "      file: docker-compose.base.yml",
            "      service: personhog-replica",
            f"    image: ghcr.io/posthog/posthog/personhog-replica:{personhog_tag}",
            "    environment:",
            "      PRIMARY_DATABASE_URL: postgres://posthog:posthog@db:5432/posthog",
            "  personhog-router:",
            "    extends:",
            "      file: docker-compose.base.yml",
            "      service: personhog-router",
            f"    image: ghcr.io/posthog/posthog/personhog-router:{personhog_tag}",
        ]
        # Temporal — what lets a preview actually RUN data-warehouse materialized
        # views instead of just rendering the UI for them. Unlike personhog,
        # dev-full.yml ALREADY defines both services (each extending base), so
        # these blocks only OVERRIDE fields; do NOT re-extend base here or the
        # definition gets pulled in twice and the merge order stops being obvious.
        # Deliberately absent: elasticsearch (see ENABLE_ES), temporal-ui and
        # temporal-admin-tools — the UI isn't reachable through the box edge
        # anyway (custom guest ports are unimplemented), so they'd be dead weight.
        lines += [
            "  temporal:",
            "    environment:",
            # base sets ENABLE_ES=true + ES_SEEDS=elasticsearch, which would drag
            # a whole ES container into every preview just to run workflows.
            # Standard (postgres) visibility is plenty — only advanced
            # list/search queries need ES. Same shape hobby uses. auto-setup
            # creates the temporal / temporal_visibility DBs in the existing db
            # service (POSTGRES_SEEDS=db), so no extra storage either.
            "      - ENABLE_ES=false",
            # ENABLE_ES=false alone is NOT enough. compose `extends` DOES
            # propagate depends_on, and depends_on is a mapping, so it MERGES
            # across files instead of being replaced — dev-full's temporal
            # inherits base's `elasticsearch: service_healthy` edge and every
            # `up` touching temporal would start and block on a ~1 GiB ES
            # container we just told temporal not to use. `!override` is the
            # only way to drop an inherited edge; keep the db one (auto-setup
            # writes its schemas there). Verified against the merged config
            # (compose 5.1.2): temporal's closure is {db, temporal} and the
            # worker's is {clickhouse, db, kafka, objectstorage, redis7,
            # temporal, zookeeper} — no elasticsearch anywhere.
            "    depends_on: !override",
            "      db:",
            "        condition: service_healthy",
            # The auto-setup image does NOT ship the dynamicconfig file base
            # points DYNAMIC_CONFIG_FILE_PATH at — without this mount
            # temporal-server exits on boot ("unable to validate dynamic
            # config: ... no such file or directory") and restart-on-failure
            # loops it forever, invisibly: `up -d` succeeds and the container
            # just never turns healthy. Cost a full golden bake on 2026-08-07.
            # Mount the checkout's copy, like hobby (whose ./posthog/docker/...
            # prefix is its own deploy layout — the repo path is docker/...).
            "    volumes:",
            "      - ./docker/temporal/dynamicconfig:/etc/temporal/config/dynamicconfig",
            "  temporal-django-worker:",
        ]
        if self.image:
            # dev-full's definition carries `build: .` — pinning the image is the
            # only thing keeping `up --no-build` from turning into the 20-min
            # build, exactly the same trap as web above.
            lines.append(f"    image: {self.image}")
        if self.image and self.mount:
            # The whole point of a per-PR worker: it has to execute the PR's
            # code, not the golden image's :master. Without these mounts it
            # silently materializes views against stale master and the feature is
            # theatre. Same backend mounts as web — but NOT the frontend
            # dist/staticfiles ones: the worker serves no HTTP.
            lines.append("    volumes:")
            lines += [f"      - ./{src}:{dst}" for src, dst in self.MOUNTS]
        lines += [
            "    environment:",
            # One worker process serves exactly ONE task queue (a temporal
            # limit), and posthog only collapses everything onto
            # development-task-queue when DEBUG=1. A preview runs DEBUG=0, so the
            # real queue names apply and we have to pick one. Materialized-view
            # runs are scheduled onto DATA_MODELING_TASK_QUEUE by web
            # (products/data_modeling/backend/logic/node_materialization.py +
            # schedule_reconcile.py, products/data_warehouse .../
            # saved_query_service.py), so that's the queue worth serving.
            # Workflows on every OTHER queue (batch exports, max-ai, warehouse
            # imports, ...) sit unstarted — accepted v1 scope; widening it means
            # one more worker service per queue.
            "      - TEMPORAL_TASK_QUEUE=data-modeling-task-queue",
            # MUST be web's key, byte for byte. TEMPORAL_SECRET_KEY defaults to
            # SECRET_KEY and keys the payload codec, so a mismatch makes every
            # workflow payload undecodable across the web/worker boundary. We
            # mint one key per preview (see self.secret_key) and write the SAME
            # one into both services.
            f"      - SECRET_KEY={self.secret_key}",
            # Same hard requirement as web — the worker imports the same Django
            # app, and require_personhog_client() raises without it.
            "      - PERSONHOG_ADDR=personhog-router:50052",
            # Same story as web's, and the worker is the side that actually
            # writes the delta files to MinIO.
            "      - USE_LOCAL_SETUP=1",
            # The health server only starts when BOTH of these are set:
            # start_temporal_worker.py gates on `health_port and
            # health_max_idle_seconds` and both settings default to None, so
            # setting just the port gets you no endpoint and a log warning.
            # /healthz and /readyz are the same handler — "did this worker run
            # anything within max_idle_seconds", with the idle clock starting at
            # process import, not first work. A preview worker is idle by
            # construction (and after a warm restore the clock resumes from bake
            # time), so a long window degenerates it into the signal we want:
            # the process got past create_worker() and is serving HTTP. Nothing
            # here gates on it — a slow worker shouldn't sink an otherwise-usable
            # preview — but the endpoint is what you curl when a materialization
            # doesn't start. Values mirror the bake.
            "      - TEMPORAL_HEALTH_PORT=7999",
            "      - TEMPORAL_HEALTH_MAX_IDLE_SECONDS=86400",
            f"      - FLAGS_REDIS_URL={self.FLAGS_REDIS_URL}",
            "      - OTEL_SERVICE_NAME=posthog-temporal-worker",
            "      - TEMPORAL_OTEL_PLUGIN_ENABLED=true",
            *_OTEL_ENV,
        ]
        self.backend.write_file(f"{self.repo_dir}/{self.OVERRIDE}", "\n".join(lines) + "\n")

    def _celery_worker_service(self) -> list[str]:
        lines = ["  worker:"]
        if self.image:
            lines.append(f"    image: {self.image}")
        lines.append("    command: ./bin/docker-worker-celery")
        if self.image and self.mount:
            lines.append("    volumes:")
            lines += [f"      - ./{src}:{dst}" for src, dst in self.MOUNTS]
        return [
            *lines,
            "    environment:",
            f"      - SITE_URL={self.backend.web_url}",
            "      - DEBUG=0",
            f"      - SECRET_KEY={self.secret_key}",
            f"      - OIDC_RSA_PRIVATE_KEY={self.oidc_private_key}",
            "      - PERSONHOG_ADDR=personhog-router:50052",
            "      - USE_LOCAL_SETUP=1",
            "      - SELF_CAPTURE=1",
            "      - SELF_CAPTURE_HOST=http://static-proxy:8000",
            f"      - WEB_CONCURRENCY={self.CELERY_CONCURRENCY}",
            f"      - FLAGS_REDIS_URL={self.FLAGS_REDIS_URL}",
            "      - OTEL_SERVICE_NAME=posthog-celery-worker",
            *_OTEL_ENV,
        ]

    def _self_capture_services(self) -> list[str]:
        return [
            "  feature-flags:",
            "    environment:",
            f"      - FLAGS_REDIS_URL={self.FLAGS_REDIS_URL}",
            "      - REDIS_COMPRESSION_ENABLED=true",
            "      - OBJECT_STORAGE_ENDPOINT=http://objectstorage:19000",
            "      - AWS_ACCESS_KEY_ID=object_storage_root_user",
            "      - AWS_SECRET_ACCESS_KEY=object_storage_root_password",
            "  ingestion-general:",
            "    extends:",
            "      file: docker-compose.base.yml",
            "      service: ingestion-general",
            f"    image: {self.CDP_IMAGE}",
            "    environment:",
            "      - PERSONHOG_ADDR=personhog-router:50052",
            "      - PERSONHOG_ENABLED=true",
            *_OTEL_ENV,
            "    depends_on:",
            "      - db",
            "      - redis7",
            "      - clickhouse",
            "      - kafka",
            "      - objectstorage",
            "      - personhog-router",
        ]

    def _telemetry_services(self) -> list[str]:
        node_consumer = ["    depends_on:", "      - db", "      - redis7", "      - kafka"]
        return [
            "  capture-logs:",
            "    extends:",
            "      file: docker-compose.base.yml",
            "      service: capture-logs",
            "    depends_on:",
            "      - kafka",
            "  ingestion-logs:",
            "    extends:",
            "      file: docker-compose.base.yml",
            "      service: ingestion-logs",
            f"    image: {self.CDP_IMAGE}",
            "    environment:",
            "      - LOGS_REDIS_TLS=false",
            *node_consumer,
            "  ingestion-traces:",
            "    extends:",
            "      file: docker-compose.base.yml",
            "      service: ingestion-traces",
            f"    image: {self.CDP_IMAGE}",
            "    environment:",
            "      - TRACES_REDIS_TLS=false",
            *node_consumer,
            "  ingestion-metrics:",
            f"    image: {self.CDP_IMAGE}",
            "    command: node nodejs/dist/index.js",
            "    restart: on-failure",
            "    environment:",
            "      - PLUGIN_SERVER_MODE=ingestion-metrics",
            "      - DATABASE_URL=postgres://posthog:posthog@db:5432/posthog",
            "      - KAFKA_HOSTS=kafka:9092",
            "      - REDIS_URL=redis://redis7:6379/",
            "      - METRICS_REDIS_HOST=redis7",
            "      - METRICS_REDIS_TLS=false",
            *node_consumer,
            "  otel-collector:",
            "    depends_on: !reset []",
            "    volumes:",
            f"      - ./{self.OTEL_COLLECTOR_CONFIG}:/etc/otel-collector-config.yaml:ro",
            "    environment:",
            f"      - COMPOSE_PROJECT={self.COMPOSE_PROJECT}",
            "    networks:",
            "      - default",
            "      - otel_network",
        ]

    def _demo_project_token(self) -> str:
        query = (
            "SELECT api_token FROM posthog_team ORDER BY "
            f"id = (SELECT current_team_id FROM posthog_user WHERE email = '{_DEMO_EMAIL}') DESC NULLS LAST, id LIMIT 1"
        )
        result = self.backend.exec(
            self._compose(f'exec -T db psql -U posthog -d posthog -tAc "{query}"'),
            timeout=120,
        )
        token = result.stdout.strip()
        if not token:
            sys.stderr.write(f"[hogbox-preview] demo project token lookup returned nothing: {result.stderr.strip()}\n")
        return token

    def write_otel_collector_config(self) -> None:
        token = self._demo_project_token()
        config = _OTEL_COLLECTOR_CONFIG.format(pipeline="|".join(_TELEMETRY_PIPELINE), token=token)
        self.backend.write_file(f"{self.repo_dir}/{self.OTEL_COLLECTOR_CONFIG}", config)

    def build_app(self) -> None:
        # `build: .` from the checkout — bakes the PR's code (front + back) into
        # the image. Cold this is ~15-25 min; on a restored golden the Docker
        # layer + turbo cache make it incremental.
        services = " ".join(self.BUILD_SERVICES)
        self.backend.run_long(self._compose(f"build {services}"), name="build", timeout=2700)

    def pull_image(self, *, attempts: int = 3) -> None:
        # The default path: fetch the ready-made images. Fast/no-op if a golden
        # already baked them in; otherwise pulls. ghcr pulls flake (TLS
        # handshake timeouts mid-layer); retry because docker resumes from
        # pulled layers, so a retry is cheap.
        assert self.image is not None
        for image in (self.image, self.CDP_IMAGE):
            last: Exception | None = None
            for _ in range(attempts):
                try:
                    self.backend.run_long(f"docker pull {image}", name="pull", timeout=1800)
                    break
                except RuntimeError as e:
                    last = e
            else:
                raise RuntimeError(f"docker pull {image} failed after {attempts} attempts: {last}")

    def reset_database(self) -> None:
        # Drop the snapshot's pre-migrated DB so migrate() rebuilds it fresh and
        # coherent with the built code. Postgres keeps its data in a named volume
        # that survives container recreation, so the volume must be removed by
        # name; ClickHouse is volume-less, so recreating its container is enough.
        # temporal must be recreated too: its temporal/temporal_visibility DBs
        # live in that same postgres volume, and only its auto-setup ENTRYPOINT
        # recreates the schema — a surviving container would just error against
        # the wiped postgres forever. The worker follows so it reconnects cleanly.
        cmd = (
            f"cd {self.repo_dir} && "
            f"docker compose -f {self.COMPOSE} -f {self.OVERRIDE} stop db clickhouse web temporal temporal-django-worker ; "
            f"docker compose -f {self.COMPOSE} -f {self.OVERRIDE} rm -f db clickhouse web temporal temporal-django-worker ; "
            "docker volume rm posthog_postgres-15-data || true"
        )
        self.backend.run_long(cmd, name="reset-db", timeout=300)

    def up_deps(self) -> None:
        # Start the dependency services and wait for postgres to report healthy
        # before migrating. Idempotent: on a restored golden whose stack is
        # already running, `up` is a no-op for already-current containers.
        timing.stage("start dependency services")
        services = " ".join(self.DEPS)
        script = (
            f"cd {self.repo_dir} && docker compose -f {self.COMPOSE} -f {self.OVERRIDE} up -d --no-build {services} && "
            "healthy=0; for _ in $(seq 1 60); do "
            f"docker compose -f {self.COMPOSE} -f {self.OVERRIDE} ps --format '{{{{.Service}}}} {{{{.Status}}}}' "
            "| grep '^db ' | grep -q healthy && { healthy=1; break; }; sleep 4; done; "
            '[ "$healthy" = 1 ] || { echo "db never became healthy" >&2; exit 1; }'
        )
        self.backend.run_long(script, name="up-deps", timeout=900)

    def up_web(self) -> None:
        # --no-build reuses the pulled image; the override mounts PR source over its /code.
        #
        # The temporal worker comes up here, alongside web and for the same
        # reason: you can't add a bind mount to a running container, so the
        # golden's warm worker has to be recreated from the override or it keeps
        # running :master code. It's also the right ordering — the worker imports
        # Django and touches the schema, so it must land after migrate(), which
        # is why it isn't in DEPS. `up` on the two together lets the two heavy
        # Django imports overlap.
        #
        # swap_frontend_only() calls this too. Recreating the worker there is
        # redundant but harmless: it re-reads the same override (same key, same
        # mounts — the frontend mounts are web-only), so it just costs one more
        # container recreate on the deferred-swap path.
        timing.stage("start web + temporal worker containers")
        self.write_otel_collector_config()
        services = " ".join(
            [
                "web",
                "temporal-django-worker",
                "static-proxy",
                *self.CELERY_SERVICES,
                *self.SELF_CAPTURE_SERVICES,
                *self.TELEMETRY_SERVICES,
            ]
        )
        self.backend.run_long(self._compose(f"up -d --no-build {services}"), name="up-web", timeout=1800)

    def migrate(self) -> None:
        # PostHog needs both: Postgres (schema) and ClickHouse (events DB +
        # tables). ClickHouse must be migrated before demo-data generation,
        # which writes events to it.
        timing.stage("migrate start (postgres + clickhouse)")
        self.backend.run_long(
            self._compose("run --rm -T web python manage.py migrate --noinput"),
            name="migrate",
            timeout=1800,
        )
        self.backend.run_long(
            self._compose("run --rm -T web python manage.py apply_persons_migrations --hobby"),
            name="migrate-persons",
            timeout=900,
        )
        index = "CREATE UNIQUE INDEX IF NOT EXISTS posthog_person_team_id_uuid_uniq ON posthog_person (team_id, uuid)"
        self.backend.run_long(
            self._compose(f'exec -T db psql -U posthog -d posthog -v ON_ERROR_STOP=1 -c "{index}"'),
            name="person-uuid-index",
            timeout=900,
        )
        self.backend.run_long(
            f"{self._compose('run --rm -T cyclotron-node-migrate')} || "
            'echo "WARN: cyclotron node migrations failed, so plugins cannot dequeue jobs in this preview" >&2',
            name="migrate-cyclotron-node",
            timeout=900,
        )
        self.backend.run_long(
            self._compose("run --rm -T web python manage.py migrate_clickhouse"),
            name="migrate-clickhouse",
            timeout=1800,
        )
        # Custom search attributes (PostHogDagId & co.) must exist in the
        # temporal namespace or /materialize/ 500s ("Namespace default has no
        # mapping defined for search attribute PostHogDagId" — hit live
        # 2026-08-07). The bake registers them too, but run it here as well:
        # it's idempotent and ~seconds, it heals goldens baked before the bake
        # learned to, and it covers --reset-db (reset_database recreates the
        # temporal container so auto-setup rebuilds the schema in the wiped
        # postgres — this then re-registers the attributes in it).
        #
        # Retried (up_deps only gates on db healthy, so temporal can still be
        # warming on a cold boot) and NON-fatal: a PR branch that predates the
        # management command would otherwise hard-abort bring-up with "Unknown
        # command". Such a preview still serves; it just can't materialize —
        # same as before temporal existed here — so warn and move on.
        register = self._compose("run --rm -T web python manage.py register_temporal_search_attributes")
        self.backend.run_long(
            f"ok=0; for _ in 1 2 3; do {register} && ok=1 && break; sleep 10; done; "
            '[ "$ok" = 1 ] || echo "WARN: register_temporal_search_attributes failed'
            " (temporal still starting, or the PR predates the command) —"
            ' materialized views may not run in this preview" >&2',
            name="register-search-attrs",
            timeout=600,
        )
        timing.stage("migrate done")

    def start_cdp_service(self) -> None:
        timing.stage("start CDP service")
        ready_probe = self._compose("exec -T plugins curl -fsS http://localhost:6738/_ready")
        script = (
            f"set -e; {self._compose('up -d --no-build plugins')}; "
            "ready=0; for _ in $(seq 1 60); do "
            f"{ready_probe} >/dev/null 2>&1 && {{ ready=1; break; }}; sleep 4; done; "
            '[ "$ready" = 1 ] || { echo "CDP service never became ready" >&2; exit 1; }'
        )
        self.backend.run_long(script, name="up-cdp", timeout=900)

    def sync_hog_function_templates(self) -> None:
        timing.stage("sync HogFunction templates")
        self.backend.run_long(
            self._compose("run --rm -T web python manage.py sync_hog_function_templates"),
            name="sync-templates",
            timeout=900,
        )

    def sync_feature_flags(self) -> None:
        timing.stage("sync feature flags")
        self.backend.run_long(
            self._compose("run --rm -T web python manage.py sync_feature_flags"),
            name="sync-flags",
            timeout=600,
        )

    def warm_flag_caches(self) -> None:
        timing.stage("warm team and flag caches")
        commands = ("warm_team_metadata_cache", "warm_flags_cache")
        self.backend.run_long(
            " && ".join(self._compose(f"run --rm -T web python manage.py {command}") for command in commands),
            name="warm-flag-caches",
            timeout=600,
        )

    def generate_demo_data(self) -> None:
        # Same command hobby-ci uses (bin/hobby-ci.py). Seeds a demo org + the
        # [email protected] / 12345678 login so the preview opens populated.
        # (`manage.py n` is a dev-only alias, not registered in this image.)
        timing.stage("seed demo data")
        self.backend.run_long(
            self._compose("run --rm -T web python manage.py generate_demo_data"),
            name="seed",
            timeout=1800,
        )

    def swap_frontend(self) -> None:
        # Serve the PR's OWN frontend. The dist is built in CI (Depot + Turbo
        # cache) and handed in as a gzipped tar; drop it into the box's
        # frontend/dist (bind-mounted over /code/frontend/dist by write_override)
        # and re-run collectstatic so WhiteNoise serves the PR's hashed chunks
        # from the (also-mounted) staticfiles/. Static is served from
        # staticfiles/, NOT frontend/dist — so collectstatic is mandatory.
        # STATIC_COLLECTION=1 skips the prod SECRET_KEY guard; the version skip
        # mirrors the image's own collectstatic. Deps are already up (up_deps ran)
        # so settings import is fine; collectstatic itself needs no live DB.
        import pathlib

        # Time upload and collectstatic separately because the exec transfer and
        # static processing require different fixes.
        with timing.span("frontend-dist-upload"):
            tar = pathlib.Path(self.frontend_dist_tar).read_bytes()
            self.backend.write_file(f"{self.repo_dir}/frontend/dist.tgz", tar)
        compose = f"docker compose -f {self.COMPOSE} -f {self.OVERRIDE}"
        # CI tars the dist with `-C frontend dist`, so its members are rooted at
        # `dist/`; strip that leading level on extract or the SPA double-nests to
        # frontend/dist/dist/ and collectstatic finds nothing.
        script = (
            f"cd {self.repo_dir} && "
            "rm -rf frontend/dist && mkdir -p frontend/dist staticfiles && "
            "tar xzf frontend/dist.tgz -C frontend/dist --strip-components=1 && "
            f"{compose} run --rm -T "
            "-e STATIC_COLLECTION=1 -e STATIC_PRECOMPRESS=0 -e SKIP_SERVICE_VERSION_REQUIREMENTS=1 "
            "web python manage.py collectstatic --noinput"
        )
        self.backend.run_long(script, name="frontend", timeout=900)
        timing.stage("frontend swap done (collectstatic done)")

    def export_image_files(self, *, static: bool) -> None:
        assert self.image is not None
        timing.stage("export image files for the static proxy and feature-flags")
        copies = ['docker cp "$cid":/code/share/GeoLite2-City.mmdb ./share/GeoLite2-City.mmdb']
        if static:
            copies.insert(0, 'rm -rf staticfiles && docker cp "$cid":/code/staticfiles ./staticfiles')
        self.backend.run_long(
            f"cd {self.repo_dir} && mkdir -p share && cid=$(docker create {self.image}) && "
            f"{{ {' && '.join(copies)}; }}; status=$?; "
            'docker rm "$cid" >/dev/null; exit $status',
            name="export-image-files",
            timeout=600,
        )

    def wait_for_health(self) -> None:
        # up_services already brought web up cleanly — just wait for it to serve.
        # Django is a heavy import; first health can take ~7 min.
        with timing.span("health-poll"):
            self.backend.wait_http_ok("/_health", expect=200, timeout=900)

    def report_pipeline_health(self, settle_seconds: int = 90) -> None:
        services = " ".join([*self.CELERY_SERVICES, *self.SELF_CAPTURE_SERVICES, *self.TELEMETRY_SERVICES])
        compose = f"docker compose -f {self.COMPOSE} -f {self.OVERRIDE}"
        report = "/tmp/hogbox-pipeline-health.txt"
        script = (
            f"cd {self.repo_dir} && sleep {settle_seconds}; exec > {report} 2>&1; "
            f"for svc in {services}; do "
            f'cid=$({compose} ps -aq "$svc"); '
            'if [ -z "$cid" ]; then echo "PIPELINE $svc missing"; continue; fi; '
            "state=$(docker inspect -f '{{.State.Status}} restarts={{.RestartCount}} exit={{.State.ExitCode}} oom={{.State.OOMKilled}}' \"$cid\"); "
            'echo "PIPELINE $svc $state"; '
            'case "$state" in "running restarts=0 "*) ;; *) docker logs --tail 25 "$cid" 2>&1 | cut -c1-400 | sed "s/^/  $svc | /";; esac; '
            "done; free -m | sed 's/^/  memory | /'"
        )
        try:
            self.backend.run_long(script, name="pipeline-health", timeout=settle_seconds + 300)
            sys.stderr.write(f"{self.backend.exec(f'cat {report}', timeout=60).stdout}\n")
        except RuntimeError as e:
            sys.stderr.write(f"[hogbox-preview] pipeline health report failed: {e}\n")

    def deep_health(self) -> None:
        # /_health is UNAUTHENTICATED — it passed the whole time previews were
        # 500ing on /api/projects/@current/ (the personhog drift), so "healthy"
        # meant "process is up", not "app is usable". This gate logs into the
        # seeded demo user and hits the endpoints that actually broke, so a
        # regression like that fails the bring-up instead of shipping a dead box.
        #
        # ALWAYS attempt the probe — including on --no-seed. The CI preview
        # workflow calls `up --no-seed` because the golden is pre-seeded, so the
        # demo user exists even when this run skipped the seed step; keying the
        # whole gate off seed_demo_data would disable it in exactly the
        # production path it was built for. The only concession to --no-seed is
        # a failed LOGIN (a genuinely unseeded box has no demo user): that
        # soft-skips with a note instead of failing. Any failure past login —
        # and a failed login on a seeded run — is fatal.
        with timing.span("deep-health"):
            self._run_authed_probe()

    def _run_authed_probe(self) -> None:
        # Everything runs INSIDE the box (curl against localhost:8000), so it's
        # independent of external networking — same posture as wait_http_ok. One
        # bash script does the whole login+probe flow with a shared cookie jar:
        #   1. GET  /login          -> seed the CSRF cookie
        #   2. POST /api/login/     -> authenticate the demo user
        #   3. GET  /api/projects/@current/               expect 200
        #   4. POST /api/environments/@current/query/     expect 200 (HogQL)
        # It prints "STEP <name> <http_code>" per step and the body of the first
        # non-2xx, so the Python side can raise with the exact failure.
        base = f"http://localhost:{self.backend.web_port}"
        script = f"""
set -u
jar=$(mktemp)
probe() {{ # name method path [json]
  name=$1; method=$2; path=$3; data=${{4:-}}
  if [ -n "$data" ]; then
    csrf=$(awk '/csrftoken/ {{print $7}}' "$jar" | tail -n1)
    code=$(curl -s -o /tmp/dh_body -w '%{{http_code}}' -m 30 -b "$jar" -c "$jar" \
      -X "$method" -H 'Content-Type: application/json' -H "X-CSRFToken: $csrf" \
      -H 'Referer: {base}/' -d "$data" "{base}$path")
  else
    code=$(curl -s -o /tmp/dh_body -w '%{{http_code}}' -m 30 -b "$jar" -c "$jar" "{base}$path")
  fi
  echo "STEP $name $code"
  case "$code" in 2*) ;; *) echo "BODY_START"; head -c 300 /tmp/dh_body; echo; echo "BODY_END"; return 1;; esac
}}
probe login GET /login || exit 1
probe api_login POST /api/login/ '{{"email":"{_DEMO_EMAIL}","password":"{_DEMO_PASSWORD}"}}' || exit 1
probe projects GET /api/projects/@current/ || exit 1
probe template GET /api/projects/@current/hog_function_templates/template-slack/ || exit 1
probe hogql POST /api/environments/@current/query/ '{{"query":{{"kind":"HogQLQuery","query":"select 1"}}}}' || exit 1
echo "DEEP_HEALTH_OK"
"""
        r = self.backend.exec(script, timeout=180)
        if "DEEP_HEALTH_OK" in r.stdout:
            timing.stage("deep health pass")
            return
        # Login failed on a run that skipped seeding: the box may genuinely have
        # no demo user (e.g. `up --no-seed` on a non-golden image), which isn't
        # an app-health failure. Everything past a successful login stays fatal.
        failed_step = ([ln.split()[1] for ln in r.stdout.splitlines() if ln.startswith("STEP ")] or ["?"])[-1]
        if not self.seed_demo_data and failed_step == "api_login":
            sys.stderr.write(
                "[hogbox-preview] deep health: demo login failed on a --no-seed run — "
                "assuming an unseeded box and skipping the authed probes\n"
            )
            timing.stage("deep health skipped (unseeded box)")
            return
        # Failed: surface the step + status + body, and the web log tail. The
        # missing traceback is what cost hours on 2026-07-06 — dump it here so
        # CI shows the Django error directly instead of just "not 200".
        logs = self.backend.exec("docker logs --tail 40 posthog-web-1 2>&1", timeout=60)
        raise RuntimeError(
            "deep health (authed api) failed — the app is up but not usable:\n"
            f"{r.stdout.strip()}\n{r.stderr.strip()}\n"
            f"--- docker logs --tail 40 posthog-web-1 ---\n{logs.stdout.strip()}"
        )

    # --- internal ------------------------------------------------------------
    def _compose(self, args: str) -> str:
        return f"cd {self.repo_dir} && docker compose -f {self.COMPOSE} -f {self.OVERRIDE} {args}"