:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ci): seed migration defaults after migrate, not during restore (#123430)
Raúl Negrón-Oterofa281fe1f596
diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.ymlindex 0cfe6ef8741b..caf1d31c0931 100644--- a/.depot/workflows/ci-backend.yml+++ b/.depot/workflows/ci-backend.yml@@ -1840,10 +1840,9 @@ jobs:fi# The load is atomic: on failure the transaction rolls back and# the fallback below recreates an empty DB for a full migrate.- # ensure_migration_defaults re-seeds data-migration rows the+ # The migrate in the next step seeds the data-migration rows the# schema-only dump lacks.- if gunzip -c schema.sql.gz | docker compose -f docker-compose.dev.yml exec -T db psql -q -v ON_ERROR_STOP=1 --single-transaction -U posthog posthog > /dev/null \- && (cd upstream-master && .venv-master/bin/python manage.py ensure_migration_defaults); then+ if gunzip -c schema.sql.gz | docker compose -f docker-compose.dev.yml exec -T db psql -q -v ON_ERROR_STOP=1 --single-transaction -U posthog posthog > /dev/null; thenecho "Primed posthog DB from cached master schema"elseecho "::warning::Cached schema restore failed — recreating empty DB; the next step runs all migrations from scratch"diff --git a/.github/workflows/cd-sandbox-base-image.yml b/.github/workflows/cd-sandbox-base-image.ymlindex a6b1a64a8e48..8a00ac9556df 100644--- a/.github/workflows/cd-sandbox-base-image.yml+++ b/.github/workflows/cd-sandbox-base-image.yml@@ -426,9 +426,9 @@ jobs:${{ steps.schema-key.outputs.prefix_key }}- name: Prime posthog from cached schema- # db:restore-schema-fresh restores the dump and seeds the RunPython data a- # schema-only dump lacks; the migrate step below layers anything newer on- # top. On failure it leaves a fresh empty db, so migrate runs the full history.+ # db:restore-schema-fresh restores the dump. The migrate step below layers+ # anything newer on top and seeds the RunPython data a schema-only dump+ # lacks. On failure it leaves a fresh empty db, so migrate runs the full history.env:SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da'DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'diff --git a/.github/workflows/ci-agent-skills.yml b/.github/workflows/ci-agent-skills.ymlindex 149a6a436e8b..c7339da94030 100644--- a/.github/workflows/ci-agent-skills.yml+++ b/.github/workflows/ci-agent-skills.yml@@ -244,9 +244,9 @@ jobs:${{ steps.schema-key.outputs.prefix_key }}- name: Prime posthog from cached schema- # db:restore-schema-fresh restores the dump and seeds the RunPython data a- # schema-only dump lacks; the migrate step below layers anything newer on- # top. On failure it leaves a fresh empty db, so migrate runs the full history.+ # db:restore-schema-fresh restores the dump. The migrate step below layers+ # anything newer on top and seeds the RunPython data a schema-only dump+ # lacks. On failure it leaves a fresh empty db, so migrate runs the full history.env:SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da'DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'diff --git a/.github/workflows/ci-e2e-playwright.yml b/.github/workflows/ci-e2e-playwright.ymlindex 5b4b24e656c8..bf6334afcc72 100644--- a/.github/workflows/ci-e2e-playwright.yml+++ b/.github/workflows/ci-e2e-playwright.yml@@ -834,10 +834,9 @@ jobs:- name: Prime posthog_e2e_test from cached schema# Schema dump is from master's `posthog` db but schemas are db-name agnostic.- # Hogli's db:restore-schema-fresh DROPs+CREATEs the target db, restores the- # schema, and runs ensure_migration_defaults to seed RunPython data that's- # missing from a schema-only dump. The migrate step below still runs and- # layers any newer migrations on top — the cache just skips the bulk of+ # Hogli's db:restore-schema-fresh DROPs+CREATEs the target db and restores the+ # schema. The migrate step below layers any newer migrations on top and+ # seeds the RunPython data a schema-only dump lacks. The cache just skips the bulk of# the historical migration work. Treat cache/restore problems as misses:# restore-schema-fresh leaves a fresh empty db on failure, so the migrate# step below runs the full history.diff --git a/.github/workflows/ci-rust-flags-integration.yml b/.github/workflows/ci-rust-flags-integration.ymlindex a1d109f72918..c1cb19ca48e4 100644--- a/.github/workflows/ci-rust-flags-integration.yml+++ b/.github/workflows/ci-rust-flags-integration.yml@@ -205,11 +205,11 @@ jobs:# The default shell is `bash -e`, so without this psql's exit code hides a failed# gunzip and a truncated dump commits as a half-primed schema.set -o pipefail+ # The dump is schema-only. The migrate in the next step seeds the rows that+ # data migrations write, after it adds any column the dump predates.prime() {gunzip -c schema.sql.gz \| psql -q -v ON_ERROR_STOP=1 --single-transaction test_posthog || return 1- # The dump is schema-only, so rows written by data migrations are absent.- python manage.py ensure_migration_defaults || return 1}if [ ! -f schema.sql.gz ]; thenecho "::notice::schema cache miss, so migrate runs from scratch"diff --git a/tools/hogli-commands/hogli_commands/db_schema.py b/tools/hogli-commands/hogli_commands/db_schema.pyindex 3245f95b1c98..59221d43ba7d 100644--- a/tools/hogli-commands/hogli_commands/db_schema.py+++ b/tools/hogli-commands/hogli_commands/db_schema.py@@ -176,18 +176,6 @@ def _recreate_database(target_db: str) -> None:_psql_admin(f"CREATE DATABASE {target_db};")-def _manage(target_db: str, *args: str) -> None:- """Run a management command against target_db."""- _run(- ["python", "manage.py", *args],- env={"DATABASE_URL": f"postgres://posthog:posthog@localhost:5432/{target_db}"},- )---def _ensure_migration_defaults(target_db: str) -> None:- _manage(target_db, "ensure_migration_defaults")--def _psql_write(target_db: str, sql: str) -> None:"""Run a one-shot statement against target_db."""_run(@@ -247,8 +235,12 @@ def restore_schema_dump(target_db: str,recreate: bool,schema_path: Path = LOCAL_SCHEMA_PATH,- ensure_defaults: bool = True,) -> None:+ """Load a schema dump. The caller must run `migrate` next, which seeds the rows the dump lacks.++ Do not seed here: the seed writes through the checkout's models, so a dump that predates a+ model column fails the restore with `UndefinedColumn` and forces a full migrate.+ """target_db = _validate_db_identifier(target_db)resolved_schema_path = _resolve_repo_path(schema_path)if not resolved_schema_path.is_file():@@ -262,9 +254,6 @@ def restore_schema_dump(try:_run_psql_with_gzip_input(resolved_schema_path, target_db)_forget_product_app_migrations(target_db)-- if ensure_defaults:- _ensure_migration_defaults(target_db)except Exception as exc:if recreate:try:@@ -532,7 +521,7 @@ def restore_schema_if_fresh(*, target_db: str, mode: ArtifactMode) -> bool:click.echo(f"Database {target_db} is no longer empty; running migrations normally")return False- restore_schema_dump(target_db=target_db, recreate=True, ensure_defaults=True)+ restore_schema_dump(target_db=target_db, recreate=True)return True@@ -633,7 +622,7 @@ def db_download_schema(base_branch: str | None) -> None:def _recreate_test_db() -> None:try:- restore_schema_dump(target_db=os.environ.get("TARGET_DB", "test_posthog"), recreate=True, ensure_defaults=True)+ restore_schema_dump(target_db=os.environ.get("TARGET_DB", "test_posthog"), recreate=True)except Exception as exc:raise click.ClickException(str(exc)) from exc@@ -682,8 +671,7 @@ def db_restore_schema(yes: bool) -> None:try:download_latest_compatible_schema(base_branch=_effective_base_branch(None))_create_postgres_backup()- restore_schema_dump(target_db="posthog", recreate=False, ensure_defaults=False)+ restore_schema_dump(target_db="posthog", recreate=False)_run([str(REPO_ROOT / "bin" / "migrate")])- _ensure_migration_defaults("posthog")except Exception as exc:raise click.ClickException(str(exc)) from excdiff --git a/tools/hogli-commands/hogli_commands/tests/test_db_schema.py b/tools/hogli-commands/hogli_commands/tests/test_db_schema.pyindex fbbd1355537a..c8d9b0c6650c 100644--- a/tools/hogli-commands/hogli_commands/tests/test_db_schema.py+++ b/tools/hogli-commands/hogli_commands/tests/test_db_schema.py@@ -297,7 +297,7 @@ def test_restore_schema_if_fresh_recreates_empty_db(monkeypatch: pytest.MonkeyPamonkeypatch.setattr(db_schema, "restore_schema_dump", lambda **kwargs: restore_calls.append(kwargs))assert db_schema.restore_schema_if_fresh(target_db="posthog", mode="auto") is True- assert restore_calls == [{"target_db": "posthog", "recreate": True, "ensure_defaults": True}]+ assert restore_calls == [{"target_db": "posthog", "recreate": True}]def test_restore_schema_if_fresh_skips_when_db_populated_after_download(@@ -364,7 +364,6 @@ def test_restore_schema_dump_recreate_drops_and_creates(tmp_path: Path, monkeypa_write_schema(schema_path)commands: list[list[str]] = []restored: list[tuple[Path, str]] = []- defaults: list[str] = []monkeypatch.setattr(db_schema, "_run", lambda command, env=None: commands.append(command))monkeypatch.setattr(@@ -372,14 +371,13 @@ def test_restore_schema_dump_recreate_drops_and_creates(tmp_path: Path, monkeypa"_run_psql_with_gzip_input",lambda gzip_path, target_db: restored.append((gzip_path, target_db)),)- monkeypatch.setattr(db_schema, "_ensure_migration_defaults", lambda target_db: defaults.append(target_db))db_schema.restore_schema_dump(target_db="test_posthog", recreate=True, schema_path=schema_path)assert any("DROP DATABASE IF EXISTS test_posthog;" in command for call in commands for command in call)assert any("CREATE DATABASE test_posthog;" in command for call in commands for command in call)assert restored == [(schema_path, "test_posthog")]- assert defaults == ["test_posthog"]+ assert not [command for command in commands if command[:2] == ["python", "manage.py"]]def test_restore_schema_dump_forgets_product_app_migrations(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:@@ -393,7 +391,6 @@ def test_restore_schema_dump_forgets_product_app_migrations(tmp_path: Path, monkmonkeypatch.setattr(db_schema, "_run", lambda command, env=None: commands.append(command))monkeypatch.setattr(db_schema, "_run_psql_with_gzip_input", lambda gzip_path, target_db: None)- monkeypatch.setattr(db_schema, "_ensure_migration_defaults", lambda target_db: None)monkeypatch.setattr(db_schema, "_product_routed_app_labels", lambda: ["stamphog", "visual_review"])db_schema.restore_schema_dump(target_db="test_posthog", recreate=False, schema_path=schema_path)@@ -414,11 +411,6 @@ def test_restore_schema_dump_recreate_cleans_up_after_failure(tmp_path: Path, mo"_run_psql_with_gzip_input",lambda gzip_path, target_db: (_ for _ in ()).throw(db_schema.SchemaRestoreError("restore failed")),)- monkeypatch.setattr(- db_schema,- "_ensure_migration_defaults",- lambda target_db: pytest.fail("defaults should not run after restore failure"),- )with pytest.raises(db_schema.SchemaRestoreError, match="restore failed"):db_schema.restore_schema_dump(target_db="test_posthog", recreate=True, schema_path=schema_path)@@ -445,7 +437,6 @@ def test_restore_schema_dump_without_recreate_does_not_drop(tmp_path: Path, monkmonkeypatch.setattr(db_schema, "_run", lambda command, env=None: commands.append(command))monkeypatch.setattr(db_schema, "_run_psql_with_gzip_input", lambda gzip_path, target_db: None)- monkeypatch.setattr(db_schema, "_ensure_migration_defaults", lambda target_db: None)db_schema.restore_schema_dump(target_db="posthog", recreate=False, schema_path=schema_path)@@ -499,7 +490,7 @@ def test_restore_schema_fresh_recreates_target_db(monkeypatch: pytest.MonkeyPatcresult = runner.invoke(db_schema.db_restore_schema_fresh)assert result.exit_code == 0- assert calls == [{"target_db": "posthog_e2e_test", "recreate": True, "ensure_defaults": True}]+ assert calls == [{"target_db": "posthog_e2e_test", "recreate": True}]@pytest.mark.parametrize(