Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ci): seed migration defaults after migrate, not during restore (#123430)

Raúl Negrón-Oterofa281fe1f596
diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.yml
index 0cfe6ef8741b..caf1d31c0931 100644
--- a/.depot/workflows/ci-backend.yml
+++ b/.depot/workflows/ci-backend.yml
@@ -1840,10 +1840,9 @@ jobs:
fi
# The load is atomic: on failure the transaction rolls back and
# the fallback below recreates an empty DB for a full migrate.
- # ensure_migration_defaults re-seeds data-migration rows the
+ # The migrate in the next step seeds the data-migration rows the
# schema-only dump lacks.
- if gunzip -c schema.sql.gz | docker compose -f docker-compose.dev.yml exec -T db psql -q -v ON_ERROR_STOP=1 --single-transaction -U posthog posthog > /dev/null \
- && (cd upstream-master && .venv-master/bin/python manage.py ensure_migration_defaults); then
+ if gunzip -c schema.sql.gz | docker compose -f docker-compose.dev.yml exec -T db psql -q -v ON_ERROR_STOP=1 --single-transaction -U posthog posthog > /dev/null; then
echo "Primed posthog DB from cached master schema"
else
echo "::warning::Cached schema restore failed — recreating empty DB; the next step runs all migrations from scratch"
diff --git a/.github/workflows/cd-sandbox-base-image.yml b/.github/workflows/cd-sandbox-base-image.yml
index a6b1a64a8e48..8a00ac9556df 100644
--- a/.github/workflows/cd-sandbox-base-image.yml
+++ b/.github/workflows/cd-sandbox-base-image.yml
@@ -426,9 +426,9 @@ jobs:
${{ steps.schema-key.outputs.prefix_key }}
- name: Prime posthog from cached schema
- # db:restore-schema-fresh restores the dump and seeds the RunPython data a
- # schema-only dump lacks; the migrate step below layers anything newer on
- # top. On failure it leaves a fresh empty db, so migrate runs the full history.
+ # db:restore-schema-fresh restores the dump. The migrate step below layers
+ # anything newer on top and seeds the RunPython data a schema-only dump
+ # lacks. On failure it leaves a fresh empty db, so migrate runs the full history.
env:
SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da'
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
diff --git a/.github/workflows/ci-agent-skills.yml b/.github/workflows/ci-agent-skills.yml
index 149a6a436e8b..c7339da94030 100644
--- a/.github/workflows/ci-agent-skills.yml
+++ b/.github/workflows/ci-agent-skills.yml
@@ -244,9 +244,9 @@ jobs:
${{ steps.schema-key.outputs.prefix_key }}
- name: Prime posthog from cached schema
- # db:restore-schema-fresh restores the dump and seeds the RunPython data a
- # schema-only dump lacks; the migrate step below layers anything newer on
- # top. On failure it leaves a fresh empty db, so migrate runs the full history.
+ # db:restore-schema-fresh restores the dump. The migrate step below layers
+ # anything newer on top and seeds the RunPython data a schema-only dump
+ # lacks. On failure it leaves a fresh empty db, so migrate runs the full history.
env:
SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da'
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
diff --git a/.github/workflows/ci-e2e-playwright.yml b/.github/workflows/ci-e2e-playwright.yml
index 5b4b24e656c8..bf6334afcc72 100644
--- a/.github/workflows/ci-e2e-playwright.yml
+++ b/.github/workflows/ci-e2e-playwright.yml
@@ -834,10 +834,9 @@ jobs:
- name: Prime posthog_e2e_test from cached schema
# Schema dump is from master's `posthog` db but schemas are db-name agnostic.
- # Hogli's db:restore-schema-fresh DROPs+CREATEs the target db, restores the
- # schema, and runs ensure_migration_defaults to seed RunPython data that's
- # missing from a schema-only dump. The migrate step below still runs and
- # layers any newer migrations on top — the cache just skips the bulk of
+ # Hogli's db:restore-schema-fresh DROPs+CREATEs the target db and restores the
+ # schema. The migrate step below layers any newer migrations on top and
+ # seeds the RunPython data a schema-only dump lacks. The cache just skips the bulk of
# the historical migration work. Treat cache/restore problems as misses:
# restore-schema-fresh leaves a fresh empty db on failure, so the migrate
# step below runs the full history.
diff --git a/.github/workflows/ci-rust-flags-integration.yml b/.github/workflows/ci-rust-flags-integration.yml
index a1d109f72918..c1cb19ca48e4 100644
--- a/.github/workflows/ci-rust-flags-integration.yml
+++ b/.github/workflows/ci-rust-flags-integration.yml
@@ -205,11 +205,11 @@ jobs:
# The default shell is `bash -e`, so without this psql's exit code hides a failed
# gunzip and a truncated dump commits as a half-primed schema.
set -o pipefail
+ # The dump is schema-only. The migrate in the next step seeds the rows that
+ # data migrations write, after it adds any column the dump predates.
prime() {
gunzip -c schema.sql.gz \
| psql -q -v ON_ERROR_STOP=1 --single-transaction test_posthog || return 1
- # The dump is schema-only, so rows written by data migrations are absent.
- python manage.py ensure_migration_defaults || return 1
}
if [ ! -f schema.sql.gz ]; then
echo "::notice::schema cache miss, so migrate runs from scratch"
diff --git a/tools/hogli-commands/hogli_commands/db_schema.py b/tools/hogli-commands/hogli_commands/db_schema.py
index 3245f95b1c98..59221d43ba7d 100644
--- a/tools/hogli-commands/hogli_commands/db_schema.py
+++ b/tools/hogli-commands/hogli_commands/db_schema.py
@@ -176,18 +176,6 @@ def _recreate_database(target_db: str) -> None:
_psql_admin(f"CREATE DATABASE {target_db};")
-def _manage(target_db: str, *args: str) -> None:
- """Run a management command against target_db."""
- _run(
- ["python", "manage.py", *args],
- env={"DATABASE_URL": f"postgres://posthog:posthog@localhost:5432/{target_db}"},
- )
-
-
-def _ensure_migration_defaults(target_db: str) -> None:
- _manage(target_db, "ensure_migration_defaults")
-
-
def _psql_write(target_db: str, sql: str) -> None:
"""Run a one-shot statement against target_db."""
_run(
@@ -247,8 +235,12 @@ def restore_schema_dump(
target_db: str,
recreate: bool,
schema_path: Path = LOCAL_SCHEMA_PATH,
- ensure_defaults: bool = True,
) -> None:
+ """Load a schema dump. The caller must run `migrate` next, which seeds the rows the dump lacks.
+
+ Do not seed here: the seed writes through the checkout's models, so a dump that predates a
+ model column fails the restore with `UndefinedColumn` and forces a full migrate.
+ """
target_db = _validate_db_identifier(target_db)
resolved_schema_path = _resolve_repo_path(schema_path)
if not resolved_schema_path.is_file():
@@ -262,9 +254,6 @@ def restore_schema_dump(
try:
_run_psql_with_gzip_input(resolved_schema_path, target_db)
_forget_product_app_migrations(target_db)
-
- if ensure_defaults:
- _ensure_migration_defaults(target_db)
except Exception as exc:
if recreate:
try:
@@ -532,7 +521,7 @@ def restore_schema_if_fresh(*, target_db: str, mode: ArtifactMode) -> bool:
click.echo(f"Database {target_db} is no longer empty; running migrations normally")
return False
- restore_schema_dump(target_db=target_db, recreate=True, ensure_defaults=True)
+ restore_schema_dump(target_db=target_db, recreate=True)
return True
@@ -633,7 +622,7 @@ def db_download_schema(base_branch: str | None) -> None:
def _recreate_test_db() -> None:
try:
- restore_schema_dump(target_db=os.environ.get("TARGET_DB", "test_posthog"), recreate=True, ensure_defaults=True)
+ restore_schema_dump(target_db=os.environ.get("TARGET_DB", "test_posthog"), recreate=True)
except Exception as exc:
raise click.ClickException(str(exc)) from exc
@@ -682,8 +671,7 @@ def db_restore_schema(yes: bool) -> None:
try:
download_latest_compatible_schema(base_branch=_effective_base_branch(None))
_create_postgres_backup()
- restore_schema_dump(target_db="posthog", recreate=False, ensure_defaults=False)
+ restore_schema_dump(target_db="posthog", recreate=False)
_run([str(REPO_ROOT / "bin" / "migrate")])
- _ensure_migration_defaults("posthog")
except Exception as exc:
raise click.ClickException(str(exc)) from exc
diff --git a/tools/hogli-commands/hogli_commands/tests/test_db_schema.py b/tools/hogli-commands/hogli_commands/tests/test_db_schema.py
index fbbd1355537a..c8d9b0c6650c 100644
--- a/tools/hogli-commands/hogli_commands/tests/test_db_schema.py
+++ b/tools/hogli-commands/hogli_commands/tests/test_db_schema.py
@@ -297,7 +297,7 @@ def test_restore_schema_if_fresh_recreates_empty_db(monkeypatch: pytest.MonkeyPa
monkeypatch.setattr(db_schema, "restore_schema_dump", lambda **kwargs: restore_calls.append(kwargs))
assert db_schema.restore_schema_if_fresh(target_db="posthog", mode="auto") is True
- assert restore_calls == [{"target_db": "posthog", "recreate": True, "ensure_defaults": True}]
+ assert restore_calls == [{"target_db": "posthog", "recreate": True}]
def test_restore_schema_if_fresh_skips_when_db_populated_after_download(
@@ -364,7 +364,6 @@ def test_restore_schema_dump_recreate_drops_and_creates(tmp_path: Path, monkeypa
_write_schema(schema_path)
commands: list[list[str]] = []
restored: list[tuple[Path, str]] = []
- defaults: list[str] = []
monkeypatch.setattr(db_schema, "_run", lambda command, env=None: commands.append(command))
monkeypatch.setattr(
@@ -372,14 +371,13 @@ def test_restore_schema_dump_recreate_drops_and_creates(tmp_path: Path, monkeypa
"_run_psql_with_gzip_input",
lambda gzip_path, target_db: restored.append((gzip_path, target_db)),
)
- monkeypatch.setattr(db_schema, "_ensure_migration_defaults", lambda target_db: defaults.append(target_db))
db_schema.restore_schema_dump(target_db="test_posthog", recreate=True, schema_path=schema_path)
assert any("DROP DATABASE IF EXISTS test_posthog;" in command for call in commands for command in call)
assert any("CREATE DATABASE test_posthog;" in command for call in commands for command in call)
assert restored == [(schema_path, "test_posthog")]
- assert defaults == ["test_posthog"]
+ assert not [command for command in commands if command[:2] == ["python", "manage.py"]]
def test_restore_schema_dump_forgets_product_app_migrations(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
@@ -393,7 +391,6 @@ def test_restore_schema_dump_forgets_product_app_migrations(tmp_path: Path, monk
monkeypatch.setattr(db_schema, "_run", lambda command, env=None: commands.append(command))
monkeypatch.setattr(db_schema, "_run_psql_with_gzip_input", lambda gzip_path, target_db: None)
- monkeypatch.setattr(db_schema, "_ensure_migration_defaults", lambda target_db: None)
monkeypatch.setattr(db_schema, "_product_routed_app_labels", lambda: ["stamphog", "visual_review"])
db_schema.restore_schema_dump(target_db="test_posthog", recreate=False, schema_path=schema_path)
@@ -414,11 +411,6 @@ def test_restore_schema_dump_recreate_cleans_up_after_failure(tmp_path: Path, mo
"_run_psql_with_gzip_input",
lambda gzip_path, target_db: (_ for _ in ()).throw(db_schema.SchemaRestoreError("restore failed")),
)
- monkeypatch.setattr(
- db_schema,
- "_ensure_migration_defaults",
- lambda target_db: pytest.fail("defaults should not run after restore failure"),
- )
with pytest.raises(db_schema.SchemaRestoreError, match="restore failed"):
db_schema.restore_schema_dump(target_db="test_posthog", recreate=True, schema_path=schema_path)
@@ -445,7 +437,6 @@ def test_restore_schema_dump_without_recreate_does_not_drop(tmp_path: Path, monk
monkeypatch.setattr(db_schema, "_run", lambda command, env=None: commands.append(command))
monkeypatch.setattr(db_schema, "_run_psql_with_gzip_input", lambda gzip_path, target_db: None)
- monkeypatch.setattr(db_schema, "_ensure_migration_defaults", lambda target_db: None)
db_schema.restore_schema_dump(target_db="posthog", recreate=False, schema_path=schema_path)
@@ -499,7 +490,7 @@ def test_restore_schema_fresh_recreates_target_db(monkeypatch: pytest.MonkeyPatc
result = runner.invoke(db_schema.db_restore_schema_fresh)
assert result.exit_code == 0
- assert calls == [{"target_db": "posthog_e2e_test", "recreate": True, "ensure_defaults": True}]
+ assert calls == [{"target_db": "posthog_e2e_test", "recreate": True}]
@pytest.mark.parametrize(