:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(ci): restore newest schema dump on queue miss (#122755)
Raúl Negrón-Otero81596e9aa9e1
diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.ymlindex 136dfe8708a4..a3c934237325 100644--- a/.depot/workflows/ci-backend.yml+++ b/.depot/workflows/ci-backend.yml@@ -748,6 +748,8 @@ jobs:# postgres image is folded in so a server-version bump invalidates the key.# Non-Django migration frameworks are excluded from this Postgres schema dump.# SCHEMA_CACHE_EPOCH (workflow env) prefixes the key as a manual cache-bust knob.+ # SchemaInputs.cache_key in .github/scripts/master_schema_cache.py names the saved+ # dumps, so this hash must stay byte-identical to it.MIG_FILES=$(git ls-tree -r --format='%(objectname) %(path)' "$BASE_SHA" \| grep -E '/migrations/[^/]+\.py$|^[0-9a-f]+ uv\.lock$' \| grep -vE '/(clickhouse|async_migrations)/migrations/' \@@ -1790,39 +1792,35 @@ jobs:run: |UV_PROJECT_ENVIRONMENT=.venv-master timeout --kill-after=10s "$PY_DEPS_SYNC_TIMEOUT" uv sync --frozen --dev --no-install-package pytest-split- name: Compute master schema cache key- # Must match the migration-set computation in turbo-discover and the- # push-side save below (inputs, hash, and key prefix). Keyed off the- # master checkout (HEAD here) so a restored dump is exactly the state+ # Keyed off the master checkout (HEAD here), so an exact hit is the state# that "Run migrations up to master" would build from scratch.if: github.event_name != 'push'id: master-schema-key- working-directory: upstream-masterrun: |- MIG_FILES=$(git ls-tree -r --format='%(objectname) %(path)' HEAD \- | grep -E '/migrations/[^/]+\.py$|^[0-9a-f]+ uv\.lock$' \- | grep -vE '/(clickhouse|async_migrations)/migrations/' \- | LC_ALL=C sort) || true- # Routing decides which app labels are in the dump, so these inputs belong in the key.- DB_ROUTING=$(git ls-tree -r --format='%(objectname) %(path)' HEAD -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py)- PG_IMAGE=$(git show "HEAD:docker-compose.base.yml" 2>/dev/null \- | grep -m1 -E '^[[:space:]]*image:[[:space:]]*postgres:' | tr -d '[:space:]')- if [ -z "$MIG_FILES" ]; then- echo "key=" >> $GITHUB_OUTPUT- else- MIG_HASH=$(printf '%s\n%s\n%s' "$MIG_FILES" "$PG_IMAGE" "$DB_ROUTING" | sha256sum | cut -c1-40)- echo "key=posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-${MIG_HASH}" >> $GITHUB_OUTPUT- fi+ KEY=$(upstream-master/.venv-master/bin/python .github/scripts/master_schema_cache.py \+ --repository upstream-master --epoch "$SCHEMA_CACHE_EPOCH" key)+ echo "key=$KEY" >> "$GITHUB_OUTPUT"- name: Restore master schema dump from cache+ id: master-schema-cacheif: github.event_name != 'push' && steps.master-schema-key.outputs.key != ''uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3with:path: schema.sql.gzkey: ${{ steps.master-schema-key.outputs.key }}- # The exact key misses in the minutes between a master migration- # landing and its dump being saved. The PR base commit's dump is- # always older than the checkout, so migrate just tops it up.+ # The newest master snapshot is the one most likely to pass the check below.+ # A merge queue or stacked base has no saved dump, and an older PR base often+ # predates a uv.lock change that the check rejects.restore-keys: |- posthog-schema-master-${{ github.event.pull_request.base.sha }}+ posthog-schema-master-+ - name: Validate fallback master schema+ id: validate-master-schema+ if: steps.master-schema-cache.outputs.cache-matched-key != '' && steps.master-schema-cache.outputs.cache-matched-key != steps.master-schema-key.outputs.key+ env:+ CACHED_KEY: ${{ steps.master-schema-cache.outputs.cache-matched-key }}+ run: >-+ upstream-master/.venv-master/bin/python .github/scripts/master_schema_cache.py+ --repository upstream-master --epoch "$SCHEMA_CACHE_EPOCH"+ validate --dump schema.sql.gz --cache-key "$CACHED_KEY"- name: Wait for Docker servicesenv:COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml@@ -1830,8 +1828,8 @@ jobs:bin/ci-wait-for-docker wait --only db redis7 clickhouse \|| { echo "::group::compose launch log"; cat "${RUNNER_TEMP:-/tmp}/compose-launch.log" 2>/dev/null || true; echo "::endgroup::"; exit 1; }- name: Prime posthog DB from cached schema- # Turns "Run migrations up to master" below into a no-op instead- # of a full migration run.+ # Leaves "Run migrations up to master" below to apply only the migrations+ # the dump lacks, instead of a full migration run.if: github.event_name != 'push'run: |if [ ! -f schema.sql.gz ]; then@@ -2256,25 +2254,12 @@ jobs:gunzip -t schema.sql.gz- name: Compute migration-set schema key- # Must match the PR-base computation in turbo-discover and the- # master restore key above (inputs, hash, and key prefix).if: github.event_name == 'push'id: mig-schema-keyrun: |- MIG_FILES=$(git ls-tree -r --format='%(objectname) %(path)' HEAD \- | grep -E '/migrations/[^/]+\.py$|^[0-9a-f]+ uv\.lock$' \- | grep -vE '/(clickhouse|async_migrations)/migrations/' \- | LC_ALL=C sort) || true- # Routing decides which app labels are in the dump, so these inputs belong in the key.- DB_ROUTING=$(git ls-tree -r --format='%(objectname) %(path)' HEAD -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py)- PG_IMAGE=$(git show "HEAD:docker-compose.base.yml" 2>/dev/null \- | grep -m1 -E '^[[:space:]]*image:[[:space:]]*postgres:' | tr -d '[:space:]')- KEY=""- if [ -z "$MIG_FILES" ]; then- echo "::notice::no migration files matched — skipping migration-set schema save"- else- MIG_HASH=$(printf '%s\n%s\n%s' "$MIG_FILES" "$PG_IMAGE" "$DB_ROUTING" | sha256sum | cut -c1-40)- KEY="posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-${MIG_HASH}"+ KEY=$(python .github/scripts/master_schema_cache.py --repository . --epoch "$SCHEMA_CACHE_EPOCH" key)+ if [ -z "$KEY" ]; then+ echo "::notice::no migration files matched, so the migration-set schema save is skipped"fiecho "key=$KEY" >> $GITHUB_OUTPUTprintf '%s' "$KEY" > schema-cache-key@@ -2292,7 +2277,8 @@ jobs:# adds pages to the artifacts listing that db:download-schema walks.retention-days: 7- name: Save schema to Actions cache for PR shards- # Kept as the exact PR-base key that every consumer's restore-keys falls back to.+ # Consumers restore this key by their PR base or merge base. check-migrations+ # also restores the newest one by prefix and validates it first.if: github.event_name == 'push'uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3with:diff --git a/.github/scripts/master_schema_cache.py b/.github/scripts/master_schema_cache.pynew file mode 100644index 000000000000..933795dd5444--- /dev/null+++ b/.github/scripts/master_schema_cache.py@@ -0,0 +1,167 @@+import re+import sys+import hashlib+import argparse+import subprocess+from collections.abc import Mapping+from dataclasses import dataclass+from pathlib import Path++import yaml++WORKFLOW_PATH = ".depot/workflows/ci-backend.yml"+DB_ROUTING_PATHS = frozenset(+ {"products/db_routing.yaml", "posthog/product_db_config.py", "posthog/product_db_router.py"}+)+DJANGO_MIGRATION = re.compile(r"/migrations/[^/]+\.py$")+NON_DJANGO_MIGRATION = re.compile(r"/(clickhouse|async_migrations)/migrations/")+POSTGRES_IMAGE = re.compile(r"[ \t\n\v\f\r]*image:[ \t\n\v\f\r]*postgres:")+WHITESPACE = re.compile(r"[ \t\n\v\f\r]")+MASTER_SNAPSHOT_KEY = re.compile(r"posthog-schema-master-([0-9a-f]{40})")+++class GitError(Exception):+ pass+++class GitRepository:+ def __init__(self, path: Path) -> None:+ self.path = path++ def run(self, *args: str) -> str:+ try:+ return subprocess.run(+ ["git", *args], cwd=self.path, check=True, capture_output=True, text=True, timeout=60+ ).stdout+ except subprocess.CalledProcessError as error:+ raise GitError(f"git {args[0]} failed: {error.stderr.strip()}") from error++ def has_commit(self, revision: str) -> bool:+ probe = ["git", "cat-file", "-e", f"{revision}^{{commit}}"]+ return subprocess.run(probe, cwd=self.path, capture_output=True, timeout=60).returncode == 0++ def files(self, revision: str) -> dict[str, str]:+ files: dict[str, str] = {}+ for entry in self.run("ls-tree", "-r", "-z", "--format=%(objectname) %(path)", revision).split("\0"):+ if entry:+ blob, path = entry.split(" ", 1)+ files[path] = blob+ return files+++@dataclass(frozen=True, kw_only=True, slots=True)+class SchemaInputs:+ migrations: Mapping[str, str]+ uv_lock: str | None+ postgres_image: str+ db_routing: str++ @classmethod+ def read(cls, repository: GitRepository, revision: str) -> "SchemaInputs":+ files = repository.files(revision)+ compose = (+ repository.run("cat-file", "blob", files["docker-compose.base.yml"])+ if "docker-compose.base.yml" in files+ else ""+ )+ postgres_image = next((line for line in compose.split("\n") if POSTGRES_IMAGE.match(line)), "")+ return cls(+ migrations={+ path: blob+ for path, blob in files.items()+ if DJANGO_MIGRATION.search(path) and not NON_DJANGO_MIGRATION.search(path)+ },+ uv_lock=files.get("uv.lock"),+ postgres_image=WHITESPACE.sub("", postgres_image),+ db_routing="\n".join(f"{blob} {path}" for path, blob in files.items() if path in DB_ROUTING_PATHS),+ )++ def cache_key(self, epoch: str) -> str | None:+ # Other workflows compute this key in shell to restore the same dumps, so a change to+ # these bytes makes their exact restores miss.+ locked_files = [f"{blob} {path}" for path, blob in self.migrations.items()]+ if self.uv_lock is not None:+ locked_files.append(f"{self.uv_lock} uv.lock")+ if not locked_files:+ return None+ fingerprint = "\n".join(["\n".join(sorted(locked_files)), self.postgres_image, self.db_routing])+ return f"posthog-schema-mig-{epoch}-{hashlib.sha256(fingerprint.encode()).hexdigest()[:40]}"++ def incompatibility_with(self, checkout: "SchemaInputs") -> str | None:+ if self.postgres_image != checkout.postgres_image:+ return "Postgres image changed"+ if self.uv_lock != checkout.uv_lock:+ return "uv.lock changed"+ if self.db_routing != checkout.db_routing:+ return "database routing changed"+ # Reverts and squash retirement can change the meaning of recorded migrations.+ # New migrations are safe only when every cached migration still exists unchanged.+ for path, blob in self.migrations.items():+ if checkout.migrations.get(path) != blob:+ return f"cached migration changed or was removed: {path}"+ return None+++class MasterSchemaCache:+ def __init__(self, checkout: Path) -> None:+ self.repository = GitRepository(checkout)++ def key(self, *, epoch: str) -> str | None:+ return SchemaInputs.read(self.repository, "HEAD").cache_key(epoch)++ def validate(self, *, key: str, epoch: str, dump: Path) -> bool:+ try:+ reason = self._incompatibility_reason(key, epoch)+ # Discarding is always safe and a crash fails the job, so any error that stops+ # the proof of compatibility discards the dump.+ except Exception as error:+ reason = f"cannot read cached schema inputs: {type(error).__name__}: {error}"+ if reason is not None:+ # The restore step treats an existing dump as a cache hit.+ dump.unlink(missing_ok=True)+ sys.stdout.write(f"::notice::Discarded cached schema: {reason}; migrate will run from scratch\n")+ return False+ sys.stdout.write("Cached master schema is compatible; migrate will apply the missing migrations\n")+ return True++ def _incompatibility_reason(self, key: str, epoch: str) -> str | None:+ match = MASTER_SNAPSHOT_KEY.fullmatch(key)+ if match is None:+ return "cache key is not a master snapshot"+ source = match.group(1)+ if not self.repository.has_commit(source):+ # A --filter fetch would turn the checkout into a partial clone for every later step.+ # A depth-1 fetch only transfers the objects that HEAD's tree lacks.+ self.repository.run("fetch", "--no-tags", "--depth=1", "origin", source)+ if self._epoch(source) != epoch:+ return "cache epoch changed"+ return SchemaInputs.read(self.repository, source).incompatibility_with(+ SchemaInputs.read(self.repository, "HEAD")+ )++ def _epoch(self, revision: str) -> str | None:+ workflow = yaml.safe_load(self.repository.run("show", f"{revision}:{WORKFLOW_PATH}"))+ epoch = workflow.get("env", {}).get("SCHEMA_CACHE_EPOCH")+ return None if epoch is None else str(epoch)+++def main() -> None:+ parser = argparse.ArgumentParser(description="Compute and validate master schema dump cache keys")+ parser.add_argument("--repository", type=Path, required=True)+ parser.add_argument("--epoch", required=True)+ commands = parser.add_subparsers(dest="command", required=True)+ commands.add_parser("key", help="Print the migration-set cache key of HEAD, or an empty line")+ validate = commands.add_parser("validate", help="Discard a master snapshot that HEAD cannot migrate forward")+ validate.add_argument("--dump", type=Path, required=True)+ validate.add_argument("--cache-key", required=True)+ args = parser.parse_args()++ cache = MasterSchemaCache(args.repository)+ if args.command == "key":+ sys.stdout.write(f"{cache.key(epoch=args.epoch) or ''}\n")+ else:+ cache.validate(key=args.cache_key, epoch=args.epoch, dump=args.dump)+++if __name__ == "__main__":+ main()diff --git a/.github/scripts/test_master_schema_cache.py b/.github/scripts/test_master_schema_cache.pynew file mode 100644index 000000000000..6796be7ea8da--- /dev/null+++ b/.github/scripts/test_master_schema_cache.py@@ -0,0 +1,120 @@+import subprocess+from pathlib import Path++import pytest++from master_schema_cache import MasterSchemaCache+++def git(repository: Path, *args: str) -> str:+ return subprocess.check_output(["git", "-C", str(repository), *args], text=True).strip()+++def repository(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:+ monkeypatch.setenv("GIT_CONFIG_GLOBAL", "/dev/null")+ monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1")+ git(tmp_path, "init", "-q")+ git(tmp_path, "config", "user.name", "Schema check")+ git(tmp_path, "config", "user.email", "[email protected]")+ git(tmp_path, "config", "core.hooksPath", "/dev/null")+ for filename, content in {+ "posthog/migrations/0001_initial.py": "initial = True",+ "posthog/migrations/0002_squashed.py": "replaces = [('posthog', '0001_initial')]",+ "posthog/clickhouse/migrations/0001_initial.py": "initial = True",+ "uv.lock": "version = 1",+ "docker-compose.base.yml": "services:\n db:\n image: postgres:15\n",+ "products/db_routing.yaml": "{}",+ "posthog/product_db_config.py": "config = {}",+ "posthog/product_db_router.py": "router = None",+ ".depot/workflows/ci-backend.yml": "env:\n SCHEMA_CACHE_EPOCH: 'v2' # bump to abandon cached dumps\n",+ }.items():+ path = tmp_path / filename+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(content)+ git(tmp_path, "add", ".")+ git(tmp_path, "commit", "-qm", "baseline")+ git(tmp_path, "config", "uploadpack.allowFilter", "true")+ git(tmp_path, "remote", "add", "origin", str(tmp_path))+ return tmp_path+++def test_key_matches_the_shell_key_other_workflows_restore(repository: Path) -> None:+ assert (+ MasterSchemaCache(repository).key(epoch="v2")+ == "posthog-schema-mig-v2-52d5a77f1c27d98c132143f281c4cde90dd58d74"+ )+++ "filename,content,keep",+ [+ pytest.param(None, "", True, id="identical-schema-inputs"),+ pytest.param("posthog/migrations/0003_column.py", "new_column = True", True, id="added-column"),+ pytest.param("posthog/migrations/0001_initial.py", None, False, id="reverted-migration"),+ pytest.param("posthog/migrations/0002_squashed.py", "replaces = []", False, id="retired-squash"),+ pytest.param("products/db_routing.yaml", "new_product: separate", False, id="routing-change"),+ pytest.param(+ "docker-compose.base.yml", "services:\n db:\n image: postgres:16\n", False, id="postgres-change"+ ),+ pytest.param(+ "docker-compose.base.yml",+ "services:\n db:\n image: postgres:15\n redis:\n image: redis:7\n",+ True,+ id="unrelated-compose-change",+ ),+ pytest.param("uv.lock", "version = 2", False, id="dependency-change"),+ pytest.param("posthog/models.py", "unrelated = True", True, id="unrelated-source-change"),+ pytest.param("posthog/clickhouse/migrations/0001_initial.py", "changed = True", True, id="clickhouse-change"),+ ],+)+def test_validate_keeps_only_compatible_dumps(+ repository: Path, filename: str | None, content: str | None, keep: bool+) -> None:+ source = git(repository, "rev-parse", "HEAD")+ if filename is not None:+ path = repository / filename+ if content is None:+ path.unlink()+ else:+ path.write_text(content)+ git(repository, "add", ".")+ git(repository, "commit", "--allow-empty", "-qm", "checkout")+ dump = repository / "schema.sql.gz"+ dump.write_text("cached dump")++ assert MasterSchemaCache(repository).validate(key=f"posthog-schema-master-{source}", epoch="v2", dump=dump) is keep+ assert dump.exists() is keep+++ "key,epoch",+ [+ pytest.param("posthog-schema-mig-v2-unmerged", "v2", id="unmerged-pr-cache"),+ pytest.param(f"posthog-schema-master-{'f' * 40}", "v2", id="unavailable-source"),+ pytest.param(None, "v3", id="different-epoch"),+ ],+)+def test_validate_discards_unusable_snapshots(repository: Path, key: str | None, epoch: str) -> None:+ if key is None:+ key = f"posthog-schema-master-{git(repository, 'rev-parse', 'HEAD')}"+ dump = repository / "schema.sql.gz"+ dump.write_text("cached dump")++ assert MasterSchemaCache(repository).validate(key=key, epoch=epoch, dump=dump) is False+ assert not dump.exists()+++def test_validate_fetches_source_in_shallow_checkout(repository: Path, tmp_path: Path) -> None:+ source = git(repository, "rev-parse", "HEAD")+ (repository / "posthog/migrations/0003_column.py").write_text("new_column = True")+ git(repository, "add", ".")+ git(repository, "commit", "-qm", "add column")+ checkout = tmp_path / "checkout"+ git(repository, "clone", "-q", "--depth=1", repository.as_uri(), str(checkout))+ dump = repository / "schema.sql.gz"+ dump.write_text("cached dump")++ assert MasterSchemaCache(checkout).validate(key=f"posthog-schema-master-{source}", epoch="v2", dump=dump) is True+ assert dump.exists()+ assert "promisor" not in git(checkout, "config", "--list", "--local")diff --git a/tools/workflow-plan/tests/workflows.test.ts b/tools/workflow-plan/tests/workflows.test.tsindex 1976bad9508d..c200b67fde62 100644--- a/tools/workflow-plan/tests/workflows.test.ts+++ b/tools/workflow-plan/tests/workflows.test.ts@@ -451,6 +451,27 @@ const pathsFilterJobs = workflowFiles.flatMap((file) =>)const STEP_EXPECTATIONS: StepExpectation[] = [+ ...[+ { name: 'exact schema hit', matched: 'posthog-schema-mig-v2-exact', runs: false },+ { name: 'schema cache miss', matched: '', runs: false },+ { name: 'master snapshot fallback', matched: `posthog-schema-master-${'a'.repeat(40)}`, runs: true },+ ].map(({ name, matched, runs }) => ({+ file: DEPOT_BACKEND,+ job: 'check-migrations',+ step: 'validate-master-schema',+ scenario: {+ name,+ github: mergeQueue(),+ steps: {+ ...allFiltersChanged(workflow(DEPOT_BACKEND)),+ 'check-migrations': {+ 'master-schema-key': { outputs: { key: 'posthog-schema-mig-v2-exact' } },+ 'master-schema-cache': { outputs: { 'cache-matched-key': matched } },+ },+ },+ },+ runs,+ })),...pathsFilterJobs.flatMap(({ file, job }) => [{file,