Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(ci): restore newest schema dump on queue miss (#122755)

Raúl Negrón-Otero81596e9aa9e1
diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.yml
index 136dfe8708a4..a3c934237325 100644
--- a/.depot/workflows/ci-backend.yml
+++ b/.depot/workflows/ci-backend.yml
@@ -748,6 +748,8 @@ jobs:
# postgres image is folded in so a server-version bump invalidates the key.
# Non-Django migration frameworks are excluded from this Postgres schema dump.
# SCHEMA_CACHE_EPOCH (workflow env) prefixes the key as a manual cache-bust knob.
+ # SchemaInputs.cache_key in .github/scripts/master_schema_cache.py names the saved
+ # dumps, so this hash must stay byte-identical to it.
MIG_FILES=$(git ls-tree -r --format='%(objectname) %(path)' "$BASE_SHA" \
| grep -E '/migrations/[^/]+\.py$|^[0-9a-f]+ uv\.lock$' \
| grep -vE '/(clickhouse|async_migrations)/migrations/' \
@@ -1790,39 +1792,35 @@ jobs:
run: |
UV_PROJECT_ENVIRONMENT=.venv-master timeout --kill-after=10s "$PY_DEPS_SYNC_TIMEOUT" uv sync --frozen --dev --no-install-package pytest-split
- name: Compute master schema cache key
- # Must match the migration-set computation in turbo-discover and the
- # push-side save below (inputs, hash, and key prefix). Keyed off the
- # master checkout (HEAD here) so a restored dump is exactly the state
+ # Keyed off the master checkout (HEAD here), so an exact hit is the state
# that "Run migrations up to master" would build from scratch.
if: github.event_name != 'push'
id: master-schema-key
- working-directory: upstream-master
run: |
- MIG_FILES=$(git ls-tree -r --format='%(objectname) %(path)' HEAD \
- | grep -E '/migrations/[^/]+\.py$|^[0-9a-f]+ uv\.lock$' \
- | grep -vE '/(clickhouse|async_migrations)/migrations/' \
- | LC_ALL=C sort) || true
- # Routing decides which app labels are in the dump, so these inputs belong in the key.
- DB_ROUTING=$(git ls-tree -r --format='%(objectname) %(path)' HEAD -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py)
- PG_IMAGE=$(git show "HEAD:docker-compose.base.yml" 2>/dev/null \
- | grep -m1 -E '^[[:space:]]*image:[[:space:]]*postgres:' | tr -d '[:space:]')
- if [ -z "$MIG_FILES" ]; then
- echo "key=" >> $GITHUB_OUTPUT
- else
- MIG_HASH=$(printf '%s\n%s\n%s' "$MIG_FILES" "$PG_IMAGE" "$DB_ROUTING" | sha256sum | cut -c1-40)
- echo "key=posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-${MIG_HASH}" >> $GITHUB_OUTPUT
- fi
+ KEY=$(upstream-master/.venv-master/bin/python .github/scripts/master_schema_cache.py \
+ --repository upstream-master --epoch "$SCHEMA_CACHE_EPOCH" key)
+ echo "key=$KEY" >> "$GITHUB_OUTPUT"
- name: Restore master schema dump from cache
+ id: master-schema-cache
if: github.event_name != 'push' && steps.master-schema-key.outputs.key != ''
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: schema.sql.gz
key: ${{ steps.master-schema-key.outputs.key }}
- # The exact key misses in the minutes between a master migration
- # landing and its dump being saved. The PR base commit's dump is
- # always older than the checkout, so migrate just tops it up.
+ # The newest master snapshot is the one most likely to pass the check below.
+ # A merge queue or stacked base has no saved dump, and an older PR base often
+ # predates a uv.lock change that the check rejects.
restore-keys: |
- posthog-schema-master-${{ github.event.pull_request.base.sha }}
+ posthog-schema-master-
+ - name: Validate fallback master schema
+ id: validate-master-schema
+ if: steps.master-schema-cache.outputs.cache-matched-key != '' && steps.master-schema-cache.outputs.cache-matched-key != steps.master-schema-key.outputs.key
+ env:
+ CACHED_KEY: ${{ steps.master-schema-cache.outputs.cache-matched-key }}
+ run: >-
+ upstream-master/.venv-master/bin/python .github/scripts/master_schema_cache.py
+ --repository upstream-master --epoch "$SCHEMA_CACHE_EPOCH"
+ validate --dump schema.sql.gz --cache-key "$CACHED_KEY"
- name: Wait for Docker services
env:
COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
@@ -1830,8 +1828,8 @@ jobs:
bin/ci-wait-for-docker wait --only db redis7 clickhouse \
|| { echo "::group::compose launch log"; cat "${RUNNER_TEMP:-/tmp}/compose-launch.log" 2>/dev/null || true; echo "::endgroup::"; exit 1; }
- name: Prime posthog DB from cached schema
- # Turns "Run migrations up to master" below into a no-op instead
- # of a full migration run.
+ # Leaves "Run migrations up to master" below to apply only the migrations
+ # the dump lacks, instead of a full migration run.
if: github.event_name != 'push'
run: |
if [ ! -f schema.sql.gz ]; then
@@ -2256,25 +2254,12 @@ jobs:
gunzip -t schema.sql.gz
- name: Compute migration-set schema key
- # Must match the PR-base computation in turbo-discover and the
- # master restore key above (inputs, hash, and key prefix).
if: github.event_name == 'push'
id: mig-schema-key
run: |
- MIG_FILES=$(git ls-tree -r --format='%(objectname) %(path)' HEAD \
- | grep -E '/migrations/[^/]+\.py$|^[0-9a-f]+ uv\.lock$' \
- | grep -vE '/(clickhouse|async_migrations)/migrations/' \
- | LC_ALL=C sort) || true
- # Routing decides which app labels are in the dump, so these inputs belong in the key.
- DB_ROUTING=$(git ls-tree -r --format='%(objectname) %(path)' HEAD -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py)
- PG_IMAGE=$(git show "HEAD:docker-compose.base.yml" 2>/dev/null \
- | grep -m1 -E '^[[:space:]]*image:[[:space:]]*postgres:' | tr -d '[:space:]')
- KEY=""
- if [ -z "$MIG_FILES" ]; then
- echo "::notice::no migration files matched — skipping migration-set schema save"
- else
- MIG_HASH=$(printf '%s\n%s\n%s' "$MIG_FILES" "$PG_IMAGE" "$DB_ROUTING" | sha256sum | cut -c1-40)
- KEY="posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-${MIG_HASH}"
+ KEY=$(python .github/scripts/master_schema_cache.py --repository . --epoch "$SCHEMA_CACHE_EPOCH" key)
+ if [ -z "$KEY" ]; then
+ echo "::notice::no migration files matched, so the migration-set schema save is skipped"
fi
echo "key=$KEY" >> $GITHUB_OUTPUT
printf '%s' "$KEY" > schema-cache-key
@@ -2292,7 +2277,8 @@ jobs:
# adds pages to the artifacts listing that db:download-schema walks.
retention-days: 7
- name: Save schema to Actions cache for PR shards
- # Kept as the exact PR-base key that every consumer's restore-keys falls back to.
+ # Consumers restore this key by their PR base or merge base. check-migrations
+ # also restores the newest one by prefix and validates it first.
if: github.event_name == 'push'
uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
diff --git a/.github/scripts/master_schema_cache.py b/.github/scripts/master_schema_cache.py
new file mode 100644
index 000000000000..933795dd5444
--- /dev/null
+++ b/.github/scripts/master_schema_cache.py
@@ -0,0 +1,167 @@
+import re
+import sys
+import hashlib
+import argparse
+import subprocess
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+WORKFLOW_PATH = ".depot/workflows/ci-backend.yml"
+DB_ROUTING_PATHS = frozenset(
+ {"products/db_routing.yaml", "posthog/product_db_config.py", "posthog/product_db_router.py"}
+)
+DJANGO_MIGRATION = re.compile(r"/migrations/[^/]+\.py$")
+NON_DJANGO_MIGRATION = re.compile(r"/(clickhouse|async_migrations)/migrations/")
+POSTGRES_IMAGE = re.compile(r"[ \t\n\v\f\r]*image:[ \t\n\v\f\r]*postgres:")
+WHITESPACE = re.compile(r"[ \t\n\v\f\r]")
+MASTER_SNAPSHOT_KEY = re.compile(r"posthog-schema-master-([0-9a-f]{40})")
+
+
+class GitError(Exception):
+ pass
+
+
+class GitRepository:
+ def __init__(self, path: Path) -> None:
+ self.path = path
+
+ def run(self, *args: str) -> str:
+ try:
+ return subprocess.run(
+ ["git", *args], cwd=self.path, check=True, capture_output=True, text=True, timeout=60
+ ).stdout
+ except subprocess.CalledProcessError as error:
+ raise GitError(f"git {args[0]} failed: {error.stderr.strip()}") from error
+
+ def has_commit(self, revision: str) -> bool:
+ probe = ["git", "cat-file", "-e", f"{revision}^{{commit}}"]
+ return subprocess.run(probe, cwd=self.path, capture_output=True, timeout=60).returncode == 0
+
+ def files(self, revision: str) -> dict[str, str]:
+ files: dict[str, str] = {}
+ for entry in self.run("ls-tree", "-r", "-z", "--format=%(objectname) %(path)", revision).split("\0"):
+ if entry:
+ blob, path = entry.split(" ", 1)
+ files[path] = blob
+ return files
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class SchemaInputs:
+ migrations: Mapping[str, str]
+ uv_lock: str | None
+ postgres_image: str
+ db_routing: str
+
+ @classmethod
+ def read(cls, repository: GitRepository, revision: str) -> "SchemaInputs":
+ files = repository.files(revision)
+ compose = (
+ repository.run("cat-file", "blob", files["docker-compose.base.yml"])
+ if "docker-compose.base.yml" in files
+ else ""
+ )
+ postgres_image = next((line for line in compose.split("\n") if POSTGRES_IMAGE.match(line)), "")
+ return cls(
+ migrations={
+ path: blob
+ for path, blob in files.items()
+ if DJANGO_MIGRATION.search(path) and not NON_DJANGO_MIGRATION.search(path)
+ },
+ uv_lock=files.get("uv.lock"),
+ postgres_image=WHITESPACE.sub("", postgres_image),
+ db_routing="\n".join(f"{blob} {path}" for path, blob in files.items() if path in DB_ROUTING_PATHS),
+ )
+
+ def cache_key(self, epoch: str) -> str | None:
+ # Other workflows compute this key in shell to restore the same dumps, so a change to
+ # these bytes makes their exact restores miss.
+ locked_files = [f"{blob} {path}" for path, blob in self.migrations.items()]
+ if self.uv_lock is not None:
+ locked_files.append(f"{self.uv_lock} uv.lock")
+ if not locked_files:
+ return None
+ fingerprint = "\n".join(["\n".join(sorted(locked_files)), self.postgres_image, self.db_routing])
+ return f"posthog-schema-mig-{epoch}-{hashlib.sha256(fingerprint.encode()).hexdigest()[:40]}"
+
+ def incompatibility_with(self, checkout: "SchemaInputs") -> str | None:
+ if self.postgres_image != checkout.postgres_image:
+ return "Postgres image changed"
+ if self.uv_lock != checkout.uv_lock:
+ return "uv.lock changed"
+ if self.db_routing != checkout.db_routing:
+ return "database routing changed"
+ # Reverts and squash retirement can change the meaning of recorded migrations.
+ # New migrations are safe only when every cached migration still exists unchanged.
+ for path, blob in self.migrations.items():
+ if checkout.migrations.get(path) != blob:
+ return f"cached migration changed or was removed: {path}"
+ return None
+
+
+class MasterSchemaCache:
+ def __init__(self, checkout: Path) -> None:
+ self.repository = GitRepository(checkout)
+
+ def key(self, *, epoch: str) -> str | None:
+ return SchemaInputs.read(self.repository, "HEAD").cache_key(epoch)
+
+ def validate(self, *, key: str, epoch: str, dump: Path) -> bool:
+ try:
+ reason = self._incompatibility_reason(key, epoch)
+ # Discarding is always safe and a crash fails the job, so any error that stops
+ # the proof of compatibility discards the dump.
+ except Exception as error:
+ reason = f"cannot read cached schema inputs: {type(error).__name__}: {error}"
+ if reason is not None:
+ # The restore step treats an existing dump as a cache hit.
+ dump.unlink(missing_ok=True)
+ sys.stdout.write(f"::notice::Discarded cached schema: {reason}; migrate will run from scratch\n")
+ return False
+ sys.stdout.write("Cached master schema is compatible; migrate will apply the missing migrations\n")
+ return True
+
+ def _incompatibility_reason(self, key: str, epoch: str) -> str | None:
+ match = MASTER_SNAPSHOT_KEY.fullmatch(key)
+ if match is None:
+ return "cache key is not a master snapshot"
+ source = match.group(1)
+ if not self.repository.has_commit(source):
+ # A --filter fetch would turn the checkout into a partial clone for every later step.
+ # A depth-1 fetch only transfers the objects that HEAD's tree lacks.
+ self.repository.run("fetch", "--no-tags", "--depth=1", "origin", source)
+ if self._epoch(source) != epoch:
+ return "cache epoch changed"
+ return SchemaInputs.read(self.repository, source).incompatibility_with(
+ SchemaInputs.read(self.repository, "HEAD")
+ )
+
+ def _epoch(self, revision: str) -> str | None:
+ workflow = yaml.safe_load(self.repository.run("show", f"{revision}:{WORKFLOW_PATH}"))
+ epoch = workflow.get("env", {}).get("SCHEMA_CACHE_EPOCH")
+ return None if epoch is None else str(epoch)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description="Compute and validate master schema dump cache keys")
+ parser.add_argument("--repository", type=Path, required=True)
+ parser.add_argument("--epoch", required=True)
+ commands = parser.add_subparsers(dest="command", required=True)
+ commands.add_parser("key", help="Print the migration-set cache key of HEAD, or an empty line")
+ validate = commands.add_parser("validate", help="Discard a master snapshot that HEAD cannot migrate forward")
+ validate.add_argument("--dump", type=Path, required=True)
+ validate.add_argument("--cache-key", required=True)
+ args = parser.parse_args()
+
+ cache = MasterSchemaCache(args.repository)
+ if args.command == "key":
+ sys.stdout.write(f"{cache.key(epoch=args.epoch) or ''}\n")
+ else:
+ cache.validate(key=args.cache_key, epoch=args.epoch, dump=args.dump)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.github/scripts/test_master_schema_cache.py b/.github/scripts/test_master_schema_cache.py
new file mode 100644
index 000000000000..6796be7ea8da
--- /dev/null
+++ b/.github/scripts/test_master_schema_cache.py
@@ -0,0 +1,120 @@
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from master_schema_cache import MasterSchemaCache
+
+
+def git(repository: Path, *args: str) -> str:
+ return subprocess.check_output(["git", "-C", str(repository), *args], text=True).strip()
+
+
+def repository(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+ monkeypatch.setenv("GIT_CONFIG_GLOBAL", "/dev/null")
+ monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1")
+ git(tmp_path, "init", "-q")
+ git(tmp_path, "config", "user.name", "Schema check")
+ git(tmp_path, "config", "user.email", "[email protected]")
+ git(tmp_path, "config", "core.hooksPath", "/dev/null")
+ for filename, content in {
+ "posthog/migrations/0001_initial.py": "initial = True",
+ "posthog/migrations/0002_squashed.py": "replaces = [('posthog', '0001_initial')]",
+ "posthog/clickhouse/migrations/0001_initial.py": "initial = True",
+ "uv.lock": "version = 1",
+ "docker-compose.base.yml": "services:\n db:\n image: postgres:15\n",
+ "products/db_routing.yaml": "{}",
+ "posthog/product_db_config.py": "config = {}",
+ "posthog/product_db_router.py": "router = None",
+ ".depot/workflows/ci-backend.yml": "env:\n SCHEMA_CACHE_EPOCH: 'v2' # bump to abandon cached dumps\n",
+ }.items():
+ path = tmp_path / filename
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(content)
+ git(tmp_path, "add", ".")
+ git(tmp_path, "commit", "-qm", "baseline")
+ git(tmp_path, "config", "uploadpack.allowFilter", "true")
+ git(tmp_path, "remote", "add", "origin", str(tmp_path))
+ return tmp_path
+
+
+def test_key_matches_the_shell_key_other_workflows_restore(repository: Path) -> None:
+ assert (
+ MasterSchemaCache(repository).key(epoch="v2")
+ == "posthog-schema-mig-v2-52d5a77f1c27d98c132143f281c4cde90dd58d74"
+ )
+
+
+ "filename,content,keep",
+ [
+ pytest.param(None, "", True, id="identical-schema-inputs"),
+ pytest.param("posthog/migrations/0003_column.py", "new_column = True", True, id="added-column"),
+ pytest.param("posthog/migrations/0001_initial.py", None, False, id="reverted-migration"),
+ pytest.param("posthog/migrations/0002_squashed.py", "replaces = []", False, id="retired-squash"),
+ pytest.param("products/db_routing.yaml", "new_product: separate", False, id="routing-change"),
+ pytest.param(
+ "docker-compose.base.yml", "services:\n db:\n image: postgres:16\n", False, id="postgres-change"
+ ),
+ pytest.param(
+ "docker-compose.base.yml",
+ "services:\n db:\n image: postgres:15\n redis:\n image: redis:7\n",
+ True,
+ id="unrelated-compose-change",
+ ),
+ pytest.param("uv.lock", "version = 2", False, id="dependency-change"),
+ pytest.param("posthog/models.py", "unrelated = True", True, id="unrelated-source-change"),
+ pytest.param("posthog/clickhouse/migrations/0001_initial.py", "changed = True", True, id="clickhouse-change"),
+ ],
+)
+def test_validate_keeps_only_compatible_dumps(
+ repository: Path, filename: str | None, content: str | None, keep: bool
+) -> None:
+ source = git(repository, "rev-parse", "HEAD")
+ if filename is not None:
+ path = repository / filename
+ if content is None:
+ path.unlink()
+ else:
+ path.write_text(content)
+ git(repository, "add", ".")
+ git(repository, "commit", "--allow-empty", "-qm", "checkout")
+ dump = repository / "schema.sql.gz"
+ dump.write_text("cached dump")
+
+ assert MasterSchemaCache(repository).validate(key=f"posthog-schema-master-{source}", epoch="v2", dump=dump) is keep
+ assert dump.exists() is keep
+
+
+ "key,epoch",
+ [
+ pytest.param("posthog-schema-mig-v2-unmerged", "v2", id="unmerged-pr-cache"),
+ pytest.param(f"posthog-schema-master-{'f' * 40}", "v2", id="unavailable-source"),
+ pytest.param(None, "v3", id="different-epoch"),
+ ],
+)
+def test_validate_discards_unusable_snapshots(repository: Path, key: str | None, epoch: str) -> None:
+ if key is None:
+ key = f"posthog-schema-master-{git(repository, 'rev-parse', 'HEAD')}"
+ dump = repository / "schema.sql.gz"
+ dump.write_text("cached dump")
+
+ assert MasterSchemaCache(repository).validate(key=key, epoch=epoch, dump=dump) is False
+ assert not dump.exists()
+
+
+def test_validate_fetches_source_in_shallow_checkout(repository: Path, tmp_path: Path) -> None:
+ source = git(repository, "rev-parse", "HEAD")
+ (repository / "posthog/migrations/0003_column.py").write_text("new_column = True")
+ git(repository, "add", ".")
+ git(repository, "commit", "-qm", "add column")
+ checkout = tmp_path / "checkout"
+ git(repository, "clone", "-q", "--depth=1", repository.as_uri(), str(checkout))
+ dump = repository / "schema.sql.gz"
+ dump.write_text("cached dump")
+
+ assert MasterSchemaCache(checkout).validate(key=f"posthog-schema-master-{source}", epoch="v2", dump=dump) is True
+ assert dump.exists()
+ assert "promisor" not in git(checkout, "config", "--list", "--local")
diff --git a/tools/workflow-plan/tests/workflows.test.ts b/tools/workflow-plan/tests/workflows.test.ts
index 1976bad9508d..c200b67fde62 100644
--- a/tools/workflow-plan/tests/workflows.test.ts
+++ b/tools/workflow-plan/tests/workflows.test.ts
@@ -451,6 +451,27 @@ const pathsFilterJobs = workflowFiles.flatMap((file) =>
)
const STEP_EXPECTATIONS: StepExpectation[] = [
+ ...[
+ { name: 'exact schema hit', matched: 'posthog-schema-mig-v2-exact', runs: false },
+ { name: 'schema cache miss', matched: '', runs: false },
+ { name: 'master snapshot fallback', matched: `posthog-schema-master-${'a'.repeat(40)}`, runs: true },
+ ].map(({ name, matched, runs }) => ({
+ file: DEPOT_BACKEND,
+ job: 'check-migrations',
+ step: 'validate-master-schema',
+ scenario: {
+ name,
+ github: mergeQueue(),
+ steps: {
+ ...allFiltersChanged(workflow(DEPOT_BACKEND)),
+ 'check-migrations': {
+ 'master-schema-key': { outputs: { key: 'posthog-schema-mig-v2-exact' } },
+ 'master-schema-cache': { outputs: { 'cache-matched-key': matched } },
+ },
+ },
+ },
+ runs,
+ })),
...pathsFilterJobs.flatMap(({ file, job }) => [
{
file,