Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ci): flag fields removed from cached query models (#124050)

Vasco de Krijger064959ad8b5b
diff --git a/.github/scripts/check_cached_schema_removals.py b/.github/scripts/check_cached_schema_removals.py
new file mode 100644
index 000000000000..306e7d228338
--- /dev/null
+++ b/.github/scripts/check_cached_schema_removals.py
@@ -0,0 +1,257 @@
+#!/usr/bin/env python3
+# ruff: noqa: T201 allow print statements
+"""
+Flag fields removed from the models that a cached query response stores.
+
+PostHog stores each query response, with its modifiers, in the query cache.
+The generated models reject unknown keys, so a cached entry that still holds a removed field
+fails to parse and counts as a cache miss.
+This script compares two copies of frontend/src/queries/schema.json and lists those removals.
+It checks named definitions and the objects inline in them, and follows a $ref when only one side has it.
+It looks through a nullable wrapper (an anyOf of one type and null) and skips every other anyOf or oneOf union.
+
+Usage:
+ check_cached_schema_removals.py BASE_SCHEMA HEAD_SCHEMA
+
+Environment:
+ PR_LABELS The pull request labels, one per line. A cache-impact label acknowledges a removal.
+ GITHUB_OUTPUT Optional. Receives "removed", "acknowledged", "status", "summary" and "body".
+ The last three describe the CI report section. Without it, the script prints them.
+"""
+
+import os
+import sys
+import json
+import uuid
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+ACKNOWLEDGEMENT_LABELS = {
+ "cache-impact-deprecated": "deprecated",
+ "cache-impact-accepted": "accepted",
+ "cache-impact-bumped": "bumped",
+}
+CACHED_ROOT_PREFIX = "Cached"
+DEFINITION_REF_PREFIX = "#/definitions/"
+NULL_BRANCH = {"type": "null"}
+
+SECTION_BODY = """\
+PostHog stores each query response, with its modifiers, in the query cache.
+The generated models reject unknown keys.
+When PostHog reads a cached entry that holds a removed field, the entry fails to parse, counts as a cache miss, and sends one exception to error tracking.
+
+**Removed from `frontend/src/queries/schema.json`:**
+
+{removals}
+
+**Choose one option.**
+Apply the matching label to the PR, then re-run the failed `Frontend typechecking` job. A label does not start CI by itself.
+
+| Option | Label | What happens | Cost |
+| --- | --- | --- | --- |
+| Keep the field as deprecated | `cache-impact-deprecated` | The field stays in the schema with `@deprecated`, and the code stops writing it to cached responses. A later PR removes it after the entries written before that expire (up to 7 days). | No cache misses, if nothing writes the field during the wait. Needs a second PR. |
+| Accept the misses | `cache-impact-accepted` | The field is removed now. | Each cached entry that holds the field is a miss until the query runs again or the entry expires (up to 7 days). A caller that calculates on a miss waits for ClickHouse. ClickHouse load and the exception rate go up, and the cache hit rate drops. Use this only when the field did not change query results. |
+| Bump the cache version | `cache-impact-bumped` | Change `"version"` in `get_cache_payload` in `posthog/hogql_queries/query_runner.py`. | Every cached query result for every team is a miss, not only the entries that hold the field. Use this only when the field changed query results and old entries must not be served. |
+
+Not sure which option fits your change? Ask in [this Slack channel](https://posthog.slack.com/archives/C076E99B152). The team is happy to help.
+
+{status}
+"""
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class Removal:
+ model: str
+ location: str
+ kind: Literal["property", "enum value"]
+ name: str
+
+ def describe(self) -> str:
+ where = f"`{self.model}`"
+ if self.location:
+ where += f" at `{self.location}`"
+ return f"{where}: {self.kind} `{self.name}`"
+
+
+def reference_name(node: Any) -> str | None:
+ ref = node.get("$ref") if isinstance(node, dict) else None
+ if isinstance(ref, str) and ref.startswith(DEFINITION_REF_PREFIX):
+ return ref.removeprefix(DEFINITION_REF_PREFIX)
+ return None
+
+
+def referenced_definitions(node: Any) -> Iterator[str]:
+ if isinstance(node, dict):
+ ref = reference_name(node)
+ if ref is not None:
+ yield ref
+ for value in node.values():
+ yield from referenced_definitions(value)
+ elif isinstance(node, list):
+ for item in node:
+ yield from referenced_definitions(item)
+
+
+def allowed_values(node: dict[str, Any]) -> list[Any] | None:
+ if isinstance(node.get("enum"), list):
+ return node["enum"]
+ if "const" in node:
+ return [node["const"]]
+ return None
+
+
+def without_null(node: Any) -> Any:
+ branches = node.get("anyOf") if isinstance(node, dict) else None
+ if isinstance(branches, list) and NULL_BRANCH in branches:
+ others = [branch for branch in branches if branch != NULL_BRANCH]
+ if len(others) == 1:
+ return others[0]
+ return node
+
+
+def is_union(node: dict[str, Any]) -> bool:
+ return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list)
+
+
+def cached_definitions(definitions: dict[str, Any]) -> set[str]:
+ pending = [name for name in definitions if name.startswith(CACHED_ROOT_PREFIX)]
+ seen: set[str] = set()
+ while pending:
+ name = pending.pop()
+ if name in seen or name not in definitions:
+ continue
+ seen.add(name)
+ pending.extend(referenced_definitions(definitions[name]))
+ return seen
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class RefPair:
+ base: str
+ head: str
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class SchemaPair:
+ base_definitions: dict[str, Any]
+ head_definitions: dict[str, Any]
+ compared: set[RefPair] = field(default_factory=set)
+
+ def removed_fields(self, model: str, base: Any, head: Any, location: str) -> Iterator[Removal]:
+ base = without_null(base)
+ head = without_null(head)
+ base_ref = reference_name(base)
+ head_ref = reference_name(head)
+ if base_ref is not None and base_ref == head_ref:
+ return
+ if base_ref is not None and head_ref is not None:
+ ref_pair = RefPair(base=base_ref, head=head_ref)
+ if ref_pair in self.compared:
+ return
+ self.compared.add(ref_pair)
+ if base_ref is not None or head_ref is not None:
+ base = self.base_definitions.get(base_ref) if base_ref is not None else base
+ head = self.head_definitions.get(head_ref) if head_ref is not None else head
+ if not isinstance(base, dict) or not isinstance(head, dict):
+ return
+ if is_union(base) or is_union(head):
+ return
+ base_values = allowed_values(base)
+ head_values = allowed_values(head)
+ if base_values is not None and head_values is not None:
+ for value in base_values:
+ if value not in head_values:
+ yield Removal(model=model, location=location, kind="enum value", name=str(value))
+ for key, base_value in base.items():
+ head_value = head.get(key)
+ if key == "properties" and isinstance(base_value, dict):
+ head_properties = head_value if isinstance(head_value, dict) else {}
+ for name, base_property in base_value.items():
+ child_location = f"{location}.{name}" if location else name
+ if name not in head_properties:
+ yield Removal(model=model, location=location, kind="property", name=name)
+ else:
+ yield from self.removed_fields(model, base_property, head_properties[name], child_location)
+ elif isinstance(base_value, dict):
+ yield from self.removed_fields(model, base_value, head_value, location)
+
+
+def find_removals(base_schema: dict[str, Any], head_schema: dict[str, Any]) -> list[Removal]:
+ pair = SchemaPair(
+ base_definitions=base_schema.get("definitions", {}), head_definitions=head_schema.get("definitions", {})
+ )
+ removals: list[Removal] = []
+ for name in sorted(cached_definitions(pair.base_definitions)):
+ if name in pair.head_definitions:
+ removals.extend(pair.removed_fields(name, pair.base_definitions[name], pair.head_definitions[name], ""))
+ return removals
+
+
+def find_acknowledgement(labels: list[str]) -> str | None:
+ for label, choice in ACKNOWLEDGEMENT_LABELS.items():
+ if label in labels:
+ return choice
+ return None
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class Section:
+ status: Literal["warn", "alert"]
+ summary: str
+ body: str
+
+
+def render_body(removals: list[Removal], status: str) -> str:
+ return SECTION_BODY.format(removals="\n".join(f"- {removal.describe()}" for removal in removals), status=status)
+
+
+def render_section(removals: list[Removal], acknowledgement: str | None) -> Section:
+ count = f"{len(removals)} removed field{'' if len(removals) == 1 else 's'}"
+ if acknowledgement:
+ status = f"Acknowledged with the `cache-impact-{acknowledgement}` label."
+ return Section(
+ status="warn",
+ summary=f"{count}, acknowledged ({acknowledgement})",
+ body=render_body(removals, status),
+ )
+ status = "Not acknowledged yet. `Frontend typechecking` fails until the PR has one of these labels."
+ return Section(status="alert", summary=f"{count}, not acknowledged", body=render_body(removals, status))
+
+
+def write_outputs(removals: list[Removal], acknowledgement: str | None, section: Section | None) -> None:
+ fields = {
+ "removed": str(bool(removals)).lower(),
+ "acknowledged": str(acknowledgement is not None).lower(),
+ "status": section.status if section else "ok",
+ "summary": section.summary if section else "no removed fields",
+ }
+ lines = [f"{name}={value}\n" for name, value in fields.items()]
+ if section:
+ delimiter = f"EOF-{uuid.uuid4()}"
+ lines.append(f"body<<{delimiter}\n{section.body}\n{delimiter}\n")
+ output_path = os.environ.get("GITHUB_OUTPUT")
+ if output_path:
+ with Path(output_path).open("a") as output:
+ output.writelines(lines)
+ else:
+ print("".join(lines), end="")
+
+
+def main(argv: list[str]) -> int:
+ base_path, head_path = argv[1:3]
+ removals = find_removals(json.loads(Path(base_path).read_text()), json.loads(Path(head_path).read_text()))
+ acknowledgement = find_acknowledgement(os.environ.get("PR_LABELS", "").splitlines())
+ section = render_section(removals, acknowledgement) if removals else None
+
+ for removal in removals:
+ print(f"Removed: {removal.describe()}")
+ if not removals:
+ print("No fields removed from cached response models.")
+ write_outputs(removals, acknowledgement, section)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/.github/scripts/post-cached-schema-section.mjs b/.github/scripts/post-cached-schema-section.mjs
new file mode 100644
index 000000000000..a6cc4784bde3
--- /dev/null
+++ b/.github/scripts/post-cached-schema-section.mjs
@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+import { pathToFileURL } from 'node:url'
+
+import { clearSectionIfPresent, postSection } from '../../frontend/bin/ci-report/update-ci-report.mjs'
+
+const SECTION_ID = 'cached-schema-removals'
+
+async function main() {
+ const { REMOVED, STATUS, SUMMARY, BODY } = process.env
+ if (REMOVED === 'true') {
+ await postSection({ id: SECTION_ID, status: STATUS, summary: SUMMARY, body: BODY })
+ return
+ }
+ await clearSectionIfPresent({
+ id: SECTION_ID,
+ summary: 'no removed fields',
+ body: 'No fields were removed from the models that cached query responses store.',
+ })
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ await main()
+}
diff --git a/.github/scripts/test_check_cached_schema_removals.py b/.github/scripts/test_check_cached_schema_removals.py
new file mode 100644
index 000000000000..a10534bcfe17
--- /dev/null
+++ b/.github/scripts/test_check_cached_schema_removals.py
@@ -0,0 +1,159 @@
+import importlib.util
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+SCRIPT_PATH = Path(__file__).with_name("check_cached_schema_removals.py")
+SPEC = importlib.util.spec_from_file_location("check_cached_schema_removals", SCRIPT_PATH)
+assert SPEC is not None
+assert SPEC.loader is not None
+check_cached_schema_removals = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(check_cached_schema_removals)
+
+ROOT = {"CachedFooResponse": {"properties": {"modifiers": {"$ref": "#/definitions/Modifiers"}}}}
+
+
+def schema(**definitions: Any) -> dict[str, Any]:
+ return {"definitions": {**ROOT, **definitions}}
+
+
+ "base,head,expected",
+ [
+ pytest.param(
+ schema(Modifiers={"properties": {"keep": {}, "gone": {}}}),
+ schema(Modifiers={"properties": {"keep": {}}}),
+ ["`Modifiers`: property `gone`"],
+ id="property-removed-from-model-reached-through-a-ref",
+ ),
+ pytest.param(
+ schema(Modifiers={"properties": {"mode": {"enum": ["a", "b"]}}}),
+ schema(Modifiers={"properties": {"mode": {"enum": ["a"]}}}),
+ ["`Modifiers` at `mode`: enum value `b`"],
+ id="enum-value-removed",
+ ),
+ pytest.param(
+ schema(Modifiers={"properties": {"mode": {"enum": ["a", "b"]}}}),
+ schema(Modifiers={"properties": {"mode": {"const": "a"}}}),
+ ["`Modifiers` at `mode`: enum value `b`"],
+ id="enum-narrowed-to-a-single-const",
+ ),
+ pytest.param(
+ schema(Modifiers={"properties": {"mode": {"const": "a"}}}),
+ schema(Modifiers={"properties": {"mode": {"const": "b"}}}),
+ ["`Modifiers` at `mode`: enum value `a`"],
+ id="const-value-changed",
+ ),
+ pytest.param(
+ schema(Modifiers={"properties": {"inner": {"properties": {"gone": {}}}}}),
+ schema(Modifiers={"properties": {"inner": {"properties": {}}}}),
+ ["`Modifiers` at `inner`: property `gone`"],
+ id="property-removed-from-inline-object",
+ ),
+ pytest.param(
+ schema(Modifiers={"properties": {"inner": {"properties": {"kept": {}}}}}),
+ schema(
+ Modifiers={"properties": {"inner": {"$ref": "#/definitions/Inner"}}}, Inner={"properties": {"kept": {}}}
+ ),
+ [],
+ id="inline-object-extracted-to-a-ref-unchanged",
+ ),
+ pytest.param(
+ schema(Modifiers={"properties": {"inner": {"properties": {"kept": {}, "gone": {}}}}}),
+ schema(
+ Modifiers={"properties": {"inner": {"$ref": "#/definitions/Inner"}}}, Inner={"properties": {"kept": {}}}
+ ),
+ ["`Modifiers` at `inner`: property `gone`"],
+ id="inline-object-extracted-to-a-ref-with-a-removal",
+ ),
+ pytest.param(
+ schema(Modifiers={"properties": {"a": {"properties": {"y": {"properties": {"gone": {}}}}}}}),
+ schema(
+ Modifiers={"properties": {"a": {"$ref": "#/definitions/Inner"}}},
+ Inner={"properties": {"y": {"$ref": "#/definitions/Inner"}}},
+ ),
+ ["`Modifiers` at `a.y`: property `gone`"],
+ id="inline-object-extracted-to-a-recursive-ref",
+ ),
+ pytest.param(
+ schema(
+ Modifiers={"properties": {"tree": {"$ref": "#/definitions/Old"}}},
+ Old={"properties": {"gone": {}, "child": {"$ref": "#/definitions/Old"}}},
+ ),
+ schema(
+ Modifiers={"properties": {"tree": {"$ref": "#/definitions/New"}}},
+ New={"properties": {"child": {"$ref": "#/definitions/New"}}},
+ ),
+ ["`Modifiers` at `tree`: property `gone`"],
+ id="renamed-recursive-model",
+ ),
+ pytest.param(
+ schema(
+ Modifiers={"properties": {"a": {"$ref": "#/definitions/Old"}, "b": {"$ref": "#/definitions/Old"}}},
+ Old={"properties": {"kept": {}, "gone": {}}},
+ ),
+ schema(
+ Modifiers={"properties": {"a": {"$ref": "#/definitions/New"}, "b": {"$ref": "#/definitions/New"}}},
+ New={"properties": {"kept": {}}},
+ ),
+ ["`Modifiers` at `a`: property `gone`"],
+ id="renamed-model-reached-twice-is-reported-once",
+ ),
+ pytest.param(
+ schema(Modifiers={"properties": {"inner": {"properties": {"kept": {}}}}}),
+ schema(Modifiers={"properties": {"inner": {"anyOf": [{"properties": {"kept": {}}}, {"type": "null"}]}}}),
+ [],
+ id="object-made-nullable-unchanged",
+ ),
+ pytest.param(
+ schema(Modifiers={"properties": {"inner": {"properties": {"kept": {}, "gone": {}}}}}),
+ schema(Modifiers={"properties": {"inner": {"anyOf": [{"properties": {"kept": {}}}, {"type": "null"}]}}}),
+ ["`Modifiers` at `inner`: property `gone`"],
+ id="object-made-nullable-with-a-removal",
+ ),
+ pytest.param(
+ schema(Modifiers={"properties": {"days": {"anyOf": [{"items": {"enum": [1, 2, 7]}}, {"type": "null"}]}}}),
+ schema(Modifiers={"properties": {"days": {"anyOf": [{"items": {"enum": [1, 2]}}, {"type": "null"}]}}}),
+ ["`Modifiers` at `days`: enum value `7`"],
+ id="enum-value-removed-from-a-nullable-array",
+ ),
+ pytest.param(
+ schema(Modifiers={"properties": {"inner": {"properties": {"kept": {}}}}}),
+ schema(Modifiers={"properties": {"inner": {"anyOf": [{"properties": {"kept": {}}}, {"type": "string"}]}}}),
+ [],
+ id="union-with-two-types-is-skipped",
+ ),
+ pytest.param(
+ schema(Modifiers={"properties": {"keep": {}}}, Request={"properties": {"gone": {}}}),
+ schema(Modifiers={"properties": {"keep": {}}}, Request={"properties": {}}),
+ [],
+ id="model-that-no-cached-response-reaches",
+ ),
+ pytest.param(
+ schema(Modifiers={"properties": {"mode": {"enum": ["a"]}}}),
+ schema(Modifiers={"properties": {"mode": {"enum": ["a", "b"]}, "new": {}}}),
+ [],
+ id="additions-only",
+ ),
+ ],
+)
+def test_find_removals(base: dict[str, Any], head: dict[str, Any], expected: list[str]) -> None:
+ removals = check_cached_schema_removals.find_removals(base, head)
+
+ assert [removal.describe() for removal in removals] == expected
+
+
+ "labels,expected",
+ [
+ pytest.param(["cache-impact-accepted"], "accepted", id="accepted"),
+ pytest.param(["cache-impact-deprecated"], "deprecated", id="deprecated"),
+ pytest.param(["cache-impact-bumped"], "bumped", id="bumped"),
+ pytest.param(["frontend", "cache-impact-accepted", "stamphog"], "accepted", id="among-other-labels"),
+ pytest.param(["cache-impact-maybe", "cache-impact", "frontend"], None, id="unrelated-labels"),
+ pytest.param([], None, id="no-labels"),
+ ],
+)
+def test_find_acknowledgement(labels: list[str], expected: str | None) -> None:
+ assert check_cached_schema_removals.find_acknowledgement(labels) == expected
diff --git a/.github/workflows/ci-frontend.yml b/.github/workflows/ci-frontend.yml
index 4068ed89cd18..29d99fd4a863 100644
--- a/.github/workflows/ci-frontend.yml
+++ b/.github/workflows/ci-frontend.yml
@@ -778,6 +778,46 @@ jobs:
- name: Validate SetupTaskId completions
run: node bin/validate-setup-tasks.mjs
+ # A field removed from a model that a cached query response stores makes every old
+ # cache entry that holds it fail to parse. The author acknowledges the removal with a
+ # cache-impact label, which is read live so a re-run of this job sees a new label.
+ # An unreadable base schema skips the check rather than failing the required gate.
+ - name: Check for fields removed from cached response models
+ id: cached-schema
+ if: ${{ !cancelled() && github.event_name == 'pull_request' && !startsWith(github.head_ref, 'trunk-merge/') }}
+ env:
+ GH_TOKEN: ${{ github.token }}
+ PR_NUMBER: ${{ github.event.pull_request.number }}
+ run: |
+ if ! base_sha=$(gh api "repos/$GITHUB_REPOSITORY/git/commits/$GITHUB_SHA" --jq '.parents[0].sha') ||
+ ! gh api "repos/$GITHUB_REPOSITORY/contents/frontend/src/queries/schema.json?ref=$base_sha" \
+ -H "Accept: application/vnd.github.raw" > "$RUNNER_TEMP/base-schema.json" ||
+ ! PR_LABELS=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels" --jq '.[].name'); then
+ echo "::notice::Could not read the base schema or the PR labels. Skipping the cached schema check."
+ exit 0
+ fi
+ export PR_LABELS
+ python3 .github/scripts/check_cached_schema_removals.py "$RUNNER_TEMP/base-schema.json" frontend/src/queries/schema.json
+
+ - name: Post CI report — cached schema removals
+ if: ${{ !cancelled() && github.event_name == 'pull_request' && !startsWith(github.head_ref, 'trunk-merge/') && steps.cached-schema.outputs.removed != '' }}
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ REMOVED: ${{ steps.cached-schema.outputs.removed }}
+ STATUS: ${{ steps.cached-schema.outputs.status }}
+ SUMMARY: ${{ steps.cached-schema.outputs.summary }}
+ BODY: ${{ steps.cached-schema.outputs.body }}
+ run: node .github/scripts/post-cached-schema-section.mjs
+
+ - name: Require a cache impact label
+ if: ${{ !cancelled() && steps.cached-schema.outputs.removed == 'true' && steps.cached-schema.outputs.acknowledged != 'true' }}
+ env:
+ BODY: ${{ steps.cached-schema.outputs.body }}
+ run: |
+ printf '%s\n' "$BODY" >> "$GITHUB_STEP_SUMMARY"
+ echo "::error::This PR removes fields that cached query responses can still hold. Apply the cache-impact-deprecated, cache-impact-accepted or cache-impact-bumped label, then re-run this job. The job summary lists the cost of each option."
+ exit 1
+
jest:
runs-on: ubuntu-24.04
# PRs run as many selective shards as the tests reachable from the changed
diff --git a/frontend/bin/ci-report/update-ci-report.mjs b/frontend/bin/ci-report/update-ci-report.mjs
index a45038484400..daeefc27482e 100644
--- a/frontend/bin/ci-report/update-ci-report.mjs
+++ b/frontend/bin/ci-report/update-ci-report.mjs
@@ -20,6 +20,7 @@ export const SECTIONS = [
{ id: 'duplication-python', title: 'Duplication (Python)' },
{ id: 'duplication-ts', title: 'Duplication (TypeScript)' },
{ id: 'comment-density', title: 'Comment density' },
+ { id: 'cached-schema-removals', title: 'Cached schema removals' },
{ id: 'bundle-size', title: 'Bundle size' },
{ id: 'eager-graph', title: 'Eager graph' },
{ id: 'toolbar-size', title: 'Toolbar bundle' },