:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ci): flag fields removed from cached query models (#124050)
Vasco de Krijger064959ad8b5b
diff --git a/.github/scripts/check_cached_schema_removals.py b/.github/scripts/check_cached_schema_removals.pynew file mode 100644index 000000000000..306e7d228338--- /dev/null+++ b/.github/scripts/check_cached_schema_removals.py@@ -0,0 +1,257 @@+#!/usr/bin/env python3+# ruff: noqa: T201 allow print statements+"""+Flag fields removed from the models that a cached query response stores.++PostHog stores each query response, with its modifiers, in the query cache.+The generated models reject unknown keys, so a cached entry that still holds a removed field+fails to parse and counts as a cache miss.+This script compares two copies of frontend/src/queries/schema.json and lists those removals.+It checks named definitions and the objects inline in them, and follows a $ref when only one side has it.+It looks through a nullable wrapper (an anyOf of one type and null) and skips every other anyOf or oneOf union.++Usage:+ check_cached_schema_removals.py BASE_SCHEMA HEAD_SCHEMA++Environment:+ PR_LABELS The pull request labels, one per line. A cache-impact label acknowledges a removal.+ GITHUB_OUTPUT Optional. Receives "removed", "acknowledged", "status", "summary" and "body".+ The last three describe the CI report section. Without it, the script prints them.+"""++import os+import sys+import json+import uuid+from collections.abc import Iterator+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any, Literal++ACKNOWLEDGEMENT_LABELS = {+ "cache-impact-deprecated": "deprecated",+ "cache-impact-accepted": "accepted",+ "cache-impact-bumped": "bumped",+}+CACHED_ROOT_PREFIX = "Cached"+DEFINITION_REF_PREFIX = "#/definitions/"+NULL_BRANCH = {"type": "null"}++SECTION_BODY = """\+PostHog stores each query response, with its modifiers, in the query cache.+The generated models reject unknown keys.+When PostHog reads a cached entry that holds a removed field, the entry fails to parse, counts as a cache miss, and sends one exception to error tracking.++**Removed from `frontend/src/queries/schema.json`:**++{removals}++**Choose one option.**+Apply the matching label to the PR, then re-run the failed `Frontend typechecking` job. A label does not start CI by itself.++| Option | Label | What happens | Cost |+| --- | --- | --- | --- |+| Keep the field as deprecated | `cache-impact-deprecated` | The field stays in the schema with `@deprecated`, and the code stops writing it to cached responses. A later PR removes it after the entries written before that expire (up to 7 days). | No cache misses, if nothing writes the field during the wait. Needs a second PR. |+| Accept the misses | `cache-impact-accepted` | The field is removed now. | Each cached entry that holds the field is a miss until the query runs again or the entry expires (up to 7 days). A caller that calculates on a miss waits for ClickHouse. ClickHouse load and the exception rate go up, and the cache hit rate drops. Use this only when the field did not change query results. |+| Bump the cache version | `cache-impact-bumped` | Change `"version"` in `get_cache_payload` in `posthog/hogql_queries/query_runner.py`. | Every cached query result for every team is a miss, not only the entries that hold the field. Use this only when the field changed query results and old entries must not be served. |++Not sure which option fits your change? Ask in [this Slack channel](https://posthog.slack.com/archives/C076E99B152). The team is happy to help.++{status}+"""+++@dataclass(frozen=True, kw_only=True, slots=True)+class Removal:+ model: str+ location: str+ kind: Literal["property", "enum value"]+ name: str++ def describe(self) -> str:+ where = f"`{self.model}`"+ if self.location:+ where += f" at `{self.location}`"+ return f"{where}: {self.kind} `{self.name}`"+++def reference_name(node: Any) -> str | None:+ ref = node.get("$ref") if isinstance(node, dict) else None+ if isinstance(ref, str) and ref.startswith(DEFINITION_REF_PREFIX):+ return ref.removeprefix(DEFINITION_REF_PREFIX)+ return None+++def referenced_definitions(node: Any) -> Iterator[str]:+ if isinstance(node, dict):+ ref = reference_name(node)+ if ref is not None:+ yield ref+ for value in node.values():+ yield from referenced_definitions(value)+ elif isinstance(node, list):+ for item in node:+ yield from referenced_definitions(item)+++def allowed_values(node: dict[str, Any]) -> list[Any] | None:+ if isinstance(node.get("enum"), list):+ return node["enum"]+ if "const" in node:+ return [node["const"]]+ return None+++def without_null(node: Any) -> Any:+ branches = node.get("anyOf") if isinstance(node, dict) else None+ if isinstance(branches, list) and NULL_BRANCH in branches:+ others = [branch for branch in branches if branch != NULL_BRANCH]+ if len(others) == 1:+ return others[0]+ return node+++def is_union(node: dict[str, Any]) -> bool:+ return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list)+++def cached_definitions(definitions: dict[str, Any]) -> set[str]:+ pending = [name for name in definitions if name.startswith(CACHED_ROOT_PREFIX)]+ seen: set[str] = set()+ while pending:+ name = pending.pop()+ if name in seen or name not in definitions:+ continue+ seen.add(name)+ pending.extend(referenced_definitions(definitions[name]))+ return seen+++@dataclass(frozen=True, kw_only=True, slots=True)+class RefPair:+ base: str+ head: str+++@dataclass(frozen=True, kw_only=True, slots=True)+class SchemaPair:+ base_definitions: dict[str, Any]+ head_definitions: dict[str, Any]+ compared: set[RefPair] = field(default_factory=set)++ def removed_fields(self, model: str, base: Any, head: Any, location: str) -> Iterator[Removal]:+ base = without_null(base)+ head = without_null(head)+ base_ref = reference_name(base)+ head_ref = reference_name(head)+ if base_ref is not None and base_ref == head_ref:+ return+ if base_ref is not None and head_ref is not None:+ ref_pair = RefPair(base=base_ref, head=head_ref)+ if ref_pair in self.compared:+ return+ self.compared.add(ref_pair)+ if base_ref is not None or head_ref is not None:+ base = self.base_definitions.get(base_ref) if base_ref is not None else base+ head = self.head_definitions.get(head_ref) if head_ref is not None else head+ if not isinstance(base, dict) or not isinstance(head, dict):+ return+ if is_union(base) or is_union(head):+ return+ base_values = allowed_values(base)+ head_values = allowed_values(head)+ if base_values is not None and head_values is not None:+ for value in base_values:+ if value not in head_values:+ yield Removal(model=model, location=location, kind="enum value", name=str(value))+ for key, base_value in base.items():+ head_value = head.get(key)+ if key == "properties" and isinstance(base_value, dict):+ head_properties = head_value if isinstance(head_value, dict) else {}+ for name, base_property in base_value.items():+ child_location = f"{location}.{name}" if location else name+ if name not in head_properties:+ yield Removal(model=model, location=location, kind="property", name=name)+ else:+ yield from self.removed_fields(model, base_property, head_properties[name], child_location)+ elif isinstance(base_value, dict):+ yield from self.removed_fields(model, base_value, head_value, location)+++def find_removals(base_schema: dict[str, Any], head_schema: dict[str, Any]) -> list[Removal]:+ pair = SchemaPair(+ base_definitions=base_schema.get("definitions", {}), head_definitions=head_schema.get("definitions", {})+ )+ removals: list[Removal] = []+ for name in sorted(cached_definitions(pair.base_definitions)):+ if name in pair.head_definitions:+ removals.extend(pair.removed_fields(name, pair.base_definitions[name], pair.head_definitions[name], ""))+ return removals+++def find_acknowledgement(labels: list[str]) -> str | None:+ for label, choice in ACKNOWLEDGEMENT_LABELS.items():+ if label in labels:+ return choice+ return None+++@dataclass(frozen=True, kw_only=True, slots=True)+class Section:+ status: Literal["warn", "alert"]+ summary: str+ body: str+++def render_body(removals: list[Removal], status: str) -> str:+ return SECTION_BODY.format(removals="\n".join(f"- {removal.describe()}" for removal in removals), status=status)+++def render_section(removals: list[Removal], acknowledgement: str | None) -> Section:+ count = f"{len(removals)} removed field{'' if len(removals) == 1 else 's'}"+ if acknowledgement:+ status = f"Acknowledged with the `cache-impact-{acknowledgement}` label."+ return Section(+ status="warn",+ summary=f"{count}, acknowledged ({acknowledgement})",+ body=render_body(removals, status),+ )+ status = "Not acknowledged yet. `Frontend typechecking` fails until the PR has one of these labels."+ return Section(status="alert", summary=f"{count}, not acknowledged", body=render_body(removals, status))+++def write_outputs(removals: list[Removal], acknowledgement: str | None, section: Section | None) -> None:+ fields = {+ "removed": str(bool(removals)).lower(),+ "acknowledged": str(acknowledgement is not None).lower(),+ "status": section.status if section else "ok",+ "summary": section.summary if section else "no removed fields",+ }+ lines = [f"{name}={value}\n" for name, value in fields.items()]+ if section:+ delimiter = f"EOF-{uuid.uuid4()}"+ lines.append(f"body<<{delimiter}\n{section.body}\n{delimiter}\n")+ output_path = os.environ.get("GITHUB_OUTPUT")+ if output_path:+ with Path(output_path).open("a") as output:+ output.writelines(lines)+ else:+ print("".join(lines), end="")+++def main(argv: list[str]) -> int:+ base_path, head_path = argv[1:3]+ removals = find_removals(json.loads(Path(base_path).read_text()), json.loads(Path(head_path).read_text()))+ acknowledgement = find_acknowledgement(os.environ.get("PR_LABELS", "").splitlines())+ section = render_section(removals, acknowledgement) if removals else None++ for removal in removals:+ print(f"Removed: {removal.describe()}")+ if not removals:+ print("No fields removed from cached response models.")+ write_outputs(removals, acknowledgement, section)+ return 0+++if __name__ == "__main__":+ sys.exit(main(sys.argv))diff --git a/.github/scripts/post-cached-schema-section.mjs b/.github/scripts/post-cached-schema-section.mjsnew file mode 100644index 000000000000..a6cc4784bde3--- /dev/null+++ b/.github/scripts/post-cached-schema-section.mjs@@ -0,0 +1,23 @@+#!/usr/bin/env node+import { pathToFileURL } from 'node:url'++import { clearSectionIfPresent, postSection } from '../../frontend/bin/ci-report/update-ci-report.mjs'++const SECTION_ID = 'cached-schema-removals'++async function main() {+ const { REMOVED, STATUS, SUMMARY, BODY } = process.env+ if (REMOVED === 'true') {+ await postSection({ id: SECTION_ID, status: STATUS, summary: SUMMARY, body: BODY })+ return+ }+ await clearSectionIfPresent({+ id: SECTION_ID,+ summary: 'no removed fields',+ body: 'No fields were removed from the models that cached query responses store.',+ })+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ await main()+}diff --git a/.github/scripts/test_check_cached_schema_removals.py b/.github/scripts/test_check_cached_schema_removals.pynew file mode 100644index 000000000000..a10534bcfe17--- /dev/null+++ b/.github/scripts/test_check_cached_schema_removals.py@@ -0,0 +1,159 @@+import importlib.util+from pathlib import Path+from typing import Any++import pytest++SCRIPT_PATH = Path(__file__).with_name("check_cached_schema_removals.py")+SPEC = importlib.util.spec_from_file_location("check_cached_schema_removals", SCRIPT_PATH)+assert SPEC is not None+assert SPEC.loader is not None+check_cached_schema_removals = importlib.util.module_from_spec(SPEC)+SPEC.loader.exec_module(check_cached_schema_removals)++ROOT = {"CachedFooResponse": {"properties": {"modifiers": {"$ref": "#/definitions/Modifiers"}}}}+++def schema(**definitions: Any) -> dict[str, Any]:+ return {"definitions": {**ROOT, **definitions}}+++ "base,head,expected",+ [+ pytest.param(+ schema(Modifiers={"properties": {"keep": {}, "gone": {}}}),+ schema(Modifiers={"properties": {"keep": {}}}),+ ["`Modifiers`: property `gone`"],+ id="property-removed-from-model-reached-through-a-ref",+ ),+ pytest.param(+ schema(Modifiers={"properties": {"mode": {"enum": ["a", "b"]}}}),+ schema(Modifiers={"properties": {"mode": {"enum": ["a"]}}}),+ ["`Modifiers` at `mode`: enum value `b`"],+ id="enum-value-removed",+ ),+ pytest.param(+ schema(Modifiers={"properties": {"mode": {"enum": ["a", "b"]}}}),+ schema(Modifiers={"properties": {"mode": {"const": "a"}}}),+ ["`Modifiers` at `mode`: enum value `b`"],+ id="enum-narrowed-to-a-single-const",+ ),+ pytest.param(+ schema(Modifiers={"properties": {"mode": {"const": "a"}}}),+ schema(Modifiers={"properties": {"mode": {"const": "b"}}}),+ ["`Modifiers` at `mode`: enum value `a`"],+ id="const-value-changed",+ ),+ pytest.param(+ schema(Modifiers={"properties": {"inner": {"properties": {"gone": {}}}}}),+ schema(Modifiers={"properties": {"inner": {"properties": {}}}}),+ ["`Modifiers` at `inner`: property `gone`"],+ id="property-removed-from-inline-object",+ ),+ pytest.param(+ schema(Modifiers={"properties": {"inner": {"properties": {"kept": {}}}}}),+ schema(+ Modifiers={"properties": {"inner": {"$ref": "#/definitions/Inner"}}}, Inner={"properties": {"kept": {}}}+ ),+ [],+ id="inline-object-extracted-to-a-ref-unchanged",+ ),+ pytest.param(+ schema(Modifiers={"properties": {"inner": {"properties": {"kept": {}, "gone": {}}}}}),+ schema(+ Modifiers={"properties": {"inner": {"$ref": "#/definitions/Inner"}}}, Inner={"properties": {"kept": {}}}+ ),+ ["`Modifiers` at `inner`: property `gone`"],+ id="inline-object-extracted-to-a-ref-with-a-removal",+ ),+ pytest.param(+ schema(Modifiers={"properties": {"a": {"properties": {"y": {"properties": {"gone": {}}}}}}}),+ schema(+ Modifiers={"properties": {"a": {"$ref": "#/definitions/Inner"}}},+ Inner={"properties": {"y": {"$ref": "#/definitions/Inner"}}},+ ),+ ["`Modifiers` at `a.y`: property `gone`"],+ id="inline-object-extracted-to-a-recursive-ref",+ ),+ pytest.param(+ schema(+ Modifiers={"properties": {"tree": {"$ref": "#/definitions/Old"}}},+ Old={"properties": {"gone": {}, "child": {"$ref": "#/definitions/Old"}}},+ ),+ schema(+ Modifiers={"properties": {"tree": {"$ref": "#/definitions/New"}}},+ New={"properties": {"child": {"$ref": "#/definitions/New"}}},+ ),+ ["`Modifiers` at `tree`: property `gone`"],+ id="renamed-recursive-model",+ ),+ pytest.param(+ schema(+ Modifiers={"properties": {"a": {"$ref": "#/definitions/Old"}, "b": {"$ref": "#/definitions/Old"}}},+ Old={"properties": {"kept": {}, "gone": {}}},+ ),+ schema(+ Modifiers={"properties": {"a": {"$ref": "#/definitions/New"}, "b": {"$ref": "#/definitions/New"}}},+ New={"properties": {"kept": {}}},+ ),+ ["`Modifiers` at `a`: property `gone`"],+ id="renamed-model-reached-twice-is-reported-once",+ ),+ pytest.param(+ schema(Modifiers={"properties": {"inner": {"properties": {"kept": {}}}}}),+ schema(Modifiers={"properties": {"inner": {"anyOf": [{"properties": {"kept": {}}}, {"type": "null"}]}}}),+ [],+ id="object-made-nullable-unchanged",+ ),+ pytest.param(+ schema(Modifiers={"properties": {"inner": {"properties": {"kept": {}, "gone": {}}}}}),+ schema(Modifiers={"properties": {"inner": {"anyOf": [{"properties": {"kept": {}}}, {"type": "null"}]}}}),+ ["`Modifiers` at `inner`: property `gone`"],+ id="object-made-nullable-with-a-removal",+ ),+ pytest.param(+ schema(Modifiers={"properties": {"days": {"anyOf": [{"items": {"enum": [1, 2, 7]}}, {"type": "null"}]}}}),+ schema(Modifiers={"properties": {"days": {"anyOf": [{"items": {"enum": [1, 2]}}, {"type": "null"}]}}}),+ ["`Modifiers` at `days`: enum value `7`"],+ id="enum-value-removed-from-a-nullable-array",+ ),+ pytest.param(+ schema(Modifiers={"properties": {"inner": {"properties": {"kept": {}}}}}),+ schema(Modifiers={"properties": {"inner": {"anyOf": [{"properties": {"kept": {}}}, {"type": "string"}]}}}),+ [],+ id="union-with-two-types-is-skipped",+ ),+ pytest.param(+ schema(Modifiers={"properties": {"keep": {}}}, Request={"properties": {"gone": {}}}),+ schema(Modifiers={"properties": {"keep": {}}}, Request={"properties": {}}),+ [],+ id="model-that-no-cached-response-reaches",+ ),+ pytest.param(+ schema(Modifiers={"properties": {"mode": {"enum": ["a"]}}}),+ schema(Modifiers={"properties": {"mode": {"enum": ["a", "b"]}, "new": {}}}),+ [],+ id="additions-only",+ ),+ ],+)+def test_find_removals(base: dict[str, Any], head: dict[str, Any], expected: list[str]) -> None:+ removals = check_cached_schema_removals.find_removals(base, head)++ assert [removal.describe() for removal in removals] == expected+++ "labels,expected",+ [+ pytest.param(["cache-impact-accepted"], "accepted", id="accepted"),+ pytest.param(["cache-impact-deprecated"], "deprecated", id="deprecated"),+ pytest.param(["cache-impact-bumped"], "bumped", id="bumped"),+ pytest.param(["frontend", "cache-impact-accepted", "stamphog"], "accepted", id="among-other-labels"),+ pytest.param(["cache-impact-maybe", "cache-impact", "frontend"], None, id="unrelated-labels"),+ pytest.param([], None, id="no-labels"),+ ],+)+def test_find_acknowledgement(labels: list[str], expected: str | None) -> None:+ assert check_cached_schema_removals.find_acknowledgement(labels) == expecteddiff --git a/.github/workflows/ci-frontend.yml b/.github/workflows/ci-frontend.ymlindex 4068ed89cd18..29d99fd4a863 100644--- a/.github/workflows/ci-frontend.yml+++ b/.github/workflows/ci-frontend.yml@@ -778,6 +778,46 @@ jobs:- name: Validate SetupTaskId completionsrun: node bin/validate-setup-tasks.mjs+ # A field removed from a model that a cached query response stores makes every old+ # cache entry that holds it fail to parse. The author acknowledges the removal with a+ # cache-impact label, which is read live so a re-run of this job sees a new label.+ # An unreadable base schema skips the check rather than failing the required gate.+ - name: Check for fields removed from cached response models+ id: cached-schema+ if: ${{ !cancelled() && github.event_name == 'pull_request' && !startsWith(github.head_ref, 'trunk-merge/') }}+ env:+ GH_TOKEN: ${{ github.token }}+ PR_NUMBER: ${{ github.event.pull_request.number }}+ run: |+ if ! base_sha=$(gh api "repos/$GITHUB_REPOSITORY/git/commits/$GITHUB_SHA" --jq '.parents[0].sha') ||+ ! gh api "repos/$GITHUB_REPOSITORY/contents/frontend/src/queries/schema.json?ref=$base_sha" \+ -H "Accept: application/vnd.github.raw" > "$RUNNER_TEMP/base-schema.json" ||+ ! PR_LABELS=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels" --jq '.[].name'); then+ echo "::notice::Could not read the base schema or the PR labels. Skipping the cached schema check."+ exit 0+ fi+ export PR_LABELS+ python3 .github/scripts/check_cached_schema_removals.py "$RUNNER_TEMP/base-schema.json" frontend/src/queries/schema.json++ - name: Post CI report — cached schema removals+ if: ${{ !cancelled() && github.event_name == 'pull_request' && !startsWith(github.head_ref, 'trunk-merge/') && steps.cached-schema.outputs.removed != '' }}+ env:+ GITHUB_TOKEN: ${{ github.token }}+ REMOVED: ${{ steps.cached-schema.outputs.removed }}+ STATUS: ${{ steps.cached-schema.outputs.status }}+ SUMMARY: ${{ steps.cached-schema.outputs.summary }}+ BODY: ${{ steps.cached-schema.outputs.body }}+ run: node .github/scripts/post-cached-schema-section.mjs++ - name: Require a cache impact label+ if: ${{ !cancelled() && steps.cached-schema.outputs.removed == 'true' && steps.cached-schema.outputs.acknowledged != 'true' }}+ env:+ BODY: ${{ steps.cached-schema.outputs.body }}+ run: |+ printf '%s\n' "$BODY" >> "$GITHUB_STEP_SUMMARY"+ echo "::error::This PR removes fields that cached query responses can still hold. Apply the cache-impact-deprecated, cache-impact-accepted or cache-impact-bumped label, then re-run this job. The job summary lists the cost of each option."+ exit 1+jest:runs-on: ubuntu-24.04# PRs run as many selective shards as the tests reachable from the changeddiff --git a/frontend/bin/ci-report/update-ci-report.mjs b/frontend/bin/ci-report/update-ci-report.mjsindex a45038484400..daeefc27482e 100644--- a/frontend/bin/ci-report/update-ci-report.mjs+++ b/frontend/bin/ci-report/update-ci-report.mjs@@ -20,6 +20,7 @@ export const SECTIONS = [{ id: 'duplication-python', title: 'Duplication (Python)' },{ id: 'duplication-ts', title: 'Duplication (TypeScript)' },{ id: 'comment-density', title: 'Comment density' },+ { id: 'cached-schema-removals', title: 'Cached schema removals' },{ id: 'bundle-size', title: 'Bundle size' },{ id: 'eager-graph', title: 'Eager graph' },{ id: 'toolbar-size', title: 'Toolbar bundle' },