:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(dags): load staged dictionaries from the snapshot on the source cluster too (#126349)
Paweł Szczur690799e9c235
diff --git a/posthog/dags/common/staged_dictionary.py b/posthog/dags/common/staged_dictionary.pyindex b4caa88b7294..6a2f43040537 100644--- a/posthog/dags/common/staged_dictionary.py+++ b/posthog/dags/common/staged_dictionary.py@@ -86,35 +86,31 @@ def create_on_every_cluster() -> None:"""Create ``dictionary`` on every cluster a mutation joining it will run on.- ``clusters`` leads with the handle that can see the source table; the rest read a staged object,- because they share no Keeper with it and so never receive the replicated table.+ ``clusters`` leads with the handle that can see the source table; the rest share no Keeper with+ it and so never receive the replicated table. With more than one cluster, every cluster reads+ the staged object, the first one too. Hosts load one at a time, so a host that read the source+ table would see rows written after the export, and the hosts would not agree."""first, *rest = clusters- first.map_all_hosts(- partial(- dictionary.create,- shards=shards,- max_execution_time=max_execution_time,- max_memory_usage=max_memory_usage,- )- ).result()- if not rest:- return+ staged = dictionary.staged() if rest else None+ if staged is not None:+ first.any_host_by_role(partial(staged.export, source_query=dictionary.query), NodeRole.DATA).result()- staged = dictionary.staged()- first.any_host_by_role(partial(staged.export, source_query=dictionary.query), NodeRole.DATA).result()- for cluster in rest:+ # Recreate, never CREATE ... IF NOT EXISTS: a failed run leaves its dictionary behind, and a+ # later run with fewer clusters would keep that definition and read a stale staged object.+ for cluster in clusters:cluster.map_all_hosts(partial(dictionary.recreate,shards=shards,max_execution_time=max_execution_time,max_memory_usage=max_memory_usage,- query=staged.query,+ query=staged.query if staged is not None else None,)).result()- context.log.info(f"Staged {dictionary.name} to {staged.key} for {[c.data_cluster_name for c in rest]}")+ if staged is not None:+ context.log.info(f"Staged {dictionary.name} to {staged.key} for {[c.data_cluster_name for c in clusters]}")def load_and_verify_on_every_cluster(clusters: Sequence[ClickhouseCluster], dictionary: StageableDictionary) -> None:diff --git a/posthog/dags/tests/test_staged_dictionary.py b/posthog/dags/tests/test_staged_dictionary.pyindex 0fc44258b86b..9f73274934c2 100644--- a/posthog/dags/tests/test_staged_dictionary.py+++ b/posthog/dags/tests/test_staged_dictionary.py@@ -1,6 +1,7 @@from typing import Any, castimport pytest+from unittest.mock import MagicMock, patchimport dagsterfrom parameterized import parameterized@@ -9,6 +10,7 @@ from posthog.clickhouse.cluster import ClickhouseClusterfrom posthog.dags.common.staged_dictionary import (StageableDictionary,StagedDictionary,+ create_on_every_cluster,load_and_verify_on_every_cluster,)@@ -75,3 +77,75 @@ class TestLoadAndVerifyOnEveryCluster:assert "posthog=[1]" in str(failure.value.description)assert "events=[2]" in str(failure.value.description)+++class _Done:+ def __init__(self, value: Any = None) -> None:+ self._value = value++ def result(self) -> Any:+ return self._value+++class _RecordingCluster:+ """A handle whose single host is a client named after the cluster, so calls say where they ran."""++ def __init__(self, name: str) -> None:+ self.data_cluster_name = name++ def map_all_hosts(self, fn: Any, concurrency: int | None = None) -> _Done:+ return _Done({self.data_cluster_name: fn(self.data_cluster_name)})++ def any_host_by_role(self, fn: Any, role: Any) -> _Done:+ return _Done(fn(self.data_cluster_name))+++_STAGED = StagedDictionary(key="snapshot_dictionary.parquet", columns="id", structure="id Int64")+++class _RecordingDictionary:+ name = "snapshot_dictionary"+ query = "SELECT FROM source"++ def __init__(self, left_behind: dict[str, str]) -> None:+ self.source_by_cluster = dict(left_behind)++ def staged(self) -> StagedDictionary:+ return _STAGED++ def create(self, client: Any, shards: int, max_execution_time: int, max_memory_usage: int, query: Any = None):+ self.source_by_cluster.setdefault(client, query or self.query)++ def recreate(self, client: Any, shards: int, max_execution_time: int, max_memory_usage: int, query: Any = None):+ self.source_by_cluster.pop(client, None)+ self.create(client, shards, max_execution_time, max_memory_usage, query)+++class TestCreateOnEveryCluster:+ @parameterized.expand(+ [+ ("one cluster reads the source table", ["posthog"], False, False),+ ("every cluster reads the staged object, the source cluster too", ["posthog", "events"], True, False),+ ("one cluster replaces a staged definition a failed run left", ["posthog"], False, True),+ ]+ )+ def test_the_dictionary_source_on_each_cluster(+ self, _name: str, names: list[str], staged: bool, left_behind: bool+ ) -> None:+ # A source cluster that reads the live table sees rows written after the export while its+ # hosts load one by one, so its hosts disagree with each other and with the other cluster.+ # A kept staged definition loads a stale object, so the run deletes an old key set.+ dictionary = _RecordingDictionary({"posthog": _STAGED.query} if left_behind else {})+ expected = dictionary.staged().query if staged else dictionary.query++ with patch.object(StagedDictionary, "export"):+ create_on_every_cluster(+ MagicMock(),+ cast(list[ClickhouseCluster], [_RecordingCluster(name) for name in names]),+ cast(StageableDictionary, dictionary),+ shards=1,+ max_execution_time=0,+ max_memory_usage=0,+ )++ assert dictionary.source_by_cluster == dict.fromkeys(names, expected)