Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(dags): load staged dictionaries from the snapshot on the source cluster too (#126349)

Paweł Szczur690799e9c235
diff --git a/posthog/dags/common/staged_dictionary.py b/posthog/dags/common/staged_dictionary.py
index b4caa88b7294..6a2f43040537 100644
--- a/posthog/dags/common/staged_dictionary.py
+++ b/posthog/dags/common/staged_dictionary.py
@@ -86,35 +86,31 @@ def create_on_every_cluster(
) -> None:
"""Create ``dictionary`` on every cluster a mutation joining it will run on.
- ``clusters`` leads with the handle that can see the source table; the rest read a staged object,
- because they share no Keeper with it and so never receive the replicated table.
+ ``clusters`` leads with the handle that can see the source table; the rest share no Keeper with
+ it and so never receive the replicated table. With more than one cluster, every cluster reads
+ the staged object, the first one too. Hosts load one at a time, so a host that read the source
+ table would see rows written after the export, and the hosts would not agree.
"""
first, *rest = clusters
- first.map_all_hosts(
- partial(
- dictionary.create,
- shards=shards,
- max_execution_time=max_execution_time,
- max_memory_usage=max_memory_usage,
- )
- ).result()
- if not rest:
- return
+ staged = dictionary.staged() if rest else None
+ if staged is not None:
+ first.any_host_by_role(partial(staged.export, source_query=dictionary.query), NodeRole.DATA).result()
- staged = dictionary.staged()
- first.any_host_by_role(partial(staged.export, source_query=dictionary.query), NodeRole.DATA).result()
- for cluster in rest:
+ # Recreate, never CREATE ... IF NOT EXISTS: a failed run leaves its dictionary behind, and a
+ # later run with fewer clusters would keep that definition and read a stale staged object.
+ for cluster in clusters:
cluster.map_all_hosts(
partial(
dictionary.recreate,
shards=shards,
max_execution_time=max_execution_time,
max_memory_usage=max_memory_usage,
- query=staged.query,
+ query=staged.query if staged is not None else None,
)
).result()
- context.log.info(f"Staged {dictionary.name} to {staged.key} for {[c.data_cluster_name for c in rest]}")
+ if staged is not None:
+ context.log.info(f"Staged {dictionary.name} to {staged.key} for {[c.data_cluster_name for c in clusters]}")
def load_and_verify_on_every_cluster(clusters: Sequence[ClickhouseCluster], dictionary: StageableDictionary) -> None:
diff --git a/posthog/dags/tests/test_staged_dictionary.py b/posthog/dags/tests/test_staged_dictionary.py
index 0fc44258b86b..9f73274934c2 100644
--- a/posthog/dags/tests/test_staged_dictionary.py
+++ b/posthog/dags/tests/test_staged_dictionary.py
@@ -1,6 +1,7 @@
from typing import Any, cast
import pytest
+from unittest.mock import MagicMock, patch
import dagster
from parameterized import parameterized
@@ -9,6 +10,7 @@ from posthog.clickhouse.cluster import ClickhouseCluster
from posthog.dags.common.staged_dictionary import (
StageableDictionary,
StagedDictionary,
+ create_on_every_cluster,
load_and_verify_on_every_cluster,
)
@@ -75,3 +77,75 @@ class TestLoadAndVerifyOnEveryCluster:
assert "posthog=[1]" in str(failure.value.description)
assert "events=[2]" in str(failure.value.description)
+
+
+class _Done:
+ def __init__(self, value: Any = None) -> None:
+ self._value = value
+
+ def result(self) -> Any:
+ return self._value
+
+
+class _RecordingCluster:
+ """A handle whose single host is a client named after the cluster, so calls say where they ran."""
+
+ def __init__(self, name: str) -> None:
+ self.data_cluster_name = name
+
+ def map_all_hosts(self, fn: Any, concurrency: int | None = None) -> _Done:
+ return _Done({self.data_cluster_name: fn(self.data_cluster_name)})
+
+ def any_host_by_role(self, fn: Any, role: Any) -> _Done:
+ return _Done(fn(self.data_cluster_name))
+
+
+_STAGED = StagedDictionary(key="snapshot_dictionary.parquet", columns="id", structure="id Int64")
+
+
+class _RecordingDictionary:
+ name = "snapshot_dictionary"
+ query = "SELECT FROM source"
+
+ def __init__(self, left_behind: dict[str, str]) -> None:
+ self.source_by_cluster = dict(left_behind)
+
+ def staged(self) -> StagedDictionary:
+ return _STAGED
+
+ def create(self, client: Any, shards: int, max_execution_time: int, max_memory_usage: int, query: Any = None):
+ self.source_by_cluster.setdefault(client, query or self.query)
+
+ def recreate(self, client: Any, shards: int, max_execution_time: int, max_memory_usage: int, query: Any = None):
+ self.source_by_cluster.pop(client, None)
+ self.create(client, shards, max_execution_time, max_memory_usage, query)
+
+
+class TestCreateOnEveryCluster:
+ @parameterized.expand(
+ [
+ ("one cluster reads the source table", ["posthog"], False, False),
+ ("every cluster reads the staged object, the source cluster too", ["posthog", "events"], True, False),
+ ("one cluster replaces a staged definition a failed run left", ["posthog"], False, True),
+ ]
+ )
+ def test_the_dictionary_source_on_each_cluster(
+ self, _name: str, names: list[str], staged: bool, left_behind: bool
+ ) -> None:
+ # A source cluster that reads the live table sees rows written after the export while its
+ # hosts load one by one, so its hosts disagree with each other and with the other cluster.
+ # A kept staged definition loads a stale object, so the run deletes an old key set.
+ dictionary = _RecordingDictionary({"posthog": _STAGED.query} if left_behind else {})
+ expected = dictionary.staged().query if staged else dictionary.query
+
+ with patch.object(StagedDictionary, "export"):
+ create_on_every_cluster(
+ MagicMock(),
+ cast(list[ClickhouseCluster], [_RecordingCluster(name) for name in names]),
+ cast(StageableDictionary, dictionary),
+ shards=1,
+ max_execution_time=0,
+ max_memory_usage=0,
+ )
+
+ assert dictionary.source_by_cluster == dict.fromkeys(names, expected)