:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(clickhouse): decode date32 values outside 1900-2299 (#124044)
Jovan Sakovic7f98ce543e92
diff --git a/posthog/clickhouse/client/connection.py b/posthog/clickhouse/client/connection.pyindex 1b40bbac2c3d..fd641c59afc7 100644--- a/posthog/clickhouse/client/connection.py+++ b/posthog/clickhouse/client/connection.py@@ -6,6 +6,7 @@ import loggingfrom collections.abc import Callable, Mappingfrom contextlib import contextmanagerfrom dataclasses import field+from datetime import date, timedeltafrom enum import StrEnumfrom functools import cachefrom pathlib import Path@@ -14,8 +15,9 @@ from typing import TYPE_CHECKING, Anyfrom django.conf import settingsfrom clickhouse_driver import Client as SyncClient+from clickhouse_driver.columns.datecolumn import DateColumnfrom clickhouse_pool import ChPool-from prometheus_client import Counter+from prometheus_client import Counter, Gaugefrom posthog.dataclasses import frozen@@ -32,6 +34,41 @@ from posthog.utils import patchableMAX_QUERY_SIZE_BYTES = 8 * 1024 * 1024+_EPOCH = date(1970, 1, 1)++OUT_OF_RANGE_DATES_DECODED_COUNTER = Counter(+ "posthog_clickhouse_out_of_range_dates_decoded",+ "Date and Date32 values decoded from a day outside the driver's 1900-2299 lookup table.",+)+OUT_OF_RANGE_DISTINCT_DAYS_GAUGE = Gauge(+ "posthog_clickhouse_out_of_range_distinct_days",+ "Distinct out-of-range days this process has decoded, up to the tracking cap.",+ multiprocess_mode="livemax",+)+# Bounds the tracking set. A gauge at the cap means a process saw at least this many distinct days.+_DISTINCT_DAYS_TRACKING_CAP = 10_000+++class _DateLookup(dict[int, date]):+ # Computes days outside the prefilled range without storing them. The table is shared by every query in the+ # process, so a cache here would grow with whatever dates any team's data holds.+ def __init__(self, prefilled: dict[int, date]) -> None:+ super().__init__(prefilled)+ self._seen_days: set[int] = set()++ def __missing__(self, day: int) -> date:+ OUT_OF_RANGE_DATES_DECODED_COUNTER.inc()+ if day not in self._seen_days and len(self._seen_days) < _DISTINCT_DAYS_TRACKING_CAP:+ self._seen_days.add(day)+ OUT_OF_RANGE_DISTINCT_DAYS_GAUGE.set(len(self._seen_days))+ return _EPOCH + timedelta(days=day)+++# clickhouse_driver decodes Date and Date32 through a lookup table that covers 1900-2299 and raises KeyError for any+# other day. ClickHouse 26.9 returns days outside that range from Parquet reads and casts.+DateColumn.date_lut = _DateLookup(DateColumn.date_lut)++class NodeRole(StrEnum):# Roles of nodes for a particular NodeType. These are meant to# match the CH macro hostClusterRolediff --git a/posthog/clickhouse/client/test/test_execute.py b/posthog/clickhouse/client/test/test_execute.pyindex 1003a571562a..b978ab5677fb 100644--- a/posthog/clickhouse/client/test/test_execute.py+++ b/posthog/clickhouse/client/test/test_execute.py@@ -1,3 +1,4 @@+from datetime import datefrom ipaddress import IPv4Address, IPv6Addressfrom types import SimpleNamespacefrom typing import Any@@ -5,6 +6,8 @@ from typing import Anyimport pytestfrom unittest.mock import MagicMock, patch+from clickhouse_driver.columns.datecolumn import DateColumn+from posthog.hogql.query_stats import query_stats_scopefrom posthog.clickhouse.client.connection import (@@ -255,3 +258,22 @@ def test_query_with_columns_removes_columns_by_type(types, row, expected):rows = query_with_columns("SELECT *", column_types_to_remove=("IPv4", "IPv6"))assert rows == [expected]+++ "raw_day_number_hex,expected",+ [+ ("aeba0500", date(2998, 1, 1)),+ ("8c29f5ff", date(25, 5, 17)),+ ],+)+def test_sync_execute_decodes_date32_outside_driver_default_range(raw_day_number_hex, expected):+ # RowBinary keeps the raw day number, the same way ClickHouse 26.9+ passes through out-of-range+ # Parquet and CAST values. The driver's default lookup table stops at 1900-2299.+ lookup_size = len(DateColumn.date_lut)++ rows = sync_execute(f"SELECT d FROM format(RowBinary, 'd Date32', unhex('{raw_day_number_hex}'))", flush=False)++ assert rows == [(expected,)]+ # The lookup is shared by every query in the process, so out-of-range days must not be cached in it.+ assert len(DateColumn.date_lut) == lookup_sizediff --git a/posthog/settings/data_stores.py b/posthog/settings/data_stores.pyindex 2185b49cbfea..cf5e288bacea 100644--- a/posthog/settings/data_stores.py+++ b/posthog/settings/data_stores.py@@ -298,6 +298,7 @@ elif TEST:SUFFIX = "_test" + XDIST_SUFFIX# Clickhouse Settings+CLICKHOUSE_TEST_DB: str = "posthog" + SUFFIXCLICKHOUSE_HOST: str = os.getenv("CLICKHOUSE_HOST", "clickhouse")