:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(clickhouse): don't capture transient egress proxy blips during duplicate-key probe (#122956)
Tom Owers3e116bd4f721
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/clickhouse.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/clickhouse.pyindex 756b630d5396..a0bbc40e557d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/clickhouse.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/clickhouse.py@@ -1083,6 +1083,9 @@ _DUPLICATE_PK_CHECK_SETTINGS: dict[str, Any] = {# execution, not ClickHouse Cloud's cold-resume wake-up latency or a scan# the server hasn't yet gotten around to capping — so the HTTP client's own# read timeout can fire first. Same designed fallback as the budget caps.+# - A transient egress-proxy blip (`is_transient_egress_proxy_error`) is our own+# infrastructure being briefly unreachable, not a sign the probe's query itself+# is suspect. Same designed-fallback reasoning as the budget caps above._EXPECTED_PROBE_FAILURE_SUBSTRINGS: tuple[str, ...] = ("is unknown or readonly","MEMORY_LIMIT_EXCEEDED",@@ -1093,7 +1096,9 @@ _EXPECTED_PROBE_FAILURE_SUBSTRINGS: tuple[str, ...] = (def _is_expected_probe_failure(message: str) -> bool:"""Whether a duplicate-PK probe failure is an expected environment limit or designed fallback."""- return any(substring in message for substring in _EXPECTED_PROBE_FAILURE_SUBSTRINGS)+ return any(+ substring in message for substring in _EXPECTED_PROBE_FAILURE_SUBSTRINGS+ ) or is_transient_egress_proxy_error(message)def _has_duplicate_primary_keys(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/test_clickhouse.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/test_clickhouse.pyindex bd69bce70055..d82b1727996d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/test_clickhouse.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/test_clickhouse.py@@ -756,6 +756,33 @@ class TestHasDuplicatePrimaryKeys:mock_capture.assert_called_once()+ @parameterized.expand(+ [+ (+ "remote_disconnected",+ "Caused by ProxyError('Cannot connect to proxy.', "+ "RemoteDisconnected('Remote end closed connection without response'))",+ ),+ ("timeout", "Caused by ProxyError('Cannot connect to proxy.', TimeoutError('timed out'))"),+ (+ "new_connection_error",+ "Caused by ProxyError('Cannot connect to proxy.', NewConnectionError('could not connect'))",+ ),+ ]+ )+ def test_egress_proxy_blip_leaves_key_unverified_without_capturing(self, _name, cause):+ from products.warehouse_sources.backend.temporal.data_imports.sources.clickhouse import clickhouse as ch_module++ client = MagicMock()+ client.query.side_effect = ClickHouseError(+ f"HTTPSConnectionPool(host='ch.example.com', port=8443): Max retries exceeded with url: / ({cause})"+ )++ with patch.object(ch_module, "capture_exception") as mock_capture:+ assert _has_duplicate_primary_keys(client, "db", "t", ["id"], self._logger()) is None++ mock_capture.assert_not_called()+class TestDuplicateProbeTargetsMergeKey:_MEMORY_LIMIT = ClickHouseError("Code: 241. DB::Exception: Query memory limit exceeded. (MEMORY_LIMIT_EXCEEDED)")