Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(clickhouse): don't capture transient egress proxy blips during duplicate-key probe (#122956)

Tom Owers3e116bd4f721
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/clickhouse.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/clickhouse.py
index 756b630d5396..a0bbc40e557d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/clickhouse.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/clickhouse.py
@@ -1083,6 +1083,9 @@ _DUPLICATE_PK_CHECK_SETTINGS: dict[str, Any] = {
# execution, not ClickHouse Cloud's cold-resume wake-up latency or a scan
# the server hasn't yet gotten around to capping — so the HTTP client's own
# read timeout can fire first. Same designed fallback as the budget caps.
+# - A transient egress-proxy blip (`is_transient_egress_proxy_error`) is our own
+# infrastructure being briefly unreachable, not a sign the probe's query itself
+# is suspect. Same designed-fallback reasoning as the budget caps above.
_EXPECTED_PROBE_FAILURE_SUBSTRINGS: tuple[str, ...] = (
"is unknown or readonly",
"MEMORY_LIMIT_EXCEEDED",
@@ -1093,7 +1096,9 @@ _EXPECTED_PROBE_FAILURE_SUBSTRINGS: tuple[str, ...] = (
def _is_expected_probe_failure(message: str) -> bool:
"""Whether a duplicate-PK probe failure is an expected environment limit or designed fallback."""
- return any(substring in message for substring in _EXPECTED_PROBE_FAILURE_SUBSTRINGS)
+ return any(
+ substring in message for substring in _EXPECTED_PROBE_FAILURE_SUBSTRINGS
+ ) or is_transient_egress_proxy_error(message)
def _has_duplicate_primary_keys(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/test_clickhouse.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/test_clickhouse.py
index bd69bce70055..d82b1727996d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/test_clickhouse.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse/test_clickhouse.py
@@ -756,6 +756,33 @@ class TestHasDuplicatePrimaryKeys:
mock_capture.assert_called_once()
+ @parameterized.expand(
+ [
+ (
+ "remote_disconnected",
+ "Caused by ProxyError('Cannot connect to proxy.', "
+ "RemoteDisconnected('Remote end closed connection without response'))",
+ ),
+ ("timeout", "Caused by ProxyError('Cannot connect to proxy.', TimeoutError('timed out'))"),
+ (
+ "new_connection_error",
+ "Caused by ProxyError('Cannot connect to proxy.', NewConnectionError('could not connect'))",
+ ),
+ ]
+ )
+ def test_egress_proxy_blip_leaves_key_unverified_without_capturing(self, _name, cause):
+ from products.warehouse_sources.backend.temporal.data_imports.sources.clickhouse import clickhouse as ch_module
+
+ client = MagicMock()
+ client.query.side_effect = ClickHouseError(
+ f"HTTPSConnectionPool(host='ch.example.com', port=8443): Max retries exceeded with url: / ({cause})"
+ )
+
+ with patch.object(ch_module, "capture_exception") as mock_capture:
+ assert _has_duplicate_primary_keys(client, "db", "t", ["id"], self._logger()) is None
+
+ mock_capture.assert_not_called()
+
class TestDuplicateProbeTargetsMergeKey:
_MEMORY_LIMIT = ClickHouseError("Code: 241. DB::Exception: Query memory limit exceeded. (MEMORY_LIMIT_EXCEEDED)")