Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): stop racing a doomed request near a trino query deadline (#129077)

Tom Owers36164ca25f7d
diff --git a/posthog/hogql/direct_sql/test/test_trino_adapter.py b/posthog/hogql/direct_sql/test/test_trino_adapter.py
index 323e1b4817cc..823bd1fb8670 100644
--- a/posthog/hogql/direct_sql/test/test_trino_adapter.py
+++ b/posthog/hogql/direct_sql/test/test_trino_adapter.py
@@ -1,9 +1,16 @@
+import time
+
import pytest
from unittest.mock import MagicMock, patch
from django.test import override_settings
-from posthog.hogql.direct_sql.trino_adapter import TrinoAdapter, ensure_read_only_raw_trino_statement
+from posthog.hogql.direct_sql.trino_adapter import (
+ DIRECT_TRINO_TIMEOUT_ERROR,
+ TrinoAdapter,
+ bounded_trino_cursor,
+ ensure_read_only_raw_trino_statement,
+)
from posthog.hogql.errors import ExposedHogQLError
from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTarget
@@ -136,6 +143,23 @@ def test_cancellation_during_prepare_prevents_starting_the_main_query() -> None:
assert statements[0].startswith("PREPARE ")
+def test_bounded_request_times_out_instead_of_racing_a_near_expired_deadline() -> None:
+ # A request issued with little time left on the deadline used to get a sub-second socket
+ # timeout and fail with a confusing network error instead of the intended clean message.
+ connection = MagicMock()
+ connection.cursor.return_value = MagicMock()
+ session = MagicMock()
+ connection._http_session = session
+ original_request = session.request
+
+ with bounded_trino_cursor(connection, timeout_seconds=60) as (_, watchdog):
+ watchdog.deadline = time.monotonic() + 0.5
+ with pytest.raises(ExposedHogQLError, match=DIRECT_TRINO_TIMEOUT_ERROR):
+ session.request("GET", "https://trino.example.com/v1/statement/abc")
+
+ original_request.assert_not_called()
+
+
@override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS="10.1.0.1")
def test_validate_source_config_accepts_the_teams_managed_warehouse() -> None:
# A managed warehouse host resolves to a private load balancer address. The host check
diff --git a/posthog/hogql/direct_sql/trino_adapter.py b/posthog/hogql/direct_sql/trino_adapter.py
index 77e136b02961..a0da63225aa2 100644
--- a/posthog/hogql/direct_sql/trino_adapter.py
+++ b/posthog/hogql/direct_sql/trino_adapter.py
@@ -41,6 +41,9 @@ DIRECT_TRINO_MAX_ROWS = 1_000_000
RAW_TRINO_READ_ONLY_ERROR = "Raw Trino queries must be read-only SELECT statements."
DIRECT_TRINO_ROW_CAP_ERROR = f"Trino query returned more than {DIRECT_TRINO_MAX_ROWS:,} rows. Add a LIMIT clause."
DIRECT_TRINO_TIMEOUT_ERROR = "Trino query timed out."
+# Below this, a request has no realistic chance to complete before the deadline anyway; racing it
+# with a sub-second socket timeout only trades a clean timeout error for a confusing network one.
+_MIN_REQUEST_BUDGET_SECONDS = 1.0
def ensure_read_only_raw_trino_statement(sql: str) -> str:
@@ -112,6 +115,10 @@ class _CancelWatchdog:
if self.fired:
raise ExposedHogQLError(DIRECT_TRINO_TIMEOUT_ERROR)
+ def expire(self) -> None:
+ self._fired.set()
+ self.check()
+
@property
def cancelled(self) -> bool:
return self._cancelled.is_set()
@@ -137,7 +144,10 @@ def bounded_trino_cursor(
remaining = 1.0
else:
watchdog.check()
- remaining = max(watchdog.deadline - time.monotonic(), 0.001)
+ remaining = watchdog.deadline - time.monotonic()
+ if remaining < _MIN_REQUEST_BUDGET_SECONDS:
+ watchdog.expire()
+ remaining = max(remaining, 0.001)
timeout = kwargs.get("timeout")
if isinstance(timeout, tuple):
kwargs["timeout"] = tuple(min(value or remaining / 2, remaining / 2) for value in timeout)