Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(logs): give patterns_diff one deadline below the api slot ttl (#119059)

Jimmy Zhu1d30aa093997
diff --git a/posthog/clickhouse/client/limit.py b/posthog/clickhouse/client/limit.py
index 5f6fa19f2af6..bd194995b2a9 100644
--- a/posthog/clickhouse/client/limit.py
+++ b/posthog/clickhouse/client/limit.py
@@ -396,7 +396,8 @@ def get_logs_api_team_rate_limiter() -> RateLimit:
get_task_id=lambda *args, **kwargs: kwargs.get("task_id") or generate_short_id(),
# A slot is held for a whole request. The longest request is patterns_diff, which runs six sequential
# queries (three per window) of up to 60 seconds each, the default max_execution_time. The ttl must
- # outlast that, or the slot expires while the request still queries ClickHouse. A worker that dies
+ # outlast that, or the slot expires while the request still queries ClickHouse. patterns_diff derives a
+ # request-wide deadline from this ttl, so a longer per-query limit cannot outlast it. A worker that dies
# without releasing its slot keeps it for the full ttl, so the ttl stays close to that bound.
ttl=450,
retry=0.134,
diff --git a/products/logs/backend/pattern_diff.py b/products/logs/backend/pattern_diff.py
index 9ffb7566f82d..61da97fd0079 100644
--- a/products/logs/backend/pattern_diff.py
+++ b/products/logs/backend/pattern_diff.py
@@ -1,5 +1,6 @@
import datetime as dt
from collections.abc import Callable
+from time import monotonic
from typing import TYPE_CHECKING
from posthog.schema import DateRange, LogsQuery
@@ -192,6 +193,7 @@ def run_patterns_diff(
*,
user: "User | None" = None,
limit: int | None = None,
+ deadline: float | None = None,
) -> dict:
"""Mine the query's window and a baseline window, and return classified diff entries.
@@ -213,6 +215,9 @@ def run_patterns_diff(
baseline_runner = PatternsQueryRunner(team=team, query=baseline_query, user=user)
baseline_from = baseline_runner.query_date_range.date_from()
baseline_to = baseline_runner.query_date_range.date_to()
+ # Both windows and the first-seen lookup share one deadline, so the whole comparison stays inside the
+ # caller's time budget.
+ current_runner.request_deadline = baseline_runner.request_deadline = deadline
versions = {runner.stored_version() for runner in (current_runner, baseline_runner)}
version = versions.pop() if len(versions) == 1 else None
@@ -234,7 +239,7 @@ def run_patterns_diff(
)
truncated = limit is not None and len(entries) > limit
entries = entries[:limit]
- first_seen = _first_seen_ever(team, query, entries, version, current_to) if version else {}
+ first_seen = _first_seen_ever(team, query, entries, version, current_to, deadline) if version else {}
history_failed = first_seen is None
first_seen = first_seen or {}
for entry in entries:
@@ -276,8 +281,19 @@ def run_patterns_diff(
def _first_seen_ever(
- team: "Team", query: LogsQuery, entries: list[dict], version: int, current_to: dt.datetime
+ team: "Team",
+ query: LogsQuery,
+ entries: list[dict],
+ version: int,
+ current_to: dt.datetime,
+ deadline: float | None = None,
) -> dict[str, dt.datetime] | None:
+ settings = fail_fast_aggregate_settings()
+ if deadline is not None:
+ remaining = int(deadline - monotonic())
+ if remaining < 1:
+ return None
+ settings.max_execution_time = min(remaining, settings.max_execution_time or remaining)
members = list(dict.fromkeys(member for entry in entries for member in entry["pattern"]["match_patterns"]))
members = members[:_FIRST_SEEN_MAX_MEMBERS] or [""]
where = "pattern_version = {version} AND pattern IN {members} AND time_bucket >= {date_from} AND time_bucket < {date_to}"
@@ -303,7 +319,7 @@ def _first_seen_ever(
query=select,
team=team,
workload=Workload.LOGS,
- settings=fail_fast_aggregate_settings(),
+ settings=settings,
)
except (CHQueryErrorTooManyBytes, ClickHouseQueryTimeOut):
return None
diff --git a/products/logs/backend/patterns_query_runner.py b/products/logs/backend/patterns_query_runner.py
index 298bde559d2a..dac8561efd63 100644
--- a/products/logs/backend/patterns_query_runner.py
+++ b/products/logs/backend/patterns_query_runner.py
@@ -34,6 +34,7 @@ from products.logs.backend.logs_query_runner import LogsQueryResponse, LogsQuery
if TYPE_CHECKING:
from posthog.models import User
+ from posthog.query_cache import QueryCache
_TimeSlice = tuple[dt.datetime, dt.datetime]
STORED_PATTERN_HEAD_MAX_BYTES = 65536
@@ -76,6 +77,8 @@ class PatternsQueryRunner(AnalyticsQueryRunner[LogsQueryResponse], LogsQueryRunn
cached_response: CachedLogsQueryResponse
use_stored_patterns: bool = True
_query_deadline: float | None = None
+ request_deadline: float | None = None
+ _request_deadline_binds: bool = False
def get_cache_payload(self) -> dict:
payload = super().get_cache_payload()
@@ -359,11 +362,16 @@ class PatternsQueryRunner(AnalyticsQueryRunner[LogsQueryResponse], LogsQueryRunn
def _execute(self, query: ast.SelectQuery | ast.SelectSetQuery):
settings = self.settings.model_copy()
- if self._query_deadline is not None:
- remaining = int(self._query_deadline - monotonic())
+ deadline = min((d for d in (self._query_deadline, self.request_deadline) if d is not None), default=None)
+ self._request_deadline_binds = False
+ if deadline is not None:
+ remaining = int(deadline - monotonic())
+ own_limit = settings.max_execution_time or remaining
+ self._request_deadline_binds = deadline == self.request_deadline and remaining <= own_limit
if remaining < 1:
raise ClickHouseQueryTimeOut()
- settings.max_execution_time = remaining
+ # The deadline bounds the whole run. It must not give one query more time than its own limit.
+ settings.max_execution_time = min(remaining, own_limit)
response = execute_hogql_query(
query_type="LogsQuery",
query=query,
@@ -374,10 +382,17 @@ class PatternsQueryRunner(AnalyticsQueryRunner[LogsQueryResponse], LogsQueryRunn
limit_context=self.limit_context,
settings=settings,
)
- if self._query_deadline is not None and monotonic() >= self._query_deadline:
+ if deadline is not None and monotonic() >= deadline:
raise ClickHouseQueryTimeOut()
return response
+ def _record_breaker_failure(self, cache_manager: "QueryCache", exc: Exception) -> None:
+ # A failure under the request deadline depends on how long the earlier queries in the request took.
+ # It does not repeat on a fresh request, so it must not open the breaker for this query.
+ if self._request_deadline_binds:
+ return
+ super()._record_breaker_failure(cache_manager, exc)
+
def _count(self, slices: list[_TimeSlice] | None = None) -> int:
response = self._execute(
parse_select(
diff --git a/products/logs/backend/presentation/views/api.py b/products/logs/backend/presentation/views/api.py
index 89ec84ac9c59..f111aa788e14 100644
--- a/products/logs/backend/presentation/views/api.py
+++ b/products/logs/backend/presentation/views/api.py
@@ -2,6 +2,7 @@ import re
import json
import base64
import datetime as dt
+from time import monotonic
from typing import Any
from django.db import models
@@ -104,6 +105,7 @@ __all__ = [
tracer = trace.get_tracer(__name__)
logger = structlog.get_logger(__name__)
LOGS_MAX_EXPORT_ROWS = 10_000
+LOGS_API_SLOT_TTL_MARGIN_SECONDS = 30
MAX_ATTRIBUTE_KEYS = 100
@@ -1987,6 +1989,9 @@ class LogsViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet):
baseline_date_range,
user=request.user if isinstance(request.user, User) else None,
limit=options.validated_data.get("limit"),
+ # The comparison runs several queries in sequence. End it before the concurrency slot's ttl, so the
+ # slot cannot expire while the request still queries ClickHouse.
+ deadline=monotonic() + get_logs_api_team_rate_limiter().ttl - LOGS_API_SLOT_TTL_MARGIN_SECONDS,
)
report_user_action(
diff --git a/products/logs/backend/test/test_pattern_diff.py b/products/logs/backend/test/test_pattern_diff.py
index 073d018ce1ef..e6b86133b218 100644
--- a/products/logs/backend/test/test_pattern_diff.py
+++ b/products/logs/backend/test/test_pattern_diff.py
@@ -19,6 +19,8 @@ from posthog.schema import (
PropertyOperator,
)
+from posthog.hogql.query import execute_hogql_query
+
from posthog.clickhouse.client import sync_execute
from posthog.errors import CHQueryErrorTooManyBytes
from posthog.exceptions import ClickHouseQueryTimeOut
@@ -249,6 +251,43 @@ class TestRunPatternsDiff(ClickhouseTestMixin, APIBaseTest):
assert results["baseline"]["date_from"].startswith("2026-06-20T12:00:00")
assert results["baseline"]["total_count"] == 2
+ @time_machine.travel(_FROZEN_NOW, tick=False)
+ def test_both_windows_share_one_deadline(self) -> None:
+ self._insert([self._log("deploy marker line", day=23), self._log("deploy marker line", day=16)])
+ query = LogsQuery(
+ dateRange=DateRange(date_from="2026-06-23T12:00:00Z", date_to="2026-06-23T13:00:00Z"),
+ filterGroup=PropertyGroupFilter(type=FilterLogicalOperator.AND_, values=[]),
+ severityLevels=[],
+ serviceNames=[],
+ searchTerm=None,
+ )
+
+ with patch("products.logs.backend.patterns_query_runner.monotonic", return_value=100.0) as clock:
+ budgets = []
+
+ def execute_with_elapsed_time(*args, **kwargs):
+ budgets.append(kwargs["settings"].max_execution_time)
+ result = execute_hogql_query(*args, **kwargs)
+ clock.return_value += 200
+ return result
+
+ with patch(
+ "products.logs.backend.patterns_query_runner.execute_hogql_query",
+ side_effect=execute_with_elapsed_time,
+ ):
+ # Three timeouts in a row would open the query failure breaker if they counted.
+ for _ in range(3):
+ budgets.clear()
+ with self.assertRaises(ClickHouseQueryTimeOut):
+ run_patterns_diff(self.team, query, None, deadline=clock.return_value + 420)
+ # Two queries per window. The third query gets the 20 seconds left, and no query gets
+ # more than its own 60-second limit.
+ assert budgets == [60, 60, 20]
+
+ results = run_patterns_diff(self.team, query, None)
+
+ assert results["baseline"]["total_count"] == 1
+
class TestStoredPatternKeys(TestCase):
def test_groups_sharing_a_stored_member_compare_as_one_pattern(self) -> None:
@@ -348,6 +387,20 @@ class TestFirstSeenEver(TestCase):
assert result is None
+ @parameterized.expand([("time_left", 110.0, 10), ("deadline_passed", 100.5, None)])
+ def test_stays_inside_the_request_deadline(self, _name, deadline, expected_limit) -> None:
+ with (
+ patch("products.logs.backend.pattern_diff.monotonic", return_value=100.0),
+ patch("products.logs.backend.pattern_diff.execute_hogql_query") as execute,
+ ):
+ execute.return_value.results = []
+ _first_seen_ever(MagicMock(), _bare_query(), self._ENTRIES, 1, dt.datetime.now(dt.UTC), deadline)
+
+ if expected_limit is None:
+ execute.assert_not_called()
+ else:
+ assert execute.call_args.kwargs["settings"].max_execution_time == expected_limit
+
class TestRunPatternsDiffV2(ClickhouseTestMixin, APIBaseTest):
CLASS_DATA_LEVEL_SETUP = False