:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(logs): give patterns_diff one deadline below the api slot ttl (#119059)
Jimmy Zhu1d30aa093997
diff --git a/posthog/clickhouse/client/limit.py b/posthog/clickhouse/client/limit.pyindex 5f6fa19f2af6..bd194995b2a9 100644--- a/posthog/clickhouse/client/limit.py+++ b/posthog/clickhouse/client/limit.py@@ -396,7 +396,8 @@ def get_logs_api_team_rate_limiter() -> RateLimit:get_task_id=lambda *args, **kwargs: kwargs.get("task_id") or generate_short_id(),# A slot is held for a whole request. The longest request is patterns_diff, which runs six sequential# queries (three per window) of up to 60 seconds each, the default max_execution_time. The ttl must- # outlast that, or the slot expires while the request still queries ClickHouse. A worker that dies+ # outlast that, or the slot expires while the request still queries ClickHouse. patterns_diff derives a+ # request-wide deadline from this ttl, so a longer per-query limit cannot outlast it. A worker that dies# without releasing its slot keeps it for the full ttl, so the ttl stays close to that bound.ttl=450,retry=0.134,diff --git a/products/logs/backend/pattern_diff.py b/products/logs/backend/pattern_diff.pyindex 9ffb7566f82d..61da97fd0079 100644--- a/products/logs/backend/pattern_diff.py+++ b/products/logs/backend/pattern_diff.py@@ -1,5 +1,6 @@import datetime as dtfrom collections.abc import Callable+from time import monotonicfrom typing import TYPE_CHECKINGfrom posthog.schema import DateRange, LogsQuery@@ -192,6 +193,7 @@ def run_patterns_diff(*,user: "User | None" = None,limit: int | None = None,+ deadline: float | None = None,) -> dict:"""Mine the query's window and a baseline window, and return classified diff entries.@@ -213,6 +215,9 @@ def run_patterns_diff(baseline_runner = PatternsQueryRunner(team=team, query=baseline_query, user=user)baseline_from = baseline_runner.query_date_range.date_from()baseline_to = baseline_runner.query_date_range.date_to()+ # Both windows and the first-seen lookup share one deadline, so the whole comparison stays inside the+ # caller's time budget.+ current_runner.request_deadline = baseline_runner.request_deadline = deadlineversions = {runner.stored_version() for runner in (current_runner, baseline_runner)}version = versions.pop() if len(versions) == 1 else None@@ -234,7 +239,7 @@ def run_patterns_diff()truncated = limit is not None and len(entries) > limitentries = entries[:limit]- first_seen = _first_seen_ever(team, query, entries, version, current_to) if version else {}+ first_seen = _first_seen_ever(team, query, entries, version, current_to, deadline) if version else {}history_failed = first_seen is Nonefirst_seen = first_seen or {}for entry in entries:@@ -276,8 +281,19 @@ def run_patterns_diff(def _first_seen_ever(- team: "Team", query: LogsQuery, entries: list[dict], version: int, current_to: dt.datetime+ team: "Team",+ query: LogsQuery,+ entries: list[dict],+ version: int,+ current_to: dt.datetime,+ deadline: float | None = None,) -> dict[str, dt.datetime] | None:+ settings = fail_fast_aggregate_settings()+ if deadline is not None:+ remaining = int(deadline - monotonic())+ if remaining < 1:+ return None+ settings.max_execution_time = min(remaining, settings.max_execution_time or remaining)members = list(dict.fromkeys(member for entry in entries for member in entry["pattern"]["match_patterns"]))members = members[:_FIRST_SEEN_MAX_MEMBERS] or [""]where = "pattern_version = {version} AND pattern IN {members} AND time_bucket >= {date_from} AND time_bucket < {date_to}"@@ -303,7 +319,7 @@ def _first_seen_ever(query=select,team=team,workload=Workload.LOGS,- settings=fail_fast_aggregate_settings(),+ settings=settings,)except (CHQueryErrorTooManyBytes, ClickHouseQueryTimeOut):return Nonediff --git a/products/logs/backend/patterns_query_runner.py b/products/logs/backend/patterns_query_runner.pyindex 298bde559d2a..dac8561efd63 100644--- a/products/logs/backend/patterns_query_runner.py+++ b/products/logs/backend/patterns_query_runner.py@@ -34,6 +34,7 @@ from products.logs.backend.logs_query_runner import LogsQueryResponse, LogsQueryif TYPE_CHECKING:from posthog.models import User+ from posthog.query_cache import QueryCache_TimeSlice = tuple[dt.datetime, dt.datetime]STORED_PATTERN_HEAD_MAX_BYTES = 65536@@ -76,6 +77,8 @@ class PatternsQueryRunner(AnalyticsQueryRunner[LogsQueryResponse], LogsQueryRunncached_response: CachedLogsQueryResponseuse_stored_patterns: bool = True_query_deadline: float | None = None+ request_deadline: float | None = None+ _request_deadline_binds: bool = Falsedef get_cache_payload(self) -> dict:payload = super().get_cache_payload()@@ -359,11 +362,16 @@ class PatternsQueryRunner(AnalyticsQueryRunner[LogsQueryResponse], LogsQueryRunndef _execute(self, query: ast.SelectQuery | ast.SelectSetQuery):settings = self.settings.model_copy()- if self._query_deadline is not None:- remaining = int(self._query_deadline - monotonic())+ deadline = min((d for d in (self._query_deadline, self.request_deadline) if d is not None), default=None)+ self._request_deadline_binds = False+ if deadline is not None:+ remaining = int(deadline - monotonic())+ own_limit = settings.max_execution_time or remaining+ self._request_deadline_binds = deadline == self.request_deadline and remaining <= own_limitif remaining < 1:raise ClickHouseQueryTimeOut()- settings.max_execution_time = remaining+ # The deadline bounds the whole run. It must not give one query more time than its own limit.+ settings.max_execution_time = min(remaining, own_limit)response = execute_hogql_query(query_type="LogsQuery",query=query,@@ -374,10 +382,17 @@ class PatternsQueryRunner(AnalyticsQueryRunner[LogsQueryResponse], LogsQueryRunnlimit_context=self.limit_context,settings=settings,)- if self._query_deadline is not None and monotonic() >= self._query_deadline:+ if deadline is not None and monotonic() >= deadline:raise ClickHouseQueryTimeOut()return response+ def _record_breaker_failure(self, cache_manager: "QueryCache", exc: Exception) -> None:+ # A failure under the request deadline depends on how long the earlier queries in the request took.+ # It does not repeat on a fresh request, so it must not open the breaker for this query.+ if self._request_deadline_binds:+ return+ super()._record_breaker_failure(cache_manager, exc)+def _count(self, slices: list[_TimeSlice] | None = None) -> int:response = self._execute(parse_select(diff --git a/products/logs/backend/presentation/views/api.py b/products/logs/backend/presentation/views/api.pyindex 89ec84ac9c59..f111aa788e14 100644--- a/products/logs/backend/presentation/views/api.py+++ b/products/logs/backend/presentation/views/api.py@@ -2,6 +2,7 @@ import reimport jsonimport base64import datetime as dt+from time import monotonicfrom typing import Anyfrom django.db import models@@ -104,6 +105,7 @@ __all__ = [tracer = trace.get_tracer(__name__)logger = structlog.get_logger(__name__)LOGS_MAX_EXPORT_ROWS = 10_000+LOGS_API_SLOT_TTL_MARGIN_SECONDS = 30MAX_ATTRIBUTE_KEYS = 100@@ -1987,6 +1989,9 @@ class LogsViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet):baseline_date_range,user=request.user if isinstance(request.user, User) else None,limit=options.validated_data.get("limit"),+ # The comparison runs several queries in sequence. End it before the concurrency slot's ttl, so the+ # slot cannot expire while the request still queries ClickHouse.+ deadline=monotonic() + get_logs_api_team_rate_limiter().ttl - LOGS_API_SLOT_TTL_MARGIN_SECONDS,)report_user_action(diff --git a/products/logs/backend/test/test_pattern_diff.py b/products/logs/backend/test/test_pattern_diff.pyindex 073d018ce1ef..e6b86133b218 100644--- a/products/logs/backend/test/test_pattern_diff.py+++ b/products/logs/backend/test/test_pattern_diff.py@@ -19,6 +19,8 @@ from posthog.schema import (PropertyOperator,)+from posthog.hogql.query import execute_hogql_query+from posthog.clickhouse.client import sync_executefrom posthog.errors import CHQueryErrorTooManyBytesfrom posthog.exceptions import ClickHouseQueryTimeOut@@ -249,6 +251,43 @@ class TestRunPatternsDiff(ClickhouseTestMixin, APIBaseTest):assert results["baseline"]["date_from"].startswith("2026-06-20T12:00:00")assert results["baseline"]["total_count"] == 2+ @time_machine.travel(_FROZEN_NOW, tick=False)+ def test_both_windows_share_one_deadline(self) -> None:+ self._insert([self._log("deploy marker line", day=23), self._log("deploy marker line", day=16)])+ query = LogsQuery(+ dateRange=DateRange(date_from="2026-06-23T12:00:00Z", date_to="2026-06-23T13:00:00Z"),+ filterGroup=PropertyGroupFilter(type=FilterLogicalOperator.AND_, values=[]),+ severityLevels=[],+ serviceNames=[],+ searchTerm=None,+ )++ with patch("products.logs.backend.patterns_query_runner.monotonic", return_value=100.0) as clock:+ budgets = []++ def execute_with_elapsed_time(*args, **kwargs):+ budgets.append(kwargs["settings"].max_execution_time)+ result = execute_hogql_query(*args, **kwargs)+ clock.return_value += 200+ return result++ with patch(+ "products.logs.backend.patterns_query_runner.execute_hogql_query",+ side_effect=execute_with_elapsed_time,+ ):+ # Three timeouts in a row would open the query failure breaker if they counted.+ for _ in range(3):+ budgets.clear()+ with self.assertRaises(ClickHouseQueryTimeOut):+ run_patterns_diff(self.team, query, None, deadline=clock.return_value + 420)+ # Two queries per window. The third query gets the 20 seconds left, and no query gets+ # more than its own 60-second limit.+ assert budgets == [60, 60, 20]++ results = run_patterns_diff(self.team, query, None)++ assert results["baseline"]["total_count"] == 1+class TestStoredPatternKeys(TestCase):def test_groups_sharing_a_stored_member_compare_as_one_pattern(self) -> None:@@ -348,6 +387,20 @@ class TestFirstSeenEver(TestCase):assert result is None+ @parameterized.expand([("time_left", 110.0, 10), ("deadline_passed", 100.5, None)])+ def test_stays_inside_the_request_deadline(self, _name, deadline, expected_limit) -> None:+ with (+ patch("products.logs.backend.pattern_diff.monotonic", return_value=100.0),+ patch("products.logs.backend.pattern_diff.execute_hogql_query") as execute,+ ):+ execute.return_value.results = []+ _first_seen_ever(MagicMock(), _bare_query(), self._ENTRIES, 1, dt.datetime.now(dt.UTC), deadline)++ if expected_limit is None:+ execute.assert_not_called()+ else:+ assert execute.call_args.kwargs["settings"].max_execution_time == expected_limit+class TestRunPatternsDiffV2(ClickhouseTestMixin, APIBaseTest):CLASS_DATA_LEVEL_SETUP = False