:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(replay-vision): stop the completion sweep before its query budget runs out (#119285)
posthog[bot]73d2b5360461
diff --git a/products/replay_vision/backend/observation_requests.py b/products/replay_vision/backend/observation_requests.pyindex ad65ec7ba335..ff5545ccccd5 100644--- a/products/replay_vision/backend/observation_requests.py+++ b/products/replay_vision/backend/observation_requests.py@@ -12,12 +12,13 @@ from typing import Any, Literalfrom uuid import NAMESPACE_URL, UUID, uuid5from django.core.cache import cache-from django.db import IntegrityError, connection, models, transaction+from django.db import IntegrityError, OperationalError, connection, models, transactionfrom django.db.models import F, Qfrom django.utils import timezoneimport structlog+from posthog.api.statement_timeout import is_query_canceledfrom posthog.dataclasses import frozenfrom posthog.kafka_client.client import ProduceResultfrom posthog.models.organization import OrganizationMembership@@ -61,8 +62,8 @@ MAX_STARTS_PER_TICK = 50# How many waiting requests one tick checks for ended sessions.MAX_CHECKS_PER_TICK = 200-# Each sweep gets its own budget, and together they leave headroom inside the reconciler activity's 45-second-# start-to-close timeout, so a slow start sweep can't starve the completion sweep or the reverse.+# Each sweep gets its own budget, and together they leave headroom inside the completion activity's query+# budget (75% of its timeout), so a slow start sweep can't starve the completion sweep or the reverse.START_SWEEP_BUDGET_SECONDS = 15COMPLETION_SWEEP_BUDGET_SECONDS = 20@@ -71,6 +72,8 @@ COMPLETION_SWEEP_BUDGET_SECONDS = 20WORKFLOW_STEP_MAX_WAIT = timedelta(hours=8)_DELIVERY_TIMEOUT_SECONDS = 10+# The completion sweep stops this long before the query budget ends, so its last page can flush and stamp.+_STAMP_HEADROOM_SECONDS = _DELIVERY_TIMEOUT_SECONDS + 5_SWEEP_CURSOR_KEY = "replay_vision:completion_sweep_cursor"_WRITER = "default"_SWEEP_CURSOR_TTL_SECONDS = 60 * 60@@ -317,6 +320,8 @@ def start_waiting_requests(*, now: datetime | None = None, budget_seconds: floatlogger.exception("replay_vision.observation_request.deferred_start_failed", request_id=str(request.id))continuestarted += 1+ if (overrun := time.monotonic() - deadline) > 0:+ logger.warning("replay_vision.observation_request.start_sweep_overran", overrun_seconds=round(overrun, 1))return started@@ -461,7 +466,10 @@ def _session(outcome: dict[str, str], observation: ReplayObservation | None, expdef complete_settled_requests(- *, now: datetime | None = None, budget_seconds: float = COMPLETION_SWEEP_BUDGET_SECONDS+ *,+ now: datetime | None = None,+ budget_seconds: float = COMPLETION_SWEEP_BUDGET_SECONDS,+ query_deadline: float | None = None,) -> int:"""Mark every open request whose sessions have all settled as completed, announcing each one first.@@ -469,8 +477,13 @@ def complete_settled_requests(tick rather than never; a destination can therefore see a request complete twice. Open requests arewalked oldest first in pages. A tick that runs out of time leaves a cursor, and the next tick startsthere and wraps around, so requests whose wakes keep failing never hold back the ones behind them.++ `query_deadline` is the wall-clock time at which Postgres starts to cancel statements. The sweep stops early+ enough before it that a canceled stamp can't re-announce a page whose events already went out."""now = now or timezone.now()+ if query_deadline is not None:+ budget_seconds = min(budget_seconds, query_deadline - time.time() - _STAMP_HEADROOM_SECONDS)deadline = time.monotonic() + budget_secondsopen_requests = (ReplayObservationRequest.objects.unscoped()@@ -482,26 +495,33 @@ def complete_settled_requests(cursor = startwrapped = start is Nonecompleted = 0- while time.monotonic() < deadline:- page_qs = open_requests- if cursor is not None:- page_qs = page_qs.filter(_after(cursor))- if wrapped and start is not None:- page_qs = page_qs.exclude(_after(start))- page = list(page_qs[:_SWEEP_PAGE_SIZE])- if not page:- if wrapped:- _store_sweep_cursor(None)- return completed- wrapped, cursor = True, None- continue- page_completed, stopped_at = _complete_page(page, now, deadline)- completed += page_completed- if stopped_at is not None:- if stopped_at:- cursor = (page[stopped_at - 1].created_at, page[stopped_at - 1].id)- break- cursor = (page[-1].created_at, page[-1].id)+ try:+ while time.monotonic() < deadline:+ page_qs = open_requests+ if cursor is not None:+ page_qs = page_qs.filter(_after(cursor))+ if wrapped and start is not None:+ page_qs = page_qs.exclude(_after(start))+ page = list(page_qs[:_SWEEP_PAGE_SIZE])+ if not page:+ if wrapped:+ _store_sweep_cursor(None)+ return completed+ wrapped, cursor = True, None+ continue+ page_completed, stopped_at = _complete_page(page, now, deadline)+ completed += page_completed+ if stopped_at is not None:+ if stopped_at:+ cursor = (page[stopped_at - 1].created_at, page[stopped_at - 1].id)+ break+ cursor = (page[-1].created_at, page[-1].id)+ except OperationalError as err:+ # The activity's query budget can run out before this sweep's own budget does. The page it cancels+ # starts the next tick, so stop here instead of failing the activity.+ if not is_query_canceled(err):+ raise+ logger.warning("replay_vision.observation_request.completion_sweep_canceled", completed=completed)_store_sweep_cursor(cursor)return completeddiff --git a/products/replay_vision/backend/temporal/activities/complete_observation_requests.py b/products/replay_vision/backend/temporal/activities/complete_observation_requests.pyindex ec3edf1a3a1d..e566a657f1b0 100644--- a/products/replay_vision/backend/temporal/activities/complete_observation_requests.py+++ b/products/replay_vision/backend/temporal/activities/complete_observation_requests.py@@ -18,12 +18,12 @@ def _complete_observation_requests() -> int:)# Isolated so a failure in one sweep still lets the other run; each also keeps to its own time budget.- with bounded_queries(COMPLETE_OBSERVATION_REQUESTS_TIMEOUT):+ with bounded_queries(COMPLETE_OBSERVATION_REQUESTS_TIMEOUT) as query_deadline:try:start_waiting_requests()except Exception:activity.logger.exception("Starting waiting observation requests failed")- return complete_settled_requests()+ return complete_settled_requests(query_deadline=query_deadline)@activity.defndiff --git a/products/replay_vision/backend/temporal/query_budget.py b/products/replay_vision/backend/temporal/query_budget.pyindex 77335688e653..ff438fa07ce4 100644--- a/products/replay_vision/backend/temporal/query_budget.py+++ b/products/replay_vision/backend/temporal/query_budget.py@@ -70,16 +70,16 @@ def _capped(connection: BaseDatabaseWrapper, deadline: float) -> Iterator[None]:@contextmanager-def bounded_queries(attempt_timeout: dt.timedelta, *, from_attempt_start: bool = True) -> Iterator[None]:+def bounded_queries(attempt_timeout: dt.timedelta, *, from_attempt_start: bool = True) -> Iterator[float]:"""Cancel Postgres work in the block once it runs past most of the timeout that ends the attempt.A timed-out attempt leaves its thread and query running, so each retry would add another copy holding locks.Every block in one attempt shares the budget counted from the attempt's start; pass `from_attempt_start=False`- to give each block its own. No connection opens until a statement runs on it.+ to give each block its own. No connection opens until a statement runs on it. Yields the wall-clock deadline."""started = _attempt_started_at() if from_attempt_start else time.time()deadline = started + attempt_timeout.total_seconds() * _BUDGET_SHAREwith ExitStack() as stack:for alias in _aliases():stack.enter_context(_capped(connections[alias], deadline))- yield+ yield deadlinediff --git a/products/replay_vision/backend/tests/test_observation_requests_api.py b/products/replay_vision/backend/tests/test_observation_requests_api.pyindex 453a7ad48faa..66a36505d680 100644--- a/products/replay_vision/backend/tests/test_observation_requests_api.py+++ b/products/replay_vision/backend/tests/test_observation_requests_api.py@@ -1,12 +1,16 @@+import timefrom datetime import timedeltafrom typing import Any+import pytestfrom posthog.test.base import APIBaseTestfrom unittest.mock import MagicMock, patchfrom django.core.cache import cache+from django.db import OperationalError, connectionfrom django.utils import timezone+import psycopgfrom parameterized import parameterizedfrom rest_framework.test import APIClient@@ -433,6 +437,53 @@ class TestCompleteSettledRequests(APIBaseTest):assert ReplayObservationRequest.objects.for_team(self.team.id).filter(completed_at__isnull=True).count() == 0assert cache.get(_SWEEP_CURSOR_KEY) is None+ def _fail_second_page_read(self, cause: type[Exception]) -> Any:+ page_reads = 0++ def wrapper(execute: Any, sql: str, params: Any, many: bool, context: Any) -> Any:+ nonlocal page_reads+ if sql.startswith("SELECT") and '"completed_at" IS NULL' in sql and "LIMIT" in sql:+ page_reads += 1+ if page_reads == 2:+ raise OperationalError("statement failed") from cause()+ return execute(sql, params, many, context)++ return connection.execute_wrapper(wrapper)++ @patch("products.replay_vision.backend.observation_requests.produce_internal_event")+ def test_a_nearly_spent_query_budget_announces_nothing(self, produce: MagicMock) -> None:+ request = self._request({"s1": "skipped_quota"})++ assert complete_settled_requests(query_deadline=time.time() + 5) == 0++ produce.assert_not_called()+ request.refresh_from_db()+ assert request.completed_at is None++ @patch("products.replay_vision.backend.observation_requests._SWEEP_PAGE_SIZE", 1)+ @patch("products.replay_vision.backend.observation_requests.produce_internal_event")+ def test_a_canceled_statement_ends_the_tick_and_keeps_its_cursor(self, _produce: MagicMock) -> None:+ first = self._request({"s1": "skipped_quota"})+ second = self._request({"s2": "skipped_quota"})++ with self._fail_second_page_read(psycopg.errors.QueryCanceled):+ assert complete_settled_requests() == 1++ first.refresh_from_db()+ second.refresh_from_db()+ assert first.completed_at is not None+ assert second.completed_at is None+ assert cache.get(_SWEEP_CURSOR_KEY) == (first.created_at.isoformat(), str(first.id))++ @patch("products.replay_vision.backend.observation_requests._SWEEP_PAGE_SIZE", 1)+ @patch("products.replay_vision.backend.observation_requests.produce_internal_event")+ def test_other_database_errors_still_fail_the_sweep(self, _produce: MagicMock) -> None:+ self._request({"s1": "skipped_quota"})+ self._request({"s2": "skipped_quota"})++ with self._fail_second_page_read(psycopg.errors.AdminShutdown), pytest.raises(OperationalError):+ complete_settled_requests()+@parameterized.expand([("settled_and_delivered", "skipped_quota", None, True),