Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(replay-vision): stop the completion sweep before its query budget runs out (#119285)

posthog[bot]73d2b5360461
diff --git a/products/replay_vision/backend/observation_requests.py b/products/replay_vision/backend/observation_requests.py
index ad65ec7ba335..ff5545ccccd5 100644
--- a/products/replay_vision/backend/observation_requests.py
+++ b/products/replay_vision/backend/observation_requests.py
@@ -12,12 +12,13 @@ from typing import Any, Literal
from uuid import NAMESPACE_URL, UUID, uuid5
from django.core.cache import cache
-from django.db import IntegrityError, connection, models, transaction
+from django.db import IntegrityError, OperationalError, connection, models, transaction
from django.db.models import F, Q
from django.utils import timezone
import structlog
+from posthog.api.statement_timeout import is_query_canceled
from posthog.dataclasses import frozen
from posthog.kafka_client.client import ProduceResult
from posthog.models.organization import OrganizationMembership
@@ -61,8 +62,8 @@ MAX_STARTS_PER_TICK = 50
# How many waiting requests one tick checks for ended sessions.
MAX_CHECKS_PER_TICK = 200
-# Each sweep gets its own budget, and together they leave headroom inside the reconciler activity's 45-second
-# start-to-close timeout, so a slow start sweep can't starve the completion sweep or the reverse.
+# Each sweep gets its own budget, and together they leave headroom inside the completion activity's query
+# budget (75% of its timeout), so a slow start sweep can't starve the completion sweep or the reverse.
START_SWEEP_BUDGET_SECONDS = 15
COMPLETION_SWEEP_BUDGET_SECONDS = 20
@@ -71,6 +72,8 @@ COMPLETION_SWEEP_BUDGET_SECONDS = 20
WORKFLOW_STEP_MAX_WAIT = timedelta(hours=8)
_DELIVERY_TIMEOUT_SECONDS = 10
+# The completion sweep stops this long before the query budget ends, so its last page can flush and stamp.
+_STAMP_HEADROOM_SECONDS = _DELIVERY_TIMEOUT_SECONDS + 5
_SWEEP_CURSOR_KEY = "replay_vision:completion_sweep_cursor"
_WRITER = "default"
_SWEEP_CURSOR_TTL_SECONDS = 60 * 60
@@ -317,6 +320,8 @@ def start_waiting_requests(*, now: datetime | None = None, budget_seconds: float
logger.exception("replay_vision.observation_request.deferred_start_failed", request_id=str(request.id))
continue
started += 1
+ if (overrun := time.monotonic() - deadline) > 0:
+ logger.warning("replay_vision.observation_request.start_sweep_overran", overrun_seconds=round(overrun, 1))
return started
@@ -461,7 +466,10 @@ def _session(outcome: dict[str, str], observation: ReplayObservation | None, exp
def complete_settled_requests(
- *, now: datetime | None = None, budget_seconds: float = COMPLETION_SWEEP_BUDGET_SECONDS
+ *,
+ now: datetime | None = None,
+ budget_seconds: float = COMPLETION_SWEEP_BUDGET_SECONDS,
+ query_deadline: float | None = None,
) -> int:
"""Mark every open request whose sessions have all settled as completed, announcing each one first.
@@ -469,8 +477,13 @@ def complete_settled_requests(
tick rather than never; a destination can therefore see a request complete twice. Open requests are
walked oldest first in pages. A tick that runs out of time leaves a cursor, and the next tick starts
there and wraps around, so requests whose wakes keep failing never hold back the ones behind them.
+
+ `query_deadline` is the wall-clock time at which Postgres starts to cancel statements. The sweep stops early
+ enough before it that a canceled stamp can't re-announce a page whose events already went out.
"""
now = now or timezone.now()
+ if query_deadline is not None:
+ budget_seconds = min(budget_seconds, query_deadline - time.time() - _STAMP_HEADROOM_SECONDS)
deadline = time.monotonic() + budget_seconds
open_requests = (
ReplayObservationRequest.objects.unscoped()
@@ -482,26 +495,33 @@ def complete_settled_requests(
cursor = start
wrapped = start is None
completed = 0
- while time.monotonic() < deadline:
- page_qs = open_requests
- if cursor is not None:
- page_qs = page_qs.filter(_after(cursor))
- if wrapped and start is not None:
- page_qs = page_qs.exclude(_after(start))
- page = list(page_qs[:_SWEEP_PAGE_SIZE])
- if not page:
- if wrapped:
- _store_sweep_cursor(None)
- return completed
- wrapped, cursor = True, None
- continue
- page_completed, stopped_at = _complete_page(page, now, deadline)
- completed += page_completed
- if stopped_at is not None:
- if stopped_at:
- cursor = (page[stopped_at - 1].created_at, page[stopped_at - 1].id)
- break
- cursor = (page[-1].created_at, page[-1].id)
+ try:
+ while time.monotonic() < deadline:
+ page_qs = open_requests
+ if cursor is not None:
+ page_qs = page_qs.filter(_after(cursor))
+ if wrapped and start is not None:
+ page_qs = page_qs.exclude(_after(start))
+ page = list(page_qs[:_SWEEP_PAGE_SIZE])
+ if not page:
+ if wrapped:
+ _store_sweep_cursor(None)
+ return completed
+ wrapped, cursor = True, None
+ continue
+ page_completed, stopped_at = _complete_page(page, now, deadline)
+ completed += page_completed
+ if stopped_at is not None:
+ if stopped_at:
+ cursor = (page[stopped_at - 1].created_at, page[stopped_at - 1].id)
+ break
+ cursor = (page[-1].created_at, page[-1].id)
+ except OperationalError as err:
+ # The activity's query budget can run out before this sweep's own budget does. The page it cancels
+ # starts the next tick, so stop here instead of failing the activity.
+ if not is_query_canceled(err):
+ raise
+ logger.warning("replay_vision.observation_request.completion_sweep_canceled", completed=completed)
_store_sweep_cursor(cursor)
return completed
diff --git a/products/replay_vision/backend/temporal/activities/complete_observation_requests.py b/products/replay_vision/backend/temporal/activities/complete_observation_requests.py
index ec3edf1a3a1d..e566a657f1b0 100644
--- a/products/replay_vision/backend/temporal/activities/complete_observation_requests.py
+++ b/products/replay_vision/backend/temporal/activities/complete_observation_requests.py
@@ -18,12 +18,12 @@ def _complete_observation_requests() -> int:
)
# Isolated so a failure in one sweep still lets the other run; each also keeps to its own time budget.
- with bounded_queries(COMPLETE_OBSERVATION_REQUESTS_TIMEOUT):
+ with bounded_queries(COMPLETE_OBSERVATION_REQUESTS_TIMEOUT) as query_deadline:
try:
start_waiting_requests()
except Exception:
activity.logger.exception("Starting waiting observation requests failed")
- return complete_settled_requests()
+ return complete_settled_requests(query_deadline=query_deadline)
@activity.defn
diff --git a/products/replay_vision/backend/temporal/query_budget.py b/products/replay_vision/backend/temporal/query_budget.py
index 77335688e653..ff438fa07ce4 100644
--- a/products/replay_vision/backend/temporal/query_budget.py
+++ b/products/replay_vision/backend/temporal/query_budget.py
@@ -70,16 +70,16 @@ def _capped(connection: BaseDatabaseWrapper, deadline: float) -> Iterator[None]:
@contextmanager
-def bounded_queries(attempt_timeout: dt.timedelta, *, from_attempt_start: bool = True) -> Iterator[None]:
+def bounded_queries(attempt_timeout: dt.timedelta, *, from_attempt_start: bool = True) -> Iterator[float]:
"""Cancel Postgres work in the block once it runs past most of the timeout that ends the attempt.
A timed-out attempt leaves its thread and query running, so each retry would add another copy holding locks.
Every block in one attempt shares the budget counted from the attempt's start; pass `from_attempt_start=False`
- to give each block its own. No connection opens until a statement runs on it.
+ to give each block its own. No connection opens until a statement runs on it. Yields the wall-clock deadline.
"""
started = _attempt_started_at() if from_attempt_start else time.time()
deadline = started + attempt_timeout.total_seconds() * _BUDGET_SHARE
with ExitStack() as stack:
for alias in _aliases():
stack.enter_context(_capped(connections[alias], deadline))
- yield
+ yield deadline
diff --git a/products/replay_vision/backend/tests/test_observation_requests_api.py b/products/replay_vision/backend/tests/test_observation_requests_api.py
index 453a7ad48faa..66a36505d680 100644
--- a/products/replay_vision/backend/tests/test_observation_requests_api.py
+++ b/products/replay_vision/backend/tests/test_observation_requests_api.py
@@ -1,12 +1,16 @@
+import time
from datetime import timedelta
from typing import Any
+import pytest
from posthog.test.base import APIBaseTest
from unittest.mock import MagicMock, patch
from django.core.cache import cache
+from django.db import OperationalError, connection
from django.utils import timezone
+import psycopg
from parameterized import parameterized
from rest_framework.test import APIClient
@@ -433,6 +437,53 @@ class TestCompleteSettledRequests(APIBaseTest):
assert ReplayObservationRequest.objects.for_team(self.team.id).filter(completed_at__isnull=True).count() == 0
assert cache.get(_SWEEP_CURSOR_KEY) is None
+ def _fail_second_page_read(self, cause: type[Exception]) -> Any:
+ page_reads = 0
+
+ def wrapper(execute: Any, sql: str, params: Any, many: bool, context: Any) -> Any:
+ nonlocal page_reads
+ if sql.startswith("SELECT") and '"completed_at" IS NULL' in sql and "LIMIT" in sql:
+ page_reads += 1
+ if page_reads == 2:
+ raise OperationalError("statement failed") from cause()
+ return execute(sql, params, many, context)
+
+ return connection.execute_wrapper(wrapper)
+
+ @patch("products.replay_vision.backend.observation_requests.produce_internal_event")
+ def test_a_nearly_spent_query_budget_announces_nothing(self, produce: MagicMock) -> None:
+ request = self._request({"s1": "skipped_quota"})
+
+ assert complete_settled_requests(query_deadline=time.time() + 5) == 0
+
+ produce.assert_not_called()
+ request.refresh_from_db()
+ assert request.completed_at is None
+
+ @patch("products.replay_vision.backend.observation_requests._SWEEP_PAGE_SIZE", 1)
+ @patch("products.replay_vision.backend.observation_requests.produce_internal_event")
+ def test_a_canceled_statement_ends_the_tick_and_keeps_its_cursor(self, _produce: MagicMock) -> None:
+ first = self._request({"s1": "skipped_quota"})
+ second = self._request({"s2": "skipped_quota"})
+
+ with self._fail_second_page_read(psycopg.errors.QueryCanceled):
+ assert complete_settled_requests() == 1
+
+ first.refresh_from_db()
+ second.refresh_from_db()
+ assert first.completed_at is not None
+ assert second.completed_at is None
+ assert cache.get(_SWEEP_CURSOR_KEY) == (first.created_at.isoformat(), str(first.id))
+
+ @patch("products.replay_vision.backend.observation_requests._SWEEP_PAGE_SIZE", 1)
+ @patch("products.replay_vision.backend.observation_requests.produce_internal_event")
+ def test_other_database_errors_still_fail_the_sweep(self, _produce: MagicMock) -> None:
+ self._request({"s1": "skipped_quota"})
+ self._request({"s2": "skipped_quota"})
+
+ with self._fail_second_page_read(psycopg.errors.AdminShutdown), pytest.raises(OperationalError):
+ complete_settled_requests()
+
@parameterized.expand(
[
("settled_and_delivered", "skipped_quota", None, True),