:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-vision): harden the watch rank consumer and add the metrics its alerts need (#124617)
Tue Haulund3777ff0843fb
diff --git a/products/replay_vision/backend/tests/test_watch_rank_consumer.py b/products/replay_vision/backend/tests/test_watch_rank_consumer.pyindex 3bb5e9a5277f..1edcc2f07e08 100644--- a/products/replay_vision/backend/tests/test_watch_rank_consumer.py+++ b/products/replay_vision/backend/tests/test_watch_rank_consumer.py@@ -1,16 +1,21 @@import jsonimport mathimport time+import uuid+import asynciofrom typing import Anyfrom posthog.test.base import BaseTestfrom unittest.mock import AsyncMock, MagicMock, patch+from django.db import OperationalErrorfrom django.test import SimpleTestCase+from django.utils import timezonefrom asgiref.sync import async_to_syncfrom confluent_kafka import KafkaExceptionfrom parameterized import parameterized+from prometheus_client import REGISTRYfrom products.replay_vision.backend.embeddings import (EMBEDDING_DOCUMENT_TYPE,@@ -123,6 +128,7 @@ class TestWatchRankConsumer(BaseTest):with (patch(f"{_WATCH_RANK}.anchor_vectors", AsyncMock(side_effect=anchor_error, return_value=self._ANCHORS)),patch(f"{_CONSUMER}._RETRY_WINDOW_S", 0),+ patch(f"{_CONSUMER}._ERROR_RETRY_WINDOW_S", 0),# It would close the connection that holds the test transaction.patch(f"{_CONSUMER}.close_old_connections"),):@@ -185,3 +191,40 @@ class TestWatchRankConsumer(BaseTest):):for _ in range(2):assert async_to_sync(consumer.handle)(message, time.monotonic()) == "deferred"+++class TestWatchRankConsumerRetries(SimpleTestCase):+ @parameterized.expand(+ [+ # The database being away waits out the full window, so a short outage costs no ranks.+ ("database_error_keeps_retrying", OperationalError("server closed the connection"), "deferred"),+ # Any other error holds every partition behind it, so it gives up after the short window.+ ("other_error_gives_up_sooner", ValueError("bad vector"), "failed"),+ ]+ )+ def test_a_failure_gives_up_after_the_window_for_its_kind(self, _name: str, error: Exception, outcome: str) -> None:+ consumer = WatchRankConsumer(MagicMock())+ consumer._first_failed_at["obs-1"] = time.monotonic() - 90++ with (+ patch(f"{_CONSUMER}.rank_observation", AsyncMock(side_effect=error)),+ patch(f"{_CONSUMER}.close_old_connections"),+ ):+ assert async_to_sync(consumer.handle)(_result_message(), time.monotonic()) == outcome++ def test_a_hung_naming_call_counts_a_timeout(self) -> None:+ async def never_returns(*_args: Any) -> None:+ await asyncio.Event().wait()++ def timeouts() -> float:+ labels = {"outcome": "timeout"}+ return REGISTRY.get_sample_value("replay_vision_watch_rank_consumer_theme_naming_total", labels) or 0.0++ before = timeouts()+ with (+ patch(f"{_CONSUMER}.name_theme", never_returns),+ patch(f"{_CONSUMER}._NAMING_TIMEOUT_S", 0.01),+ ):+ async_to_sync(WatchRankConsumer._name)(7, uuid.uuid4(), timezone.now())++ assert timeouts() == before + 1diff --git a/products/replay_vision/backend/watch_rank_consumer.py b/products/replay_vision/backend/watch_rank_consumer.pyindex e63104e2199c..390f9730e86b 100644--- a/products/replay_vision/backend/watch_rank_consumer.py+++ b/products/replay_vision/backend/watch_rank_consumer.py@@ -14,14 +14,14 @@ from datetime import datetimefrom typing import Any, Literal, Protocolfrom django.conf import settings-from django.db import close_old_connections+from django.db import InterfaceError, OperationalError, close_old_connections, connection, transactionfrom django.utils import timezoneimport orjsonimport structlogfrom asgiref.sync import sync_to_asyncfrom confluent_kafka import Consumer, KafkaException, Message, TopicPartition-from prometheus_client import Counter+from prometheus_client import Counter, Gauge, Histogramfrom posthog.kafka_client import helperfrom posthog.kafka_client.client import _build_sasl_config, _KafkaSecurityProtocol@@ -34,8 +34,9 @@ from products.replay_vision.backend.embeddings import (EMBEDDING_PRODUCT,OBSERVATION_EMBEDDING_MODEL,)+from products.replay_vision.backend.watch_feed_groups import feed_group_cachefrom products.replay_vision.backend.watch_rank import RankRequest, rank_observation-from products.replay_vision.backend.watch_semantics import assign_semantics+from products.replay_vision.backend.watch_semantics import SemanticAssignment, assign_semanticsfrom products.replay_vision.backend.watch_themes import name_themelogger = structlog.get_logger(__name__)@@ -49,6 +50,11 @@ _BATCH_BUDGET_S = 240.0_MAX_POLL_INTERVAL_S = 360.0# How long one observation keeps failing, across batches, before the consumer moves past it._RETRY_WINDOW_S = 600.0+# Any other error is unlikely to clear by waiting, and every partition waits behind it, so it gives up sooner.+_ERROR_RETRY_WINDOW_S = 60.0+# Long enough for a naming call that waits behind ranking on the shared database thread, short enough that a hung+# model call frees its slot.+_NAMING_TIMEOUT_S = 120.0_NAMING_DRAIN_S = 20.0# A due theme left unnamed while this many are in flight is named when its next member arrives._MAX_NAMING_TASKS = 8@@ -59,9 +65,23 @@ Outcome = Literal["skipped", "ranked", "already_ranked", "missing", "failed", "dMESSAGES_TOTAL = Counter("replay_vision_watch_rank_consumer_messages_total",- "Embedding results the What to watch rank consumer read, by outcome.",+ "Embedding results the What to watch rank consumer handled, by outcome. A deferred result counts once per batch it is deferred in.",labelnames=["outcome"],)+BATCH_DURATION = Histogram(+ "replay_vision_watch_rank_consumer_batch_duration_seconds",+ "Time the What to watch rank consumer spent on one non-empty batch.",+ buckets=(0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 240, 300),+)+THEME_NAMING_TOTAL = Counter(+ "replay_vision_watch_rank_consumer_theme_naming_total",+ "Theme naming outcomes of the What to watch rank consumer, including names dropped while every slot was busy.",+ labelnames=["outcome"],+)+NAMING_IN_FLIGHT = Gauge(+ "replay_vision_watch_rank_consumer_naming_in_flight",+ "Theme naming calls the What to watch rank consumer has in flight.",+)# The worker writes compact JSON, and a quote inside a string value is escaped, so this marks our results._DOCUMENT_TYPE_MARKER = f'"document_type":"{EMBEDDING_DOCUMENT_TYPE}"'.encode()@@ -102,6 +122,26 @@ def parse_rank_request(value: bytes | None) -> RankRequest | None:return RankRequest(team_id=team_id, scanner_id=scanner_id, observation_id=observation_id, vector=vector)+def _assign_with_timeouts(+ *, team_id: int, scanner_id: str, observation_id: str, vector: list[float], now: datetime+) -> SemanticAssignment | None:+ """Assign under transaction-local timeouts, which survive transaction pooling where a session setting would not."""+ try:+ with transaction.atomic():+ # A slow statement or a long wait on the scanner's lock fails and retries instead of hanging.+ with connection.cursor() as cursor:+ cursor.execute(+ "SELECT set_config('lock_timeout', '30s', true), set_config('statement_timeout', '60s', true)"+ )+ return assign_semantics(+ team_id=team_id, scanner_id=scanner_id, observation_id=observation_id, vector=vector, now=now+ )+ except Exception:+ # The commit happens here, outside the assignment's own guard, so a failed one must still drop the cache.+ feed_group_cache.forget(scanner_id)+ raise++class BatchConsumer(Protocol):def store_offsets(self, *, message: Message) -> None: ...@@ -116,6 +156,7 @@ class WatchRankConsumer:self._first_failed_at: dict[str, float] = {}# Naming calls run beside the loop, so a slow model call never holds up ranking. Kept to stop collection.self._naming: set[asyncio.Task[Any]] = set()+ NAMING_IN_FLIGHT.set_function(lambda: len(self._naming))async def _wait(self, seconds: float) -> bool:"""Sleep, unless shutdown comes first. True when the full time passed."""@@ -140,15 +181,19 @@ class WatchRankConsumer:await self._semantics(request)self._first_failed_at.pop(request.observation_id, None)return outcome- except Exception:+ except Exception as error:logger.exception("replay_vision.watch_rank_consumer.rank_failed",team_id=request.team_id,observation_id=request.observation_id,)+ # A lock or statement timeout is an OperationalError, so it waits out the database like an outage.+ window = (+ _RETRY_WINDOW_S if isinstance(error, OperationalError | InterfaceError) else _ERROR_RETRY_WINDOW_S+ )self._liveness.record_heartbeat()now = time.monotonic()- give_up_at = self._first_failed_at.setdefault(request.observation_id, now) + _RETRY_WINDOW_S+ give_up_at = self._first_failed_at.setdefault(request.observation_id, now) + windowif now + backoff > give_up_at:# The backfill command can write a rank this gave up on.self._first_failed_at.pop(request.observation_id, None)@@ -162,7 +207,7 @@ class WatchRankConsumer:async def _semantics(self, request: RankRequest) -> None:"""Put a ranked observation in a feed group and a theme, and name the theme in the background once it is due."""now = timezone.now()- assignment = await sync_to_async(assign_semantics)(+ assignment = await sync_to_async(_assign_with_timeouts)(team_id=request.team_id,scanner_id=request.scanner_id,observation_id=request.observation_id,@@ -172,6 +217,7 @@ class WatchRankConsumer:if assignment is None or assignment.theme is None or not assignment.theme.name_due:returnif len(self._naming) >= _MAX_NAMING_TASKS:+ THEME_NAMING_TOTAL.labels(outcome="dropped").inc()returntask = asyncio.create_task(self._name(request.team_id, assignment.theme.theme_id, now))self._naming.add(task)@@ -180,10 +226,16 @@ class WatchRankConsumer:@staticmethodasync def _name(team_id: int, theme_id: uuid.UUID, now: datetime) -> None:try:- await name_theme(team_id, theme_id, now)+ name = await asyncio.wait_for(name_theme(team_id, theme_id, now), timeout=_NAMING_TIMEOUT_S)+ except TimeoutError:+ THEME_NAMING_TOTAL.labels(outcome="timeout").inc()+ logger.warning("replay_vision.watch_rank_consumer.theme_naming_timed_out", team_id=team_id)except Exception:+ THEME_NAMING_TOTAL.labels(outcome="failed").inc()# A theme that missed its name is named again when it next doubles.logger.exception("replay_vision.watch_rank_consumer.theme_naming_failed", team_id=team_id)+ else:+ THEME_NAMING_TOTAL.labels(outcome="named" if name else "unnamed").inc()async def process_batch(self, consumer: BatchConsumer, messages: Sequence[Message]) -> None:batch_deadline = time.monotonic() + _BATCH_BUDGET_S@@ -195,12 +247,12 @@ class WatchRankConsumer:continueoutcome = await self.handle(message.value(), batch_deadline)self._liveness.record_heartbeat()+ MESSAGES_TOTAL.labels(outcome=outcome).inc()if outcome == "interrupted":returnif outcome == "deferred":self._rewind(consumer, messages[index:])return- MESSAGES_TOTAL.labels(outcome=outcome).inc()try:consumer.store_offsets(message=message)except KafkaException:@@ -262,7 +314,10 @@ class WatchRankConsumer:while not self._stop.is_set():self._liveness.record_heartbeat()messages = await asyncio.to_thread(consumer.consume, _BATCH_SIZE, _POLL_TIMEOUT_S)- await self.process_batch(consumer, messages)+ if not messages:+ continue+ with BATCH_DURATION.time():+ await self.process_batch(consumer, messages)finally:consumer.close()# A theme claimed for naming stays unnamed until it doubles, so in-flight names get a moment to finish.