Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-vision): harden the watch rank consumer and add the metrics its alerts need (#124617)

Tue Haulund3777ff0843fb
diff --git a/products/replay_vision/backend/tests/test_watch_rank_consumer.py b/products/replay_vision/backend/tests/test_watch_rank_consumer.py
index 3bb5e9a5277f..1edcc2f07e08 100644
--- a/products/replay_vision/backend/tests/test_watch_rank_consumer.py
+++ b/products/replay_vision/backend/tests/test_watch_rank_consumer.py
@@ -1,16 +1,21 @@
import json
import math
import time
+import uuid
+import asyncio
from typing import Any
from posthog.test.base import BaseTest
from unittest.mock import AsyncMock, MagicMock, patch
+from django.db import OperationalError
from django.test import SimpleTestCase
+from django.utils import timezone
from asgiref.sync import async_to_sync
from confluent_kafka import KafkaException
from parameterized import parameterized
+from prometheus_client import REGISTRY
from products.replay_vision.backend.embeddings import (
EMBEDDING_DOCUMENT_TYPE,
@@ -123,6 +128,7 @@ class TestWatchRankConsumer(BaseTest):
with (
patch(f"{_WATCH_RANK}.anchor_vectors", AsyncMock(side_effect=anchor_error, return_value=self._ANCHORS)),
patch(f"{_CONSUMER}._RETRY_WINDOW_S", 0),
+ patch(f"{_CONSUMER}._ERROR_RETRY_WINDOW_S", 0),
# It would close the connection that holds the test transaction.
patch(f"{_CONSUMER}.close_old_connections"),
):
@@ -185,3 +191,40 @@ class TestWatchRankConsumer(BaseTest):
):
for _ in range(2):
assert async_to_sync(consumer.handle)(message, time.monotonic()) == "deferred"
+
+
+class TestWatchRankConsumerRetries(SimpleTestCase):
+ @parameterized.expand(
+ [
+ # The database being away waits out the full window, so a short outage costs no ranks.
+ ("database_error_keeps_retrying", OperationalError("server closed the connection"), "deferred"),
+ # Any other error holds every partition behind it, so it gives up after the short window.
+ ("other_error_gives_up_sooner", ValueError("bad vector"), "failed"),
+ ]
+ )
+ def test_a_failure_gives_up_after_the_window_for_its_kind(self, _name: str, error: Exception, outcome: str) -> None:
+ consumer = WatchRankConsumer(MagicMock())
+ consumer._first_failed_at["obs-1"] = time.monotonic() - 90
+
+ with (
+ patch(f"{_CONSUMER}.rank_observation", AsyncMock(side_effect=error)),
+ patch(f"{_CONSUMER}.close_old_connections"),
+ ):
+ assert async_to_sync(consumer.handle)(_result_message(), time.monotonic()) == outcome
+
+ def test_a_hung_naming_call_counts_a_timeout(self) -> None:
+ async def never_returns(*_args: Any) -> None:
+ await asyncio.Event().wait()
+
+ def timeouts() -> float:
+ labels = {"outcome": "timeout"}
+ return REGISTRY.get_sample_value("replay_vision_watch_rank_consumer_theme_naming_total", labels) or 0.0
+
+ before = timeouts()
+ with (
+ patch(f"{_CONSUMER}.name_theme", never_returns),
+ patch(f"{_CONSUMER}._NAMING_TIMEOUT_S", 0.01),
+ ):
+ async_to_sync(WatchRankConsumer._name)(7, uuid.uuid4(), timezone.now())
+
+ assert timeouts() == before + 1
diff --git a/products/replay_vision/backend/watch_rank_consumer.py b/products/replay_vision/backend/watch_rank_consumer.py
index e63104e2199c..390f9730e86b 100644
--- a/products/replay_vision/backend/watch_rank_consumer.py
+++ b/products/replay_vision/backend/watch_rank_consumer.py
@@ -14,14 +14,14 @@ from datetime import datetime
from typing import Any, Literal, Protocol
from django.conf import settings
-from django.db import close_old_connections
+from django.db import InterfaceError, OperationalError, close_old_connections, connection, transaction
from django.utils import timezone
import orjson
import structlog
from asgiref.sync import sync_to_async
from confluent_kafka import Consumer, KafkaException, Message, TopicPartition
-from prometheus_client import Counter
+from prometheus_client import Counter, Gauge, Histogram
from posthog.kafka_client import helper
from posthog.kafka_client.client import _build_sasl_config, _KafkaSecurityProtocol
@@ -34,8 +34,9 @@ from products.replay_vision.backend.embeddings import (
EMBEDDING_PRODUCT,
OBSERVATION_EMBEDDING_MODEL,
)
+from products.replay_vision.backend.watch_feed_groups import feed_group_cache
from products.replay_vision.backend.watch_rank import RankRequest, rank_observation
-from products.replay_vision.backend.watch_semantics import assign_semantics
+from products.replay_vision.backend.watch_semantics import SemanticAssignment, assign_semantics
from products.replay_vision.backend.watch_themes import name_theme
logger = structlog.get_logger(__name__)
@@ -49,6 +50,11 @@ _BATCH_BUDGET_S = 240.0
_MAX_POLL_INTERVAL_S = 360.0
# How long one observation keeps failing, across batches, before the consumer moves past it.
_RETRY_WINDOW_S = 600.0
+# Any other error is unlikely to clear by waiting, and every partition waits behind it, so it gives up sooner.
+_ERROR_RETRY_WINDOW_S = 60.0
+# Long enough for a naming call that waits behind ranking on the shared database thread, short enough that a hung
+# model call frees its slot.
+_NAMING_TIMEOUT_S = 120.0
_NAMING_DRAIN_S = 20.0
# A due theme left unnamed while this many are in flight is named when its next member arrives.
_MAX_NAMING_TASKS = 8
@@ -59,9 +65,23 @@ Outcome = Literal["skipped", "ranked", "already_ranked", "missing", "failed", "d
MESSAGES_TOTAL = Counter(
"replay_vision_watch_rank_consumer_messages_total",
- "Embedding results the What to watch rank consumer read, by outcome.",
+ "Embedding results the What to watch rank consumer handled, by outcome. A deferred result counts once per batch it is deferred in.",
labelnames=["outcome"],
)
+BATCH_DURATION = Histogram(
+ "replay_vision_watch_rank_consumer_batch_duration_seconds",
+ "Time the What to watch rank consumer spent on one non-empty batch.",
+ buckets=(0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 240, 300),
+)
+THEME_NAMING_TOTAL = Counter(
+ "replay_vision_watch_rank_consumer_theme_naming_total",
+ "Theme naming outcomes of the What to watch rank consumer, including names dropped while every slot was busy.",
+ labelnames=["outcome"],
+)
+NAMING_IN_FLIGHT = Gauge(
+ "replay_vision_watch_rank_consumer_naming_in_flight",
+ "Theme naming calls the What to watch rank consumer has in flight.",
+)
# The worker writes compact JSON, and a quote inside a string value is escaped, so this marks our results.
_DOCUMENT_TYPE_MARKER = f'"document_type":"{EMBEDDING_DOCUMENT_TYPE}"'.encode()
@@ -102,6 +122,26 @@ def parse_rank_request(value: bytes | None) -> RankRequest | None:
return RankRequest(team_id=team_id, scanner_id=scanner_id, observation_id=observation_id, vector=vector)
+def _assign_with_timeouts(
+ *, team_id: int, scanner_id: str, observation_id: str, vector: list[float], now: datetime
+) -> SemanticAssignment | None:
+ """Assign under transaction-local timeouts, which survive transaction pooling where a session setting would not."""
+ try:
+ with transaction.atomic():
+ # A slow statement or a long wait on the scanner's lock fails and retries instead of hanging.
+ with connection.cursor() as cursor:
+ cursor.execute(
+ "SELECT set_config('lock_timeout', '30s', true), set_config('statement_timeout', '60s', true)"
+ )
+ return assign_semantics(
+ team_id=team_id, scanner_id=scanner_id, observation_id=observation_id, vector=vector, now=now
+ )
+ except Exception:
+ # The commit happens here, outside the assignment's own guard, so a failed one must still drop the cache.
+ feed_group_cache.forget(scanner_id)
+ raise
+
+
class BatchConsumer(Protocol):
def store_offsets(self, *, message: Message) -> None: ...
@@ -116,6 +156,7 @@ class WatchRankConsumer:
self._first_failed_at: dict[str, float] = {}
# Naming calls run beside the loop, so a slow model call never holds up ranking. Kept to stop collection.
self._naming: set[asyncio.Task[Any]] = set()
+ NAMING_IN_FLIGHT.set_function(lambda: len(self._naming))
async def _wait(self, seconds: float) -> bool:
"""Sleep, unless shutdown comes first. True when the full time passed."""
@@ -140,15 +181,19 @@ class WatchRankConsumer:
await self._semantics(request)
self._first_failed_at.pop(request.observation_id, None)
return outcome
- except Exception:
+ except Exception as error:
logger.exception(
"replay_vision.watch_rank_consumer.rank_failed",
team_id=request.team_id,
observation_id=request.observation_id,
)
+ # A lock or statement timeout is an OperationalError, so it waits out the database like an outage.
+ window = (
+ _RETRY_WINDOW_S if isinstance(error, OperationalError | InterfaceError) else _ERROR_RETRY_WINDOW_S
+ )
self._liveness.record_heartbeat()
now = time.monotonic()
- give_up_at = self._first_failed_at.setdefault(request.observation_id, now) + _RETRY_WINDOW_S
+ give_up_at = self._first_failed_at.setdefault(request.observation_id, now) + window
if now + backoff > give_up_at:
# The backfill command can write a rank this gave up on.
self._first_failed_at.pop(request.observation_id, None)
@@ -162,7 +207,7 @@ class WatchRankConsumer:
async def _semantics(self, request: RankRequest) -> None:
"""Put a ranked observation in a feed group and a theme, and name the theme in the background once it is due."""
now = timezone.now()
- assignment = await sync_to_async(assign_semantics)(
+ assignment = await sync_to_async(_assign_with_timeouts)(
team_id=request.team_id,
scanner_id=request.scanner_id,
observation_id=request.observation_id,
@@ -172,6 +217,7 @@ class WatchRankConsumer:
if assignment is None or assignment.theme is None or not assignment.theme.name_due:
return
if len(self._naming) >= _MAX_NAMING_TASKS:
+ THEME_NAMING_TOTAL.labels(outcome="dropped").inc()
return
task = asyncio.create_task(self._name(request.team_id, assignment.theme.theme_id, now))
self._naming.add(task)
@@ -180,10 +226,16 @@ class WatchRankConsumer:
@staticmethod
async def _name(team_id: int, theme_id: uuid.UUID, now: datetime) -> None:
try:
- await name_theme(team_id, theme_id, now)
+ name = await asyncio.wait_for(name_theme(team_id, theme_id, now), timeout=_NAMING_TIMEOUT_S)
+ except TimeoutError:
+ THEME_NAMING_TOTAL.labels(outcome="timeout").inc()
+ logger.warning("replay_vision.watch_rank_consumer.theme_naming_timed_out", team_id=team_id)
except Exception:
+ THEME_NAMING_TOTAL.labels(outcome="failed").inc()
# A theme that missed its name is named again when it next doubles.
logger.exception("replay_vision.watch_rank_consumer.theme_naming_failed", team_id=team_id)
+ else:
+ THEME_NAMING_TOTAL.labels(outcome="named" if name else "unnamed").inc()
async def process_batch(self, consumer: BatchConsumer, messages: Sequence[Message]) -> None:
batch_deadline = time.monotonic() + _BATCH_BUDGET_S
@@ -195,12 +247,12 @@ class WatchRankConsumer:
continue
outcome = await self.handle(message.value(), batch_deadline)
self._liveness.record_heartbeat()
+ MESSAGES_TOTAL.labels(outcome=outcome).inc()
if outcome == "interrupted":
return
if outcome == "deferred":
self._rewind(consumer, messages[index:])
return
- MESSAGES_TOTAL.labels(outcome=outcome).inc()
try:
consumer.store_offsets(message=message)
except KafkaException:
@@ -262,7 +314,10 @@ class WatchRankConsumer:
while not self._stop.is_set():
self._liveness.record_heartbeat()
messages = await asyncio.to_thread(consumer.consume, _BATCH_SIZE, _POLL_TIMEOUT_S)
- await self.process_batch(consumer, messages)
+ if not messages:
+ continue
+ with BATCH_DURATION.time():
+ await self.process_batch(consumer, messages)
finally:
consumer.close()
# A theme claimed for naming stays unnamed until it doubles, so in-flight names get a moment to finish.