Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(replay-vision): sort pick rows once and retry backfill at capacity (#123264)

Tue Haulundcf4967dd4619
diff --git a/products/replay_vision/backend/api/scanners.py b/products/replay_vision/backend/api/scanners.py
index c67f5d107f1c..8be666cfea7c 100644
--- a/products/replay_vision/backend/api/scanners.py
+++ b/products/replay_vision/backend/api/scanners.py
@@ -2728,10 +2728,8 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi
# Before ranking, so a row the viewer marked not useful neither takes a slot nor shifts the window baselines.
# Views and dismissals after `as_of` wait for the next load, so later pages of this one do not shift.
walkable = candidates.exclude(
- Exists(
- ReplayObservationFeedDismissal.objects.filter(
- observation_id=OuterRef("id"), user_id=viewer_id, created_at__lte=as_of
- )
+ id__in=ReplayObservationFeedDismissal.objects.filter(user_id=viewer_id, created_at__lte=as_of).values(
+ "observation_id"
)
)
# Row-gate on each row's snapshot experiment before ranking, so a restricted row can't take a feed slot.
diff --git a/products/replay_vision/backend/tests/test_api.py b/products/replay_vision/backend/tests/test_api.py
index eb97417985dc..8f05841ad8e9 100644
--- a/products/replay_vision/backend/tests/test_api.py
+++ b/products/replay_vision/backend/tests/test_api.py
@@ -5284,6 +5284,38 @@ class TestWatchFeedAPI(_VisionAPITestCase):
picked = {item["observation"]["session_id"] for item in items if item["reason"]["kind"] == "embedding_pick"}
self.assertEqual(picked, {"busy-0", "quiet-0"})
+ @parameterized.expand(
+ [
+ ("group_limit", 3, 4),
+ # The bounded fetch holds only copies, so the other finding needs the unbounded retry.
+ ("bounded_fetch_full_of_copies", 1, 2),
+ ]
+ )
+ def test_a_duplicate_group_cannot_fill_the_pick_cap(self, _name: str, group_candidates: int, cap: int) -> None:
+ scanner = self._create_scanner(name="m")
+ root_id: str | None = None
+ # Of 30 rows only the top six clear the percentile cut: five copies of one finding, then one other finding.
+ for index in range(30):
+ obs = self._succeeded_observation(scanner, f"row-{index}", index, self._monitor_result("yes"))
+ ReplayObservation.objects.filter(id=obs.id).update(
+ watch_rank={
+ "category": "failure",
+ "contrast": 1 - index / 100,
+ "duplicate_of": root_id if 0 < index < 5 else None,
+ "anchor_version": ANCHOR_VERSION,
+ }
+ )
+ root_id = root_id or str(obs.id)
+
+ with (
+ patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value="embedding"),
+ patch("products.replay_vision.backend.api.scanners.WATCH_FEED_CANDIDATE_CAP", cap),
+ patch("products.replay_vision.backend.watch_rank.WATCH_PICK_GROUP_CANDIDATES", group_candidates),
+ ):
+ items = self.client.get(self.feed_url).json()["results"]
+ picked = {item["observation"]["session_id"] for item in items if item["reason"]["kind"] == "embedding_pick"}
+ self.assertIn("row-5", picked)
+
def test_impressions_lower_an_embedding_pick_only_after_the_load_that_showed_it(self) -> None:
scanner = self._create_scanner(name="m")
rows = {}
diff --git a/products/replay_vision/backend/tests/test_watch_rank.py b/products/replay_vision/backend/tests/test_watch_rank.py
index f86e76633462..3207d697d3aa 100644
--- a/products/replay_vision/backend/tests/test_watch_rank.py
+++ b/products/replay_vision/backend/tests/test_watch_rank.py
@@ -1,11 +1,16 @@
from datetime import UTC, datetime, timedelta
from typing import Any
+from unittest.mock import patch
+
from django.test import SimpleTestCase
import numpy as np
from parameterized import parameterized
+from posthog.exceptions import ClickHouseAtCapacity
+
+from products.replay_vision.backend import watch_rank_backfill
from products.replay_vision.backend.watch_feed import SESSION_SPACING
from products.replay_vision.backend.watch_rank import (
ANCHOR_VERSION,
@@ -14,7 +19,7 @@ from products.replay_vision.backend.watch_rank import (
duplicate_root,
rank_watch_feed_by_embedding,
)
-from products.replay_vision.backend.watch_rank_backfill import _rank_scanner, _RecentVectors
+from products.replay_vision.backend.watch_rank_backfill import _rank_scanner, _RecentVectors, _stored_vectors
_NOW = datetime(2026, 10, 23, 12, 0, tzinfo=UTC)
@@ -206,3 +211,33 @@ class TestBackfillRankScanner(SimpleTestCase):
assert written["stale"]["duplicate_of"] == "root"
assert written["calm"]["category"] == "routine"
assert written["calm"]["duplicate_of"] is None
+
+ @parameterized.expand(
+ [
+ ("cluster_frees_up", 2, {"obs-1"}),
+ ("cluster_stays_full", 99, None),
+ ]
+ )
+ def test_vector_read_waits_out_a_cluster_at_capacity(
+ self, _name: str, refusals: int, expected: set[str] | None
+ ) -> None:
+ calls = 0
+
+ def execute(*args: Any, **kwargs: Any) -> list[tuple[str, list[float]]]:
+ nonlocal calls
+ calls += 1
+ if calls <= refusals:
+ raise ClickHouseAtCapacity()
+ return [("obs-1", [1.0, 0.0, 0.0])]
+
+ with (
+ patch.object(watch_rank_backfill, "sync_execute", side_effect=execute),
+ patch.object(watch_rank_backfill.time, "sleep") as sleep,
+ ):
+ if expected is None:
+ with self.assertRaises(ClickHouseAtCapacity):
+ _stored_vectors(2, "scanner-a", ["obs-1"], _NOW)
+ assert calls == watch_rank_backfill._CAPACITY_RETRY_ATTEMPTS
+ else:
+ assert set(_stored_vectors(2, "scanner-a", ["obs-1"], _NOW)) == expected
+ assert sleep.call_count == refusals
diff --git a/products/replay_vision/backend/watch_rank.py b/products/replay_vision/backend/watch_rank.py
index ac2f5ca0611d..33c52e2386ce 100644
--- a/products/replay_vision/backend/watch_rank.py
+++ b/products/replay_vision/backend/watch_rank.py
@@ -9,6 +9,7 @@ errors all the time does not fill the feed.
import math
import asyncio
+from collections import Counter
from dataclasses import asdict
from datetime import datetime, timedelta
from typing import Any, Literal
@@ -362,6 +363,23 @@ def _embedding_group_root() -> Coalesce:
return Coalesce(KeyTextTransform("duplicate_of", "watch_rank"), Cast("id", CharField()), output_field=CharField())
+def _cap_duplicate_groups(rows: list[dict[str, Any]], cap: int) -> dict[Any, dict[str, Any]]:
+ """The first `cap` rows, keeping a few per duplicate group so the ranker can still pick a readable, unseen one.
+
+ A group never spans scanners, so the pick order meets each group's rows best first.
+ """
+ picked: dict[Any, dict[str, Any]] = {}
+ group_rows: Counter[str] = Counter()
+ for row in rows:
+ if group_rows[row["watch_group_root"]] >= WATCH_PICK_GROUP_CANDIDATES:
+ continue
+ group_rows[row["watch_group_root"]] += 1
+ picked[row["id"]] = row
+ if len(picked) >= cap:
+ break
+ return picked
+
+
def select_embedding_picks(
walkable: QuerySet[ReplayObservation],
scanner_ids: list[Any],
@@ -393,26 +411,17 @@ def select_embedding_picks(
)
.filter(contrast_type="number")
)
- is_issue = Case(
- When(watch_category__in=ISSUE_CATEGORIES, then=Value(1)), default=Value(0), output_field=IntegerField()
- )
- ranked = (
+ qualifying = (
current_ranks.annotate(
scanner_rows=Window(Count("id"), partition_by=[F("scanner_id")]),
scanner_position=Window(
RowNumber(), partition_by=[F("scanner_id")], order_by=[contrast.asc(), F("id").asc()]
),
- # Each scanner's best row ranks 1, so the cap takes every scanner's best before any second-best.
- scanner_rank=Window(
- RowNumber(), partition_by=[F("scanner_id")], order_by=[contrast.desc(), F("id").desc()]
- ),
- # A few rows per duplicate group reach the cap, enough for the ranker to pick a readable, unseen one.
- group_position=Window(
- RowNumber(), partition_by=[group_root], order_by=[is_issue.desc(), contrast.desc(), F("id").desc()]
- ),
)
.annotate(
watch_percentile=(F("scanner_position") - Value(0.5)) / Cast(F("scanner_rows"), FloatField()),
+ # Each scanner's best row ranks 1, so the cap takes every scanner's best before any second-best.
+ scanner_rank=F("scanner_rows") - F("scanner_position") + Value(1),
# Reads a window value, so Django filters on it after the windows run rather than before.
issue_scanner_rows=Case(
When(watch_category__in=ISSUE_CATEGORIES, then=F("scanner_rows")),
@@ -421,15 +430,16 @@ def select_embedding_picks(
),
watch_group_root=group_root,
)
- .filter(
- issue_scanner_rows__gte=WATCH_PICK_MIN_SCANNER_ROWS,
- watch_percentile__gte=WATCH_PICK_MIN_PERCENTILE,
- group_position__lte=WATCH_PICK_GROUP_CANDIDATES,
- )
- .order_by("scanner_rank", "-watch_percentile", "-created_at")
- .values("id", "watch_percentile", "watch_group_root")[:cap]
+ .filter(issue_scanner_rows__gte=WATCH_PICK_MIN_SCANNER_ROWS, watch_percentile__gte=WATCH_PICK_MIN_PERCENTILE)
+ .order_by("scanner_rank", "-watch_percentile", "-created_at", "-id")
+ .values("id", "watch_percentile", "watch_group_root")
)
- window_values = {row["id"]: row for row in ranked}
+ # Bounds the fetch, since a client-side cursor buffers every row; only a window full of copies needs the rest.
+ fetch_limit = cap * (WATCH_PICK_GROUP_CANDIDATES + 1)
+ rows = list(qualifying[:fetch_limit])
+ window_values = _cap_duplicate_groups(rows, cap)
+ if len(window_values) < cap and len(rows) == fetch_limit:
+ window_values = _cap_duplicate_groups(list(qualifying), cap)
if not window_values:
return []
# Counted over readable rows only, so a group never reveals how many restricted sessions it holds. Duplicates
diff --git a/products/replay_vision/backend/watch_rank_backfill.py b/products/replay_vision/backend/watch_rank_backfill.py
index de0e70a07fb1..45dc261f1a55 100644
--- a/products/replay_vision/backend/watch_rank_backfill.py
+++ b/products/replay_vision/backend/watch_rank_backfill.py
@@ -6,6 +6,8 @@ up. It reads stored vectors from ClickHouse rather than embedding again, and fin
memory per scanner, oldest first, the same way the consumer would have found them.
"""
+import time
+import random
from collections import defaultdict
from datetime import datetime
from itertools import batched
@@ -22,6 +24,7 @@ from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import Product, tags_context
from posthog.clickhouse.workload import Workload
from posthog.dataclasses import frozen
+from posthog.exceptions import ClickHouseAtCapacity
from products.replay_vision.backend.embeddings import (
OBSERVATION_EMBEDDINGS_TABLE,
@@ -45,6 +48,8 @@ logger = structlog.get_logger(__name__)
_VECTOR_CHUNK = 500
_QUERY_TIMEOUT_S = 60
+_CAPACITY_RETRY_BASE_S = 15
+_CAPACITY_RETRY_ATTEMPTS = 6
@frozen
@@ -68,7 +73,21 @@ def teams_with_observations(since: datetime) -> list[int]:
def _stored_vectors(
team_id: int, scanner_id: str, observation_ids: list[str], since: datetime
) -> dict[str, np.ndarray]:
- """Each observation's stored unit vector, for one batch of a scanner's observations."""
+ """Each observation's stored unit vector, for one batch of a scanner's observations, waiting out a full cluster."""
+ for attempt in range(_CAPACITY_RETRY_ATTEMPTS - 1):
+ try:
+ return _read_stored_vectors(team_id, scanner_id, observation_ids, since)
+ except ClickHouseAtCapacity:
+ delay = _CAPACITY_RETRY_BASE_S * 2**attempt
+ wait = random.uniform(delay, 2 * delay)
+ logger.info("replay_vision.watch_rank_backfill.capacity_wait", team_id=team_id, attempt=attempt, wait=wait)
+ time.sleep(wait)
+ return _read_stored_vectors(team_id, scanner_id, observation_ids, since)
+
+
+def _read_stored_vectors(
+ team_id: int, scanner_id: str, observation_ids: list[str], since: datetime
+) -> dict[str, np.ndarray]:
with tags_context(product=Product.REPLAY_VISION, query_type="replay_vision_watch_rank_backfill"):
# nosemgrep: clickhouse-fstring-param-audit - table name and scope are module constants
rows = sync_execute(