:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(replay-vision): sort pick rows once and retry backfill at capacity (#123264)
Tue Haulundcf4967dd4619
diff --git a/products/replay_vision/backend/api/scanners.py b/products/replay_vision/backend/api/scanners.pyindex c67f5d107f1c..8be666cfea7c 100644--- a/products/replay_vision/backend/api/scanners.py+++ b/products/replay_vision/backend/api/scanners.py@@ -2728,10 +2728,8 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi# Before ranking, so a row the viewer marked not useful neither takes a slot nor shifts the window baselines.# Views and dismissals after `as_of` wait for the next load, so later pages of this one do not shift.walkable = candidates.exclude(- Exists(- ReplayObservationFeedDismissal.objects.filter(- observation_id=OuterRef("id"), user_id=viewer_id, created_at__lte=as_of- )+ id__in=ReplayObservationFeedDismissal.objects.filter(user_id=viewer_id, created_at__lte=as_of).values(+ "observation_id"))# Row-gate on each row's snapshot experiment before ranking, so a restricted row can't take a feed slot.diff --git a/products/replay_vision/backend/tests/test_api.py b/products/replay_vision/backend/tests/test_api.pyindex eb97417985dc..8f05841ad8e9 100644--- a/products/replay_vision/backend/tests/test_api.py+++ b/products/replay_vision/backend/tests/test_api.py@@ -5284,6 +5284,38 @@ class TestWatchFeedAPI(_VisionAPITestCase):picked = {item["observation"]["session_id"] for item in items if item["reason"]["kind"] == "embedding_pick"}self.assertEqual(picked, {"busy-0", "quiet-0"})+ @parameterized.expand(+ [+ ("group_limit", 3, 4),+ # The bounded fetch holds only copies, so the other finding needs the unbounded retry.+ ("bounded_fetch_full_of_copies", 1, 2),+ ]+ )+ def test_a_duplicate_group_cannot_fill_the_pick_cap(self, _name: str, group_candidates: int, cap: int) -> None:+ scanner = self._create_scanner(name="m")+ root_id: str | None = None+ # Of 30 rows only the top six clear the percentile cut: five copies of one finding, then one other finding.+ for index in range(30):+ obs = self._succeeded_observation(scanner, f"row-{index}", index, self._monitor_result("yes"))+ ReplayObservation.objects.filter(id=obs.id).update(+ watch_rank={+ "category": "failure",+ "contrast": 1 - index / 100,+ "duplicate_of": root_id if 0 < index < 5 else None,+ "anchor_version": ANCHOR_VERSION,+ }+ )+ root_id = root_id or str(obs.id)++ with (+ patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value="embedding"),+ patch("products.replay_vision.backend.api.scanners.WATCH_FEED_CANDIDATE_CAP", cap),+ patch("products.replay_vision.backend.watch_rank.WATCH_PICK_GROUP_CANDIDATES", group_candidates),+ ):+ items = self.client.get(self.feed_url).json()["results"]+ picked = {item["observation"]["session_id"] for item in items if item["reason"]["kind"] == "embedding_pick"}+ self.assertIn("row-5", picked)+def test_impressions_lower_an_embedding_pick_only_after_the_load_that_showed_it(self) -> None:scanner = self._create_scanner(name="m")rows = {}diff --git a/products/replay_vision/backend/tests/test_watch_rank.py b/products/replay_vision/backend/tests/test_watch_rank.pyindex f86e76633462..3207d697d3aa 100644--- a/products/replay_vision/backend/tests/test_watch_rank.py+++ b/products/replay_vision/backend/tests/test_watch_rank.py@@ -1,11 +1,16 @@from datetime import UTC, datetime, timedeltafrom typing import Any+from unittest.mock import patch+from django.test import SimpleTestCaseimport numpy as npfrom parameterized import parameterized+from posthog.exceptions import ClickHouseAtCapacity++from products.replay_vision.backend import watch_rank_backfillfrom products.replay_vision.backend.watch_feed import SESSION_SPACINGfrom products.replay_vision.backend.watch_rank import (ANCHOR_VERSION,@@ -14,7 +19,7 @@ from products.replay_vision.backend.watch_rank import (duplicate_root,rank_watch_feed_by_embedding,)-from products.replay_vision.backend.watch_rank_backfill import _rank_scanner, _RecentVectors+from products.replay_vision.backend.watch_rank_backfill import _rank_scanner, _RecentVectors, _stored_vectors_NOW = datetime(2026, 10, 23, 12, 0, tzinfo=UTC)@@ -206,3 +211,33 @@ class TestBackfillRankScanner(SimpleTestCase):assert written["stale"]["duplicate_of"] == "root"assert written["calm"]["category"] == "routine"assert written["calm"]["duplicate_of"] is None++ @parameterized.expand(+ [+ ("cluster_frees_up", 2, {"obs-1"}),+ ("cluster_stays_full", 99, None),+ ]+ )+ def test_vector_read_waits_out_a_cluster_at_capacity(+ self, _name: str, refusals: int, expected: set[str] | None+ ) -> None:+ calls = 0++ def execute(*args: Any, **kwargs: Any) -> list[tuple[str, list[float]]]:+ nonlocal calls+ calls += 1+ if calls <= refusals:+ raise ClickHouseAtCapacity()+ return [("obs-1", [1.0, 0.0, 0.0])]++ with (+ patch.object(watch_rank_backfill, "sync_execute", side_effect=execute),+ patch.object(watch_rank_backfill.time, "sleep") as sleep,+ ):+ if expected is None:+ with self.assertRaises(ClickHouseAtCapacity):+ _stored_vectors(2, "scanner-a", ["obs-1"], _NOW)+ assert calls == watch_rank_backfill._CAPACITY_RETRY_ATTEMPTS+ else:+ assert set(_stored_vectors(2, "scanner-a", ["obs-1"], _NOW)) == expected+ assert sleep.call_count == refusalsdiff --git a/products/replay_vision/backend/watch_rank.py b/products/replay_vision/backend/watch_rank.pyindex ac2f5ca0611d..33c52e2386ce 100644--- a/products/replay_vision/backend/watch_rank.py+++ b/products/replay_vision/backend/watch_rank.py@@ -9,6 +9,7 @@ errors all the time does not fill the feed.import mathimport asyncio+from collections import Counterfrom dataclasses import asdictfrom datetime import datetime, timedeltafrom typing import Any, Literal@@ -362,6 +363,23 @@ def _embedding_group_root() -> Coalesce:return Coalesce(KeyTextTransform("duplicate_of", "watch_rank"), Cast("id", CharField()), output_field=CharField())+def _cap_duplicate_groups(rows: list[dict[str, Any]], cap: int) -> dict[Any, dict[str, Any]]:+ """The first `cap` rows, keeping a few per duplicate group so the ranker can still pick a readable, unseen one.++ A group never spans scanners, so the pick order meets each group's rows best first.+ """+ picked: dict[Any, dict[str, Any]] = {}+ group_rows: Counter[str] = Counter()+ for row in rows:+ if group_rows[row["watch_group_root"]] >= WATCH_PICK_GROUP_CANDIDATES:+ continue+ group_rows[row["watch_group_root"]] += 1+ picked[row["id"]] = row+ if len(picked) >= cap:+ break+ return picked++def select_embedding_picks(walkable: QuerySet[ReplayObservation],scanner_ids: list[Any],@@ -393,26 +411,17 @@ def select_embedding_picks().filter(contrast_type="number"))- is_issue = Case(- When(watch_category__in=ISSUE_CATEGORIES, then=Value(1)), default=Value(0), output_field=IntegerField()- )- ranked = (+ qualifying = (current_ranks.annotate(scanner_rows=Window(Count("id"), partition_by=[F("scanner_id")]),scanner_position=Window(RowNumber(), partition_by=[F("scanner_id")], order_by=[contrast.asc(), F("id").asc()]),- # Each scanner's best row ranks 1, so the cap takes every scanner's best before any second-best.- scanner_rank=Window(- RowNumber(), partition_by=[F("scanner_id")], order_by=[contrast.desc(), F("id").desc()]- ),- # A few rows per duplicate group reach the cap, enough for the ranker to pick a readable, unseen one.- group_position=Window(- RowNumber(), partition_by=[group_root], order_by=[is_issue.desc(), contrast.desc(), F("id").desc()]- ),).annotate(watch_percentile=(F("scanner_position") - Value(0.5)) / Cast(F("scanner_rows"), FloatField()),+ # Each scanner's best row ranks 1, so the cap takes every scanner's best before any second-best.+ scanner_rank=F("scanner_rows") - F("scanner_position") + Value(1),# Reads a window value, so Django filters on it after the windows run rather than before.issue_scanner_rows=Case(When(watch_category__in=ISSUE_CATEGORIES, then=F("scanner_rows")),@@ -421,15 +430,16 @@ def select_embedding_picks(),watch_group_root=group_root,)- .filter(- issue_scanner_rows__gte=WATCH_PICK_MIN_SCANNER_ROWS,- watch_percentile__gte=WATCH_PICK_MIN_PERCENTILE,- group_position__lte=WATCH_PICK_GROUP_CANDIDATES,- )- .order_by("scanner_rank", "-watch_percentile", "-created_at")- .values("id", "watch_percentile", "watch_group_root")[:cap]+ .filter(issue_scanner_rows__gte=WATCH_PICK_MIN_SCANNER_ROWS, watch_percentile__gte=WATCH_PICK_MIN_PERCENTILE)+ .order_by("scanner_rank", "-watch_percentile", "-created_at", "-id")+ .values("id", "watch_percentile", "watch_group_root"))- window_values = {row["id"]: row for row in ranked}+ # Bounds the fetch, since a client-side cursor buffers every row; only a window full of copies needs the rest.+ fetch_limit = cap * (WATCH_PICK_GROUP_CANDIDATES + 1)+ rows = list(qualifying[:fetch_limit])+ window_values = _cap_duplicate_groups(rows, cap)+ if len(window_values) < cap and len(rows) == fetch_limit:+ window_values = _cap_duplicate_groups(list(qualifying), cap)if not window_values:return []# Counted over readable rows only, so a group never reveals how many restricted sessions it holds. Duplicatesdiff --git a/products/replay_vision/backend/watch_rank_backfill.py b/products/replay_vision/backend/watch_rank_backfill.pyindex de0e70a07fb1..45dc261f1a55 100644--- a/products/replay_vision/backend/watch_rank_backfill.py+++ b/products/replay_vision/backend/watch_rank_backfill.py@@ -6,6 +6,8 @@ up. It reads stored vectors from ClickHouse rather than embedding again, and finmemory per scanner, oldest first, the same way the consumer would have found them."""+import time+import randomfrom collections import defaultdictfrom datetime import datetimefrom itertools import batched@@ -22,6 +24,7 @@ from posthog.clickhouse.client.connection import ClickHouseUserfrom posthog.clickhouse.query_tagging import Product, tags_contextfrom posthog.clickhouse.workload import Workloadfrom posthog.dataclasses import frozen+from posthog.exceptions import ClickHouseAtCapacityfrom products.replay_vision.backend.embeddings import (OBSERVATION_EMBEDDINGS_TABLE,@@ -45,6 +48,8 @@ logger = structlog.get_logger(__name__)_VECTOR_CHUNK = 500_QUERY_TIMEOUT_S = 60+_CAPACITY_RETRY_BASE_S = 15+_CAPACITY_RETRY_ATTEMPTS = 6@frozen@@ -68,7 +73,21 @@ def teams_with_observations(since: datetime) -> list[int]:def _stored_vectors(team_id: int, scanner_id: str, observation_ids: list[str], since: datetime) -> dict[str, np.ndarray]:- """Each observation's stored unit vector, for one batch of a scanner's observations."""+ """Each observation's stored unit vector, for one batch of a scanner's observations, waiting out a full cluster."""+ for attempt in range(_CAPACITY_RETRY_ATTEMPTS - 1):+ try:+ return _read_stored_vectors(team_id, scanner_id, observation_ids, since)+ except ClickHouseAtCapacity:+ delay = _CAPACITY_RETRY_BASE_S * 2**attempt+ wait = random.uniform(delay, 2 * delay)+ logger.info("replay_vision.watch_rank_backfill.capacity_wait", team_id=team_id, attempt=attempt, wait=wait)+ time.sleep(wait)+ return _read_stored_vectors(team_id, scanner_id, observation_ids, since)+++def _read_stored_vectors(+ team_id: int, scanner_id: str, observation_ids: list[str], since: datetime+) -> dict[str, np.ndarray]:with tags_context(product=Product.REPLAY_VISION, query_type="replay_vision_watch_rank_backfill"):# nosemgrep: clickhouse-fstring-param-audit - table name and scope are module constantsrows = sync_execute(