:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(replay-vision): read each scanner's top picks instead of sorting the window (#123468)
Tue Haulund7b14c4b77078
diff --git a/products/replay_vision/backend/api/scanners.py b/products/replay_vision/backend/api/scanners.pyindex 8e99638080a3..82367a3cb850 100644--- a/products/replay_vision/backend/api/scanners.py+++ b/products/replay_vision/backend/api/scanners.py@@ -148,6 +148,7 @@ from products.replay_vision.backend.scanner_access import (accessible_experiment_ids,accessible_observations,is_experiment_accessible,+ readable_experiment_ids,readable_observation_scanner_ids,scanner_experiment_scope_q,)@@ -1671,8 +1672,10 @@ WATCH_FEED_PER_SCANNER_CAP = 100# each id__in list bounded while the walk stops as soon as the page is covered.WATCH_FEED_BYPASS_CHUNK = 200WATCH_FEED_IMPRESSIONS_MAX_IDS = 100-# Long enough to cover a reader scrolling one load; a new load gets a new `as_of` and a fresh ranking.-WATCH_FEED_PICKS_CACHE_TTL_S = 15 * 60+# Loads in one bucket share their picks, so a reload or tab switch skips the pick query.+WATCH_FEED_PICKS_BUCKET_S = 10 * 60+# Outlives the bucket by long enough to cover a reader scrolling one load.+WATCH_FEED_PICKS_CACHE_TTL_S = WATCH_FEED_PICKS_BUCKET_S + 15 * 60def select_watch_feed_candidates(@@ -1709,6 +1712,16 @@ def select_watch_feed_candidates(return rows[:WATCH_FEED_CANDIDATE_CAP]+def refresh_feed_viewed(rows: list[dict[str, Any]], viewer_id: int, as_of: datetime) -> list[dict[str, Any]]:+ """Cached rows with `feed_viewed` as of `as_of`."""+ viewed = set(+ ReplayObservationView.objects.filter(+ user_id=viewer_id, created_at__lte=as_of, observation_id__in=[row["id"] for row in rows]+ ).values_list("observation_id", flat=True)+ )+ return [{**row, "feed_viewed": row["id"] in viewed} for row in rows]++def attach_scanner_results(team_id: int, rows: list[dict[str, Any]]) -> None:"""Load the scan result for each candidate row, by primary key, after the window cap has applied."""results = dict(@@ -2688,11 +2701,12 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, viviewer_id = cast(User, request.user).id# Before ranking, so a row the viewer marked not useful neither takes a slot nor shifts the window baselines.# Views and dismissals after `as_of` wait for the next load, so later pages of this one do not shift.- walkable = candidates.exclude(- id__in=ReplayObservationFeedDismissal.objects.filter(user_id=viewer_id, created_at__lte=as_of).values(- "observation_id"+ dismissed_ids = sorted(+ ReplayObservationFeedDismissal.objects.filter(user_id=viewer_id, created_at__lte=as_of).values_list(+ "observation_id", flat=True))+ walkable = candidates.exclude(id__in=dismissed_ids)# Row-gate on each row's snapshot experiment before ranking, so a restricted row can't take a feed slot.candidates = accessible_observations(self.user_access_control, self.team_id, walkable.filter(scanner_id__in=allowed_ids)@@ -2711,7 +2725,8 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vimode = ranker_mode(variant)ranker = mode if mode in ("jev", "embedding") else "weighted-score"if ranker == "embedding":- # Every page of one load ranks the same window as of the same `as_of`, so later pages reuse the first's picks.+ # Keyed on the bucket `as_of` falls in, so loads in one bucket share picks, and on the dismissals and the+ # readable experiments, so a dismissal or an access change ranks afresh.picks_key = ("replay_vision:watch_picks:"+ hashlib.sha256(@@ -2719,9 +2734,11 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi[self.team_id,viewer_id,- as_of.isoformat(),- str(date_from),- str(window_end),+ int(as_of.timestamp()) // WATCH_FEED_PICKS_BUCKET_S,+ params["date_from"],+ params.get("date_to"),+ list(map(str, dismissed_ids)),+ sorted(readable_experiment_ids(self.user_access_control, self.team_id)),sorted(map(str, allowed_ids)),params.get("search"),params.get("scanner_type"),@@ -2729,8 +2746,14 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi).encode()).hexdigest())- candidate_rows = cache.get(picks_key)- if candidate_rows is None:+ cached = cache.get(picks_key)+ # Picks ranked as of a later moment may hold rows this load's first page never saw, so they don't serve it.+ if cached is not None and cached[0] <= as_of:+ picked_as_of, candidate_rows = cached+ # An earlier load in the bucket ranked these, so its views may be older than this one's.+ if picked_as_of != as_of:+ candidate_rows = refresh_feed_viewed(candidate_rows, viewer_id, as_of)+ else:# The embedding ranker reads only a small stored rank, so it picks over the whole window in SQL.candidate_rows = select_embedding_picks(walkable, allowed_ids, viewed, self.user_access_control, self.team_id, WATCH_FEED_CANDIDATE_CAP@@ -2741,7 +2764,8 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vicandidate_rows += select_embedding_filler(walkable, allowed_ids, viewed, self.user_access_control, self.team_id, pick_roots)- cache.set(picks_key, candidate_rows, WATCH_FEED_PICKS_CACHE_TTL_S)+ # The first load in the bucket keeps the entry, so concurrent loads never replace it with later picks.+ cache.add(picks_key, (as_of, candidate_rows), WATCH_FEED_PICKS_CACHE_TTL_S)else:candidate_rows = select_watch_feed_candidates(walkable, allowed_ids, viewed, self.user_access_control, self.team_iddiff --git a/products/replay_vision/backend/migrations/0131_rename_legacy_field_indexes.py b/products/replay_vision/backend/migrations/0131_rename_legacy_field_indexes.pynew file mode 100644index 000000000000..280f26582d03--- /dev/null+++ b/products/replay_vision/backend/migrations/0131_rename_legacy_field_indexes.py@@ -0,0 +1,20 @@+from django.db import migrations+++class Migration(migrations.Migration):+ dependencies = [+ ("replay_vision", "0130_rom_small_asset_idx"),+ ]++ operations = [+ # Databases from before the replaylensobservation rename hold these under the old table's names.+ migrations.RunSQL(+ sql=[+ "ALTER INDEX IF EXISTS replay_vision_replaylensobservation_team_id_b10decc8 "+ "RENAME TO replay_vision_replayobservation_team_id_b204b330",+ "ALTER INDEX IF EXISTS replay_vision_replaylensobservation_lens_id_4a4cd5c2 "+ "RENAME TO replay_vision_replayobservation_scanner_id_f2400b26",+ ],+ reverse_sql=migrations.RunSQL.noop,+ ),+ ]diff --git a/products/replay_vision/backend/migrations/0132_ranked_pick_indexes.py b/products/replay_vision/backend/migrations/0132_ranked_pick_indexes.pynew file mode 100644index 000000000000..40952adfd164--- /dev/null+++ b/products/replay_vision/backend/migrations/0132_ranked_pick_indexes.py@@ -0,0 +1,76 @@+import django.db.models.fields.json+import django.db.models.functions.comparison+from django.db import migrations, models++from posthog.migration_helpers import (+ DropFieldIndexesConcurrently,+ SafeAddIndexConcurrently,+ SafeRemoveIndexConcurrently,+)+++class Migration(migrations.Migration):+ atomic = False++ dependencies = [+ ("replay_vision", "0131_rename_legacy_field_indexes"),+ ]++ operations = [+ SafeAddIndexConcurrently(+ model_name="replayobservation",+ index=models.Index(+ condition=models.Q(+ ("status", "succeeded"),+ ("watch_rank__anchor_version", 1),+ ("watch_rank__isnull", False),+ ),+ fields=["scanner", "created_at"],+ include=("id", "team", "completed_at"),+ name="rlo_ranked_count_idx",+ ),+ ),+ SafeAddIndexConcurrently(+ model_name="replayobservation",+ index=models.Index(+ models.F("scanner"),+ models.OrderBy(+ django.db.models.functions.comparison.Cast(+ django.db.models.fields.json.KeyTextTransform("contrast", "watch_rank"),+ models.FloatField(),+ ),+ descending=True,+ ),+ models.OrderBy(models.F("id"), descending=True),+ condition=models.Q(+ ("status", "succeeded"),+ ("watch_rank__anchor_version", 1),+ ("watch_rank__isnull", False),+ ),+ include=("team", "created_at", "completed_at", "watch_rank"),+ name="rlo_ranked_contrast_idx",+ ),+ ),+ SafeAddIndexConcurrently(+ model_name="replayobservation",+ index=models.Index(+ django.db.models.functions.comparison.Coalesce(+ django.db.models.fields.json.KeyTextTransform("duplicate_of", "watch_rank"),+ django.db.models.functions.comparison.Cast("id", models.CharField()),+ output_field=models.CharField(),+ ),+ condition=models.Q(("status", "succeeded"), ("watch_rank__isnull", False)),+ name="rlo_ranked_group_idx",+ ),+ ),+ SafeRemoveIndexConcurrently(+ model_name="replayobservation",+ name="rlo_scanner_ranked_idx",+ ),+ SafeRemoveIndexConcurrently(+ model_name="replayobservation",+ name="rlo_workflow_id_idx",+ ),+ DropFieldIndexesConcurrently(model_name="replayobservation", name="team"),+ DropFieldIndexesConcurrently(model_name="replayobservation", name="scanner"),+ ]diff --git a/products/replay_vision/backend/migrations/max_migration.txt b/products/replay_vision/backend/migrations/max_migration.txtindex bad141a4bacb..9fd42bb17ea8 100644--- a/products/replay_vision/backend/migrations/max_migration.txt+++ b/products/replay_vision/backend/migrations/max_migration.txt@@ -1 +1 @@-0130_rom_small_asset_idx+0132_ranked_pick_indexesdiff --git a/products/replay_vision/backend/models/replay_observation.py b/products/replay_vision/backend/models/replay_observation.pyindex e248f5f65a27..96aba7598d7d 100644--- a/products/replay_vision/backend/models/replay_observation.py+++ b/products/replay_vision/backend/models/replay_observation.py@@ -1,7 +1,7 @@from django.db import modelsfrom django.db.models import Case, CharField, Exists, Expression, F, FloatField, Func, OuterRef, Value, Whenfrom django.db.models.fields.json import KeyTextTransform, KeyTransform-from django.db.models.functions import Cast+from django.db.models.functions import Cast, Coalescefrom posthog.models.utils import UUIDModel@@ -42,11 +42,25 @@ class ObservationTrigger(models.TextChoices):BACKFILL = "backfill", "Backfill"+def watch_rank_contrast() -> Cast:+ """A ranked row's score; `rlo_ranked_contrast_idx` orders by it."""+ return Cast(KeyTextTransform("contrast", "watch_rank"), FloatField())+++def watch_group_root() -> Coalesce:+ """A ranked row's duplicate group: the observation it duplicates, or itself. `rlo_ranked_group_idx` indexes it."""+ return Coalesce(KeyTextTransform("duplicate_of", "watch_rank"), Cast("id", CharField()), output_field=CharField())++class ReplayObservation(UUIDModel):"""One application of a `ReplayScanner` to a session recording (see README)."""- scanner = models.ForeignKey("replay_vision.ReplayScanner", on_delete=models.CASCADE, related_name="observations")- team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+")+ # db_index=False: rlo_scanner_status_idx leads with scanner_id and serves every scanner filter.+ scanner = models.ForeignKey(+ "replay_vision.ReplayScanner", on_delete=models.CASCADE, related_name="observations", db_index=False+ )+ # db_index=False: rlo_team_created_idx leads with team_id and serves every team filter.+ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+", db_index=False)session_id = models.CharField(max_length=200, help_text="Session recording id this scanner was applied to.")distinct_id = models.CharField(max_length=400,@@ -163,11 +177,6 @@ class ReplayObservation(UUIDModel):),# Serves the per-scanner list ordering and the prev/next-neighbor lookups (both order by created_at).models.Index(fields=["scanner", "created_at"], name="rlo_scanner_created_idx"),- models.Index(- fields=["workflow_id"],- name="rlo_workflow_id_idx",- condition=~models.Q(workflow_id=""),- ),# Serves the per-team in-flight concurrency count (sweep headroom + on-demand 429). Partial on the# in-flight statuses only, since terminal rows dominate and are never counted.models.Index(@@ -175,11 +184,26 @@ class ReplayObservation(UUIDModel):name="rlo_team_in_flight_idx",condition=models.Q(status__in=("pending", "running")),),- # Serves the embedding feed's whole-window pick query, which reads only these columns of ranked rows.+ # The pick query counts each scanner's ranked rows here; the anchor version sits in the condition so the count never reads `watch_rank`.models.Index(fields=["scanner", "created_at"],- include=["id", "team", "completed_at", "watch_rank", "distinct_id", "session_id"],- name="rlo_scanner_ranked_idx",+ include=["id", "team", "completed_at"],+ name="rlo_ranked_count_idx",+ condition=models.Q(status="succeeded", watch_rank__isnull=False, watch_rank__anchor_version=1),+ ),+ # The pick query reads each scanner's top rows by score from here.+ models.Index(+ F("scanner"),+ watch_rank_contrast().desc(),+ F("id").desc(),+ include=["team", "created_at", "completed_at", "watch_rank"],+ name="rlo_ranked_contrast_idx",+ condition=models.Q(status="succeeded", watch_rank__isnull=False, watch_rank__anchor_version=1),+ ),+ # Finds the members of the feed picks' duplicate groups, to count the sessions and people each one reached.+ models.Index(+ watch_group_root(),+ name="rlo_ranked_group_idx",condition=models.Q(status="succeeded", watch_rank__isnull=False),),# Serves the per-backfill observation filter and progress counts; partial since live rows never match.diff --git a/products/replay_vision/backend/scanner_access.py b/products/replay_vision/backend/scanner_access.pyindex e46269849195..13a4b6707a19 100644--- a/products/replay_vision/backend/scanner_access.py+++ b/products/replay_vision/backend/scanner_access.py@@ -59,6 +59,11 @@ def accessible_experiment_ids(return facade_accessible_experiment_ids(access, team_id, experiment_ids=experiment_ids)+def readable_experiment_ids(access: UserAccessControl, team_id: int) -> set[int]:+ """Every experiment the caller may view in the team, the set `accessible_observations` gates rows on."""+ return accessible_experiment_ids(access, team_id, None)++def is_experiment_accessible(access: UserAccessControl | None, team_id: int, experiment_id: int) -> bool:"""Whether the caller may view this experiment, filtered by object-level access (not just team).diff --git a/products/replay_vision/backend/tests/test_api.py b/products/replay_vision/backend/tests/test_api.pyindex dc7c9ac2d725..566e38a19d9d 100644--- a/products/replay_vision/backend/tests/test_api.py+++ b/products/replay_vision/backend/tests/test_api.py@@ -34,6 +34,7 @@ from posthog.session_recordings.queries.test.session_replay_sql import produce_rfrom products.experiments.backend.models.experiment import Experimentfrom products.replay_vision.backend.api.scanners import (WATCH_FEED_CANDIDATE_CAP,+ WATCH_FEED_PICKS_BUCKET_S,ReplayScannerSerializer,WatchFeedQuerySerializer,)@@ -5251,6 +5252,34 @@ class TestWatchFeedAPI(_VisionAPITestCase):assert sessions == [f"hit-{i}" for i in range(6)]assert second["has_more"] is False+ def test_a_later_loads_cached_picks_do_not_serve_an_earlier_loads_pages(self) -> None:+ scanner = self._create_scanner(name="m")+ bucket_start = datetime.fromtimestamp(+ (int(timezone.now().timestamp()) // WATCH_FEED_PICKS_BUCKET_S - 1) * WATCH_FEED_PICKS_BUCKET_S, tz=UTC+ )+ earlier, later = bucket_start + timedelta(minutes=1), bucket_start + timedelta(minutes=5)+ for index in range(12):+ obs = self._succeeded_observation(scanner, f"row-{index}", 120 + index, self._monitor_result("no"))+ # The top-scoring row completes between the two loads, so only the later load can rank it.+ completed_at = bucket_start + timedelta(minutes=3) if index == 0 else bucket_start+ ReplayObservation.objects.filter(id=obs.id).update(+ completed_at=completed_at,+ watch_rank={+ "category": "failure",+ "contrast": 1 - index / 20,+ "duplicate_of": None,+ "anchor_version": ANCHOR_VERSION,+ },+ )++ def sessions(as_of: datetime) -> list[str]:+ response = self.client.get(self.feed_url, {"limit": "20", "as_of": as_of.isoformat()}).json()+ return [item["observation"]["session_id"] for item in response["results"]]++ with patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value="embedding"):+ assert "row-0" in sessions(later)+ assert "row-0" not in sessions(earlier)+def test_embedding_picks_reach_past_the_recency_slice_and_count_hidden_duplicates(self) -> None:scanner = self._create_scanner(name="m")@@ -5863,6 +5892,21 @@ class TestWatchFeedAPI(_VisionAPITestCase):self.assertEqual(resp.status_code, 200, resp.json())self.assertEqual(resp.json()["results"][0]["reason"]["kind"], "unviewed_recent")+ @parameterized.expand([("weighted_score", "control"), ("embedding", "embedding")])+ def test_a_scope_with_no_readable_scanner_returns_an_empty_feed(self, _name: str, variant: str) -> None:+ hidden = self._create_scanner(name="hidden")+ self._succeeded_observation(hidden, "hidden-sess", 1, self._monitor_result("no"))+ with (+ patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value=variant),+ patch(+ "products.access_control.backend.facade.user_access_control.UserAccessControl.filter_queryset_by_access_level",+ side_effect=lambda qs, **_: qs.none() if qs.model is ReplayScanner else qs,+ ),+ ):+ resp = self.client.get(self.feed_url)+ self.assertEqual(resp.status_code, 200, resp.json())+ self.assertEqual(resp.json()["results"], [])+def test_unreadable_and_cross_team_scanners_are_dropped_silently(self) -> None:visible = self._create_scanner(name="visible")hidden = self._create_scanner(name="hidden")@@ -5935,26 +5979,28 @@ class TestWatchFeedAPI(_VisionAPITestCase):second = self.client.get(self.feed_url, {"limit": 5, "offset": 1, "as_of": first["as_of"]}).json()self.assertEqual(second["results"], [])- def test_not_useful_hides_a_row_for_that_user_only_until_undone(self) -> None:- scanner = self._create_scanner(name="m")- dismissed = self._succeeded_observation(scanner, "dismissed-sess", 1, self._monitor_result("yes"))- self._succeeded_observation(scanner, "kept-sess", 2, self._monitor_result("yes"))- dismissal_url = f"/api/projects/{self.team.id}/vision/observations/{dismissed.id}/feed_dismissal/"+ @parameterized.expand([("weighted_score", "control"), ("embedding_reload_in_one_bucket", "embedding")])+ def test_not_useful_hides_a_row_for_that_user_only_until_undone(self, _name: str, variant: str) -> None:+ with patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value=variant):+ scanner = self._create_scanner(name="m")+ dismissed = self._succeeded_observation(scanner, "dismissed-sess", 1, self._monitor_result("yes"))+ self._succeeded_observation(scanner, "kept-sess", 2, self._monitor_result("yes"))+ dismissal_url = f"/api/projects/{self.team.id}/vision/observations/{dismissed.id}/feed_dismissal/"- def feed_sessions() -> list[str]:- return [item["observation"]["session_id"] for item in self.client.get(self.feed_url).json()["results"]]+ def feed_sessions() -> list[str]:+ return [item["observation"]["session_id"] for item in self.client.get(self.feed_url).json()["results"]]- self.assertEqual(self.client.post(dismissal_url, {"reason": "nonsense"}).status_code, 400)- self.assertEqual(self.client.post(dismissal_url, {"reason": "already_known"}).status_code, 204)- self.assertEqual(feed_sessions(), ["kept-sess"])+ self.assertEqual(self.client.post(dismissal_url, {"reason": "nonsense"}).status_code, 400)+ self.assertEqual(self.client.post(dismissal_url, {"reason": "already_known"}).status_code, 204)+ self.assertEqual(feed_sessions(), ["kept-sess"])- other = User.objects.create_and_join(self.organization, "[email protected]", None)- self.client.force_login(other)- self.assertEqual(sorted(feed_sessions()), ["dismissed-sess", "kept-sess"])+ other = User.objects.create_and_join(self.organization, "[email protected]", None)+ self.client.force_login(other)+ self.assertEqual(sorted(feed_sessions()), ["dismissed-sess", "kept-sess"])- self.client.force_login(self.user)- self.assertEqual(self.client.delete(dismissal_url).status_code, 204)- self.assertEqual(sorted(feed_sessions()), ["dismissed-sess", "kept-sess"])+ self.client.force_login(self.user)+ self.assertEqual(self.client.delete(dismissal_url).status_code, 204)+ self.assertEqual(sorted(feed_sessions()), ["dismissed-sess", "kept-sess"])def test_requires_session_recording_read(self) -> None:with patch(diff --git a/products/replay_vision/backend/tests/test_watch_rank.py b/products/replay_vision/backend/tests/test_watch_rank.pyindex 3207d697d3aa..41399781b463 100644--- a/products/replay_vision/backend/tests/test_watch_rank.py+++ b/products/replay_vision/backend/tests/test_watch_rank.py@@ -3,6 +3,7 @@ from typing import Anyfrom unittest.mock import patch+from django.db.models import Qfrom django.test import SimpleTestCaseimport numpy as np@@ -11,6 +12,7 @@ from parameterized import parameterizedfrom posthog.exceptions import ClickHouseAtCapacityfrom products.replay_vision.backend import watch_rank_backfill+from products.replay_vision.backend.models.replay_observation import ReplayObservationfrom products.replay_vision.backend.watch_feed import SESSION_SPACINGfrom products.replay_vision.backend.watch_rank import (ANCHOR_VERSION,@@ -77,6 +79,15 @@ class TestClassify(SimpleTestCase):assert (contrast > 0) is positive+class TestPickIndexes(SimpleTestCase):+ @parameterized.expand([("rlo_ranked_count_idx",), ("rlo_ranked_contrast_idx",)])+ def test_pick_index_matches_the_current_anchor_version(self, name: str) -> None:+ index = next(index for index in ReplayObservation._meta.indexes if index.name == name)+ assert index.condition == Q(+ status="succeeded", watch_rank__isnull=False, watch_rank__anchor_version=ANCHOR_VERSION+ )++class TestDuplicateRoot(SimpleTestCase):@parameterized.expand([diff --git a/products/replay_vision/backend/watch_rank.py b/products/replay_vision/backend/watch_rank.pyindex 33c52e2386ce..7e91cb8913a8 100644--- a/products/replay_vision/backend/watch_rank.py+++ b/products/replay_vision/backend/watch_rank.py@@ -13,26 +13,14 @@ from collections import Counterfrom dataclasses import asdictfrom datetime import datetime, timedeltafrom typing import Any, Literal-from uuid import UUIDfrom django.core.cache import cache-from django.db.models import (- Case,- CharField,- Count,- Exists,- F,- FloatField,- Func,- IntegerField,- Q,- QuerySet,- Value,- When,- Window,-)+from django.core.exceptions import EmptyResultSet+from django.db import connections+from django.db.models import CharField, Count, Exists, F, QuerySet, Value, Window+from django.db.models.expressions import RawSQLfrom django.db.models.fields.json import KeyTextTransform, KeyTransform-from django.db.models.functions import Cast, Coalesce, NullIf, RowNumber+from django.db.models.functions import Coalesce, NullIf, RowNumberfrom django.utils import timezoneimport numpy as np@@ -44,6 +32,7 @@ from posthog.clickhouse.query_tagging import Product, tags_contextfrom posthog.clickhouse.workload import Workloadfrom posthog.dataclasses import frozenfrom posthog.models.team import Team+from posthog.utils import dict_from_cursor_fetchallfrom products.access_control.backend.facade.user_access_control import UserAccessControlfrom products.replay_vision.backend.embeddings import (@@ -52,7 +41,12 @@ from products.replay_vision.backend.embeddings import (OBSERVATION_SCOPE_PREWHERE,observation_scope_params,)-from products.replay_vision.backend.models.replay_observation import ReplayObservation+from products.replay_vision.backend.models.replay_observation import (+ ReplayObservation,+ jsonb_typeof,+ watch_group_root,+ watch_rank_contrast,+)from products.replay_vision.backend.scanner_access import accessible_observationsfrom products.replay_vision.backend.watch_feed import WATCH_FEED_MIN_ITEMS, WatchFeedEntry, finish_feed@@ -350,19 +344,6 @@ def people_boost(people: int) -> float:return min(WATCH_PICK_PEOPLE_BOOST_CAP, 1 + WATCH_PICK_PEOPLE_BOOST * math.log(max(people, 1)))-def _is_uuid(value: str) -> bool:- try:- UUID(value)- except ValueError:- return False- return True---def _embedding_group_root() -> Coalesce:- """A ranked row's duplicate group: the observation it duplicates, or itself."""- return Coalesce(KeyTextTransform("duplicate_of", "watch_rank"), Cast("id", CharField()), output_field=CharField())--def _cap_duplicate_groups(rows: list[dict[str, Any]], cap: int) -> dict[Any, dict[str, Any]]:"""The first `cap` rows, keeping a few per duplicate group so the ranker can still pick a readable, unseen one.@@ -380,6 +361,65 @@ def _cap_duplicate_groups(rows: list[dict[str, Any]], cap: int) -> dict[Any, dicreturn picked+def _pick_window_rows(+ ranked: QuerySet[ReplayObservation], current_ranks: QuerySet[ReplayObservation], limit: int | None+) -> list[dict[str, Any]]:+ """Each scanner's top-fifth issue rows in pick order: every scanner's best first, then every second-best.++ A row's percentile needs only its rank from the top and its scanner's row count, so the query counts each+ scanner on a narrow index and then reads just its top fifth by score, instead of sorting the whole window.+ """+ contrast = watch_rank_contrast()+ scanner_counts = (+ ranked.values("scanner_id")+ .annotate(scanner_rows=Count("id"))+ .filter(scanner_rows__gte=WATCH_PICK_MIN_SCANNER_ROWS)+ .values("scanner_id", "scanner_rows")+ )+ scanner_top = (+ # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (a fixed column reference to the CTE below)+ current_ranks.filter(scanner_id=RawSQL('"scanner_counts"."scanner_id"', ()))+ .annotate(+ scanner_rank=Window(RowNumber(), order_by=[contrast.desc(), F("id").desc()]),+ watch_group_root=watch_group_root(),+ pick_category=F("watch_category"),+ )+ .order_by(contrast.desc(), "-id")+ .values("id", "created_at", "scanner_rank", "watch_group_root", "pick_category")+ )+ try:+ counts_sql, counts_params = scanner_counts.query.sql_with_params()+ top_sql, top_params = scanner_top.query.sql_with_params()+ except EmptyResultSet:+ # Django compiles a filter that can match nothing, such as an empty scanner list, to this.+ return []+ # Only fragments written here join the compiled ORM SQL; every value travels as a parameter.+ sql = f"""+ WITH scanner_counts AS ({counts_sql})+ SELECT id, watch_percentile, watch_group_root FROM (+ SELECT scanner_top.*,+ ((scanner_counts.scanner_rows - scanner_top.scanner_rank + 1) - 0.5)+ / scanner_counts.scanner_rows::double precision AS watch_percentile+ FROM scanner_counts+ CROSS JOIN LATERAL ({top_sql} LIMIT floor(scanner_counts.scanner_rows * %s)::int + 1) scanner_top+ ) picks+ WHERE pick_category = ANY(%s) AND watch_percentile >= %s+ ORDER BY scanner_rank, watch_percentile DESC, created_at DESC, id DESC+ LIMIT %s+ """+ params = (+ *counts_params,+ *top_params,+ 1 - WATCH_PICK_MIN_PERCENTILE,+ list(ISSUE_CATEGORIES),+ WATCH_PICK_MIN_PERCENTILE,+ limit,+ )+ with connections[ranked.db].cursor() as cursor:+ cursor.execute(sql, params)+ return dict_from_cursor_fetchall(cursor)++def select_embedding_picks(walkable: QuerySet[ReplayObservation],scanner_ids: list[Any],@@ -393,53 +433,23 @@ def select_embedding_picks(The percentile is taken over every ranked row a scanner has in the window, not over a recency slice, so alonger window reaches further back instead of re-ranking the same newest rows."""- contrast = Cast(KeyTextTransform("contrast", "watch_rank"), FloatField())- category = KeyTextTransform("category", "watch_rank")- group_root = _embedding_group_root()- current_ranks = (- walkable.filter(- # The explicit null check lets Postgres match the partial `rlo_scanner_ranked_idx`.- scanner_id__in=scanner_ids,- watch_rank__isnull=False,- watch_rank__anchor_version=ANCHOR_VERSION,- )- .alias(- watch_category=category,- contrast_type=Func(- KeyTransform("contrast", "watch_rank"), function="jsonb_typeof", output_field=CharField()- ),- )- .filter(contrast_type="number")- )- qualifying = (- current_ranks.annotate(- scanner_rows=Window(Count("id"), partition_by=[F("scanner_id")]),- scanner_position=Window(- RowNumber(), partition_by=[F("scanner_id")], order_by=[contrast.asc(), F("id").asc()]- ),- )- .annotate(- watch_percentile=(F("scanner_position") - Value(0.5)) / Cast(F("scanner_rows"), FloatField()),- # Each scanner's best row ranks 1, so the cap takes every scanner's best before any second-best.- scanner_rank=F("scanner_rows") - F("scanner_position") + Value(1),- # Reads a window value, so Django filters on it after the windows run rather than before.- issue_scanner_rows=Case(- When(watch_category__in=ISSUE_CATEGORIES, then=F("scanner_rows")),- default=Value(0),- output_field=IntegerField(),- ),- watch_group_root=group_root,- )- .filter(issue_scanner_rows__gte=WATCH_PICK_MIN_SCANNER_ROWS, watch_percentile__gte=WATCH_PICK_MIN_PERCENTILE)- .order_by("scanner_rank", "-watch_percentile", "-created_at", "-id")- .values("id", "watch_percentile", "watch_group_root")+ group_root = watch_group_root()+ ranked = walkable.filter(+ # The explicit null and anchor checks let Postgres match the partial `rlo_ranked_*` indexes.+ scanner_id__in=scanner_ids,+ watch_rank__isnull=False,+ watch_rank__anchor_version=ANCHOR_VERSION,)+ current_ranks = ranked.alias(+ watch_category=KeyTextTransform("category", "watch_rank"),+ contrast_type=jsonb_typeof(KeyTransform("contrast", "watch_rank")),+ ).filter(contrast_type="number")# Bounds the fetch, since a client-side cursor buffers every row; only a window full of copies needs the rest.fetch_limit = cap * (WATCH_PICK_GROUP_CANDIDATES + 1)- rows = list(qualifying[:fetch_limit])+ rows = _pick_window_rows(ranked, current_ranks, fetch_limit)window_values = _cap_duplicate_groups(rows, cap)if len(window_values) < cap and len(rows) == fetch_limit:- window_values = _cap_duplicate_groups(list(qualifying), cap)+ window_values = _cap_duplicate_groups(_pick_window_rows(ranked, current_ranks, None), cap)if not window_values:return []# Counted over readable rows only, so a group never reveals how many restricted sessions it holds. Duplicates@@ -451,10 +461,9 @@ def select_embedding_picks(for row in accessible_observations(access,team_id,- # Matches the picks' groups on plain columns first, so only their members reach the snapshot gate.- current_ranks.filter(- Q(id__in=[root for root in roots if _is_uuid(root)]) | Q(watch_rank__duplicate_of__in=list(roots)),- watch_category__in=ISSUE_CATEGORIES,+ # Matches the picks' groups on the indexed group root first, so only their members reach the snapshot gate.+ current_ranks.alias(member_root=group_root).filter(+ member_root__in=list(roots), watch_category__in=ISSUE_CATEGORIES),).annotate(watch_group_root=group_root)@@ -490,7 +499,7 @@ def select_embedding_filler("""The newest readable rows outside the picks' duplicate groups, unwatched first, enough to bring a thin feed up to its floor."""base = (accessible_observations(access, team_id, walkable.filter(scanner_id__in=scanner_ids))- .annotate(watch_group_root=_embedding_group_root())+ .annotate(watch_group_root=watch_group_root()).exclude(watch_group_root__in=pick_roots).annotate(feed_viewed=viewed))