Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(replay-vision): read each scanner's top picks instead of sorting the window (#123468)

Tue Haulund7b14c4b77078
diff --git a/products/replay_vision/backend/api/scanners.py b/products/replay_vision/backend/api/scanners.py
index 8e99638080a3..82367a3cb850 100644
--- a/products/replay_vision/backend/api/scanners.py
+++ b/products/replay_vision/backend/api/scanners.py
@@ -148,6 +148,7 @@ from products.replay_vision.backend.scanner_access import (
accessible_experiment_ids,
accessible_observations,
is_experiment_accessible,
+ readable_experiment_ids,
readable_observation_scanner_ids,
scanner_experiment_scope_q,
)
@@ -1671,8 +1672,10 @@ WATCH_FEED_PER_SCANNER_CAP = 100
# each id__in list bounded while the walk stops as soon as the page is covered.
WATCH_FEED_BYPASS_CHUNK = 200
WATCH_FEED_IMPRESSIONS_MAX_IDS = 100
-# Long enough to cover a reader scrolling one load; a new load gets a new `as_of` and a fresh ranking.
-WATCH_FEED_PICKS_CACHE_TTL_S = 15 * 60
+# Loads in one bucket share their picks, so a reload or tab switch skips the pick query.
+WATCH_FEED_PICKS_BUCKET_S = 10 * 60
+# Outlives the bucket by long enough to cover a reader scrolling one load.
+WATCH_FEED_PICKS_CACHE_TTL_S = WATCH_FEED_PICKS_BUCKET_S + 15 * 60
def select_watch_feed_candidates(
@@ -1709,6 +1712,16 @@ def select_watch_feed_candidates(
return rows[:WATCH_FEED_CANDIDATE_CAP]
+def refresh_feed_viewed(rows: list[dict[str, Any]], viewer_id: int, as_of: datetime) -> list[dict[str, Any]]:
+ """Cached rows with `feed_viewed` as of `as_of`."""
+ viewed = set(
+ ReplayObservationView.objects.filter(
+ user_id=viewer_id, created_at__lte=as_of, observation_id__in=[row["id"] for row in rows]
+ ).values_list("observation_id", flat=True)
+ )
+ return [{**row, "feed_viewed": row["id"] in viewed} for row in rows]
+
+
def attach_scanner_results(team_id: int, rows: list[dict[str, Any]]) -> None:
"""Load the scan result for each candidate row, by primary key, after the window cap has applied."""
results = dict(
@@ -2688,11 +2701,12 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi
viewer_id = cast(User, request.user).id
# Before ranking, so a row the viewer marked not useful neither takes a slot nor shifts the window baselines.
# Views and dismissals after `as_of` wait for the next load, so later pages of this one do not shift.
- walkable = candidates.exclude(
- id__in=ReplayObservationFeedDismissal.objects.filter(user_id=viewer_id, created_at__lte=as_of).values(
- "observation_id"
+ dismissed_ids = sorted(
+ ReplayObservationFeedDismissal.objects.filter(user_id=viewer_id, created_at__lte=as_of).values_list(
+ "observation_id", flat=True
)
)
+ walkable = candidates.exclude(id__in=dismissed_ids)
# Row-gate on each row's snapshot experiment before ranking, so a restricted row can't take a feed slot.
candidates = accessible_observations(
self.user_access_control, self.team_id, walkable.filter(scanner_id__in=allowed_ids)
@@ -2711,7 +2725,8 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi
mode = ranker_mode(variant)
ranker = mode if mode in ("jev", "embedding") else "weighted-score"
if ranker == "embedding":
- # Every page of one load ranks the same window as of the same `as_of`, so later pages reuse the first's picks.
+ # Keyed on the bucket `as_of` falls in, so loads in one bucket share picks, and on the dismissals and the
+ # readable experiments, so a dismissal or an access change ranks afresh.
picks_key = (
"replay_vision:watch_picks:"
+ hashlib.sha256(
@@ -2719,9 +2734,11 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi
[
self.team_id,
viewer_id,
- as_of.isoformat(),
- str(date_from),
- str(window_end),
+ int(as_of.timestamp()) // WATCH_FEED_PICKS_BUCKET_S,
+ params["date_from"],
+ params.get("date_to"),
+ list(map(str, dismissed_ids)),
+ sorted(readable_experiment_ids(self.user_access_control, self.team_id)),
sorted(map(str, allowed_ids)),
params.get("search"),
params.get("scanner_type"),
@@ -2729,8 +2746,14 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi
).encode()
).hexdigest()
)
- candidate_rows = cache.get(picks_key)
- if candidate_rows is None:
+ cached = cache.get(picks_key)
+ # Picks ranked as of a later moment may hold rows this load's first page never saw, so they don't serve it.
+ if cached is not None and cached[0] <= as_of:
+ picked_as_of, candidate_rows = cached
+ # An earlier load in the bucket ranked these, so its views may be older than this one's.
+ if picked_as_of != as_of:
+ candidate_rows = refresh_feed_viewed(candidate_rows, viewer_id, as_of)
+ else:
# The embedding ranker reads only a small stored rank, so it picks over the whole window in SQL.
candidate_rows = select_embedding_picks(
walkable, allowed_ids, viewed, self.user_access_control, self.team_id, WATCH_FEED_CANDIDATE_CAP
@@ -2741,7 +2764,8 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi
candidate_rows += select_embedding_filler(
walkable, allowed_ids, viewed, self.user_access_control, self.team_id, pick_roots
)
- cache.set(picks_key, candidate_rows, WATCH_FEED_PICKS_CACHE_TTL_S)
+ # The first load in the bucket keeps the entry, so concurrent loads never replace it with later picks.
+ cache.add(picks_key, (as_of, candidate_rows), WATCH_FEED_PICKS_CACHE_TTL_S)
else:
candidate_rows = select_watch_feed_candidates(
walkable, allowed_ids, viewed, self.user_access_control, self.team_id
diff --git a/products/replay_vision/backend/migrations/0131_rename_legacy_field_indexes.py b/products/replay_vision/backend/migrations/0131_rename_legacy_field_indexes.py
new file mode 100644
index 000000000000..280f26582d03
--- /dev/null
+++ b/products/replay_vision/backend/migrations/0131_rename_legacy_field_indexes.py
@@ -0,0 +1,20 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("replay_vision", "0130_rom_small_asset_idx"),
+ ]
+
+ operations = [
+ # Databases from before the replaylensobservation rename hold these under the old table's names.
+ migrations.RunSQL(
+ sql=[
+ "ALTER INDEX IF EXISTS replay_vision_replaylensobservation_team_id_b10decc8 "
+ "RENAME TO replay_vision_replayobservation_team_id_b204b330",
+ "ALTER INDEX IF EXISTS replay_vision_replaylensobservation_lens_id_4a4cd5c2 "
+ "RENAME TO replay_vision_replayobservation_scanner_id_f2400b26",
+ ],
+ reverse_sql=migrations.RunSQL.noop,
+ ),
+ ]
diff --git a/products/replay_vision/backend/migrations/0132_ranked_pick_indexes.py b/products/replay_vision/backend/migrations/0132_ranked_pick_indexes.py
new file mode 100644
index 000000000000..40952adfd164
--- /dev/null
+++ b/products/replay_vision/backend/migrations/0132_ranked_pick_indexes.py
@@ -0,0 +1,76 @@
+import django.db.models.fields.json
+import django.db.models.functions.comparison
+from django.db import migrations, models
+
+from posthog.migration_helpers import (
+ DropFieldIndexesConcurrently,
+ SafeAddIndexConcurrently,
+ SafeRemoveIndexConcurrently,
+)
+
+
+class Migration(migrations.Migration):
+ atomic = False
+
+ dependencies = [
+ ("replay_vision", "0131_rename_legacy_field_indexes"),
+ ]
+
+ operations = [
+ SafeAddIndexConcurrently(
+ model_name="replayobservation",
+ index=models.Index(
+ condition=models.Q(
+ ("status", "succeeded"),
+ ("watch_rank__anchor_version", 1),
+ ("watch_rank__isnull", False),
+ ),
+ fields=["scanner", "created_at"],
+ include=("id", "team", "completed_at"),
+ name="rlo_ranked_count_idx",
+ ),
+ ),
+ SafeAddIndexConcurrently(
+ model_name="replayobservation",
+ index=models.Index(
+ models.F("scanner"),
+ models.OrderBy(
+ django.db.models.functions.comparison.Cast(
+ django.db.models.fields.json.KeyTextTransform("contrast", "watch_rank"),
+ models.FloatField(),
+ ),
+ descending=True,
+ ),
+ models.OrderBy(models.F("id"), descending=True),
+ condition=models.Q(
+ ("status", "succeeded"),
+ ("watch_rank__anchor_version", 1),
+ ("watch_rank__isnull", False),
+ ),
+ include=("team", "created_at", "completed_at", "watch_rank"),
+ name="rlo_ranked_contrast_idx",
+ ),
+ ),
+ SafeAddIndexConcurrently(
+ model_name="replayobservation",
+ index=models.Index(
+ django.db.models.functions.comparison.Coalesce(
+ django.db.models.fields.json.KeyTextTransform("duplicate_of", "watch_rank"),
+ django.db.models.functions.comparison.Cast("id", models.CharField()),
+ output_field=models.CharField(),
+ ),
+ condition=models.Q(("status", "succeeded"), ("watch_rank__isnull", False)),
+ name="rlo_ranked_group_idx",
+ ),
+ ),
+ SafeRemoveIndexConcurrently(
+ model_name="replayobservation",
+ name="rlo_scanner_ranked_idx",
+ ),
+ SafeRemoveIndexConcurrently(
+ model_name="replayobservation",
+ name="rlo_workflow_id_idx",
+ ),
+ DropFieldIndexesConcurrently(model_name="replayobservation", name="team"),
+ DropFieldIndexesConcurrently(model_name="replayobservation", name="scanner"),
+ ]
diff --git a/products/replay_vision/backend/migrations/max_migration.txt b/products/replay_vision/backend/migrations/max_migration.txt
index bad141a4bacb..9fd42bb17ea8 100644
--- a/products/replay_vision/backend/migrations/max_migration.txt
+++ b/products/replay_vision/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0130_rom_small_asset_idx
+0132_ranked_pick_indexes
diff --git a/products/replay_vision/backend/models/replay_observation.py b/products/replay_vision/backend/models/replay_observation.py
index e248f5f65a27..96aba7598d7d 100644
--- a/products/replay_vision/backend/models/replay_observation.py
+++ b/products/replay_vision/backend/models/replay_observation.py
@@ -1,7 +1,7 @@
from django.db import models
from django.db.models import Case, CharField, Exists, Expression, F, FloatField, Func, OuterRef, Value, When
from django.db.models.fields.json import KeyTextTransform, KeyTransform
-from django.db.models.functions import Cast
+from django.db.models.functions import Cast, Coalesce
from posthog.models.utils import UUIDModel
@@ -42,11 +42,25 @@ class ObservationTrigger(models.TextChoices):
BACKFILL = "backfill", "Backfill"
+def watch_rank_contrast() -> Cast:
+ """A ranked row's score; `rlo_ranked_contrast_idx` orders by it."""
+ return Cast(KeyTextTransform("contrast", "watch_rank"), FloatField())
+
+
+def watch_group_root() -> Coalesce:
+ """A ranked row's duplicate group: the observation it duplicates, or itself. `rlo_ranked_group_idx` indexes it."""
+ return Coalesce(KeyTextTransform("duplicate_of", "watch_rank"), Cast("id", CharField()), output_field=CharField())
+
+
class ReplayObservation(UUIDModel):
"""One application of a `ReplayScanner` to a session recording (see README)."""
- scanner = models.ForeignKey("replay_vision.ReplayScanner", on_delete=models.CASCADE, related_name="observations")
- team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+")
+ # db_index=False: rlo_scanner_status_idx leads with scanner_id and serves every scanner filter.
+ scanner = models.ForeignKey(
+ "replay_vision.ReplayScanner", on_delete=models.CASCADE, related_name="observations", db_index=False
+ )
+ # db_index=False: rlo_team_created_idx leads with team_id and serves every team filter.
+ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+", db_index=False)
session_id = models.CharField(max_length=200, help_text="Session recording id this scanner was applied to.")
distinct_id = models.CharField(
max_length=400,
@@ -163,11 +177,6 @@ class ReplayObservation(UUIDModel):
),
# Serves the per-scanner list ordering and the prev/next-neighbor lookups (both order by created_at).
models.Index(fields=["scanner", "created_at"], name="rlo_scanner_created_idx"),
- models.Index(
- fields=["workflow_id"],
- name="rlo_workflow_id_idx",
- condition=~models.Q(workflow_id=""),
- ),
# Serves the per-team in-flight concurrency count (sweep headroom + on-demand 429). Partial on the
# in-flight statuses only, since terminal rows dominate and are never counted.
models.Index(
@@ -175,11 +184,26 @@ class ReplayObservation(UUIDModel):
name="rlo_team_in_flight_idx",
condition=models.Q(status__in=("pending", "running")),
),
- # Serves the embedding feed's whole-window pick query, which reads only these columns of ranked rows.
+ # The pick query counts each scanner's ranked rows here; the anchor version sits in the condition so the count never reads `watch_rank`.
models.Index(
fields=["scanner", "created_at"],
- include=["id", "team", "completed_at", "watch_rank", "distinct_id", "session_id"],
- name="rlo_scanner_ranked_idx",
+ include=["id", "team", "completed_at"],
+ name="rlo_ranked_count_idx",
+ condition=models.Q(status="succeeded", watch_rank__isnull=False, watch_rank__anchor_version=1),
+ ),
+ # The pick query reads each scanner's top rows by score from here.
+ models.Index(
+ F("scanner"),
+ watch_rank_contrast().desc(),
+ F("id").desc(),
+ include=["team", "created_at", "completed_at", "watch_rank"],
+ name="rlo_ranked_contrast_idx",
+ condition=models.Q(status="succeeded", watch_rank__isnull=False, watch_rank__anchor_version=1),
+ ),
+ # Finds the members of the feed picks' duplicate groups, to count the sessions and people each one reached.
+ models.Index(
+ watch_group_root(),
+ name="rlo_ranked_group_idx",
condition=models.Q(status="succeeded", watch_rank__isnull=False),
),
# Serves the per-backfill observation filter and progress counts; partial since live rows never match.
diff --git a/products/replay_vision/backend/scanner_access.py b/products/replay_vision/backend/scanner_access.py
index e46269849195..13a4b6707a19 100644
--- a/products/replay_vision/backend/scanner_access.py
+++ b/products/replay_vision/backend/scanner_access.py
@@ -59,6 +59,11 @@ def accessible_experiment_ids(
return facade_accessible_experiment_ids(access, team_id, experiment_ids=experiment_ids)
+def readable_experiment_ids(access: UserAccessControl, team_id: int) -> set[int]:
+ """Every experiment the caller may view in the team, the set `accessible_observations` gates rows on."""
+ return accessible_experiment_ids(access, team_id, None)
+
+
def is_experiment_accessible(access: UserAccessControl | None, team_id: int, experiment_id: int) -> bool:
"""Whether the caller may view this experiment, filtered by object-level access (not just team).
diff --git a/products/replay_vision/backend/tests/test_api.py b/products/replay_vision/backend/tests/test_api.py
index dc7c9ac2d725..566e38a19d9d 100644
--- a/products/replay_vision/backend/tests/test_api.py
+++ b/products/replay_vision/backend/tests/test_api.py
@@ -34,6 +34,7 @@ from posthog.session_recordings.queries.test.session_replay_sql import produce_r
from products.experiments.backend.models.experiment import Experiment
from products.replay_vision.backend.api.scanners import (
WATCH_FEED_CANDIDATE_CAP,
+ WATCH_FEED_PICKS_BUCKET_S,
ReplayScannerSerializer,
WatchFeedQuerySerializer,
)
@@ -5251,6 +5252,34 @@ class TestWatchFeedAPI(_VisionAPITestCase):
assert sessions == [f"hit-{i}" for i in range(6)]
assert second["has_more"] is False
+ def test_a_later_loads_cached_picks_do_not_serve_an_earlier_loads_pages(self) -> None:
+ scanner = self._create_scanner(name="m")
+ bucket_start = datetime.fromtimestamp(
+ (int(timezone.now().timestamp()) // WATCH_FEED_PICKS_BUCKET_S - 1) * WATCH_FEED_PICKS_BUCKET_S, tz=UTC
+ )
+ earlier, later = bucket_start + timedelta(minutes=1), bucket_start + timedelta(minutes=5)
+ for index in range(12):
+ obs = self._succeeded_observation(scanner, f"row-{index}", 120 + index, self._monitor_result("no"))
+ # The top-scoring row completes between the two loads, so only the later load can rank it.
+ completed_at = bucket_start + timedelta(minutes=3) if index == 0 else bucket_start
+ ReplayObservation.objects.filter(id=obs.id).update(
+ completed_at=completed_at,
+ watch_rank={
+ "category": "failure",
+ "contrast": 1 - index / 20,
+ "duplicate_of": None,
+ "anchor_version": ANCHOR_VERSION,
+ },
+ )
+
+ def sessions(as_of: datetime) -> list[str]:
+ response = self.client.get(self.feed_url, {"limit": "20", "as_of": as_of.isoformat()}).json()
+ return [item["observation"]["session_id"] for item in response["results"]]
+
+ with patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value="embedding"):
+ assert "row-0" in sessions(later)
+ assert "row-0" not in sessions(earlier)
+
def test_embedding_picks_reach_past_the_recency_slice_and_count_hidden_duplicates(self) -> None:
scanner = self._create_scanner(name="m")
@@ -5863,6 +5892,21 @@ class TestWatchFeedAPI(_VisionAPITestCase):
self.assertEqual(resp.status_code, 200, resp.json())
self.assertEqual(resp.json()["results"][0]["reason"]["kind"], "unviewed_recent")
+ @parameterized.expand([("weighted_score", "control"), ("embedding", "embedding")])
+ def test_a_scope_with_no_readable_scanner_returns_an_empty_feed(self, _name: str, variant: str) -> None:
+ hidden = self._create_scanner(name="hidden")
+ self._succeeded_observation(hidden, "hidden-sess", 1, self._monitor_result("no"))
+ with (
+ patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value=variant),
+ patch(
+ "products.access_control.backend.facade.user_access_control.UserAccessControl.filter_queryset_by_access_level",
+ side_effect=lambda qs, **_: qs.none() if qs.model is ReplayScanner else qs,
+ ),
+ ):
+ resp = self.client.get(self.feed_url)
+ self.assertEqual(resp.status_code, 200, resp.json())
+ self.assertEqual(resp.json()["results"], [])
+
def test_unreadable_and_cross_team_scanners_are_dropped_silently(self) -> None:
visible = self._create_scanner(name="visible")
hidden = self._create_scanner(name="hidden")
@@ -5935,26 +5979,28 @@ class TestWatchFeedAPI(_VisionAPITestCase):
second = self.client.get(self.feed_url, {"limit": 5, "offset": 1, "as_of": first["as_of"]}).json()
self.assertEqual(second["results"], [])
- def test_not_useful_hides_a_row_for_that_user_only_until_undone(self) -> None:
- scanner = self._create_scanner(name="m")
- dismissed = self._succeeded_observation(scanner, "dismissed-sess", 1, self._monitor_result("yes"))
- self._succeeded_observation(scanner, "kept-sess", 2, self._monitor_result("yes"))
- dismissal_url = f"/api/projects/{self.team.id}/vision/observations/{dismissed.id}/feed_dismissal/"
+ @parameterized.expand([("weighted_score", "control"), ("embedding_reload_in_one_bucket", "embedding")])
+ def test_not_useful_hides_a_row_for_that_user_only_until_undone(self, _name: str, variant: str) -> None:
+ with patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value=variant):
+ scanner = self._create_scanner(name="m")
+ dismissed = self._succeeded_observation(scanner, "dismissed-sess", 1, self._monitor_result("yes"))
+ self._succeeded_observation(scanner, "kept-sess", 2, self._monitor_result("yes"))
+ dismissal_url = f"/api/projects/{self.team.id}/vision/observations/{dismissed.id}/feed_dismissal/"
- def feed_sessions() -> list[str]:
- return [item["observation"]["session_id"] for item in self.client.get(self.feed_url).json()["results"]]
+ def feed_sessions() -> list[str]:
+ return [item["observation"]["session_id"] for item in self.client.get(self.feed_url).json()["results"]]
- self.assertEqual(self.client.post(dismissal_url, {"reason": "nonsense"}).status_code, 400)
- self.assertEqual(self.client.post(dismissal_url, {"reason": "already_known"}).status_code, 204)
- self.assertEqual(feed_sessions(), ["kept-sess"])
+ self.assertEqual(self.client.post(dismissal_url, {"reason": "nonsense"}).status_code, 400)
+ self.assertEqual(self.client.post(dismissal_url, {"reason": "already_known"}).status_code, 204)
+ self.assertEqual(feed_sessions(), ["kept-sess"])
- other = User.objects.create_and_join(self.organization, "[email protected]", None)
- self.client.force_login(other)
- self.assertEqual(sorted(feed_sessions()), ["dismissed-sess", "kept-sess"])
+ other = User.objects.create_and_join(self.organization, "[email protected]", None)
+ self.client.force_login(other)
+ self.assertEqual(sorted(feed_sessions()), ["dismissed-sess", "kept-sess"])
- self.client.force_login(self.user)
- self.assertEqual(self.client.delete(dismissal_url).status_code, 204)
- self.assertEqual(sorted(feed_sessions()), ["dismissed-sess", "kept-sess"])
+ self.client.force_login(self.user)
+ self.assertEqual(self.client.delete(dismissal_url).status_code, 204)
+ self.assertEqual(sorted(feed_sessions()), ["dismissed-sess", "kept-sess"])
def test_requires_session_recording_read(self) -> None:
with patch(
diff --git a/products/replay_vision/backend/tests/test_watch_rank.py b/products/replay_vision/backend/tests/test_watch_rank.py
index 3207d697d3aa..41399781b463 100644
--- a/products/replay_vision/backend/tests/test_watch_rank.py
+++ b/products/replay_vision/backend/tests/test_watch_rank.py
@@ -3,6 +3,7 @@ from typing import Any
from unittest.mock import patch
+from django.db.models import Q
from django.test import SimpleTestCase
import numpy as np
@@ -11,6 +12,7 @@ from parameterized import parameterized
from posthog.exceptions import ClickHouseAtCapacity
from products.replay_vision.backend import watch_rank_backfill
+from products.replay_vision.backend.models.replay_observation import ReplayObservation
from products.replay_vision.backend.watch_feed import SESSION_SPACING
from products.replay_vision.backend.watch_rank import (
ANCHOR_VERSION,
@@ -77,6 +79,15 @@ class TestClassify(SimpleTestCase):
assert (contrast > 0) is positive
+class TestPickIndexes(SimpleTestCase):
+ @parameterized.expand([("rlo_ranked_count_idx",), ("rlo_ranked_contrast_idx",)])
+ def test_pick_index_matches_the_current_anchor_version(self, name: str) -> None:
+ index = next(index for index in ReplayObservation._meta.indexes if index.name == name)
+ assert index.condition == Q(
+ status="succeeded", watch_rank__isnull=False, watch_rank__anchor_version=ANCHOR_VERSION
+ )
+
+
class TestDuplicateRoot(SimpleTestCase):
@parameterized.expand(
[
diff --git a/products/replay_vision/backend/watch_rank.py b/products/replay_vision/backend/watch_rank.py
index 33c52e2386ce..7e91cb8913a8 100644
--- a/products/replay_vision/backend/watch_rank.py
+++ b/products/replay_vision/backend/watch_rank.py
@@ -13,26 +13,14 @@ from collections import Counter
from dataclasses import asdict
from datetime import datetime, timedelta
from typing import Any, Literal
-from uuid import UUID
from django.core.cache import cache
-from django.db.models import (
- Case,
- CharField,
- Count,
- Exists,
- F,
- FloatField,
- Func,
- IntegerField,
- Q,
- QuerySet,
- Value,
- When,
- Window,
-)
+from django.core.exceptions import EmptyResultSet
+from django.db import connections
+from django.db.models import CharField, Count, Exists, F, QuerySet, Value, Window
+from django.db.models.expressions import RawSQL
from django.db.models.fields.json import KeyTextTransform, KeyTransform
-from django.db.models.functions import Cast, Coalesce, NullIf, RowNumber
+from django.db.models.functions import Coalesce, NullIf, RowNumber
from django.utils import timezone
import numpy as np
@@ -44,6 +32,7 @@ from posthog.clickhouse.query_tagging import Product, tags_context
from posthog.clickhouse.workload import Workload
from posthog.dataclasses import frozen
from posthog.models.team import Team
+from posthog.utils import dict_from_cursor_fetchall
from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.replay_vision.backend.embeddings import (
@@ -52,7 +41,12 @@ from products.replay_vision.backend.embeddings import (
OBSERVATION_SCOPE_PREWHERE,
observation_scope_params,
)
-from products.replay_vision.backend.models.replay_observation import ReplayObservation
+from products.replay_vision.backend.models.replay_observation import (
+ ReplayObservation,
+ jsonb_typeof,
+ watch_group_root,
+ watch_rank_contrast,
+)
from products.replay_vision.backend.scanner_access import accessible_observations
from products.replay_vision.backend.watch_feed import WATCH_FEED_MIN_ITEMS, WatchFeedEntry, finish_feed
@@ -350,19 +344,6 @@ def people_boost(people: int) -> float:
return min(WATCH_PICK_PEOPLE_BOOST_CAP, 1 + WATCH_PICK_PEOPLE_BOOST * math.log(max(people, 1)))
-def _is_uuid(value: str) -> bool:
- try:
- UUID(value)
- except ValueError:
- return False
- return True
-
-
-def _embedding_group_root() -> Coalesce:
- """A ranked row's duplicate group: the observation it duplicates, or itself."""
- return Coalesce(KeyTextTransform("duplicate_of", "watch_rank"), Cast("id", CharField()), output_field=CharField())
-
-
def _cap_duplicate_groups(rows: list[dict[str, Any]], cap: int) -> dict[Any, dict[str, Any]]:
"""The first `cap` rows, keeping a few per duplicate group so the ranker can still pick a readable, unseen one.
@@ -380,6 +361,65 @@ def _cap_duplicate_groups(rows: list[dict[str, Any]], cap: int) -> dict[Any, dic
return picked
+def _pick_window_rows(
+ ranked: QuerySet[ReplayObservation], current_ranks: QuerySet[ReplayObservation], limit: int | None
+) -> list[dict[str, Any]]:
+ """Each scanner's top-fifth issue rows in pick order: every scanner's best first, then every second-best.
+
+ A row's percentile needs only its rank from the top and its scanner's row count, so the query counts each
+ scanner on a narrow index and then reads just its top fifth by score, instead of sorting the whole window.
+ """
+ contrast = watch_rank_contrast()
+ scanner_counts = (
+ ranked.values("scanner_id")
+ .annotate(scanner_rows=Count("id"))
+ .filter(scanner_rows__gte=WATCH_PICK_MIN_SCANNER_ROWS)
+ .values("scanner_id", "scanner_rows")
+ )
+ scanner_top = (
+ # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (a fixed column reference to the CTE below)
+ current_ranks.filter(scanner_id=RawSQL('"scanner_counts"."scanner_id"', ()))
+ .annotate(
+ scanner_rank=Window(RowNumber(), order_by=[contrast.desc(), F("id").desc()]),
+ watch_group_root=watch_group_root(),
+ pick_category=F("watch_category"),
+ )
+ .order_by(contrast.desc(), "-id")
+ .values("id", "created_at", "scanner_rank", "watch_group_root", "pick_category")
+ )
+ try:
+ counts_sql, counts_params = scanner_counts.query.sql_with_params()
+ top_sql, top_params = scanner_top.query.sql_with_params()
+ except EmptyResultSet:
+ # Django compiles a filter that can match nothing, such as an empty scanner list, to this.
+ return []
+ # Only fragments written here join the compiled ORM SQL; every value travels as a parameter.
+ sql = f"""
+ WITH scanner_counts AS ({counts_sql})
+ SELECT id, watch_percentile, watch_group_root FROM (
+ SELECT scanner_top.*,
+ ((scanner_counts.scanner_rows - scanner_top.scanner_rank + 1) - 0.5)
+ / scanner_counts.scanner_rows::double precision AS watch_percentile
+ FROM scanner_counts
+ CROSS JOIN LATERAL ({top_sql} LIMIT floor(scanner_counts.scanner_rows * %s)::int + 1) scanner_top
+ ) picks
+ WHERE pick_category = ANY(%s) AND watch_percentile >= %s
+ ORDER BY scanner_rank, watch_percentile DESC, created_at DESC, id DESC
+ LIMIT %s
+ """
+ params = (
+ *counts_params,
+ *top_params,
+ 1 - WATCH_PICK_MIN_PERCENTILE,
+ list(ISSUE_CATEGORIES),
+ WATCH_PICK_MIN_PERCENTILE,
+ limit,
+ )
+ with connections[ranked.db].cursor() as cursor:
+ cursor.execute(sql, params)
+ return dict_from_cursor_fetchall(cursor)
+
+
def select_embedding_picks(
walkable: QuerySet[ReplayObservation],
scanner_ids: list[Any],
@@ -393,53 +433,23 @@ def select_embedding_picks(
The percentile is taken over every ranked row a scanner has in the window, not over a recency slice, so a
longer window reaches further back instead of re-ranking the same newest rows.
"""
- contrast = Cast(KeyTextTransform("contrast", "watch_rank"), FloatField())
- category = KeyTextTransform("category", "watch_rank")
- group_root = _embedding_group_root()
- current_ranks = (
- walkable.filter(
- # The explicit null check lets Postgres match the partial `rlo_scanner_ranked_idx`.
- scanner_id__in=scanner_ids,
- watch_rank__isnull=False,
- watch_rank__anchor_version=ANCHOR_VERSION,
- )
- .alias(
- watch_category=category,
- contrast_type=Func(
- KeyTransform("contrast", "watch_rank"), function="jsonb_typeof", output_field=CharField()
- ),
- )
- .filter(contrast_type="number")
- )
- qualifying = (
- current_ranks.annotate(
- scanner_rows=Window(Count("id"), partition_by=[F("scanner_id")]),
- scanner_position=Window(
- RowNumber(), partition_by=[F("scanner_id")], order_by=[contrast.asc(), F("id").asc()]
- ),
- )
- .annotate(
- watch_percentile=(F("scanner_position") - Value(0.5)) / Cast(F("scanner_rows"), FloatField()),
- # Each scanner's best row ranks 1, so the cap takes every scanner's best before any second-best.
- scanner_rank=F("scanner_rows") - F("scanner_position") + Value(1),
- # Reads a window value, so Django filters on it after the windows run rather than before.
- issue_scanner_rows=Case(
- When(watch_category__in=ISSUE_CATEGORIES, then=F("scanner_rows")),
- default=Value(0),
- output_field=IntegerField(),
- ),
- watch_group_root=group_root,
- )
- .filter(issue_scanner_rows__gte=WATCH_PICK_MIN_SCANNER_ROWS, watch_percentile__gte=WATCH_PICK_MIN_PERCENTILE)
- .order_by("scanner_rank", "-watch_percentile", "-created_at", "-id")
- .values("id", "watch_percentile", "watch_group_root")
+ group_root = watch_group_root()
+ ranked = walkable.filter(
+ # The explicit null and anchor checks let Postgres match the partial `rlo_ranked_*` indexes.
+ scanner_id__in=scanner_ids,
+ watch_rank__isnull=False,
+ watch_rank__anchor_version=ANCHOR_VERSION,
)
+ current_ranks = ranked.alias(
+ watch_category=KeyTextTransform("category", "watch_rank"),
+ contrast_type=jsonb_typeof(KeyTransform("contrast", "watch_rank")),
+ ).filter(contrast_type="number")
# Bounds the fetch, since a client-side cursor buffers every row; only a window full of copies needs the rest.
fetch_limit = cap * (WATCH_PICK_GROUP_CANDIDATES + 1)
- rows = list(qualifying[:fetch_limit])
+ rows = _pick_window_rows(ranked, current_ranks, fetch_limit)
window_values = _cap_duplicate_groups(rows, cap)
if len(window_values) < cap and len(rows) == fetch_limit:
- window_values = _cap_duplicate_groups(list(qualifying), cap)
+ window_values = _cap_duplicate_groups(_pick_window_rows(ranked, current_ranks, None), cap)
if not window_values:
return []
# Counted over readable rows only, so a group never reveals how many restricted sessions it holds. Duplicates
@@ -451,10 +461,9 @@ def select_embedding_picks(
for row in accessible_observations(
access,
team_id,
- # Matches the picks' groups on plain columns first, so only their members reach the snapshot gate.
- current_ranks.filter(
- Q(id__in=[root for root in roots if _is_uuid(root)]) | Q(watch_rank__duplicate_of__in=list(roots)),
- watch_category__in=ISSUE_CATEGORIES,
+ # Matches the picks' groups on the indexed group root first, so only their members reach the snapshot gate.
+ current_ranks.alias(member_root=group_root).filter(
+ member_root__in=list(roots), watch_category__in=ISSUE_CATEGORIES
),
)
.annotate(watch_group_root=group_root)
@@ -490,7 +499,7 @@ def select_embedding_filler(
"""The newest readable rows outside the picks' duplicate groups, unwatched first, enough to bring a thin feed up to its floor."""
base = (
accessible_observations(access, team_id, walkable.filter(scanner_id__in=scanner_ids))
- .annotate(watch_group_root=_embedding_group_root())
+ .annotate(watch_group_root=watch_group_root())
.exclude(watch_group_root__in=pick_roots)
.annotate(feed_viewed=viewed)
)