:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(replay-vision): rank highlight picks from stored entries and rebuild on a schedule (#126402)
Tue Haulund1e238660eeb2
diff --git a/posthog/temporal/schedule.py b/posthog/temporal/schedule.pyindex 7cdee4807f85..929d7d4c4e6c 100644--- a/posthog/temporal/schedule.py+++ b/posthog/temporal/schedule.py@@ -132,6 +132,7 @@ from products.replay_vision.backend.temporal.estimates import create_replay_visifrom products.replay_vision.backend.temporal.gemini_cleanup_sweep import (create_replay_vision_gemini_cleanup_sweep_schedule,)+from products.replay_vision.backend.temporal.highlight_rebuild import create_replay_vision_highlight_rebuild_schedulefrom products.replay_vision.backend.temporal.learned_rules import create_replay_vision_learned_rules_schedulefrom products.replay_vision.backend.temporal.read_meter import create_replay_vision_read_meter_schedulefrom products.replay_vision.backend.temporal.reconciler import create_replay_vision_reconciler_schedule@@ -1033,6 +1034,7 @@ schedules = [create_replay_vision_learned_rules_schedule,create_vision_alert_check_schedule,create_replay_vision_read_meter_schedule,+ create_replay_vision_highlight_rebuild_schedule,create_github_job_logs_coordinator_schedule,create_review_hog_finding_outcomes_schedule,create_ci_signals_coordinator_schedule,diff --git a/products/replay_vision/backend/api/scanners.py b/products/replay_vision/backend/api/scanners.pyindex eb6a5528de03..370ff862fb37 100644--- a/products/replay_vision/backend/api/scanners.py+++ b/products/replay_vision/backend/api/scanners.py@@ -100,7 +100,6 @@ from products.replay_vision.backend.models.replay_observation import (hydrate_for_serialization,)from products.replay_vision.backend.models.replay_observation_feed_dismissal import ReplayObservationFeedDismissal-from products.replay_vision.backend.models.replay_observation_view import ReplayObservationViewfrom products.replay_vision.backend.models.replay_scanner import (ReplayScanner,SamplingMode,@@ -185,6 +184,7 @@ from products.replay_vision.backend.watch_rank import (rank_highlight_feed_by_embedding,select_embedding_filler,select_embedding_picks,+ views_by,)from products.signals.backend.facade.api import get_outcomes_for_signal_source_slice@@ -2488,11 +2488,7 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi"observation_id", flat=True))- viewed = Exists(- ReplayObservationView.objects.filter(- observation_id=OuterRef("id"), user_id=viewer_id, created_at__lte=as_of- )- )+ viewed = Exists(views_by(viewer_id, as_of).filter(observation_id=OuterRef("id")))categories = params.get("categories") or ISSUE_CATEGORIEScandidate_rows = select_embedding_picks(feed_window(@@ -2503,7 +2499,7 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, viNone if date_to is not None else HIGHLIGHT_FEED_PRESETS.get(params["date_from"]),),allowed_ids,- viewed,+ viewer_id,self.user_access_control,self.team_id,HIGHLIGHT_FEED_CANDIDATE_CAP,diff --git a/products/replay_vision/backend/highlight_builds.py b/products/replay_vision/backend/highlight_builds.pyindex 67bad1c3961b..e4962970bed5 100644--- a/products/replay_vision/backend/highlight_builds.py+++ b/products/replay_vision/backend/highlight_builds.py@@ -6,7 +6,7 @@ preset windows from a single read of the longest one. A build keeps only rows thbounded number of places per scanner, which is all a page load can reach."""-from collections.abc import Sequence+from collections.abc import Callable, Sequencefrom dataclasses import fieldsfrom datetime import datetime, timedeltafrom typing import Any@@ -61,6 +61,7 @@ class RankedEntry:experiment_id: intpercentile: floatgroup_root: Any+ person_key: int | None@frozen@@ -92,7 +93,7 @@ def _ranked_sql(WITH bounds(window_days, window_start, window_end) AS (VALUES {window_values}),ranked_rows AS MATERIALIZED (SELECT observation_id, scanner_id, created_at, contrast, category, experiment_id,- COALESCE(feed_group_id, observation_id) AS group_root+ COALESCE(feed_group_id, observation_id) AS group_root, person_keyFROM replay_vision_replayobservationrankWHERE team_id = %s AND anchor_version = %sAND created_at >= (SELECT min(window_start) FROM bounds)@@ -114,7 +115,7 @@ def _ranked_sql() windowed)SELECT window_days, place, observation_id, scanner_id, created_at, category, experiment_id, percentile,- group_root+ group_root, person_keyFROM placedWHERE scanner_rows >= %s AND place <= %s AND percentile >= %s"""@@ -190,7 +191,7 @@ def build_highlight_feed(team_id: int, anchor_version: int, *, wait: bool = Truef"""INSERT INTO replay_vision_replayhighlightfeedentry (team_id, generation, window_days, place, observation_id, scanner_id, created_at, category,- experiment_id, percentile, group_root+ experiment_id, percentile, group_root, person_key)SELECT %s, %s, ranked.* FROM ({sql}) ranked""",@@ -226,9 +227,11 @@ def stored_generation(team_id: int, anchor_version: int, as_of: datetime) -> intreturn max(current_by_then) if current_by_then else min(build[0] for build in state.builds)-def rebuild_changed_feeds(anchor_version: int) -> int:+def rebuild_changed_feeds(anchor_version: int, *, on_team: Callable[[], bool] | None = None) -> int:"""Rebuild teams whose ranks changed since their last build, least recently tried first, skipping any team another- process is building. A team whose build fails waits behind the others. Returns how many it built."""+ process is building. A team whose build fails waits behind the others. `on_team` runs before each team's build, and+ a false return stops the sweep.+ Returns how many it built."""now = timezone.now()states = ReplayHighlightFeedState.objects.unscoped()due = list(@@ -242,6 +245,8 @@ def rebuild_changed_feeds(anchor_version: int) -> int:states.filter(team_id__in=due).update(attempted_at=now)built = 0for team_id in due:+ if on_team is not None and not on_team():+ breaktry:built += build_highlight_feed(team_id, anchor_version, wait=False) is not Noneexcept Exception:diff --git a/products/replay_vision/backend/highlight_feed.py b/products/replay_vision/backend/highlight_feed.pyindex b2b84f2d38b8..8f0a252860c3 100644--- a/products/replay_vision/backend/highlight_feed.py+++ b/products/replay_vision/backend/highlight_feed.py@@ -15,8 +15,8 @@ HIGHLIGHT_FEED_MIN_ITEMS = 3class HighlightFeedEntry:observation_id: UUIDreason: dict[str, Any]- # None opts the entry out of session spacing.- session_id: str | None = None+ # None opts the entry out of person spacing.+ person_key: int | None = None# One incident can put many near-identical sessions on one scanner, so each scanner holds at most this share of the@@ -45,24 +45,24 @@ def spread_scanners(ordered: list[tuple[Any, HighlightFeedEntry]]) -> list[Highlreturn placed + deferred-# Several scanners can pick one recording. Its cards stay this many places apart, so the feed never-# shows the same session twice in a row.-SESSION_SPACING = 5+# Several scanners can pick one recording, and one person can have many. A person's cards stay this many places+# apart, so the feed never shows the same person twice in a row.+PERSON_SPACING = 5-def space_sessions(entries: list[HighlightFeedEntry]) -> list[HighlightFeedEntry]:- """Keep each session's cards at least SESSION_SPACING places apart, reordering but never dropping one.+def space_people(entries: list[HighlightFeedEntry]) -> list[HighlightFeedEntry]:+ """Keep each person's cards at least PERSON_SPACING places apart, reordering but never dropping one.- A held card returns at the first place its session allows, so score order holds otherwise. A feed- with too few other sessions to space every repeat still ends with every card, in score order.+ A held card returns at the first place its person allows, so score order holds otherwise. A feed+ with too few other people to space every repeat still ends with every card, in score order."""placed: list[HighlightFeedEntry] = []waiting: list[HighlightFeedEntry] = []for entry in entries:waiting.append(entry)while waiting:- recent = {placed_entry.session_id for placed_entry in placed[-(SESSION_SPACING - 1) :]} - {None}- ready = next((held for held in waiting if held.session_id not in recent), None)+ recent = {placed_entry.person_key for placed_entry in placed[-(PERSON_SPACING - 1) :]} - {None}+ ready = next((held for held in waiting if held.person_key not in recent), None)if ready is None:breakwaiting.remove(ready)@@ -82,5 +82,5 @@ def finish_feed(entry = HighlightFeedEntry(observation_id=row["id"], reason={"kind": "recent" if viewed else "unviewed_recent"})filler.append((not viewed, row["created_at"], entry))filler.sort(key=lambda item: (item[0], item[1]), reverse=True)- placed = space_sessions(spread_scanners([(scanner_id, entry) for _, _, scanner_id, entry in ordered]))+ placed = space_people(spread_scanners([(scanner_id, entry) for _, _, scanner_id, entry in ordered]))return placed + [entry for *_, entry in filler][: max(0, HIGHLIGHT_FEED_MIN_ITEMS - len(placed))]diff --git a/products/replay_vision/backend/migrations/0145_replay_highlight_entry_person_key.py b/products/replay_vision/backend/migrations/0145_replay_highlight_entry_person_key.pynew file mode 100644index 000000000000..365fe8ecbace--- /dev/null+++ b/products/replay_vision/backend/migrations/0145_replay_highlight_entry_person_key.py@@ -0,0 +1,17 @@+# Generated by Django 6.1.2 on 2026-10-30 12:09++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("replay_vision", "0144_drop_watch_rank_indexes"),+ ]++ operations = [+ migrations.AddField(+ model_name="replayhighlightfeedentry",+ name="person_key",+ field=models.BigIntegerField(null=True),+ ),+ ]diff --git a/products/replay_vision/backend/migrations/max_migration.txt b/products/replay_vision/backend/migrations/max_migration.txtindex b671e392cd51..7fe38d2410cf 100644--- a/products/replay_vision/backend/migrations/max_migration.txt+++ b/products/replay_vision/backend/migrations/max_migration.txt@@ -1 +1 @@-0144_drop_watch_rank_indexes+0145_replay_highlight_entry_person_keydiff --git a/products/replay_vision/backend/models/replay_highlight_feed.py b/products/replay_vision/backend/models/replay_highlight_feed.pyindex 72bcafc9b240..33b94b0d4d92 100644--- a/products/replay_vision/backend/models/replay_highlight_feed.py+++ b/products/replay_vision/backend/models/replay_highlight_feed.py@@ -51,6 +51,8 @@ class ReplayHighlightFeedEntry(TeamScopedRootMixin, models.Model):experiment_id = models.IntegerField(default=0)percentile = models.FloatField()group_root = models.UUIDField()+ # Spaces one person's cards apart without reading the observations.+ person_key = models.BigIntegerField(null=True)class Meta:app_label = "replay_vision"diff --git a/products/replay_vision/backend/temporal/__init__.py b/products/replay_vision/backend/temporal/__init__.pyindex 44eaefeac430..9c16072fb7f2 100644--- a/products/replay_vision/backend/temporal/__init__.py+++ b/products/replay_vision/backend/temporal/__init__.py@@ -51,6 +51,7 @@ from products.replay_vision.backend.temporal.activities.benchmark import (snapshot_benchmark_labels_activity,write_benchmark_manifest_activity,)+from products.replay_vision.backend.temporal.activities.rebuild_highlight_feeds import rebuild_highlight_feeds_activityfrom products.replay_vision.backend.temporal.activities.refresh_learned_rules import (list_due_learned_rules_teams_activity,refresh_team_learned_rules_activity,@@ -62,6 +63,7 @@ from products.replay_vision.backend.temporal.gemini_cleanup_sweep import (ReplayVisionGeminiCleanupSweepWorkflow,sweep_gemini_files_activity,)+from products.replay_vision.backend.temporal.highlight_rebuild import RebuildHighlightFeedsWorkflowfrom products.replay_vision.backend.temporal.learned_rules import RefreshLearnedRulesWorkflowfrom products.replay_vision.backend.temporal.media_workflow import ObservationMediaWorkflowfrom products.replay_vision.backend.temporal.read_meter import MeterScannerReadsWorkflow@@ -85,6 +87,7 @@ WORKFLOWS = [ReconcileScannerSchedulesWorkflow,RefreshScannerEstimatesWorkflow,RefreshLearnedRulesWorkflow,+ RebuildHighlightFeedsWorkflow,ReplayVisionGeminiCleanupSweepWorkflow,SweepScannerWorkflow,VisionAlertCheckWorkflow,@@ -138,6 +141,7 @@ ACTIVITIES: list[Callable[..., Any]] = [sweep_gemini_files_activity,list_due_learned_rules_teams_activity,refresh_team_learned_rules_activity,+ rebuild_highlight_feeds_activity,snapshot_benchmark_labels_activity,load_benchmark_cases_activity,prepare_benchmark_case_activity,diff --git a/products/replay_vision/backend/temporal/activities/rebuild_highlight_feeds.py b/products/replay_vision/backend/temporal/activities/rebuild_highlight_feeds.pynew file mode 100644index 000000000000..1db7ee5a49e4--- /dev/null+++ b/products/replay_vision/backend/temporal/activities/rebuild_highlight_feeds.py@@ -0,0 +1,30 @@+import time++from temporalio import activity++from products.replay_vision.backend.highlight_builds import rebuild_changed_feeds+from products.replay_vision.backend.temporal.constants import HIGHLIGHT_REBUILD_CLAIM_BUDGET+from products.replay_vision.backend.temporal.decorators import track_activity+from products.replay_vision.backend.temporal.highlight_rebuild_types import RebuildHighlightFeedsResult+from products.replay_vision.backend.watch_rank import ANCHOR_VERSION+++@track_activity()+def rebuild_highlight_feeds_activity() -> RebuildHighlightFeedsResult:+ """Rebuild every team whose ranks changed since its last build, a batch at a time, until none is due or the claim+ budget runs out."""+ # Not `bounded_queries`: it would lift each build's own 120s statement cap to the whole budget.+ deadline = time.monotonic() + HIGHLIGHT_REBUILD_CLAIM_BUDGET.total_seconds()++ def before_team() -> bool:+ activity.heartbeat()+ return time.monotonic() < deadline++ built = 0+ while time.monotonic() < deadline:+ batch = rebuild_changed_feeds(ANCHOR_VERSION, on_team=before_team)+ if not batch:+ break+ built += batch+ return RebuildHighlightFeedsResult(built=built)diff --git a/products/replay_vision/backend/temporal/constants.py b/products/replay_vision/backend/temporal/constants.pyindex 3f627a146764..0ae46b25e5bc 100644--- a/products/replay_vision/backend/temporal/constants.py+++ b/products/replay_vision/backend/temporal/constants.py@@ -265,3 +265,16 @@ LEARNED_RULES_CONCURRENCY = 16LIST_DUE_LEARNED_RULES_TIMEOUT = dt.timedelta(seconds=60)# Covers the model call's own 120s timeout plus the database reads around it.REFRESH_TEAM_LEARNED_RULES_TIMEOUT = dt.timedelta(minutes=4)+++# Highlight feed rebuilds: a schedule starts this every minute and skips a start while one still runs.+HIGHLIGHT_REBUILD_WORKFLOW_NAME = "replay-vision-rebuild-highlight-feeds"+HIGHLIGHT_REBUILD_WORKFLOW_ID = "replay-vision-highlight-feed-rebuilder"+HIGHLIGHT_REBUILD_SCHEDULE_ID = "replay-vision-highlight-feed-rebuilder-schedule"+HIGHLIGHT_REBUILD_INTERVAL = dt.timedelta(minutes=1)+HIGHLIGHT_REBUILD_EXECUTION_TIMEOUT = dt.timedelta(minutes=10)+HIGHLIGHT_REBUILD_ACTIVITY_TIMEOUT = dt.timedelta(minutes=8)+# One team's build holds one heavy statement under its own 120s cap, so a heartbeat per team arrives well inside this.+HIGHLIGHT_REBUILD_HEARTBEAT_TIMEOUT = dt.timedelta(minutes=5)+# The activity starts no team after this, which leaves a slow last build room inside the activity timeout.+HIGHLIGHT_REBUILD_CLAIM_BUDGET = dt.timedelta(minutes=3)diff --git a/products/replay_vision/backend/temporal/highlight_rebuild.py b/products/replay_vision/backend/temporal/highlight_rebuild.pynew file mode 100644index 000000000000..aa144578a901--- /dev/null+++ b/products/replay_vision/backend/temporal/highlight_rebuild.py@@ -0,0 +1,65 @@+"""Rebuilds the stored highlight feeds of teams whose ranks changed."""++import datetime as dt+from typing import TYPE_CHECKING++from temporalio import workflow+from temporalio.common import RetryPolicy++from posthog.temporal.common.base import PostHogWorkflow++from products.replay_vision.backend.temporal.constants import (+ HIGHLIGHT_REBUILD_ACTIVITY_TIMEOUT,+ HIGHLIGHT_REBUILD_EXECUTION_TIMEOUT,+ HIGHLIGHT_REBUILD_HEARTBEAT_TIMEOUT,+ HIGHLIGHT_REBUILD_INTERVAL,+ HIGHLIGHT_REBUILD_SCHEDULE_ID,+ HIGHLIGHT_REBUILD_WORKFLOW_ID,+ HIGHLIGHT_REBUILD_WORKFLOW_NAME,+)+from products.replay_vision.backend.temporal.highlight_rebuild_types import (+ RebuildHighlightFeedsInputs,+ RebuildHighlightFeedsResult,+)++if TYPE_CHECKING:+ from temporalio.client import Client++# `activities` pulls in Django, which the workflow sandbox can't safely re-import.+with workflow.unsafe.imports_passed_through():+ from products.replay_vision.backend.temporal.activities.rebuild_highlight_feeds import (+ rebuild_highlight_feeds_activity,+ )++[email protected](name=HIGHLIGHT_REBUILD_WORKFLOW_NAME)+class RebuildHighlightFeedsWorkflow(PostHogWorkflow):+ inputs_cls = RebuildHighlightFeedsInputs+ inputs_optional = True++ @workflow.run+ async def run(self, inputs: RebuildHighlightFeedsInputs) -> RebuildHighlightFeedsResult:+ # No retry: the schedule starts the next run a minute later, and it picks up whatever this one left.+ return await workflow.execute_activity(+ rebuild_highlight_feeds_activity,+ start_to_close_timeout=HIGHLIGHT_REBUILD_ACTIVITY_TIMEOUT,+ heartbeat_timeout=HIGHLIGHT_REBUILD_HEARTBEAT_TIMEOUT,+ retry_policy=RetryPolicy(maximum_attempts=1),+ )+++async def create_replay_vision_highlight_rebuild_schedule(client: "Client") -> None:+ """Upsert the per-minute highlight feed rebuild schedule. Called on worker startup."""+ # Function-local: with `@workflow.defn` above, the Temporal sandbox can't re-import the helper's dependencies.+ from products.replay_vision.backend.temporal.schedule import upsert_interval_schedule # noqa: PLC0415++ await upsert_interval_schedule(+ client,+ schedule_id=HIGHLIGHT_REBUILD_SCHEDULE_ID,+ workflow_name=HIGHLIGHT_REBUILD_WORKFLOW_NAME,+ workflow_id=HIGHLIGHT_REBUILD_WORKFLOW_ID,+ inputs=RebuildHighlightFeedsInputs(),+ interval=HIGHLIGHT_REBUILD_INTERVAL,+ offset=dt.timedelta(0),+ execution_timeout=HIGHLIGHT_REBUILD_EXECUTION_TIMEOUT,+ )diff --git a/products/replay_vision/backend/temporal/highlight_rebuild_types.py b/products/replay_vision/backend/temporal/highlight_rebuild_types.pynew file mode 100644index 000000000000..8e7c227cee6d--- /dev/null+++ b/products/replay_vision/backend/temporal/highlight_rebuild_types.py@@ -0,0 +1,11 @@+"""Highlight feed rebuild types, kept out of `types.py` so the workflow sandbox loads only these."""++from pydantic import BaseModel+++class RebuildHighlightFeedsInputs(BaseModel, frozen=True):+ pass+++class RebuildHighlightFeedsResult(BaseModel, frozen=True):+ built: int = 0diff --git a/products/replay_vision/backend/tests/test_api.py b/products/replay_vision/backend/tests/test_api.pyindex 738e797f3b1c..b68f0a93155a 100644--- a/products/replay_vision/backend/tests/test_api.py+++ b/products/replay_vision/backend/tests/test_api.py@@ -4957,6 +4957,19 @@ class TestHighlightFeedAPI(_VisionAPITestCase):sessions = [item["observation"]["session_id"] for item in response["results"]]assert sessions and not any(session.startswith("leaving") for session in sessions)+ def test_a_deleted_observation_leaves_the_stored_picks_before_the_feed_rebuilds(self) -> None:+ scanner = self._create_scanner(name="m")+ for index in range(10):+ self._ranked_observation(scanner, f"s-{index}", 10 + index, "failure", 0.9 - index / 100)+ built = self.client.get(self.feed_url, {"date_from": "-1d", "limit": "20"}).json()["results"]+ deleted = built[0]["observation"]+ ReplayObservation.objects.filter(id=deleted["id"]).delete()++ # The first slot is where the deleted pick ranked, so a stored entry left in place empties the page.+ response = self.client.get(self.feed_url, {"date_from": "-1d", "limit": "1"}).json()["results"]++ assert [item["observation"]["id"] for item in response] == [built[1]["observation"]["id"]]+def test_an_embedding_pick_counts_its_hidden_duplicates(self) -> None:scanner = self._create_scanner(name="m")oldest = self._ranked_observation(scanner, "oldest-failure", 500, "failure", 0.9)diff --git a/products/replay_vision/backend/tests/test_highlight_builds.py b/products/replay_vision/backend/tests/test_highlight_builds.pyindex cbcd87396949..f6cd2e6ab479 100644--- a/products/replay_vision/backend/tests/test_highlight_builds.py+++ b/products/replay_vision/backend/tests/test_highlight_builds.py@@ -4,10 +4,12 @@ from typing import Anyimport time_machinefrom posthog.test.base import BaseTest+from unittest.mock import patchfrom django.utils import timezonefrom parameterized import parameterized+from temporalio.testing import ActivityEnvironmentfrom posthog.models import Team@@ -31,6 +33,7 @@ from products.replay_vision.backend.models.replay_observation import (from products.replay_vision.backend.models.replay_observation_rank import RankCategory, ReplayObservationRankfrom products.replay_vision.backend.models.replay_scanner import ReplayScanner, ScannerModel, ScannerTypefrom products.replay_vision.backend.observation_ranks import NewRank, record_ranks+from products.replay_vision.backend.temporal.activities.rebuild_highlight_feeds import rebuild_highlight_feeds_activityfrom products.replay_vision.backend.tests.helpers import snapshot_forfrom products.replay_vision.backend.watch_rank import ANCHOR_VERSION@@ -90,6 +93,11 @@ class TestHighlightBuilds(BaseTest):(entry.window_days, str(entry.observation_id)): (entry.place, round(entry.percentile, 3))for entry in ReplayHighlightFeedEntry.objects.for_team(self.team.id)}+ people = dict(ReplayObservationRank.objects.for_team(self.team.id).values_list("observation_id", "person_key"))+ assert all(+ entry.person_key is not None and entry.person_key == people[entry.observation_id]+ for entry in ReplayHighlightFeedEntry.objects.for_team(self.team.id)+ )# The day window has ten rows, so its best two clear the cut; the week window places all twenty.assert {key: value for key, value in entries.items() if key[0] == 1} == {(1, recent[0]): (1, 0.95),@@ -231,6 +239,31 @@ class TestHighlightBuilds(BaseTest):assert (first, served, built) == (1, 1, [1, 0])assert ReplayHighlightFeedState.objects.for_team(self.team.id).get().generation == 2+ def test_the_rebuild_activity_works_through_every_due_batch(self) -> None:+ other = Team.objects.create(organization=self.organization, name="other")+ other_scanner = ReplayScanner.objects.create(+ team=other,+ name="other",+ scanner_type=ScannerType.MONITOR,+ scanner_config={"prompt": "p"},+ model=ScannerModel.GEMINI_3_8_FLASH,+ )+ self._ranked(10, age=timedelta(hours=1))+ for index in range(10):+ observation = self._observation(_NOW - timedelta(hours=1, minutes=index), scanner=other_scanner, team=other)+ record_ranks(+ other.id, ANCHOR_VERSION, {str(observation.id): NewRank(contrast=index / 10, category="failure")}+ )++ with patch("products.replay_vision.backend.highlight_builds.REBUILDS_PER_SWEEP", 1):+ result = ActivityEnvironment().run(rebuild_highlight_feeds_activity)++ assert result.built == 2+ assert set(ReplayHighlightFeedState.objects.unscoped().values_list("team_id", "generation")) == {+ (self.team.id, 1),+ (other.id, 1),+ }+def test_a_load_that_began_before_a_rebuild_stays_on_the_build_it_started_on(self) -> None:self._ranked(10, age=timedelta(hours=1))diff --git a/products/replay_vision/backend/tests/test_watch_rank.py b/products/replay_vision/backend/tests/test_watch_rank.pyindex d3a8f9b5dc67..74b771a06319 100644--- a/products/replay_vision/backend/tests/test_watch_rank.py+++ b/products/replay_vision/backend/tests/test_watch_rank.py@@ -1,3 +1,4 @@+import zlibfrom datetime import UTC, datetime, timedeltafrom typing import Any@@ -11,7 +12,7 @@ from parameterized import parameterizedfrom posthog.exceptions import ClickHouseAtCapacityfrom products.replay_vision.backend import watch_rank_backfill-from products.replay_vision.backend.highlight_feed import SESSION_SPACING+from products.replay_vision.backend.highlight_feed import PERSON_SPACINGfrom products.replay_vision.backend.tests.helpers import axis_anchorsfrom products.replay_vision.backend.watch_rank import (ANCHOR_VERSION,@@ -35,13 +36,13 @@ def _row(anchor_version: int = ANCHOR_VERSION,minutes_ago: int = 0,viewed: bool = False,- session: str | None = None,+ person: int | None = None,) -> dict[str, Any]:current = contrast is not None and anchor_version == ANCHOR_VERSIONreturn {"id": observation_id,"scanner_id": scanner,- "session_id": session or f"session-{observation_id}",+ "person_key": person if person is not None else zlib.crc32(observation_id.encode()),"created_at": _NOW - timedelta(minutes=minutes_ago),"feed_viewed": viewed,"contrast": contrast if current else None,@@ -139,21 +140,21 @@ class TestRankHighlightFeedByEmbedding(SimpleTestCase):assert picks["seen-top"]["similar_count"] == 1assert "mid-copy" not in {entry.observation_id for entry in ranked}- def test_two_scanners_picking_one_session_land_apart_without_dropping_a_pick(self) -> None:+ def test_two_scanners_picking_one_person_land_apart_without_dropping_a_pick(self) -> None:rows = [_row(f"{scanner}-{index}", index / 20, scanner=scanner, minutes_ago=index)for scanner in ("scanner-a", "scanner-b", "scanner-c")for index in range(9)] + [- _row("a-top", 0.9, scanner="scanner-a", session="shared"),- _row("b-top", 0.9, scanner="scanner-b", session="shared", minutes_ago=1),+ _row("a-top", 0.9, scanner="scanner-a", person=7),+ _row("b-top", 0.9, scanner="scanner-b", person=7, minutes_ago=1),_row("c-top", 0.9, scanner="scanner-c", minutes_ago=2),]ranked = [str(entry.observation_id) for entry in rank_highlight_feed_by_embedding(_with_percentiles(rows))]assert ranked[0] == "a-top"- assert ranked.index("b-top") - ranked.index("a-top") >= SESSION_SPACING+ assert ranked.index("b-top") - ranked.index("a-top") >= PERSON_SPACINGassert {"a-top", "b-top", "c-top", "scanner-a-8", "scanner-b-8", "scanner-c-8"} <= set(ranked)def test_rows_without_a_rank_fall_back_to_the_filler_floor(self) -> None:diff --git a/products/replay_vision/backend/watch_rank.py b/products/replay_vision/backend/watch_rank.pyindex 697baa3ea183..90fe5b70fca8 100644--- a/products/replay_vision/backend/watch_rank.py+++ b/products/replay_vision/backend/watch_rank.py@@ -12,7 +12,7 @@ import asynciofrom collections import Counterfrom collections.abc import Collection, Sequencefrom datetime import datetime-from typing import Any, Literal, cast+from typing import Any, Literalfrom django.core.cache import cachefrom django.db import transaction@@ -35,6 +35,7 @@ from products.replay_vision.backend.models.replay_highlight_feed import ReplayHifrom products.replay_vision.backend.models.replay_observation import ReplayObservationfrom products.replay_vision.backend.models.replay_observation_anchor_similarity import ReplayObservationAnchorSimilarityfrom products.replay_vision.backend.models.replay_observation_rank import ReplayObservationRank+from products.replay_vision.backend.models.replay_observation_view import ReplayObservationViewfrom products.replay_vision.backend.observation_ranks import CATEGORY_CODES, CATEGORY_NAMES, NewRank, record_ranksfrom products.replay_vision.backend.scanner_access import accessible_observations, readable_experiment_ids@@ -326,6 +327,7 @@ _ENTRY_FIELDS = ("category","percentile","group_root",+ "person_key",)@@ -383,10 +385,15 @@ def _group_reach(return {str(row["root"]): GroupReach(sessions=row["sessions"], people=row["people"]) for row in members}+def views_by(viewer_id: int, as_of: datetime) -> QuerySet[ReplayObservationView]:+ """The viewer's observation views up to `as_of`, so later views wait for the next feed load."""+ return ReplayObservationView.objects.filter(user_id=viewer_id, created_at__lte=as_of)++def select_embedding_picks(window: FeedWindow,scanner_ids: Sequence[Any],- viewed: Exists,+ viewer_id: int,access: UserAccessControl,team_id: int,cap: int,@@ -428,31 +435,39 @@ def select_embedding_picks(window_values = _cap_feed_groups(rows, cap)if len(window_values) < cap and len(rows) == fetch_limit:window_values = _cap_feed_groups(window_rows(fetch_limit * HIGHLIGHT_PICK_REFILL_FACTOR), cap)+ # The entries carry everything ranking reads, so only ids load here, and the endpoint gates its page's rows on+ # experiment access. A stored entry outlives a deleted observation until its feed rebuilds.+ live = set(+ ReplayObservation.objects.filter(team_id=team_id, id__in=list(window_values)).values_list("id", flat=True)+ )+ window_values = {observation_id: entry for observation_id, entry in window_values.items() if observation_id in live}if not window_values:return []- # The entries already carry each row's experiment scope, so this skips the snapshot JSON the row-gate reads; the- # endpoint still gates the rows of the page it serves.- rows = cast(- list[dict[str, Any]],- list(- ReplayObservation.objects.filter(team_id=team_id, id__in=list(window_values))- .annotate(feed_viewed=viewed)- .values("id", "session_id", "feed_viewed")- ),+ viewed = set(+ views_by(viewer_id, window.as_of)+ .filter(observation_id__in=list(window_values))+ .values_list("observation_id", flat=True))roots = {entry["group_root"] for entry in window_values.values()}reach = _group_reach(team_id, window, roots, scanner_ids, categories, experiment_ids)- for row in rows:- entry = window_values[row["id"]]- row["scanner_id"] = entry["scanner_id"]- row["created_at"] = entry["created_at"]+ picks: list[dict[str, Any]] = []+ for observation_id, entry in window_values.items():group = reach.get(str(entry["group_root"]), GroupReach(sessions=1, people=1))- row["watch_category"] = CATEGORY_NAMES[entry["category"]]- row["watch_percentile"] = entry["percentile"]- row["watch_group_root"] = str(entry["group_root"])- row["watch_group_size"] = group.sessions- row["watch_group_people"] = group.people- return rows+ picks.append(+ {+ "id": observation_id,+ "scanner_id": entry["scanner_id"],+ "created_at": entry["created_at"],+ "person_key": entry["person_key"],+ "feed_viewed": observation_id in viewed,+ "watch_category": CATEGORY_NAMES[entry["category"]],+ "watch_percentile": entry["percentile"],+ "watch_group_root": str(entry["group_root"]),+ "watch_group_size": group.sessions,+ "watch_group_people": group.people,+ }+ )+ return picksdef select_embedding_filler(@@ -481,7 +496,7 @@ def select_embedding_filler().annotate(feed_viewed=viewed))- fields = ("id", "scanner_id", "session_id", "created_at", "feed_viewed")+ fields = ("id", "scanner_id", "created_at", "feed_viewed")# Two short newest-first walks, instead of ordering by the viewed flag, which would sort the whole window.rows = list(base.filter(feed_viewed=False).order_by("-created_at", "-id").values(*fields)[:HIGHLIGHT_FEED_MIN_ITEMS]@@ -496,12 +511,12 @@ def select_embedding_filler(def rank_highlight_feed_by_embedding(rows: list[dict[str, Any]]) -> list[HighlightFeedEntry]:- """Rank candidate rows (`id`, `scanner_id`, `session_id`, `created_at`, `feed_viewed`, optionally- `feed_impressions`, and for picks the `watch_*` fields `select_embedding_picks` adds).+ """Rank candidate rows (`id`, `scanner_id`, `created_at`, `feed_viewed`, optionally `feed_impressions`, and for+ picks `person_key` and the `watch_*` fields `select_embedding_picks` adds).A pick is a row with a percentile whose category is an issue. Rows of one feed group collapse to their best-scoringrow, which carries the size of the group. Picks sort by weighted percentile, decayed for each earlier impression,- viewed rows docked, one scanner held to a share of the picks, and one session's picks spaced apart. Every other row+ viewed rows docked, one scanner held to a share of the picks, and one person's picks spaced apart. Every other rowis filler, kept only to reach the feed's floor."""best_in_group: dict[str, tuple[float, dict[str, Any]]] = {}@@ -541,7 +556,7 @@ def rank_highlight_feed_by_embedding(rows: list[dict[str, Any]]) -> list[Highligscore,row["created_at"],row.get("scanner_id"),- HighlightFeedEntry(observation_id=row["id"], reason=reason, session_id=row["session_id"]),+ HighlightFeedEntry(observation_id=row["id"], reason=reason, person_key=row.get("person_key")),))# A pick stands for its whole feed group, so no member returns as filler.diff --git a/products/replay_vision/backend/watch_rank_consumer.py b/products/replay_vision/backend/watch_rank_consumer.pyindex c3ecc3e8c201..d8edf5c780b7 100644--- a/products/replay_vision/backend/watch_rank_consumer.py+++ b/products/replay_vision/backend/watch_rank_consumer.py@@ -34,9 +34,8 @@ from products.replay_vision.backend.embeddings import (EMBEDDING_PRODUCT,OBSERVATION_EMBEDDING_MODEL,)-from products.replay_vision.backend.highlight_builds import rebuild_changed_feedsfrom products.replay_vision.backend.highlight_feed_groups import feed_group_cache-from products.replay_vision.backend.watch_rank import ANCHOR_VERSION, RankRequest, rank_observation+from products.replay_vision.backend.watch_rank import RankRequest, rank_observationfrom products.replay_vision.backend.watch_semantics import SemanticAssignment, assign_semanticsfrom products.replay_vision.backend.watch_themes import name_theme@@ -56,8 +55,6 @@ _ERROR_RETRY_WINDOW_S = 60.0# Long enough for a naming call that waits behind ranking on the shared database thread, short enough that a hung# model call frees its slot._NAMING_TIMEOUT_S = 120.0-# How often the consumer rebuilds the highlight feeds of teams whose ranks changed.-_REBUILD_INTERVAL_S = 60.0_NAMING_DRAIN_S = 20.0# A due theme left unnamed while this many are in flight is named when its next member arrives._MAX_NAMING_TASKS = 8@@ -76,11 +73,6 @@ BATCH_DURATION = Histogram("Time the highlight feed rank consumer spent on one non-empty batch.",buckets=(0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 240, 300),)-FEED_REBUILDS_TOTAL = Counter(- "replay_vision_watch_rank_consumer_feed_rebuilds_total",- "Highlight feed rebuild sweeps of the rank consumer, by outcome.",- labelnames=["outcome"],-)THEME_NAMING_TOTAL = Counter("replay_vision_watch_rank_consumer_theme_naming_total","Theme naming outcomes of the highlight feed rank consumer, including names dropped while every slot was busy.",@@ -150,15 +142,6 @@ def _assign_with_timeouts(raise-def _rebuild_changed_feeds() -> int:- # The sweep's thread keeps its own connection, so a broken one is replaced before each sweep.- close_old_connections()- try:- return rebuild_changed_feeds(ANCHOR_VERSION)- finally:- close_old_connections()--class BatchConsumer(Protocol):def store_offsets(self, *, message: Message) -> None: ...@@ -319,17 +302,6 @@ class WatchRankConsumer:config.update(helper.ssl_cert_config())return config- async def _rebuild_feeds(self) -> None:- """Rebuild changed teams' highlight feeds beside the message loop, on a thread of its own."""- while await self._wait(_REBUILD_INTERVAL_S):- try:- built = await sync_to_async(_rebuild_changed_feeds, thread_sensitive=False)()- except Exception:- FEED_REBUILDS_TOTAL.labels(outcome="failed").inc()- logger.exception("replay_vision.watch_rank_consumer.feed_rebuild_failed")- else:- FEED_REBUILDS_TOTAL.labels(outcome="built" if built else "idle").inc()-async def run(self) -> None: # pragma: no cover - exercised in dogfoodloop = asyncio.get_running_loop()for sig in (signal.SIGINT, signal.SIGTERM):@@ -338,7 +310,6 @@ class WatchRankConsumer:consumer = Consumer(self._consumer_config())consumer.subscribe([KAFKA_DOCUMENT_EMBEDDING_RESULTS_TOPIC])logger.info("replay_vision.watch_rank_consumer.started")- rebuilds = asyncio.create_task(self._rebuild_feeds())try:while not self._stop.is_set():self._liveness.record_heartbeat()@@ -348,7 +319,6 @@ class WatchRankConsumer:with BATCH_DURATION.time():await self.process_batch(consumer, messages)finally:- rebuilds.cancel()consumer.close()# A theme claimed for naming stays unnamed until it doubles, so in-flight names get a moment to finish.if self._naming: