Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(replay-vision): rank highlight picks from stored entries and rebuild on a schedule (#126402)

Tue Haulund1e238660eeb2
diff --git a/posthog/temporal/schedule.py b/posthog/temporal/schedule.py
index 7cdee4807f85..929d7d4c4e6c 100644
--- a/posthog/temporal/schedule.py
+++ b/posthog/temporal/schedule.py
@@ -132,6 +132,7 @@ from products.replay_vision.backend.temporal.estimates import create_replay_visi
from products.replay_vision.backend.temporal.gemini_cleanup_sweep import (
create_replay_vision_gemini_cleanup_sweep_schedule,
)
+from products.replay_vision.backend.temporal.highlight_rebuild import create_replay_vision_highlight_rebuild_schedule
from products.replay_vision.backend.temporal.learned_rules import create_replay_vision_learned_rules_schedule
from products.replay_vision.backend.temporal.read_meter import create_replay_vision_read_meter_schedule
from products.replay_vision.backend.temporal.reconciler import create_replay_vision_reconciler_schedule
@@ -1033,6 +1034,7 @@ schedules = [
create_replay_vision_learned_rules_schedule,
create_vision_alert_check_schedule,
create_replay_vision_read_meter_schedule,
+ create_replay_vision_highlight_rebuild_schedule,
create_github_job_logs_coordinator_schedule,
create_review_hog_finding_outcomes_schedule,
create_ci_signals_coordinator_schedule,
diff --git a/products/replay_vision/backend/api/scanners.py b/products/replay_vision/backend/api/scanners.py
index eb6a5528de03..370ff862fb37 100644
--- a/products/replay_vision/backend/api/scanners.py
+++ b/products/replay_vision/backend/api/scanners.py
@@ -100,7 +100,6 @@ from products.replay_vision.backend.models.replay_observation import (
hydrate_for_serialization,
)
from products.replay_vision.backend.models.replay_observation_feed_dismissal import ReplayObservationFeedDismissal
-from products.replay_vision.backend.models.replay_observation_view import ReplayObservationView
from products.replay_vision.backend.models.replay_scanner import (
ReplayScanner,
SamplingMode,
@@ -185,6 +184,7 @@ from products.replay_vision.backend.watch_rank import (
rank_highlight_feed_by_embedding,
select_embedding_filler,
select_embedding_picks,
+ views_by,
)
from products.signals.backend.facade.api import get_outcomes_for_signal_source_slice
@@ -2488,11 +2488,7 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi
"observation_id", flat=True
)
)
- viewed = Exists(
- ReplayObservationView.objects.filter(
- observation_id=OuterRef("id"), user_id=viewer_id, created_at__lte=as_of
- )
- )
+ viewed = Exists(views_by(viewer_id, as_of).filter(observation_id=OuterRef("id")))
categories = params.get("categories") or ISSUE_CATEGORIES
candidate_rows = select_embedding_picks(
feed_window(
@@ -2503,7 +2499,7 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi
None if date_to is not None else HIGHLIGHT_FEED_PRESETS.get(params["date_from"]),
),
allowed_ids,
- viewed,
+ viewer_id,
self.user_access_control,
self.team_id,
HIGHLIGHT_FEED_CANDIDATE_CAP,
diff --git a/products/replay_vision/backend/highlight_builds.py b/products/replay_vision/backend/highlight_builds.py
index 67bad1c3961b..e4962970bed5 100644
--- a/products/replay_vision/backend/highlight_builds.py
+++ b/products/replay_vision/backend/highlight_builds.py
@@ -6,7 +6,7 @@ preset windows from a single read of the longest one. A build keeps only rows th
bounded number of places per scanner, which is all a page load can reach.
"""
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
from dataclasses import fields
from datetime import datetime, timedelta
from typing import Any
@@ -61,6 +61,7 @@ class RankedEntry:
experiment_id: int
percentile: float
group_root: Any
+ person_key: int | None
@frozen
@@ -92,7 +93,7 @@ def _ranked_sql(
WITH bounds(window_days, window_start, window_end) AS (VALUES {window_values}),
ranked_rows AS MATERIALIZED (
SELECT observation_id, scanner_id, created_at, contrast, category, experiment_id,
- COALESCE(feed_group_id, observation_id) AS group_root
+ COALESCE(feed_group_id, observation_id) AS group_root, person_key
FROM replay_vision_replayobservationrank
WHERE team_id = %s AND anchor_version = %s
AND created_at >= (SELECT min(window_start) FROM bounds)
@@ -114,7 +115,7 @@ def _ranked_sql(
) windowed
)
SELECT window_days, place, observation_id, scanner_id, created_at, category, experiment_id, percentile,
- group_root
+ group_root, person_key
FROM placed
WHERE scanner_rows >= %s AND place <= %s AND percentile >= %s
"""
@@ -190,7 +191,7 @@ def build_highlight_feed(team_id: int, anchor_version: int, *, wait: bool = True
f"""
INSERT INTO replay_vision_replayhighlightfeedentry (
team_id, generation, window_days, place, observation_id, scanner_id, created_at, category,
- experiment_id, percentile, group_root
+ experiment_id, percentile, group_root, person_key
)
SELECT %s, %s, ranked.* FROM ({sql}) ranked
""",
@@ -226,9 +227,11 @@ def stored_generation(team_id: int, anchor_version: int, as_of: datetime) -> int
return max(current_by_then) if current_by_then else min(build[0] for build in state.builds)
-def rebuild_changed_feeds(anchor_version: int) -> int:
+def rebuild_changed_feeds(anchor_version: int, *, on_team: Callable[[], bool] | None = None) -> int:
"""Rebuild teams whose ranks changed since their last build, least recently tried first, skipping any team another
- process is building. A team whose build fails waits behind the others. Returns how many it built."""
+ process is building. A team whose build fails waits behind the others. `on_team` runs before each team's build, and
+ a false return stops the sweep.
+ Returns how many it built."""
now = timezone.now()
states = ReplayHighlightFeedState.objects.unscoped()
due = list(
@@ -242,6 +245,8 @@ def rebuild_changed_feeds(anchor_version: int) -> int:
states.filter(team_id__in=due).update(attempted_at=now)
built = 0
for team_id in due:
+ if on_team is not None and not on_team():
+ break
try:
built += build_highlight_feed(team_id, anchor_version, wait=False) is not None
except Exception:
diff --git a/products/replay_vision/backend/highlight_feed.py b/products/replay_vision/backend/highlight_feed.py
index b2b84f2d38b8..8f0a252860c3 100644
--- a/products/replay_vision/backend/highlight_feed.py
+++ b/products/replay_vision/backend/highlight_feed.py
@@ -15,8 +15,8 @@ HIGHLIGHT_FEED_MIN_ITEMS = 3
class HighlightFeedEntry:
observation_id: UUID
reason: dict[str, Any]
- # None opts the entry out of session spacing.
- session_id: str | None = None
+ # None opts the entry out of person spacing.
+ person_key: int | None = None
# One incident can put many near-identical sessions on one scanner, so each scanner holds at most this share of the
@@ -45,24 +45,24 @@ def spread_scanners(ordered: list[tuple[Any, HighlightFeedEntry]]) -> list[Highl
return placed + deferred
-# Several scanners can pick one recording. Its cards stay this many places apart, so the feed never
-# shows the same session twice in a row.
-SESSION_SPACING = 5
+# Several scanners can pick one recording, and one person can have many. A person's cards stay this many places
+# apart, so the feed never shows the same person twice in a row.
+PERSON_SPACING = 5
-def space_sessions(entries: list[HighlightFeedEntry]) -> list[HighlightFeedEntry]:
- """Keep each session's cards at least SESSION_SPACING places apart, reordering but never dropping one.
+def space_people(entries: list[HighlightFeedEntry]) -> list[HighlightFeedEntry]:
+ """Keep each person's cards at least PERSON_SPACING places apart, reordering but never dropping one.
- A held card returns at the first place its session allows, so score order holds otherwise. A feed
- with too few other sessions to space every repeat still ends with every card, in score order.
+ A held card returns at the first place its person allows, so score order holds otherwise. A feed
+ with too few other people to space every repeat still ends with every card, in score order.
"""
placed: list[HighlightFeedEntry] = []
waiting: list[HighlightFeedEntry] = []
for entry in entries:
waiting.append(entry)
while waiting:
- recent = {placed_entry.session_id for placed_entry in placed[-(SESSION_SPACING - 1) :]} - {None}
- ready = next((held for held in waiting if held.session_id not in recent), None)
+ recent = {placed_entry.person_key for placed_entry in placed[-(PERSON_SPACING - 1) :]} - {None}
+ ready = next((held for held in waiting if held.person_key not in recent), None)
if ready is None:
break
waiting.remove(ready)
@@ -82,5 +82,5 @@ def finish_feed(
entry = HighlightFeedEntry(observation_id=row["id"], reason={"kind": "recent" if viewed else "unviewed_recent"})
filler.append((not viewed, row["created_at"], entry))
filler.sort(key=lambda item: (item[0], item[1]), reverse=True)
- placed = space_sessions(spread_scanners([(scanner_id, entry) for _, _, scanner_id, entry in ordered]))
+ placed = space_people(spread_scanners([(scanner_id, entry) for _, _, scanner_id, entry in ordered]))
return placed + [entry for *_, entry in filler][: max(0, HIGHLIGHT_FEED_MIN_ITEMS - len(placed))]
diff --git a/products/replay_vision/backend/migrations/0145_replay_highlight_entry_person_key.py b/products/replay_vision/backend/migrations/0145_replay_highlight_entry_person_key.py
new file mode 100644
index 000000000000..365fe8ecbace
--- /dev/null
+++ b/products/replay_vision/backend/migrations/0145_replay_highlight_entry_person_key.py
@@ -0,0 +1,17 @@
+# Generated by Django 6.1.2 on 2026-10-30 12:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("replay_vision", "0144_drop_watch_rank_indexes"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="replayhighlightfeedentry",
+ name="person_key",
+ field=models.BigIntegerField(null=True),
+ ),
+ ]
diff --git a/products/replay_vision/backend/migrations/max_migration.txt b/products/replay_vision/backend/migrations/max_migration.txt
index b671e392cd51..7fe38d2410cf 100644
--- a/products/replay_vision/backend/migrations/max_migration.txt
+++ b/products/replay_vision/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0144_drop_watch_rank_indexes
+0145_replay_highlight_entry_person_key
diff --git a/products/replay_vision/backend/models/replay_highlight_feed.py b/products/replay_vision/backend/models/replay_highlight_feed.py
index 72bcafc9b240..33b94b0d4d92 100644
--- a/products/replay_vision/backend/models/replay_highlight_feed.py
+++ b/products/replay_vision/backend/models/replay_highlight_feed.py
@@ -51,6 +51,8 @@ class ReplayHighlightFeedEntry(TeamScopedRootMixin, models.Model):
experiment_id = models.IntegerField(default=0)
percentile = models.FloatField()
group_root = models.UUIDField()
+ # Spaces one person's cards apart without reading the observations.
+ person_key = models.BigIntegerField(null=True)
class Meta:
app_label = "replay_vision"
diff --git a/products/replay_vision/backend/temporal/__init__.py b/products/replay_vision/backend/temporal/__init__.py
index 44eaefeac430..9c16072fb7f2 100644
--- a/products/replay_vision/backend/temporal/__init__.py
+++ b/products/replay_vision/backend/temporal/__init__.py
@@ -51,6 +51,7 @@ from products.replay_vision.backend.temporal.activities.benchmark import (
snapshot_benchmark_labels_activity,
write_benchmark_manifest_activity,
)
+from products.replay_vision.backend.temporal.activities.rebuild_highlight_feeds import rebuild_highlight_feeds_activity
from products.replay_vision.backend.temporal.activities.refresh_learned_rules import (
list_due_learned_rules_teams_activity,
refresh_team_learned_rules_activity,
@@ -62,6 +63,7 @@ from products.replay_vision.backend.temporal.gemini_cleanup_sweep import (
ReplayVisionGeminiCleanupSweepWorkflow,
sweep_gemini_files_activity,
)
+from products.replay_vision.backend.temporal.highlight_rebuild import RebuildHighlightFeedsWorkflow
from products.replay_vision.backend.temporal.learned_rules import RefreshLearnedRulesWorkflow
from products.replay_vision.backend.temporal.media_workflow import ObservationMediaWorkflow
from products.replay_vision.backend.temporal.read_meter import MeterScannerReadsWorkflow
@@ -85,6 +87,7 @@ WORKFLOWS = [
ReconcileScannerSchedulesWorkflow,
RefreshScannerEstimatesWorkflow,
RefreshLearnedRulesWorkflow,
+ RebuildHighlightFeedsWorkflow,
ReplayVisionGeminiCleanupSweepWorkflow,
SweepScannerWorkflow,
VisionAlertCheckWorkflow,
@@ -138,6 +141,7 @@ ACTIVITIES: list[Callable[..., Any]] = [
sweep_gemini_files_activity,
list_due_learned_rules_teams_activity,
refresh_team_learned_rules_activity,
+ rebuild_highlight_feeds_activity,
snapshot_benchmark_labels_activity,
load_benchmark_cases_activity,
prepare_benchmark_case_activity,
diff --git a/products/replay_vision/backend/temporal/activities/rebuild_highlight_feeds.py b/products/replay_vision/backend/temporal/activities/rebuild_highlight_feeds.py
new file mode 100644
index 000000000000..1db7ee5a49e4
--- /dev/null
+++ b/products/replay_vision/backend/temporal/activities/rebuild_highlight_feeds.py
@@ -0,0 +1,30 @@
+import time
+
+from temporalio import activity
+
+from products.replay_vision.backend.highlight_builds import rebuild_changed_feeds
+from products.replay_vision.backend.temporal.constants import HIGHLIGHT_REBUILD_CLAIM_BUDGET
+from products.replay_vision.backend.temporal.decorators import track_activity
+from products.replay_vision.backend.temporal.highlight_rebuild_types import RebuildHighlightFeedsResult
+from products.replay_vision.backend.watch_rank import ANCHOR_VERSION
+
+
+@track_activity()
+def rebuild_highlight_feeds_activity() -> RebuildHighlightFeedsResult:
+ """Rebuild every team whose ranks changed since its last build, a batch at a time, until none is due or the claim
+ budget runs out."""
+ # Not `bounded_queries`: it would lift each build's own 120s statement cap to the whole budget.
+ deadline = time.monotonic() + HIGHLIGHT_REBUILD_CLAIM_BUDGET.total_seconds()
+
+ def before_team() -> bool:
+ activity.heartbeat()
+ return time.monotonic() < deadline
+
+ built = 0
+ while time.monotonic() < deadline:
+ batch = rebuild_changed_feeds(ANCHOR_VERSION, on_team=before_team)
+ if not batch:
+ break
+ built += batch
+ return RebuildHighlightFeedsResult(built=built)
diff --git a/products/replay_vision/backend/temporal/constants.py b/products/replay_vision/backend/temporal/constants.py
index 3f627a146764..0ae46b25e5bc 100644
--- a/products/replay_vision/backend/temporal/constants.py
+++ b/products/replay_vision/backend/temporal/constants.py
@@ -265,3 +265,16 @@ LEARNED_RULES_CONCURRENCY = 16
LIST_DUE_LEARNED_RULES_TIMEOUT = dt.timedelta(seconds=60)
# Covers the model call's own 120s timeout plus the database reads around it.
REFRESH_TEAM_LEARNED_RULES_TIMEOUT = dt.timedelta(minutes=4)
+
+
+# Highlight feed rebuilds: a schedule starts this every minute and skips a start while one still runs.
+HIGHLIGHT_REBUILD_WORKFLOW_NAME = "replay-vision-rebuild-highlight-feeds"
+HIGHLIGHT_REBUILD_WORKFLOW_ID = "replay-vision-highlight-feed-rebuilder"
+HIGHLIGHT_REBUILD_SCHEDULE_ID = "replay-vision-highlight-feed-rebuilder-schedule"
+HIGHLIGHT_REBUILD_INTERVAL = dt.timedelta(minutes=1)
+HIGHLIGHT_REBUILD_EXECUTION_TIMEOUT = dt.timedelta(minutes=10)
+HIGHLIGHT_REBUILD_ACTIVITY_TIMEOUT = dt.timedelta(minutes=8)
+# One team's build holds one heavy statement under its own 120s cap, so a heartbeat per team arrives well inside this.
+HIGHLIGHT_REBUILD_HEARTBEAT_TIMEOUT = dt.timedelta(minutes=5)
+# The activity starts no team after this, which leaves a slow last build room inside the activity timeout.
+HIGHLIGHT_REBUILD_CLAIM_BUDGET = dt.timedelta(minutes=3)
diff --git a/products/replay_vision/backend/temporal/highlight_rebuild.py b/products/replay_vision/backend/temporal/highlight_rebuild.py
new file mode 100644
index 000000000000..aa144578a901
--- /dev/null
+++ b/products/replay_vision/backend/temporal/highlight_rebuild.py
@@ -0,0 +1,65 @@
+"""Rebuilds the stored highlight feeds of teams whose ranks changed."""
+
+import datetime as dt
+from typing import TYPE_CHECKING
+
+from temporalio import workflow
+from temporalio.common import RetryPolicy
+
+from posthog.temporal.common.base import PostHogWorkflow
+
+from products.replay_vision.backend.temporal.constants import (
+ HIGHLIGHT_REBUILD_ACTIVITY_TIMEOUT,
+ HIGHLIGHT_REBUILD_EXECUTION_TIMEOUT,
+ HIGHLIGHT_REBUILD_HEARTBEAT_TIMEOUT,
+ HIGHLIGHT_REBUILD_INTERVAL,
+ HIGHLIGHT_REBUILD_SCHEDULE_ID,
+ HIGHLIGHT_REBUILD_WORKFLOW_ID,
+ HIGHLIGHT_REBUILD_WORKFLOW_NAME,
+)
+from products.replay_vision.backend.temporal.highlight_rebuild_types import (
+ RebuildHighlightFeedsInputs,
+ RebuildHighlightFeedsResult,
+)
+
+if TYPE_CHECKING:
+ from temporalio.client import Client
+
+# `activities` pulls in Django, which the workflow sandbox can't safely re-import.
+with workflow.unsafe.imports_passed_through():
+ from products.replay_vision.backend.temporal.activities.rebuild_highlight_feeds import (
+ rebuild_highlight_feeds_activity,
+ )
+
+
[email protected](name=HIGHLIGHT_REBUILD_WORKFLOW_NAME)
+class RebuildHighlightFeedsWorkflow(PostHogWorkflow):
+ inputs_cls = RebuildHighlightFeedsInputs
+ inputs_optional = True
+
+ @workflow.run
+ async def run(self, inputs: RebuildHighlightFeedsInputs) -> RebuildHighlightFeedsResult:
+ # No retry: the schedule starts the next run a minute later, and it picks up whatever this one left.
+ return await workflow.execute_activity(
+ rebuild_highlight_feeds_activity,
+ start_to_close_timeout=HIGHLIGHT_REBUILD_ACTIVITY_TIMEOUT,
+ heartbeat_timeout=HIGHLIGHT_REBUILD_HEARTBEAT_TIMEOUT,
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+
+
+async def create_replay_vision_highlight_rebuild_schedule(client: "Client") -> None:
+ """Upsert the per-minute highlight feed rebuild schedule. Called on worker startup."""
+ # Function-local: with `@workflow.defn` above, the Temporal sandbox can't re-import the helper's dependencies.
+ from products.replay_vision.backend.temporal.schedule import upsert_interval_schedule # noqa: PLC0415
+
+ await upsert_interval_schedule(
+ client,
+ schedule_id=HIGHLIGHT_REBUILD_SCHEDULE_ID,
+ workflow_name=HIGHLIGHT_REBUILD_WORKFLOW_NAME,
+ workflow_id=HIGHLIGHT_REBUILD_WORKFLOW_ID,
+ inputs=RebuildHighlightFeedsInputs(),
+ interval=HIGHLIGHT_REBUILD_INTERVAL,
+ offset=dt.timedelta(0),
+ execution_timeout=HIGHLIGHT_REBUILD_EXECUTION_TIMEOUT,
+ )
diff --git a/products/replay_vision/backend/temporal/highlight_rebuild_types.py b/products/replay_vision/backend/temporal/highlight_rebuild_types.py
new file mode 100644
index 000000000000..8e7c227cee6d
--- /dev/null
+++ b/products/replay_vision/backend/temporal/highlight_rebuild_types.py
@@ -0,0 +1,11 @@
+"""Highlight feed rebuild types, kept out of `types.py` so the workflow sandbox loads only these."""
+
+from pydantic import BaseModel
+
+
+class RebuildHighlightFeedsInputs(BaseModel, frozen=True):
+ pass
+
+
+class RebuildHighlightFeedsResult(BaseModel, frozen=True):
+ built: int = 0
diff --git a/products/replay_vision/backend/tests/test_api.py b/products/replay_vision/backend/tests/test_api.py
index 738e797f3b1c..b68f0a93155a 100644
--- a/products/replay_vision/backend/tests/test_api.py
+++ b/products/replay_vision/backend/tests/test_api.py
@@ -4957,6 +4957,19 @@ class TestHighlightFeedAPI(_VisionAPITestCase):
sessions = [item["observation"]["session_id"] for item in response["results"]]
assert sessions and not any(session.startswith("leaving") for session in sessions)
+ def test_a_deleted_observation_leaves_the_stored_picks_before_the_feed_rebuilds(self) -> None:
+ scanner = self._create_scanner(name="m")
+ for index in range(10):
+ self._ranked_observation(scanner, f"s-{index}", 10 + index, "failure", 0.9 - index / 100)
+ built = self.client.get(self.feed_url, {"date_from": "-1d", "limit": "20"}).json()["results"]
+ deleted = built[0]["observation"]
+ ReplayObservation.objects.filter(id=deleted["id"]).delete()
+
+ # The first slot is where the deleted pick ranked, so a stored entry left in place empties the page.
+ response = self.client.get(self.feed_url, {"date_from": "-1d", "limit": "1"}).json()["results"]
+
+ assert [item["observation"]["id"] for item in response] == [built[1]["observation"]["id"]]
+
def test_an_embedding_pick_counts_its_hidden_duplicates(self) -> None:
scanner = self._create_scanner(name="m")
oldest = self._ranked_observation(scanner, "oldest-failure", 500, "failure", 0.9)
diff --git a/products/replay_vision/backend/tests/test_highlight_builds.py b/products/replay_vision/backend/tests/test_highlight_builds.py
index cbcd87396949..f6cd2e6ab479 100644
--- a/products/replay_vision/backend/tests/test_highlight_builds.py
+++ b/products/replay_vision/backend/tests/test_highlight_builds.py
@@ -4,10 +4,12 @@ from typing import Any
import time_machine
from posthog.test.base import BaseTest
+from unittest.mock import patch
from django.utils import timezone
from parameterized import parameterized
+from temporalio.testing import ActivityEnvironment
from posthog.models import Team
@@ -31,6 +33,7 @@ from products.replay_vision.backend.models.replay_observation import (
from products.replay_vision.backend.models.replay_observation_rank import RankCategory, ReplayObservationRank
from products.replay_vision.backend.models.replay_scanner import ReplayScanner, ScannerModel, ScannerType
from products.replay_vision.backend.observation_ranks import NewRank, record_ranks
+from products.replay_vision.backend.temporal.activities.rebuild_highlight_feeds import rebuild_highlight_feeds_activity
from products.replay_vision.backend.tests.helpers import snapshot_for
from products.replay_vision.backend.watch_rank import ANCHOR_VERSION
@@ -90,6 +93,11 @@ class TestHighlightBuilds(BaseTest):
(entry.window_days, str(entry.observation_id)): (entry.place, round(entry.percentile, 3))
for entry in ReplayHighlightFeedEntry.objects.for_team(self.team.id)
}
+ people = dict(ReplayObservationRank.objects.for_team(self.team.id).values_list("observation_id", "person_key"))
+ assert all(
+ entry.person_key is not None and entry.person_key == people[entry.observation_id]
+ for entry in ReplayHighlightFeedEntry.objects.for_team(self.team.id)
+ )
# The day window has ten rows, so its best two clear the cut; the week window places all twenty.
assert {key: value for key, value in entries.items() if key[0] == 1} == {
(1, recent[0]): (1, 0.95),
@@ -231,6 +239,31 @@ class TestHighlightBuilds(BaseTest):
assert (first, served, built) == (1, 1, [1, 0])
assert ReplayHighlightFeedState.objects.for_team(self.team.id).get().generation == 2
+ def test_the_rebuild_activity_works_through_every_due_batch(self) -> None:
+ other = Team.objects.create(organization=self.organization, name="other")
+ other_scanner = ReplayScanner.objects.create(
+ team=other,
+ name="other",
+ scanner_type=ScannerType.MONITOR,
+ scanner_config={"prompt": "p"},
+ model=ScannerModel.GEMINI_3_8_FLASH,
+ )
+ self._ranked(10, age=timedelta(hours=1))
+ for index in range(10):
+ observation = self._observation(_NOW - timedelta(hours=1, minutes=index), scanner=other_scanner, team=other)
+ record_ranks(
+ other.id, ANCHOR_VERSION, {str(observation.id): NewRank(contrast=index / 10, category="failure")}
+ )
+
+ with patch("products.replay_vision.backend.highlight_builds.REBUILDS_PER_SWEEP", 1):
+ result = ActivityEnvironment().run(rebuild_highlight_feeds_activity)
+
+ assert result.built == 2
+ assert set(ReplayHighlightFeedState.objects.unscoped().values_list("team_id", "generation")) == {
+ (self.team.id, 1),
+ (other.id, 1),
+ }
+
def test_a_load_that_began_before_a_rebuild_stays_on_the_build_it_started_on(self) -> None:
self._ranked(10, age=timedelta(hours=1))
diff --git a/products/replay_vision/backend/tests/test_watch_rank.py b/products/replay_vision/backend/tests/test_watch_rank.py
index d3a8f9b5dc67..74b771a06319 100644
--- a/products/replay_vision/backend/tests/test_watch_rank.py
+++ b/products/replay_vision/backend/tests/test_watch_rank.py
@@ -1,3 +1,4 @@
+import zlib
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -11,7 +12,7 @@ from parameterized import parameterized
from posthog.exceptions import ClickHouseAtCapacity
from products.replay_vision.backend import watch_rank_backfill
-from products.replay_vision.backend.highlight_feed import SESSION_SPACING
+from products.replay_vision.backend.highlight_feed import PERSON_SPACING
from products.replay_vision.backend.tests.helpers import axis_anchors
from products.replay_vision.backend.watch_rank import (
ANCHOR_VERSION,
@@ -35,13 +36,13 @@ def _row(
anchor_version: int = ANCHOR_VERSION,
minutes_ago: int = 0,
viewed: bool = False,
- session: str | None = None,
+ person: int | None = None,
) -> dict[str, Any]:
current = contrast is not None and anchor_version == ANCHOR_VERSION
return {
"id": observation_id,
"scanner_id": scanner,
- "session_id": session or f"session-{observation_id}",
+ "person_key": person if person is not None else zlib.crc32(observation_id.encode()),
"created_at": _NOW - timedelta(minutes=minutes_ago),
"feed_viewed": viewed,
"contrast": contrast if current else None,
@@ -139,21 +140,21 @@ class TestRankHighlightFeedByEmbedding(SimpleTestCase):
assert picks["seen-top"]["similar_count"] == 1
assert "mid-copy" not in {entry.observation_id for entry in ranked}
- def test_two_scanners_picking_one_session_land_apart_without_dropping_a_pick(self) -> None:
+ def test_two_scanners_picking_one_person_land_apart_without_dropping_a_pick(self) -> None:
rows = [
_row(f"{scanner}-{index}", index / 20, scanner=scanner, minutes_ago=index)
for scanner in ("scanner-a", "scanner-b", "scanner-c")
for index in range(9)
] + [
- _row("a-top", 0.9, scanner="scanner-a", session="shared"),
- _row("b-top", 0.9, scanner="scanner-b", session="shared", minutes_ago=1),
+ _row("a-top", 0.9, scanner="scanner-a", person=7),
+ _row("b-top", 0.9, scanner="scanner-b", person=7, minutes_ago=1),
_row("c-top", 0.9, scanner="scanner-c", minutes_ago=2),
]
ranked = [str(entry.observation_id) for entry in rank_highlight_feed_by_embedding(_with_percentiles(rows))]
assert ranked[0] == "a-top"
- assert ranked.index("b-top") - ranked.index("a-top") >= SESSION_SPACING
+ assert ranked.index("b-top") - ranked.index("a-top") >= PERSON_SPACING
assert {"a-top", "b-top", "c-top", "scanner-a-8", "scanner-b-8", "scanner-c-8"} <= set(ranked)
def test_rows_without_a_rank_fall_back_to_the_filler_floor(self) -> None:
diff --git a/products/replay_vision/backend/watch_rank.py b/products/replay_vision/backend/watch_rank.py
index 697baa3ea183..90fe5b70fca8 100644
--- a/products/replay_vision/backend/watch_rank.py
+++ b/products/replay_vision/backend/watch_rank.py
@@ -12,7 +12,7 @@ import asyncio
from collections import Counter
from collections.abc import Collection, Sequence
from datetime import datetime
-from typing import Any, Literal, cast
+from typing import Any, Literal
from django.core.cache import cache
from django.db import transaction
@@ -35,6 +35,7 @@ from products.replay_vision.backend.models.replay_highlight_feed import ReplayHi
from products.replay_vision.backend.models.replay_observation import ReplayObservation
from products.replay_vision.backend.models.replay_observation_anchor_similarity import ReplayObservationAnchorSimilarity
from products.replay_vision.backend.models.replay_observation_rank import ReplayObservationRank
+from products.replay_vision.backend.models.replay_observation_view import ReplayObservationView
from products.replay_vision.backend.observation_ranks import CATEGORY_CODES, CATEGORY_NAMES, NewRank, record_ranks
from products.replay_vision.backend.scanner_access import accessible_observations, readable_experiment_ids
@@ -326,6 +327,7 @@ _ENTRY_FIELDS = (
"category",
"percentile",
"group_root",
+ "person_key",
)
@@ -383,10 +385,15 @@ def _group_reach(
return {str(row["root"]): GroupReach(sessions=row["sessions"], people=row["people"]) for row in members}
+def views_by(viewer_id: int, as_of: datetime) -> QuerySet[ReplayObservationView]:
+ """The viewer's observation views up to `as_of`, so later views wait for the next feed load."""
+ return ReplayObservationView.objects.filter(user_id=viewer_id, created_at__lte=as_of)
+
+
def select_embedding_picks(
window: FeedWindow,
scanner_ids: Sequence[Any],
- viewed: Exists,
+ viewer_id: int,
access: UserAccessControl,
team_id: int,
cap: int,
@@ -428,31 +435,39 @@ def select_embedding_picks(
window_values = _cap_feed_groups(rows, cap)
if len(window_values) < cap and len(rows) == fetch_limit:
window_values = _cap_feed_groups(window_rows(fetch_limit * HIGHLIGHT_PICK_REFILL_FACTOR), cap)
+ # The entries carry everything ranking reads, so only ids load here, and the endpoint gates its page's rows on
+ # experiment access. A stored entry outlives a deleted observation until its feed rebuilds.
+ live = set(
+ ReplayObservation.objects.filter(team_id=team_id, id__in=list(window_values)).values_list("id", flat=True)
+ )
+ window_values = {observation_id: entry for observation_id, entry in window_values.items() if observation_id in live}
if not window_values:
return []
- # The entries already carry each row's experiment scope, so this skips the snapshot JSON the row-gate reads; the
- # endpoint still gates the rows of the page it serves.
- rows = cast(
- list[dict[str, Any]],
- list(
- ReplayObservation.objects.filter(team_id=team_id, id__in=list(window_values))
- .annotate(feed_viewed=viewed)
- .values("id", "session_id", "feed_viewed")
- ),
+ viewed = set(
+ views_by(viewer_id, window.as_of)
+ .filter(observation_id__in=list(window_values))
+ .values_list("observation_id", flat=True)
)
roots = {entry["group_root"] for entry in window_values.values()}
reach = _group_reach(team_id, window, roots, scanner_ids, categories, experiment_ids)
- for row in rows:
- entry = window_values[row["id"]]
- row["scanner_id"] = entry["scanner_id"]
- row["created_at"] = entry["created_at"]
+ picks: list[dict[str, Any]] = []
+ for observation_id, entry in window_values.items():
group = reach.get(str(entry["group_root"]), GroupReach(sessions=1, people=1))
- row["watch_category"] = CATEGORY_NAMES[entry["category"]]
- row["watch_percentile"] = entry["percentile"]
- row["watch_group_root"] = str(entry["group_root"])
- row["watch_group_size"] = group.sessions
- row["watch_group_people"] = group.people
- return rows
+ picks.append(
+ {
+ "id": observation_id,
+ "scanner_id": entry["scanner_id"],
+ "created_at": entry["created_at"],
+ "person_key": entry["person_key"],
+ "feed_viewed": observation_id in viewed,
+ "watch_category": CATEGORY_NAMES[entry["category"]],
+ "watch_percentile": entry["percentile"],
+ "watch_group_root": str(entry["group_root"]),
+ "watch_group_size": group.sessions,
+ "watch_group_people": group.people,
+ }
+ )
+ return picks
def select_embedding_filler(
@@ -481,7 +496,7 @@ def select_embedding_filler(
)
.annotate(feed_viewed=viewed)
)
- fields = ("id", "scanner_id", "session_id", "created_at", "feed_viewed")
+ fields = ("id", "scanner_id", "created_at", "feed_viewed")
# Two short newest-first walks, instead of ordering by the viewed flag, which would sort the whole window.
rows = list(
base.filter(feed_viewed=False).order_by("-created_at", "-id").values(*fields)[:HIGHLIGHT_FEED_MIN_ITEMS]
@@ -496,12 +511,12 @@ def select_embedding_filler(
def rank_highlight_feed_by_embedding(rows: list[dict[str, Any]]) -> list[HighlightFeedEntry]:
- """Rank candidate rows (`id`, `scanner_id`, `session_id`, `created_at`, `feed_viewed`, optionally
- `feed_impressions`, and for picks the `watch_*` fields `select_embedding_picks` adds).
+ """Rank candidate rows (`id`, `scanner_id`, `created_at`, `feed_viewed`, optionally `feed_impressions`, and for
+ picks `person_key` and the `watch_*` fields `select_embedding_picks` adds).
A pick is a row with a percentile whose category is an issue. Rows of one feed group collapse to their best-scoring
row, which carries the size of the group. Picks sort by weighted percentile, decayed for each earlier impression,
- viewed rows docked, one scanner held to a share of the picks, and one session's picks spaced apart. Every other row
+ viewed rows docked, one scanner held to a share of the picks, and one person's picks spaced apart. Every other row
is filler, kept only to reach the feed's floor.
"""
best_in_group: dict[str, tuple[float, dict[str, Any]]] = {}
@@ -541,7 +556,7 @@ def rank_highlight_feed_by_embedding(rows: list[dict[str, Any]]) -> list[Highlig
score,
row["created_at"],
row.get("scanner_id"),
- HighlightFeedEntry(observation_id=row["id"], reason=reason, session_id=row["session_id"]),
+ HighlightFeedEntry(observation_id=row["id"], reason=reason, person_key=row.get("person_key")),
)
)
# A pick stands for its whole feed group, so no member returns as filler.
diff --git a/products/replay_vision/backend/watch_rank_consumer.py b/products/replay_vision/backend/watch_rank_consumer.py
index c3ecc3e8c201..d8edf5c780b7 100644
--- a/products/replay_vision/backend/watch_rank_consumer.py
+++ b/products/replay_vision/backend/watch_rank_consumer.py
@@ -34,9 +34,8 @@ from products.replay_vision.backend.embeddings import (
EMBEDDING_PRODUCT,
OBSERVATION_EMBEDDING_MODEL,
)
-from products.replay_vision.backend.highlight_builds import rebuild_changed_feeds
from products.replay_vision.backend.highlight_feed_groups import feed_group_cache
-from products.replay_vision.backend.watch_rank import ANCHOR_VERSION, RankRequest, rank_observation
+from products.replay_vision.backend.watch_rank import RankRequest, rank_observation
from products.replay_vision.backend.watch_semantics import SemanticAssignment, assign_semantics
from products.replay_vision.backend.watch_themes import name_theme
@@ -56,8 +55,6 @@ _ERROR_RETRY_WINDOW_S = 60.0
# Long enough for a naming call that waits behind ranking on the shared database thread, short enough that a hung
# model call frees its slot.
_NAMING_TIMEOUT_S = 120.0
-# How often the consumer rebuilds the highlight feeds of teams whose ranks changed.
-_REBUILD_INTERVAL_S = 60.0
_NAMING_DRAIN_S = 20.0
# A due theme left unnamed while this many are in flight is named when its next member arrives.
_MAX_NAMING_TASKS = 8
@@ -76,11 +73,6 @@ BATCH_DURATION = Histogram(
"Time the highlight feed rank consumer spent on one non-empty batch.",
buckets=(0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 240, 300),
)
-FEED_REBUILDS_TOTAL = Counter(
- "replay_vision_watch_rank_consumer_feed_rebuilds_total",
- "Highlight feed rebuild sweeps of the rank consumer, by outcome.",
- labelnames=["outcome"],
-)
THEME_NAMING_TOTAL = Counter(
"replay_vision_watch_rank_consumer_theme_naming_total",
"Theme naming outcomes of the highlight feed rank consumer, including names dropped while every slot was busy.",
@@ -150,15 +142,6 @@ def _assign_with_timeouts(
raise
-def _rebuild_changed_feeds() -> int:
- # The sweep's thread keeps its own connection, so a broken one is replaced before each sweep.
- close_old_connections()
- try:
- return rebuild_changed_feeds(ANCHOR_VERSION)
- finally:
- close_old_connections()
-
-
class BatchConsumer(Protocol):
def store_offsets(self, *, message: Message) -> None: ...
@@ -319,17 +302,6 @@ class WatchRankConsumer:
config.update(helper.ssl_cert_config())
return config
- async def _rebuild_feeds(self) -> None:
- """Rebuild changed teams' highlight feeds beside the message loop, on a thread of its own."""
- while await self._wait(_REBUILD_INTERVAL_S):
- try:
- built = await sync_to_async(_rebuild_changed_feeds, thread_sensitive=False)()
- except Exception:
- FEED_REBUILDS_TOTAL.labels(outcome="failed").inc()
- logger.exception("replay_vision.watch_rank_consumer.feed_rebuild_failed")
- else:
- FEED_REBUILDS_TOTAL.labels(outcome="built" if built else "idle").inc()
-
async def run(self) -> None: # pragma: no cover - exercised in dogfood
loop = asyncio.get_running_loop()
for sig in (signal.SIGINT, signal.SIGTERM):
@@ -338,7 +310,6 @@ class WatchRankConsumer:
consumer = Consumer(self._consumer_config())
consumer.subscribe([KAFKA_DOCUMENT_EMBEDDING_RESULTS_TOPIC])
logger.info("replay_vision.watch_rank_consumer.started")
- rebuilds = asyncio.create_task(self._rebuild_feeds())
try:
while not self._stop.is_set():
self._liveness.record_heartbeat()
@@ -348,7 +319,6 @@ class WatchRankConsumer:
with BATCH_DURATION.time():
await self.process_batch(consumer, messages)
finally:
- rebuilds.cancel()
consumer.close()
# A theme claimed for naming stays unnamed until it doubles, so in-flight names get a moment to finish.
if self._naming: