Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(replay-vision): drop the watch_rank indexes and hide the column (#125982)

Tue Haulund13fcf233eb01
diff --git a/products/replay_vision/backend/api/scanners.py b/products/replay_vision/backend/api/scanners.py
index 6b287c4b6460..eb6a5528de03 100644
--- a/products/replay_vision/backend/api/scanners.py
+++ b/products/replay_vision/backend/api/scanners.py
@@ -140,6 +140,7 @@ from products.replay_vision.backend.scanner_access import (
is_experiment_accessible,
readable_observation_scanner_ids,
scanner_experiment_scope_q,
+ scanners_for_reading_observations,
)
from products.replay_vision.backend.scanner_config import (
MAX_PROMPT_LENGTH,
@@ -2474,9 +2475,9 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi
allowed_ids = [scanner_id for scanner_id in allowed_ids if scanner_id in created_ids]
if params.get("scanner_type"):
typed_ids = set(
- ReplayScanner.objects.filter(
- team_id=self.team_id, id__in=allowed_ids, scanner_type=params["scanner_type"]
- ).values_list("id", flat=True)
+ scanners_for_reading_observations(self.team_id)
+ .filter(id__in=allowed_ids, scanner_type=params["scanner_type"])
+ .values_list("id", flat=True)
)
allowed_ids = [scanner_id for scanner_id in allowed_ids if scanner_id in typed_ids]
viewer_id = cast(User, request.user).id
diff --git a/products/replay_vision/backend/management/commands/backfill_replay_scanner_prompt_questions.py b/products/replay_vision/backend/management/commands/backfill_replay_scanner_prompt_questions.py
deleted file mode 100644
index c10c3f742c62..000000000000
--- a/products/replay_vision/backend/management/commands/backfill_replay_scanner_prompt_questions.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from typing import Any
-
-from django.core.management.base import BaseCommand, CommandParser
-
-from products.replay_vision.backend.prompt_questions import backfill_prompt_questions
-
-
-class Command(BaseCommand):
- help = "Condense each Replay Vision scanner's prompt into the question the observation page shows"
-
- def add_arguments(self, parser: CommandParser) -> None:
- parser.add_argument("--team-id", type=int, default=None, help="Only this team's scanners")
- parser.add_argument(
- "--include-inline",
- action="store_true",
- help="Also give built-in inline scanners their written question; inline scanners never get a model-written one",
- )
- parser.add_argument("--limit", type=int, default=None, help="Stop after writing this many questions")
- parser.add_argument("--dry-run", action="store_true", help="Count the scanners due a question, write nothing")
-
- def handle(self, *args: Any, **options: Any) -> None:
- result = backfill_prompt_questions(
- team_id=options["team_id"],
- include_inline=bool(options["include_inline"]),
- limit=options["limit"],
- dry_run=bool(options["dry_run"]),
- )
- verb = "Would write" if options["dry_run"] else "Wrote"
- self.stdout.write(f"Checked {result.checked} scanners. {verb} {result.written} questions.")
diff --git a/products/replay_vision/backend/management/commands/backfill_replay_vision_observation_ranks.py b/products/replay_vision/backend/management/commands/backfill_replay_vision_observation_ranks.py
deleted file mode 100644
index fbb34b7d02f4..000000000000
--- a/products/replay_vision/backend/management/commands/backfill_replay_vision_observation_ranks.py
+++ /dev/null
@@ -1,36 +0,0 @@
-from datetime import timedelta
-from typing import Any
-
-from django.core.management.base import BaseCommand, CommandParser
-from django.utils import timezone
-
-from products.replay_vision.backend.highlight_builds import build_highlight_feed
-from products.replay_vision.backend.models.replay_highlight_feed import HighlightWindow
-from products.replay_vision.backend.observation_ranks import copy_team_ranks
-from products.replay_vision.backend.watch_rank import ANCHOR_VERSION
-from products.replay_vision.backend.watch_rank_backfill import teams_with_observations
-
-
-class Command(BaseCommand):
- help = "Copy Replay Vision observations' ranks into rank rows, then build each team's highlight feed from them"
-
- def add_arguments(self, parser: CommandParser) -> None:
- parser.add_argument("--team-id", type=int, action="append", help="Only this team; repeat for more")
- parser.add_argument(
- "--days", type=int, default=max(HighlightWindow.values), help="How far back to copy, in days"
- )
- parser.add_argument("--dry-run", action="store_true", help="Count the ranks it would copy, write nothing")
-
- def handle(self, *args: Any, **options: Any) -> None:
- since = timezone.now() - timedelta(days=options["days"])
- dry_run = bool(options["dry_run"])
- for team_id in options["team_id"] or teams_with_observations(since):
- try:
- copied = copy_team_ranks(team_id, ANCHOR_VERSION, since, dry_run=dry_run)
- if not dry_run:
- build_highlight_feed(team_id, ANCHOR_VERSION)
- except Exception as e:
- # One team's slow query must not stop the rest.
- self.stderr.write(f"Team {team_id}: skipped ({e.__class__.__name__}: {e})")
- continue
- self.stdout.write(f"Team {team_id}: {'would copy' if dry_run else 'copied'} {copied} ranks.")
diff --git a/products/replay_vision/backend/migrations/0144_drop_watch_rank_indexes.py b/products/replay_vision/backend/migrations/0144_drop_watch_rank_indexes.py
new file mode 100644
index 000000000000..d867dd973b39
--- /dev/null
+++ b/products/replay_vision/backend/migrations/0144_drop_watch_rank_indexes.py
@@ -0,0 +1,19 @@
+# Generated by Django 5.2.18 on 2026-10-29 22:38
+
+from django.db import migrations
+
+from posthog.migration_helpers import SafeRemoveIndexConcurrently
+
+
+class Migration(migrations.Migration):
+ atomic = False
+
+ dependencies = [
+ ("replay_vision", "0143_replay_highlight_autovacuum"),
+ ]
+
+ operations = [
+ SafeRemoveIndexConcurrently(model_name="replayobservation", name="rlo_ranked_count_idx"),
+ SafeRemoveIndexConcurrently(model_name="replayobservation", name="rlo_ranked_contrast_idx"),
+ SafeRemoveIndexConcurrently(model_name="replayobservation", name="rlo_ranked_feed_group_idx"),
+ ]
diff --git a/products/replay_vision/backend/migrations/max_migration.txt b/products/replay_vision/backend/migrations/max_migration.txt
index c62685845af1..b671e392cd51 100644
--- a/products/replay_vision/backend/migrations/max_migration.txt
+++ b/products/replay_vision/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0143_replay_highlight_autovacuum
+0144_drop_watch_rank_indexes
diff --git a/products/replay_vision/backend/models/replay_observation.py b/products/replay_vision/backend/models/replay_observation.py
index d12f22c4f51e..e93b94ea0584 100644
--- a/products/replay_vision/backend/models/replay_observation.py
+++ b/products/replay_vision/backend/models/replay_observation.py
@@ -1,8 +1,9 @@
from django.db import models
from django.db.models import Case, CharField, Exists, Expression, F, FloatField, Func, OuterRef, Value, When
from django.db.models.fields.json import KeyTextTransform, KeyTransform
-from django.db.models.functions import Cast, Coalesce
+from django.db.models.functions import Cast
+from posthog.migration_helpers import deprecate_field
from posthog.models.utils import UUIDModel
from products.replay_vision.backend.error_kinds import ERROR_REASON_HELP_TEXT
@@ -42,16 +43,6 @@ class ObservationTrigger(models.TextChoices):
BACKFILL = "backfill", "Backfill"
-def watch_rank_contrast() -> Cast:
- """A ranked row's score; `rlo_ranked_contrast_idx` orders by it."""
- return Cast(KeyTextTransform("contrast", "watch_rank"), FloatField())
-
-
-def watch_group_root() -> Coalesce:
- """A ranked row's feed group, or the row itself when it has none. `rlo_ranked_feed_group_idx` indexes it."""
- return Coalesce(KeyTextTransform("group", "watch_rank"), Cast("id", CharField()), output_field=CharField())
-
-
class ReplayObservation(UUIDModel):
"""One application of a `ReplayScanner` to a session recording (see README)."""
@@ -112,10 +103,14 @@ class ReplayObservation(UUIDModel):
default=dict,
help_text="Result data persisted on success (model output, signals count); see `temporal.types.ScannerResult`.",
)
- watch_rank = models.JSONField(
- null=True,
- blank=True,
- help_text="Highlight feed ranking derived from the observation's embedding; see `watch_rank.WatchRank`.",
+ # Ranks live in `ReplayObservationRank`.
+ watch_rank = deprecate_field(
+ models.JSONField(
+ null=True,
+ blank=True,
+ help_text="Highlight feed ranking derived from the observation's embedding; see `watch_rank.WatchRank`.",
+ ),
+ raise_on_access=True,
)
created_task_id = models.UUIDField(
null=True,
@@ -184,28 +179,6 @@ class ReplayObservation(UUIDModel):
name="rlo_team_in_flight_idx",
condition=models.Q(status__in=("pending", "running")),
),
- # The pick query counts each scanner's ranked rows here; the anchor version sits in the condition so the count never reads `watch_rank`.
- models.Index(
- fields=["scanner", "created_at"],
- include=["id", "team", "completed_at"],
- name="rlo_ranked_count_idx",
- condition=models.Q(status="succeeded", watch_rank__isnull=False, watch_rank__anchor_version=1),
- ),
- # The pick query reads each scanner's top rows by score from here.
- models.Index(
- F("scanner"),
- watch_rank_contrast().desc(),
- F("id").desc(),
- include=["team", "created_at", "completed_at", "watch_rank"],
- name="rlo_ranked_contrast_idx",
- condition=models.Q(status="succeeded", watch_rank__isnull=False, watch_rank__anchor_version=1),
- ),
- # Finds the members of the feed picks' groups, to count the sessions and people each one reached.
- models.Index(
- watch_group_root(),
- name="rlo_ranked_feed_group_idx",
- condition=models.Q(status="succeeded", watch_rank__isnull=False),
- ),
# Serves the per-backfill observation filter and progress counts; partial since live rows never match.
models.Index(
fields=["backfill"],
diff --git a/products/replay_vision/backend/observation_ranks.py b/products/replay_vision/backend/observation_ranks.py
index f5643a275bbb..354a4a38ea08 100644
--- a/products/replay_vision/backend/observation_ranks.py
+++ b/products/replay_vision/backend/observation_ranks.py
@@ -6,12 +6,10 @@ them the same way.
import uuid
from collections.abc import Mapping
-from datetime import datetime, timedelta
from itertools import batched
from typing import Any
-from django.db import connection, transaction
-from django.utils import timezone
+from django.db import connection
from posthog.dataclasses import frozen
from posthog.models.scoping.manager import resolve_effective_team_id
@@ -19,7 +17,6 @@ from posthog.models.scoping.manager import resolve_effective_team_id
from products.replay_vision.backend.models.replay_observation_rank import RankCategory
_WRITE_CHUNK = 500
-_COPY_SLICE = timedelta(days=1)
CATEGORY_CODES: dict[str, int] = {category.name.lower(): category.value for category in RankCategory}
CATEGORY_NAMES: dict[int, str] = {code: name for name, code in CATEGORY_CODES.items()}
@@ -113,87 +110,6 @@ def set_rank_clusters(team_id: int, clusters: Mapping[str, tuple[uuid.UUID, uuid
mark_changed(team_id)
-def copy_team_ranks(team_id: int, anchor_version: int, since: datetime, *, dry_run: bool = False) -> int:
- """Copy the team's current ranks since `since` from the observations' rank JSON into rank rows, filling in a
- feed group or search theme an existing row lacks. Returns how many rows it wrote, or how many ranks a write would
- read."""
- source = """
- FROM replay_vision_replayobservation o
- WHERE o.team_id = %(observation_team_id)s
- AND o.created_at >= %(since)s
- AND o.created_at < %(until)s
- AND o.status = 'succeeded'
- AND o.watch_rank IS NOT NULL
- AND o.watch_rank -> 'anchor_version' = to_jsonb(%(anchor_version)s::int)
- AND jsonb_typeof(o.watch_rank -> 'contrast') = 'number'
- AND o.watch_rank ->> 'category' = ANY(%(categories)s)
- """
- params: dict[str, Any] = {
- "team_id": resolve_effective_team_id(team_id),
- "observation_team_id": team_id,
- "since": since,
- "until": timezone.now(),
- "anchor_version": anchor_version,
- "categories": list(CATEGORY_CODES),
- }
- if dry_run:
- with connection.cursor() as cursor:
- # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (fixed SQL, every value is a parameter)
- cursor.execute(f"SELECT count(*) {source}", params)
- return cursor.fetchone()[0]
- written = 0
- end = params["until"]
- # One day per statement, so the rank rows an upsert locks are released before the next day's rows.
- while since < end:
- params |= {"since": since, "until": min(since + _COPY_SLICE, end)}
- written += _copy_slice(team_id, source, params)
- since += _COPY_SLICE
- _delete_orphaned_ranks(team_id)
- return written
-
-
-def _delete_orphaned_ranks(team_id: int) -> None:
- # A recording deletion removes observations in raw SQL, so rank rows it missed are removed here.
- with connection.cursor() as cursor:
- cursor.execute(
- """
- DELETE FROM replay_vision_replayobservationrank rank
- WHERE rank.team_id = %s
- AND NOT EXISTS (SELECT 1 FROM replay_vision_replayobservation o WHERE o.id = rank.observation_id)
- """,
- [resolve_effective_team_id(team_id)],
- )
-
-
-def _copy_slice(team_id: int, source: str, params: dict[str, Any]) -> int:
- category = "CASE o.watch_rank ->> 'category' " + " ".join(
- f"WHEN '{name}' THEN {code}" for name, code in CATEGORY_CODES.items()
- )
- with transaction.atomic(), connection.cursor() as cursor:
- cursor.execute(
- # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (fixed SQL, every value is a parameter)
- f"""
- INSERT INTO replay_vision_replayobservationrank ({_RANK_COLUMNS})
- SELECT o.id, %(team_id)s, o.scanner_id, o.created_at, %(anchor_version)s,
- (o.watch_rank ->> 'contrast')::double precision, {category} END,
- (o.watch_rank ->> 'group')::uuid, (o.watch_rank ->> 'theme')::uuid,
- {_PERSON_SQL}, {_EXPERIMENT_SQL}
- {source}
- ON CONFLICT (observation_id) DO UPDATE SET
- feed_group_id = COALESCE(replay_vision_replayobservationrank.feed_group_id, EXCLUDED.feed_group_id),
- search_theme_id = COALESCE(replay_vision_replayobservationrank.search_theme_id, EXCLUDED.search_theme_id)
- WHERE (replay_vision_replayobservationrank.feed_group_id IS NULL AND EXCLUDED.feed_group_id IS NOT NULL)
- OR (replay_vision_replayobservationrank.search_theme_id IS NULL AND EXCLUDED.search_theme_id IS NOT NULL)
- """,
- params,
- )
- written = cursor.rowcount
- # Each committed day asks for a rebuild, so a copy stopped partway still reaches the feed.
- if written:
- mark_changed(team_id)
- return written
-
-
def mark_changed(team_id: int) -> None:
"""Ask for a rebuild of the team's highlight feed, in the caller's transaction. Every ask counts, so one that lands
while a build ranks is still due after it; a build never holds the state row while it ranks."""
diff --git a/products/replay_vision/backend/prompt_questions.py b/products/replay_vision/backend/prompt_questions.py
index 3453f970e8b7..160a89cef642 100644
--- a/products/replay_vision/backend/prompt_questions.py
+++ b/products/replay_vision/backend/prompt_questions.py
@@ -24,7 +24,6 @@ from products.replay_vision.backend.gemini_client import gemini_text_models
from products.replay_vision.backend.models.replay_scanner import (
PromptValence,
ReplayScanner,
- ScannerOrigin,
ScannerType,
prompt_fingerprint,
)
@@ -247,93 +246,3 @@ def valence_for_snapshot(
if scanner_type not in _DIRECTIONAL_TYPES or not valence or not _from_snapshot_prompt(snapshot_config, source):
return None
return PromptValence(valence)
-
-
-@frozen
-class BackfillResult:
- checked: int
- written: int
-
-
-def backfill_prompt_questions(
- *,
- team_id: int | None = None,
- include_inline: bool = False,
- limit: int | None = None,
- dry_run: bool = False,
-) -> BackfillResult:
- """Give every scanner whose question is missing or came from another prompt a question for its current prompt.
- A monitor or scorer whose current question has no valence gets only a valence, so its question keeps its wording.
-
- Writes through a queryset update, so the scanner's version, updated_at and activity log stay untouched.
- The update is conditional on the prompt still being the one condensed, so an edit that lands mid-run wins.
- Each distinct prompt is condensed once per run, team and scanner type, since many scanners share one word for word.
- Inline scanners only take a built-in scan's question, the same rule `create_inline_scanner` follows.
- """
- scanners = ReplayScanner.all_origins.order_by("created_at")
- if not include_inline:
- scanners = scanners.filter(origin=ScannerOrigin.CONFIGURED)
- if team_id is not None:
- scanners = scanners.filter(team_id=team_id)
- checked = written = 0
- condensed: dict[tuple[int, str, str, str], PromptQuestion] = {}
- for scanner in scanners.only(
- "id",
- "team_id",
- "origin",
- "scanner_type",
- "scanner_config",
- "prompt_question",
- "prompt_question_source",
- "prompt_valence",
- ).iterator():
- if limit is not None and written >= limit:
- break
- checked += 1
- source = _source_of(scanner.scanner_config)
- # A question written before the scale joined the fingerprint still matches its prompt, so it keeps its wording.
- question_is_current = scanner.prompt_question_source in (
- source,
- prompt_fingerprint(_prompt_of(scanner.scanner_config)),
- )
- if question_is_current and (scanner.scanner_type not in _DIRECTIONAL_TYPES or scanner.prompt_valence):
- continue
- inline_question = (
- built_in_question(scanner.scanner_type, scanner.scanner_config)
- if scanner.origin == ScannerOrigin.INLINE
- else None
- )
- if scanner.origin == ScannerOrigin.INLINE and inline_question is None:
- continue
- if dry_run:
- written += 1
- continue
- # A kept question goes to the model too, so the valence judges the question the page shows.
- kept_question = scanner.prompt_question if question_is_current else ""
- # Keyed by team too, since the answer can echo a team's own scale label.
- key = (scanner.team_id, source, scanner.scanner_type, kept_question)
- question = inline_question or condensed.get(key)
- if question is None:
- question = condense_prompt(
- team_id=scanner.team_id,
- scanner_type=scanner.scanner_type,
- scanner_config=scanner.scanner_config,
- metered=False,
- kept_question=kept_question,
- )
- # A failed call has no valence; the next scanner with this prompt tries the model again.
- if question.valence:
- condensed[key] = question
- fields = question.as_fields()
- # Zero rows when the prompt was edited mid-run, which is then not a write of ours.
- unchanged = ReplayScanner.all_origins.filter(pk=scanner.pk, scanner_config=scanner.scanner_config)
- if question_is_current:
- # A failed model call carries no valence, and writing its question would replace a model-written one.
- fields = {"prompt_question_source": source}
- if question.valence:
- fields["prompt_valence"] = question.valence
- unchanged = unchanged.filter(prompt_valence="")
- elif scanner.prompt_question_source == source:
- continue
- written += unchanged.update(**fields)
- return BackfillResult(checked=checked, written=written)
diff --git a/products/replay_vision/backend/tests/test_api.py b/products/replay_vision/backend/tests/test_api.py
index cdbf9c46aa1c..738e797f3b1c 100644
--- a/products/replay_vision/backend/tests/test_api.py
+++ b/products/replay_vision/backend/tests/test_api.py
@@ -4943,6 +4943,20 @@ class TestHighlightFeedAPI(_VisionAPITestCase):
assert "late-top" not in sessions(datetime.fromisoformat(first["as_of"]))
assert "late-top" in sessions(later + timedelta(seconds=1))
+ def test_a_build_older_than_the_load_drops_rows_that_left_the_window(self) -> None:
+ scanner = self._create_scanner(name="m")
+ for index in range(6):
+ self._ranked_observation(scanner, f"leaving-{index}", 1430 + index, "failure", 0.9 - index / 100)
+ self._ranked_observation(scanner, f"staying-{index}", 60 + index, "failure", 0.5 - index / 100)
+ built = self.client.get(self.feed_url, {"date_from": "-1d", "limit": "20"}).json()
+ assert any(item["observation"]["session_id"].startswith("leaving") for item in built["results"])
+
+ with time_machine.travel(timezone.now() + timedelta(hours=1), tick=False):
+ response = self.client.get(self.feed_url, {"date_from": "-1d", "limit": "20"}).json()
+
+ sessions = [item["observation"]["session_id"] for item in response["results"]]
+ assert sessions and not any(session.startswith("leaving") for session in sessions)
+
def test_an_embedding_pick_counts_its_hidden_duplicates(self) -> None:
scanner = self._create_scanner(name="m")
oldest = self._ranked_observation(scanner, "oldest-failure", 500, "failure", 0.9)
@@ -5069,6 +5083,7 @@ class TestHighlightFeedAPI(_VisionAPITestCase):
summarizer = self._create_scanner(
name="sum", scanner_type=ScannerType.SUMMARIZER, scanner_config={"prompt": "p", "length": "short"}
)
+ inline_monitor = self._create_scanner(name="one-off", origin=ScannerOrigin.INLINE, inline_key="one-off")
self._succeeded_observation(monitor, "in-window", 60, self._monitor_result("no"))
old = self._succeeded_observation(monitor, "too-old", 1, self._monitor_result("yes"))
ReplayObservation.objects.filter(pk=old.pk).update(created_at=timezone.now() - timedelta(days=10))
@@ -5082,13 +5097,15 @@ class TestHighlightFeedAPI(_VisionAPITestCase):
},
)
+ self._succeeded_observation(inline_monitor, "inline", 90, self._monitor_result("no"))
+
resp = self.client.get(self.feed_url)
sessions = [item["observation"]["session_id"] for item in resp.json()["results"]]
- self.assertEqual(sessions, ["summary", "in-window"])
+ self.assertEqual(sessions, ["summary", "in-window", "inline"])
resp = self.client.get(f"{self.feed_url}?date_from=-30d&scanner_type=monitor")
sessions = [item["observation"]["session_id"] for item in resp.json()["results"]]
- self.assertEqual(sessions, ["in-window", "too-old"])
+ self.assertEqual(sessions, ["in-window", "inline", "too-old"])
resp = self.client.get(f"{self.feed_url}?limit=1")
self.assertEqual(len(resp.json()["results"]), 1)
@@ -5230,6 +5247,28 @@ class TestHighlightFeedAPI(_VisionAPITestCase):
sorted(item["observation"]["session_id"] for item in resp.json()["results"]), ["open-sess", "open-sess-2"]
)
+ def test_restricted_picks_never_take_a_page_slot(self) -> None:
+ experiment = create_experiment(self.team, "restricted-picks-flag")
+ restricted = self._create_scanner(name="restricted", experiment_targeting={"experiment_id": experiment.id})
+ open_scanner = self._create_scanner(name="open")
+ for index in range(10):
+ obs = self._succeeded_observation(restricted, f"restricted-{index}", index, self._monitor_result("yes"))
+ set_rank(obs, category="failure", contrast=index / 10, group=None)
+ obs = self._succeeded_observation(open_scanner, f"open-{index}", index, self._monitor_result("yes"))
+ set_rank(obs, category="struggle", contrast=index / 10, group=None)
+ restricted.experiment_targeting = None
+ restricted.save()
+
+ # The restricted picks outscore the open ones, so a page of two holds them unless the picks drop them first.
+ with patch(
+ "products.access_control.backend.facade.user_access_control.UserAccessControl.filter_queryset_by_access_level",
+ side_effect=lambda qs, **_: qs.exclude(pk=experiment.pk) if qs.model is Experiment else qs,
+ ):
+ resp = self.client.get(self.feed_url, {"limit": 2})
+ self.assertEqual(
+ sorted(item["observation"]["session_id"] for item in resp.json()["results"]), ["open-8", "open-9"]
+ )
+
def test_a_later_page_drops_rows_the_viewer_lost_access_to_after_the_first(self) -> None:
experiment = create_experiment(self.team, "revoked-flag")
scanner = self._create_scanner(name="retargeted", experiment_targeting={"experiment_id": experiment.id})
diff --git a/products/replay_vision/backend/tests/test_highlight_builds.py b/products/replay_vision/backend/tests/test_highlight_builds.py
index 5f2220b9e308..cbcd87396949 100644
--- a/products/replay_vision/backend/tests/test_highlight_builds.py
+++ b/products/replay_vision/backend/tests/test_highlight_builds.py
@@ -30,7 +30,7 @@ from products.replay_vision.backend.models.replay_observation import (
)
from products.replay_vision.backend.models.replay_observation_rank import RankCategory, ReplayObservationRank
from products.replay_vision.backend.models.replay_scanner import ReplayScanner, ScannerModel, ScannerType
-from products.replay_vision.backend.observation_ranks import NewRank, copy_team_ranks, record_ranks
+from products.replay_vision.backend.observation_ranks import NewRank, record_ranks
from products.replay_vision.backend.tests.helpers import snapshot_for
from products.replay_vision.backend.watch_rank import ANCHOR_VERSION
@@ -220,26 +220,6 @@ class TestHighlightBuilds(BaseTest):
assert ReplayHighlightFeedEntry.objects.for_team(self.team.id).filter(window_days=1).count() == 2
- def test_copy_fills_a_missing_group_from_the_rank_json_once(self) -> None:
- group = uuid.uuid4()
- created_at = _NOW - timedelta(minutes=1)
- observation = self._observation(
- created_at,
- watch_rank={"category": "struggle", "contrast": 0.7, "anchor_version": ANCHOR_VERSION, "group": str(group)},
- )
- self._observation(created_at, watch_rank={"category": "failure", "contrast": 0.7, "anchor_version": 99})
-
- copied = copy_team_ranks(self.team.id, ANCHOR_VERSION, _NOW - timedelta(days=1))
- again = copy_team_ranks(self.team.id, ANCHOR_VERSION, _NOW - timedelta(days=1))
-
- rank = ReplayObservationRank.objects.for_team(self.team.id).get()
- assert (copied, again) == (1, 0)
- assert (rank.observation_id, rank.category, rank.feed_group_id) == (
- observation.id,
- RankCategory.STRUGGLE,
- group,
- )
-
def test_a_stale_build_is_served_and_the_sweep_rebuilds_it_once(self) -> None:
self._ranked(10, age=timedelta(hours=1))
first = build_highlight_feed(self.team.id, ANCHOR_VERSION)
diff --git a/products/replay_vision/backend/tests/test_import_order.py b/products/replay_vision/backend/tests/test_import_order.py
index 3d54086b3066..694fde5c71c4 100644
--- a/products/replay_vision/backend/tests/test_import_order.py
+++ b/products/replay_vision/backend/tests/test_import_order.py
@@ -13,10 +13,6 @@ class TestImportOrder(SimpleTestCase):
[
("prompt_questions", "products.replay_vision.backend.prompt_questions"),
("inline_scan", "products.replay_vision.backend.inline_scan"),
- (
- "backfill_command",
- "products.replay_vision.backend.management.commands.backfill_replay_scanner_prompt_questions",
- ),
]
)
def test_module_imports_before_the_temporal_package(self, _name: str, module: str) -> None:
diff --git a/products/replay_vision/backend/tests/test_prompt_questions.py b/products/replay_vision/backend/tests/test_prompt_questions.py
index e14b668a483d..fa548b2bfd34 100644
--- a/products/replay_vision/backend/tests/test_prompt_questions.py
+++ b/products/replay_vision/backend/tests/test_prompt_questions.py
@@ -9,8 +9,6 @@ from django.utils import timezone
from parameterized import parameterized
-from posthog.models.team import Team
-
from products.replay_vision.backend.built_in_scans import BUILT_IN_SCANS, BuiltInScanKey
from products.replay_vision.backend.inline_scan import create_inline_scanner
from products.replay_vision.backend.models.replay_observation import (
@@ -28,7 +26,6 @@ from products.replay_vision.backend.prompt_questions import (
MAX_MODEL_CALLS_PER_TEAM_PER_HOUR,
MAX_QUESTION_CHARS,
_budget_key,
- backfill_prompt_questions,
condense_prompt,
scanner_question,
)
@@ -261,72 +258,6 @@ class TestPromptQuestions(APIBaseTest):
assert self.client.get(url).json()["prompt_question"] is None
assert self.client.get(url).json()["prompt_valence"] is None
- def test_backfill_fills_only_stale_questions_without_touching_the_scanner_version(self) -> None:
- stale = self._scanner(name="stale")
- copy = self._scanner(name="copy")
- ReplayScanner.objects.filter(pk__in=[stale.pk, copy.pk]).update(prompt_question="", prompt_question_source="")
- fresh = self._scanner(name="fresh")
- ReplayScanner.objects.filter(pk=fresh.pk).update(
- prompt_question="Kept as is?", prompt_question_source=prompt_fingerprint(PROMPT), prompt_valence="good"
- )
- unjudged = self._scanner(name="unjudged")
- ReplayScanner.objects.filter(pk=unjudged.pk).update(
- prompt_question="Judged before valence?", prompt_question_source=prompt_fingerprint(PROMPT)
- )
- version = ReplayScanner.objects.get(pk=stale.pk).scanner_version
-
- dry = backfill_prompt_questions(team_id=self.team.id, dry_run=True)
- assert (dry.checked, dry.written) == (4, 3)
- assert ReplayScanner.objects.get(pk=stale.pk).prompt_question == ""
- self.client_mock.return_value.generate_content.reset_mock()
-
- result = backfill_prompt_questions(team_id=self.team.id)
-
- assert result.written == 3
- # The copy shares the stale scanner's call; the unjudged scanner sends its kept question.
- calls = self.client_mock.return_value.generate_content.call_args_list
- assert ["Judged before valence?" in call.kwargs["contents"] for call in calls] == [False, True]
- assert ReplayScanner.objects.get(pk=copy.pk).prompt_question == "Did the user struggle at checkout?"
- stale.refresh_from_db()
- assert stale.prompt_question == "Did the user struggle at checkout?"
- assert stale.prompt_question_source == prompt_fingerprint(PROMPT)
- assert stale.scanner_version == version
- assert ReplayScanner.objects.get(pk=fresh.pk).prompt_question == "Kept as is?"
- unjudged.refresh_from_db()
- assert (unjudged.prompt_question, unjudged.prompt_valence) == ("Judged before valence?", "bad")
-
- def test_backfill_never_shares_an_answer_across_teams(self) -> None:
- other_team = Team.objects.create(organization=self.organization, name="Other team")
- scale = {"min": 0, "max": 10, "label": "checkout friction"}
- ours = self._scanner(scanner_type=ScannerType.SCORER, scanner_config={"prompt": PROMPT, "scale": scale})
- theirs = self._scanner(
- team=other_team, scanner_type=ScannerType.SCORER, scanner_config={"prompt": PROMPT, "scale": scale}
- )
- ReplayScanner.all_origins.filter(pk__in=[ours.pk, theirs.pk]).update(
- prompt_question="", prompt_question_source=""
- )
-
- backfill_prompt_questions()
-
- assert self.client_mock.return_value.generate_content.call_count == 2
-
- def test_backfill_retries_the_model_for_each_scanner_after_a_failed_call(self) -> None:
- unjudged = self._scanner(name="unjudged")
- retried = self._scanner(name="retried")
- ReplayScanner.objects.filter(pk__in=[unjudged.pk, retried.pk]).update(
- prompt_question="Judged before valence?", prompt_question_source=prompt_fingerprint(PROMPT)
- )
- reply = MagicMock(text=json.dumps({"question": "Did the user struggle at checkout?", "valence": "bad"}))
- self.client_mock.return_value.generate_content.side_effect = [RuntimeError("provider down"), reply]
-
- result = backfill_prompt_questions(team_id=self.team.id)
-
- assert result.written == 1
- unjudged.refresh_from_db()
- retried.refresh_from_db()
- assert (unjudged.prompt_question, unjudged.prompt_valence) == ("Judged before valence?", "")
- assert (retried.prompt_question, retried.prompt_valence) == ("Judged before valence?", "bad")
-
def test_inline_scanners_only_ever_get_a_built_in_question(self) -> None:
self.client_mock.return_value.generate_content.reset_mock()
summarize = create_inline_scanner(
@@ -344,11 +275,4 @@ class TestPromptQuestions(APIBaseTest):
model=ScannerModel.GEMINI_3_8_FLASH,
)
assert (summarize.prompt_question, custom.prompt_question) == (SUMMARY.question, "")
- ReplayScanner.all_origins.filter(pk=summarize.pk).update(prompt_question="", prompt_question_source="")
-
- result = backfill_prompt_questions(team_id=self.team.id, include_inline=True)
-
- assert result.written == 1
- assert ReplayScanner.all_origins.get(pk=summarize.pk).prompt_question == SUMMARY.question
- assert ReplayScanner.all_origins.get(pk=custom.pk).prompt_question == ""
self.client_mock.return_value.generate_content.assert_not_called()
diff --git a/products/replay_vision/backend/watch_rank.py b/products/replay_vision/backend/watch_rank.py
index a5b859f24562..697baa3ea183 100644
--- a/products/replay_vision/backend/watch_rank.py
+++ b/products/replay_vision/backend/watch_rank.py
@@ -268,6 +268,7 @@ def _stored_window_entries(
team_id: int,
generation: int,
window_days: int,
+ start: datetime,
scanner_ids: Sequence[Any],
categories: Sequence[str],
experiment_ids: Collection[int],
@@ -279,6 +280,8 @@ def _stored_window_entries(
.filter(
generation=generation,
window_days=window_days,
+ # A build ranked before this load's window moved on can hold rows that have since left it.
+ created_at__gte=start,
scanner_id__in=list(scanner_ids),
category__in=[CATEGORY_CODES[category] for category in categories],
experiment_id__in=[0, *experiment_ids],
@@ -413,6 +416,7 @@ def select_embedding_picks(
team_id,
window.generation,
window.window_days,
+ window.start,
scanner_ids,
categories,
experiment_ids,
@@ -426,21 +430,22 @@ def select_embedding_picks(
window_values = _cap_feed_groups(window_rows(fetch_limit * HIGHLIGHT_PICK_REFILL_FACTOR), cap)
if not window_values:
return []
- # The row-gate reads each row's snapshot JSON, so it runs on the picks only.
+ # The entries already carry each row's experiment scope, so this skips the snapshot JSON the row-gate reads; the
+ # endpoint still gates the rows of the page it serves.
rows = cast(
list[dict[str, Any]],
list(
- accessible_observations(
- access, team_id, ReplayObservation.objects.filter(team_id=team_id, id__in=list(window_values))
- )
+ ReplayObservation.objects.filter(team_id=team_id, id__in=list(window_values))
.annotate(feed_viewed=viewed)
- .values("id", "scanner_id", "session_id", "created_at", "feed_viewed")
+ .values("id", "session_id", "feed_viewed")
),
)
roots = {entry["group_root"] for entry in window_values.values()}
reach = _group_reach(team_id, window, roots, scanner_ids, categories, experiment_ids)
for row in rows:
entry = window_values[row["id"]]
+ row["scanner_id"] = entry["scanner_id"]
+ row["created_at"] = entry["created_at"]
group = reach.get(str(entry["group_root"]), GroupReach(sessions=1, people=1))
row["watch_category"] = CATEGORY_NAMES[entry["category"]]
row["watch_percentile"] = entry["percentile"]
diff --git a/products/replay_vision/backend/watch_rank_backfill.py b/products/replay_vision/backend/watch_rank_backfill.py
index 25e69b20c280..23fc3580b56e 100644
--- a/products/replay_vision/backend/watch_rank_backfill.py
+++ b/products/replay_vision/backend/watch_rank_backfill.py
@@ -1,9 +1,9 @@
"""Rank a window of past observations from the embeddings search already stored.
-The watch rank consumer ranks each new observation as its embedding lands. This fills in the observations from before
-that shipped, so a team can move to the embedding ranker without waiting a week for ranks to build
-up. It reads stored vectors from ClickHouse rather than embedding again. Feed groups and search themes
-come from the semantics backfill, which runs after this one.
+The watch rank consumer ranks each new observation as its embedding lands. This ranks the rest: observations whose
+results the consumer missed, and every observation again after the anchors change. It reads stored vectors from
+ClickHouse rather than embedding again. Feed groups and search themes come from the semantics backfill, which runs
+after this one.
"""
import time
@@ -33,6 +33,7 @@ from products.replay_vision.backend.embeddings import (
observation_scope_params,
team_embeddings_params,
)
+from products.replay_vision.backend.models.replay_highlight_feed import HighlightWindow
from products.replay_vision.backend.models.replay_observation import ObservationStatus, ReplayObservation
from products.replay_vision.backend.observation_ranks import NewRank, record_ranks
from products.replay_vision.backend.watch_rank import (
@@ -49,7 +50,8 @@ from products.replay_vision.backend.watch_rank import (
logger = structlog.get_logger(__name__)
_VECTOR_CHUNK = 500
-WATCH_RANK_BACKFILL_DAYS = 7
+# The longest highlight feed window, so a run covers everything the feed can show.
+WATCH_RANK_BACKFILL_DAYS = max(HighlightWindow.values)
_QUERY_TIMEOUT_S = 60
_CAPACITY_RETRY_BASE_S = 15
_CAPACITY_RETRY_ATTEMPTS = 6