:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-vision): rank highlights on embeddings only and retire the old rankers (#123987)
Tue Haulund4e98f96ab056
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex 09c221add67f..35920f34ac94 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -14697,21 +14697,13 @@ snapshots:scenes-app-replay-vision--experiment-variants-waiting-for-observations--light:hash: v1.k794b7964.a1555dab4c0006028e8c240451c56c6aed5e31c8a8e68c893e3d6a07b562bba6.VYeYPsjZ_p0Rt4cbZzF7lbxjxurMjLXff9kJRsy49gUscenes-app-replay-vision--home-watch-feed--dark:- hash: v1.k794b7964.7d0e2559c229b41c2dd34bb70bb0b89c2a4c7f2038c2482290e7b8f63b6dd261.znzm1HO_g6PbiSk7IXl6qOsnk9rYY-QEfTg000qIXF0+ hash: v1.k794b7964.8895978e3c94bcf7e223c6951c9feb12ca04177543d615e4b303a10f59ecdca9.tz7PmhgaOqvs8ZZXbuNNgbUPBnCaW1T9nCQXKf3rDYkscenes-app-replay-vision--home-watch-feed--light:- hash: v1.k794b7964.4d1c2d852d5380f72997ca12282df9c5194effa9f908e62b6086dd675e797e6f.9B9NfkA62DYoeVZKKFjbn8Dv9mg1zn7GXPZjGqnhr9Y- scenes-app-replay-vision--home-watch-feed-embedding-arm--dark:- hash: v1.k794b7964.ccaaf7d4f0753f587f0f9a925f24da96088e28acdd9e03439486bbac356478f8.yWzVWUCUSLPvu5-cOEucqXHc0xzNZUUL-FM7GmgCglQ- scenes-app-replay-vision--home-watch-feed-embedding-arm--light:- hash: v1.k794b7964.ef9bc52e0e31035e1ced8b94ed67348d4d31cff269e9a4c6bc8af2357b2117bd.cmduyszYcyvgTo8cLSgqyzbUI-Swod1Hqh72SJfaRBk+ hash: v1.k794b7964.917bf5474cc4fa89bfbbef376ec24a10c015a6a0806e562a18557628bb0652fa.zISmbhI7bi37HCk5YZh3Q3_khX6HV_vXCj8bCydkapgscenes-app-replay-vision--home-watch-feed-empty--dark:hash: v1.k794b7964.e8bf2a561963c96df9eb9dc6d005e192bdc64301c92c9654b7de4593ea8518c5.X43IWxVfHE9iboy6cY-VKIyo3R0N8Fej_PiO1JIyYZYscenes-app-replay-vision--home-watch-feed-empty--light:hash: v1.k794b7964.347da36a401fa205ffb5d2c388197fdf5a102eff3148438c2da416aeba65c2a7.TBRMQTWYMU9iHr1naonnIVacv_NqjtZOCWe3Yc6__UU- scenes-app-replay-vision--home-watch-feed-jev-arm--dark:- hash: v1.k794b7964.dded93992064551ce579a8a07c96b7414e3d83d33104497a391a88ffa0bf15bd.J9WyZjcnl0CsP0t2Ml9Ic9wv6IWl6XlpIjV9m0OOFvk- scenes-app-replay-vision--home-watch-feed-jev-arm--light:- hash: v1.k794b7964.029b6b85f20f538d24a908cd7fdb8322aef5087f69103a81b10fd3d3bfbea887.lmw6vqb-wXrFbsAMle9av0BP2tkB9CyhK1IL9xMmDa8scenes-app-replay-vision--home-watch-feed-only-newest--dark:hash: v1.k794b7964.42e9a57b761186ae53e6d6a9aad97b59bc0a71a572ee9c6977b733818930ced7.Ab1APElWbvJ9mvPg5s-ezcdcRb1VTakv0FDgE3i-VfAscenes-app-replay-vision--home-watch-feed-only-newest--light:diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsxindex fb7bc8a22efa..578481ea3b19 100644--- a/frontend/src/lib/constants.tsx+++ b/frontend/src/lib/constants.tsx@@ -575,7 +575,6 @@ export const FEATURE_FLAGS = {UX_REMOVE_SIDEPANEL: 'ux-remove-sidepanel', // owner: #team-surveysVISION_ENTRYPOINT_REPLAY_FILTERS: 'vision-entrypoint-replay-filters', // owner: #team-replay, cross-sell entry point from the replay filters panelVISION_EXPERIMENT_SCANNER: 'experiment-scanner', // owner: #team-replay, experiment scanners overall: the type in the scanner editor, creating one from experiments, and the variants tab- VISION_WATCH_FEED_RANKER: 'vision-watch-feed-ranker', // owner: #team-replay multivariate=control,jev-shadow,jev,embedding. Project-aggregated; compare the variant stringVISUAL_REVIEW: 'visual-review', // owner: #team-devexWAREHOUSE_MULTI_DESTINATION: 'warehouse-multi-destination', // owner: #team-warehouse-sourcesWAREHOUSE_PERSON_PROPERTIES: 'warehouse-person-properties', // owner: @tomowers #team-warehouse-sources, gates warehouse -> person properties (person-target custom property sources)diff --git a/posthog/models/activity_logging/activity_log.py b/posthog/models/activity_logging/activity_log.pyindex 1cf65a3a9cff..6da852b85108 100644--- a/posthog/models/activity_logging/activity_log.py+++ b/posthog/models/activity_logging/activity_log.py@@ -559,10 +559,6 @@ replay_scanner_machine_fields = ["estimated_monthly_observations","estimated_at","estimate_attempted_at",- "search_suggestions",- "search_suggestions_watermark",- "search_suggestions_generated_at",- "search_last_viewed_at","prompt_question","prompt_question_source","prompt_valence",diff --git a/posthog/temporal/schedule.py b/posthog/temporal/schedule.pyindex bf044637a667..7cdee4807f85 100644--- a/posthog/temporal/schedule.py+++ b/posthog/temporal/schedule.py@@ -132,7 +132,6 @@ from products.replay_vision.backend.temporal.estimates import create_replay_visifrom products.replay_vision.backend.temporal.gemini_cleanup_sweep import (create_replay_vision_gemini_cleanup_sweep_schedule,)-from products.replay_vision.backend.temporal.jev_watch_rank import create_replay_vision_jev_watch_rank_schedulefrom products.replay_vision.backend.temporal.learned_rules import create_replay_vision_learned_rules_schedulefrom products.replay_vision.backend.temporal.read_meter import create_replay_vision_read_meter_schedulefrom products.replay_vision.backend.temporal.reconciler import create_replay_vision_reconciler_schedule@@ -168,10 +167,15 @@ async def cleanup_sync_vectors_schedule(client: Client):await a_delete_schedule(client, "ai-sync-vectors-schedule")-async def cleanup_replay_vision_search_suggestions_schedule(client: Client):- """Retired: the search suggestion refresher is gone, and its in-flight run ends on its own execution timeout."""- if await a_schedule_exists(client, "replay-vision-search-suggestions-refresher-schedule"):- await a_delete_schedule(client, "replay-vision-search-suggestions-refresher-schedule")+async def cleanup_retired_replay_vision_schedules(client: Client):+ """Delete Replay Vision schedules whose workflows are gone. Any in-flight run ends on its own execution timeout."""+ retired_schedule_ids = [+ "replay-vision-search-suggestions-refresher-schedule",+ "replay-vision-jev-watch-rank-schedule",+ ]+ for schedule_id in retired_schedule_ids:+ if await a_schedule_exists(client, schedule_id):+ await a_delete_schedule(client, schedule_id)async def create_run_quota_limiting_schedule(client: Client):@@ -1025,11 +1029,10 @@ schedules = [create_membership_coordinator_schedule,create_replay_vision_reconciler_schedule,create_replay_vision_estimates_schedule,- cleanup_replay_vision_search_suggestions_schedule,+ cleanup_retired_replay_vision_schedules,create_replay_vision_learned_rules_schedule,create_vision_alert_check_schedule,create_replay_vision_read_meter_schedule,- create_replay_vision_jev_watch_rank_schedule,create_github_job_logs_coordinator_schedule,create_review_hog_finding_outcomes_schedule,create_ci_signals_coordinator_schedule,diff --git a/products/replay_vision/backend/api/observations.py b/products/replay_vision/backend/api/observations.pyindex 54608330fdf1..f696eed1cb24 100644--- a/products/replay_vision/backend/api/observations.py+++ b/products/replay_vision/backend/api/observations.py@@ -198,7 +198,7 @@ class ReplayObservationLabelSerializer(serializers.Serializer):class ObservationFeedDismissalSerializer(serializers.Serializer):- """Why the calling user marked an observation not useful in What to watch."""+ """Why the calling user marked an observation not useful in the Highlights feed."""reason = serializers.ChoiceField(choices=FeedDismissalReason.choices,@@ -1269,7 +1269,7 @@ class ReplayObservationViewSet(request=ObservationFeedDismissalSerializer,responses={204: None},description=(- "Mark this observation not useful for the calling user, with an optional reason. What to watch stops "+ "Mark this observation not useful for the calling user, with an optional reason. The Highlights feed stops ""showing it to that user. Sending it again replaces the reason."),)@@ -1277,7 +1277,7 @@ class ReplayObservationViewSet(methods=["DELETE"],request=None,responses={204: None},- description="Undo the calling user's not-useful mark, so What to watch can show the observation again.",+ description="Undo the calling user's not-useful mark, so the Highlights feed can show the observation again.",)@action(detail=True,diff --git a/products/replay_vision/backend/api/scanners.py b/products/replay_vision/backend/api/scanners.pyindex a6c521c723bf..6937e6434e27 100644--- a/products/replay_vision/backend/api/scanners.py+++ b/products/replay_vision/backend/api/scanners.py@@ -65,7 +65,6 @@ from posthog.rate_limit import (ReplayVisionWatchFeedSustainedRateThrottle,)-from products.access_control.backend.facade.user_access_control import UserAccessControlfrom products.access_control.backend.presentation.access_control import (AccessControlViewSetMixin,UserAccessControlSerializerMixin,@@ -94,14 +93,6 @@ from products.replay_vision.backend.impact import (compute_scanner_impact,create_affected_cohort,)-from products.replay_vision.backend.jev_watch_feed import (- JEV_WATCHABLE_MIN,- JevWatchReason,- load_watch_ranks,- rank_watch_feed_by_jev,- ranker_mode,- watch_feed_ranker_variant,-)from products.replay_vision.backend.models.replay_observation import (ObservationStatus,ObservationTrigger,@@ -188,7 +179,7 @@ from products.replay_vision.backend.session_limits import MAX_SESSION_ID_LENGTHfrom products.replay_vision.backend.temporal.constants import VISION_SIGNALS_SOURCE_PRODUCT, VISION_SIGNALS_SOURCE_TYPEfrom products.replay_vision.backend.temporal.metrics import record_estimate_outcome, record_scanner_limit_reachedfrom products.replay_vision.backend.temporal.read_meter_types import current_sweep_throttle_factor-from products.replay_vision.backend.watch_feed import WATCH_FEED_MIN_ITEMS, rank_watch_feed_candidates+from products.replay_vision.backend.watch_feed import WATCH_FEED_MIN_ITEMSfrom products.replay_vision.backend.watch_feed_impressions import feed_impression_counts, record_feed_impressionsfrom products.replay_vision.backend.watch_rank import (rank_watch_feed_by_embedding,@@ -1659,18 +1650,10 @@ class ScannerStatsResponseSerializer(serializers.Serializer):WATCH_FEED_DEFAULT_LIMIT = 20WATCH_FEED_MAX_LIMIT = 50WATCH_FEED_DEFAULT_DATE_FROM = "-7d"-# Widest window one request may rank. A scanner with few rows in the window walks its whole window before the cap stops it.+# Widest window one request may rank.WATCH_FEED_MAX_WINDOW_DAYS = 90-# Rank at most this many of the newest succeeded observations in the window. A very high-volume-# team ranks only the newest slice, which recency-biases the feed — acceptable for a preview surface.+# The most picks one load ranks, and the furthest `offset` may reach.WATCH_FEED_CANDIDATE_CAP = 1000-# Within that slice, one scanner contributes at most this many rows. Without it, a busy scanner fills-# the whole cap and quiet scanners lose not just feed slots but their own baselines — "unusual for-# this scanner lately" silently stops firing for exactly the scanners the cap crowded out.-WATCH_FEED_PER_SCANNER_CAP = 100-# Ids per query when the jev feed fetches cached watchable rows the recency slice cut off. Keeps-# each id__in list bounded while the walk stops as soon as the page is covered.-WATCH_FEED_BYPASS_CHUNK = 200WATCH_FEED_IMPRESSIONS_MAX_IDS = 100# Loads in one bucket share their picks, so a reload or tab switch skips the pick query.WATCH_FEED_PICKS_BUCKET_S = 10 * 60@@ -1678,45 +1661,6 @@ WATCH_FEED_PICKS_BUCKET_S = 10 * 60WATCH_FEED_PICKS_CACHE_TTL_S = WATCH_FEED_PICKS_BUCKET_S + 15 * 60-def select_watch_feed_candidates(- walkable: QuerySet[ReplayObservation],- scanner_ids: list[UUID],- viewed: Exists,- access: UserAccessControl,- team_id: int,-) -> list[dict[str, Any]]:- """The newest readable rows of the window, at most WATCH_FEED_PER_SCANNER_CAP per scanner, without scan results."""- # One slice per scanner stops each index walk at the cap, where a window function sorts the whole window first.- slices: list[Any] = [- walkable.filter(scanner_id=scanner_id)- .order_by("-created_at", "-id")- .values_list("id", "created_at")[:WATCH_FEED_PER_SCANNER_CAP]- for scanner_id in scanner_ids- ]- if not slices:- return []- walked = slices[0].union(*slices[1:], all=True)- newest_first = [row_id for row_id, _ in sorted(walked, key=lambda row: (row[1], row[0]), reverse=True)]- # The row-gate reads each row's snapshot JSON, so it runs on the newest walked rows only.- rows: list[dict[str, Any]] = []- for start in range(0, len(newest_first), WATCH_FEED_CANDIDATE_CAP):- chunk = newest_first[start : start + WATCH_FEED_CANDIDATE_CAP]- rows += cast(- list[dict[str, Any]],- list(- accessible_observations(- access, team_id, ReplayObservation.objects.filter(team_id=team_id, id__in=chunk)- )- .annotate(feed_viewed=viewed)- .order_by("-created_at", "-id")- .values("id", "scanner_id", "session_id", "created_at", "watch_rank", "feed_viewed")- ),- )- if len(rows) >= WATCH_FEED_CANDIDATE_CAP:- break- return rows[:WATCH_FEED_CANDIDATE_CAP]--def refresh_feed_viewed(rows: list[dict[str, Any]], viewer_id: int, as_of: datetime) -> list[dict[str, Any]]:"""Cached rows with `feed_viewed` as of `as_of`."""viewed = set(@@ -1727,23 +1671,6 @@ def refresh_feed_viewed(rows: list[dict[str, Any]], viewer_id: int, as_of: datetreturn [{**row, "feed_viewed": row["id"] in viewed} for row in rows]-def attach_scanner_results(team_id: int, rows: list[dict[str, Any]]) -> None:- """Load the scan result for each candidate row, by primary key, after the window cap has applied."""- results = dict(- ReplayObservation.objects.filter(team_id=team_id, id__in=[row["id"] for row in rows]).values_list(- "id", "scanner_result"- )- )- for row in rows:- row["scanner_result"] = results.get(row["id"])---class WatchFeedRanker(models.TextChoices):- WEIGHTED_SCORE = "weighted-score"- JEV = "jev"- EMBEDDING = "embedding"--class WatchFeedCategory(models.TextChoices):FAILURE = "failure"STRUGGLE = "struggle"@@ -1751,15 +1678,6 @@ class WatchFeedCategory(models.TextChoices):class WatchFeedReason(models.TextChoices):- SIGNAL_EMITTED = "signal_emitted"- UNUSUAL_VERDICT = "unusual_verdict"- NOTABLE = "notable"- VERDICT_YES = "verdict_yes"- OUTLIER_SCORE = "outlier_score"- RARE_TAG = "rare_tag"- NOVEL_SUMMARY = "novel_summary"- FRICTION = "friction"- JEV_WATCHABLE = "jev_watchable"EMBEDDING_PICK = "embedding_pick"UNVIEWED_RECENT = "unviewed_recent"RECENT = "recent"@@ -1804,14 +1722,6 @@ class WatchFeedQuerySerializer(serializers.Serializer):required=False,help_text="Comma-separated user IDs. Restrict the feed to observations from scanners those users created.",)- search = serializers.CharField(- required=False,- help_text=(- "Case-insensitive text to match against the scan's own words (title, summary, reasoning, and the "- "notability sentence) and the scanner's name. Applied before ranking, so it searches the whole "- "window rather than the items that would have surfaced without it."- ),- )limit = serializers.IntegerField(required=False,default=WATCH_FEED_DEFAULT_LIMIT,@@ -1858,104 +1768,17 @@ class WatchFeedQuerySerializer(serializers.Serializer):return [int(raw_id) for raw_id in raw_ids]-class WatchFeedSignalSerializer(serializers.Serializer):- """One signal an observation raised, named rather than counted."""-- problem_type = serializers.CharField(help_text="Issue type: `bug`, `crash`, `design_flaw`, or `ux_friction`.")- headline = serializers.CharField(- help_text="The finding in a few words, written by the scan. The full description lives on the signal itself."- )--class WatchFeedReasonSerializer(serializers.Serializer):"""Machine-readable reason an observation made the feed; the frontend renders the copy."""kind = serializers.ChoiceField(choices=WatchFeedReason.choices,help_text=(- "Highest-priority rule the observation satisfied: `signal_emitted` (it pushed a signal), "- "`unusual_verdict` (a monitor answer that is the minority for that scanner this window), "- "`verdict_yes` (a monitor hit, when the window is too thin to know which answer is unusual), "- "`outlier_score` (far from the scanner's window average), "- "`rare_tag` (a tag uncommon for the scanner this window), `novel_summary` (a summary that "- "reads unlike the scanner's other sessions this window), `notable` (the scan itself judged the "- "session worth watching), `friction` (the scan describes errors, retries, or dead ends), "- "`jev_watchable` (the decision model judged the session worth watching; teams on the "- "Jev ranker experiment only), "- "`embedding_pick` (the session reads like a failure, struggle, or revenue moment and ranks in "- "its scanner's top fifth; teams on the embedding ranker only), "+ "Why the observation is in the feed: `embedding_pick` (the session reads like a failure, struggle, "+ "or revenue moment and ranks in its scanner's top fifth), ""`unviewed_recent` (new to you), `recent` (nothing special, newest available)."),)- signals_count = serializers.IntegerField(- required=False, allow_null=True, help_text="Signals this observation emitted, for `signal_emitted`."- )- problem_types = serializers.ListField(- child=serializers.CharField(),- required=False,- help_text=(- "Issue type of each emitted signal (`bug`, `crash`, `design_flaw`, `ux_friction`), one entry per "- "signal in the order raised, for `signal_emitted`. Absent on signals scanned before this shipped."- ),- )- signals = WatchFeedSignalSerializer(- many=True,- required=False,- help_text=(- "Each emitted signal in the order raised, for `signal_emitted`. Carries what the card needs to name "- "the findings instead of counting them. Absent on sessions scanned before this shipped, which carry "- "`problem_types` alone."- ),- )- verdict = serializers.CharField(- required=False, allow_null=True, help_text="The monitor's answer, for `unusual_verdict`."- )- verdict_share = serializers.FloatField(- required=False,- allow_null=True,- help_text="Share (0-1) of the scanner's window observations with this answer, for `unusual_verdict`.",- )- notability = serializers.FloatField(- required=False,- allow_null=True,- help_text="The scan's own 0-1 judgment of how much a team would benefit from watching, for `notable`.",- )- jev_probability = serializers.FloatField(- required=False,- allow_null=True,- help_text="The decision model's 0-1 judgment that the session is worth watching, for `jev_watchable`.",- )- notability_reason = serializers.CharField(- required=False,- allow_null=True,- help_text=(- "The scan's own sentence naming why the session is worth watching. Present on the `notable` and "- "`jev_watchable` reason kinds when the scan itself found the session notable, and preferred over "- "copy derived from the reason kind. Absent on observations scanned before notability shipped."- ),- )- watch_reason = serializers.ChoiceField(- choices=JevWatchReason.choices,- required=False,- help_text=(- "Why the decision model rated the session worth watching, picked from a fixed list, for "- "`jev_watchable`. Absent when no reason on the list fits, or on sessions judged before reasons shipped."- ),- )- score = serializers.FloatField(- required=False, allow_null=True, help_text="The observation's score, for `outlier_score`."- )- window_mean = serializers.FloatField(- required=False, allow_null=True, help_text="The scanner's mean score in the window, for `outlier_score`."- )- tag = serializers.CharField(- required=False, allow_null=True, help_text="The rare tag that ranked the observation, for `rare_tag`."- )- tag_share = serializers.FloatField(- required=False,- allow_null=True,- help_text="Share (0-1) of the scanner's window observations carrying `tag`, for `rare_tag`.",- )watch_category = serializers.ChoiceField(choices=WatchFeedCategory.choices,required=False,@@ -1973,7 +1796,7 @@ class WatchFeedReasonSerializer(serializers.Serializer):required=False,allow_null=True,help_text=(- "Distinct people across the session's duplicate group in the window, for `embedding_pick`. A finding "+ "Distinct people across the session's feed group in the window, for `embedding_pick`. A finding ""that reached more people ranks higher."),)@@ -2004,24 +1827,6 @@ class WatchFeedResponseSerializer(serializers.Serializer):as_of = serializers.DateTimeField(help_text="The time this feed ranks as of. Send it back with later pages.",)- ranker = serializers.ChoiceField(- choices=WatchFeedRanker.choices,- help_text=(- "Which ranker ordered this feed: `jev` ranks on the decision model's cached judgments, "- "`embedding` on the rank the watch rank consumer wrote from the observation's embedding, "- "`weighted-score` on the deterministic blend. The arm is decided server-side per team, so "- "clients read it from here rather than evaluating the flag themselves."- ),- )- ranker_variant = serializers.CharField(- allow_null=True,- help_text=(- "The team's variant of the `vision-watch-feed-ranker` experiment flag (`control`, `jev-shadow`, "- "`jev`, `embedding`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from "- "control. Clients report it on the feed-viewed event as `$feature/vision-watch-feed-ranker`, "- "which is the exposure the experiment counts."- ),- )class WatchFeedImpressionsSerializer(serializers.Serializer):@@ -2035,7 +1840,7 @@ class WatchFeedImpressionsSerializer(serializers.Serializer):min_length=1,max_length=WATCH_FEED_IMPRESSIONS_MAX_IDS,help_text=(- f"Observations the calling user saw in What to watch, at most {WATCH_FEED_IMPRESSIONS_MAX_IDS}. "+ f"Observations the calling user saw in the Highlights feed, at most {WATCH_FEED_IMPRESSIONS_MAX_IDS}. ""Ids the user cannot read are ignored."),)@@ -2634,7 +2439,7 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vithrottle_classes=[ReplayVisionWatchFeedBurstRateThrottle, ReplayVisionWatchFeedSustainedRateThrottle],)def watch_feed(self, request: Request, **kwargs: Any) -> Response:- """Succeeded observations in the window worth watching, ranked — feeds the What to watch tab."""+ """Succeeded observations in the window worth watching, ranked for the Highlights tab."""query = WatchFeedQuerySerializer(data=request.query_params)query.is_valid(raise_exception=True)params = query.validated_data@@ -2681,7 +2486,7 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi).values_list("id", flat=True))allowed_ids = [scanner_id for scanner_id in allowed_ids if scanner_id in created_ids]- # The scanner list stays off this base: every per-scanner walk would otherwise carry the whole list.+ # The scanner list stays off this base: the pick query adds it once per scanner in its lateral join.candidates = ReplayObservation.objects.filter(team_id=self.team_id,status=ObservationStatus.SUCCEEDED,@@ -2691,20 +2496,8 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi)if params.get("scanner_type"):candidates = candidates.filter(scanner_snapshot__scanner_type=params["scanner_type"])- if params.get("search"):- # Applied before ranking so the box searches the whole window, not the slice that would have- # surfaced anyway. Unindexed, but the candidate query is already bounded by team, readable- # scanners, succeeded status and the date window. Lookup keys are literals, not caller input.- term = params["search"]- candidates = candidates.filter(- Q(scanner_result__model_output__reasoning__icontains=term)- | Q(scanner_result__model_output__summary__icontains=term)- | Q(scanner_result__model_output__title__icontains=term)- | Q(scanner_result__model_output__notability_reason__icontains=term)- | Q(scanner_snapshot__name__icontains=term)- )viewer_id = cast(User, request.user).id- # Before ranking, so a row the viewer marked not useful neither takes a slot nor shifts the window baselines.+ # Before ranking, so a row the viewer marked not useful never takes a slot.# Views and dismissals after `as_of` wait for the next load, so later pages of this one do not shift.dismissed_ids = sorted(ReplayObservationFeedDismissal.objects.filter(user_id=viewer_id, created_at__lte=as_of).values_list(@@ -2712,118 +2505,61 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi))walkable = candidates.exclude(id__in=dismissed_ids)- # Row-gate on each row's snapshot experiment before ranking, so a restricted row can't take a feed slot.- candidates = accessible_observations(- self.user_access_control, self.team_id, walkable.filter(scanner_id__in=allowed_ids)- )viewed = Exists(ReplayObservationView.objects.filter(observation_id=OuterRef("id"), user_id=viewer_id, created_at__lte=as_of))- # The flag selects one of two independent rankers; nothing is blended between them. Shadow- # teams rank on the weighted score too, because only the `jev` arm reads the probabilities- # the hourly sweep cached. Neither arm makes a model call here. The response names the- # ranker that ordered it, so the shadow arm reads as weighted-score to the client, and the- # flag variant separately, which the client reports as the experiment's exposure.- variant = watch_feed_ranker_variant(self.team_id, self.team.uuid)- mode = ranker_mode(variant)- ranker = mode if mode in ("jev", "embedding") else "weighted-score"- if ranker == "embedding":- # Keyed on the bucket `as_of` falls in, so loads in one bucket share picks, and on the dismissals and the- # readable experiments, so a dismissal or an access change ranks afresh.- picks_key = (- "replay_vision:watch_picks:"- + hashlib.sha256(- json.dumps(- [- self.team_id,- viewer_id,- int(as_of.timestamp()) // WATCH_FEED_PICKS_BUCKET_S,- params["date_from"],- params.get("date_to"),- list(map(str, dismissed_ids)),- sorted(readable_experiment_ids(self.user_access_control, self.team_id)),- sorted(map(str, allowed_ids)),- params.get("search"),- params.get("scanner_type"),- ]- ).encode()- ).hexdigest()- )- cached = cache.get(picks_key)- # Picks ranked as of a later moment may hold rows this load's first page never saw, so they don't serve it.- if cached is not None and cached[0] <= as_of:- picked_as_of, candidate_rows = cached- # An earlier load in the bucket ranked these, so its views may be older than this one's.- if picked_as_of != as_of:- candidate_rows = refresh_feed_viewed(candidate_rows, viewer_id, as_of)- else:- # The embedding ranker reads only a small stored rank, so it picks over the whole window in SQL.- candidate_rows = select_embedding_picks(- walkable, allowed_ids, viewed, self.user_access_control, self.team_id, WATCH_FEED_CANDIDATE_CAP- )- # Picks in one duplicate group collapse into one card, so the floor counts groups.- pick_roots = {row["watch_group_root"] for row in candidate_rows}- if len(pick_roots) < WATCH_FEED_MIN_ITEMS:- candidate_rows += select_embedding_filler(- walkable, allowed_ids, viewed, self.user_access_control, self.team_id, pick_roots- )- # The first load in the bucket keeps the entry, so concurrent loads never replace it with later picks.- cache.add(picks_key, (as_of, candidate_rows), WATCH_FEED_PICKS_CACHE_TTL_S)+ # Keyed on the bucket `as_of` falls in, so loads in one bucket share picks, and on the dismissals and the+ # readable experiments, so a dismissal or an access change ranks afresh.+ picks_key = (+ "replay_vision:watch_picks:"+ + hashlib.sha256(+ json.dumps(+ [+ self.team_id,+ viewer_id,+ int(as_of.timestamp()) // WATCH_FEED_PICKS_BUCKET_S,+ params["date_from"],+ params.get("date_to"),+ list(map(str, dismissed_ids)),+ sorted(readable_experiment_ids(self.user_access_control, self.team_id)),+ sorted(map(str, allowed_ids)),+ params.get("scanner_type"),+ ]+ ).encode()+ ).hexdigest()+ )+ cached = cache.get(picks_key)+ # Picks ranked as of a later moment may hold rows this load's first page never saw, so they don't serve it.+ if cached is not None and cached[0] <= as_of:+ picked_as_of, candidate_rows = cached+ # An earlier load in the bucket ranked these, so its views may be older than this one's.+ if picked_as_of != as_of:+ candidate_rows = refresh_feed_viewed(candidate_rows, viewer_id, as_of)else:- candidate_rows = select_watch_feed_candidates(- walkable, allowed_ids, viewed, self.user_access_control, self.team_id- )- attach_scanner_results(self.team_id, candidate_rows)- page_end = params["offset"] + params["limit"]- if ranker == "embedding":- impressions = feed_impression_counts(- team_id=self.team_id,- user_id=viewer_id,- observation_ids=[row["id"] for row in candidate_rows],- as_of=as_of,- )- for row in candidate_rows:- row["feed_impressions"] = impressions.get(row["id"], 0)- ranked = rank_watch_feed_by_embedding(candidate_rows)- elif ranker == "jev":- ranks = load_watch_ranks(self.team_id, allowed_ids)- probabilities = ranks.probabilities- jev_rows = list(candidate_rows)- # The recency slice above holds only each scanner's newest rows, which on a high-volume- # scanner covers minutes. The sweep judged the whole window, so fetch the watchable rows- # the slice cut off, walking the cache in probability-descending chunks. Each chunk goes- # through `candidates` — team, scanner, date, and search — so nothing outside the- # request's scope can enter, and a cached id that matches no candidate row cannot use up- # the page: slicing the cache before filtering would let high-probability non-matches- # push out matching rows. The walk stops once the page is covered, and the cache itself- # bounds it (at most a window's judged rows per readable scanner).- watchable_missing = sorted(- (- UUID(observation_id)- for observation_id, probability in probabilities.items()- if probability >= JEV_WATCHABLE_MIN- ),- key=lambda observation_id: probabilities[str(observation_id)],- reverse=True,+ # The embedding ranker reads only a small stored rank, so it picks over the whole window in SQL.+ candidate_rows = select_embedding_picks(+ walkable, allowed_ids, viewed, self.user_access_control, self.team_id, WATCH_FEED_CANDIDATE_CAP)- loaded_ids = {row["id"] for row in jev_rows}- missing = [observation_id for observation_id in watchable_missing if observation_id not in loaded_ids]- needed = page_end- for start in range(0, len(missing), WATCH_FEED_BYPASS_CHUNK):- if needed <= 0:- break- fetched = list(- candidates.filter(id__in=missing[start : start + WATCH_FEED_BYPASS_CHUNK])- .annotate(feed_viewed=viewed)- .values("id", "scanner_id", "created_at", "scanner_result", "feed_viewed")+ # Picks in one feed group collapse into one card, so the floor counts groups.+ pick_roots = {row["watch_group_root"] for row in candidate_rows}+ if len(pick_roots) < WATCH_FEED_MIN_ITEMS:+ candidate_rows += select_embedding_filler(+ walkable, allowed_ids, viewed, self.user_access_control, self.team_id, pick_roots)- jev_rows += fetched- needed -= len(fetched)- ranked = rank_watch_feed_by_jev(jev_rows, probabilities, ranks.reasons)- else:- ranked = rank_watch_feed_candidates(candidate_rows)+ # The first load in the bucket keeps the entry, so concurrent loads never replace it with later picks.+ cache.add(picks_key, (as_of, candidate_rows), WATCH_FEED_PICKS_CACHE_TTL_S)+ page_end = params["offset"] + params["limit"]+ impressions = feed_impression_counts(+ team_id=self.team_id,+ user_id=viewer_id,+ observation_ids=[row["id"] for row in candidate_rows],+ as_of=as_of,+ )+ for row in candidate_rows:+ row["feed_impressions"] = impressions.get(row["id"], 0)+ ranked = rank_watch_feed_by_embedding(candidate_rows)has_more = len(ranked) > page_endranked = ranked[params["offset"] : page_end]reasons_by_id = {entry.observation_id: entry.reason for entry in ranked}@@ -2855,8 +2591,6 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi"results": results,"has_more": has_more,"as_of": as_of,- "ranker": ranker,- "ranker_variant": variant,})@@ -2870,7 +2604,7 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vithrottle_classes=[ReplayVisionWatchFeedImpressionsRateThrottle],)def watch_feed_impressions(self, request: Request, **kwargs: Any) -> Response:- """Record that the calling user saw these cards in What to watch, so the feed ranks them lower next time."""+ """Record that the calling user saw these cards in the Highlights feed, so the feed ranks them lower next time."""body = WatchFeedImpressionsSerializer(data=request.data)body.is_valid(raise_exception=True)if not self.user_access_control.check_access_level_for_resource("session_recording", required_level="viewer"):diff --git a/products/replay_vision/backend/jev_watch_feed.py b/products/replay_vision/backend/jev_watch_feed.pydeleted file mode 100644index 206dd34f5947..000000000000--- a/products/replay_vision/backend/jev_watch_feed.py+++ /dev/null@@ -1,854 +0,0 @@-"""Jev ranking for the What to watch feed, as an alternative to the weighted score.--The multivariate `vision-watch-feed-ranker` flag selects one of two independent rankers per team:--- `weighted-score` (default): the deterministic blend in `watch_feed.py`, unchanged, no Jev calls.-- `jev-shadow`: the hourly sweep judges and caches probabilities, but the feed still ranks on the- weighted score. This arm exists to collect probabilities and cost data before any feed changes.-- `jev`: the feed ranks on the cached probabilities alone (`rank_watch_feed_by_jev`).--Jev, the shared decision model behind the ml_inference facade, judges each observation once, in-company: one request carries a chunk of observations as the state and one yes/no question per-observation, with already-judged rows padding a short chunk as context, so every judgment sees the-scanner's own recent sessions rather than standing alone. A second request over the same state asks-one multiple-choice question per watchable row, so the feed can say why Jev picked it-(`JevWatchReason`). The hourly Temporal sweep-(`temporal/jev_watch_rank/`) judges only the rows without a cached probability and merges the-results into a Redis cache, so coverage accumulates across sweeps at a bounded hourly cost whatever-the scanner's volume. The cache exists because the feed API is synchronous over up to 1,000 rows-and must not make model calls, and the scan pipeline must not either. The two rankers share only the-`WatchFeedEntry` shape and the feed's presentation contract (the filler floor and the notable-reason-bar), so switching the flag switches the whole ranking, not one component of it.-"""--import json-import math-from collections.abc import Callable, Collection-from datetime import UTC, datetime, timedelta-from time import monotonic, perf_counter-from typing import Any, Literal-from uuid import UUID, uuid4--from django.db import models--import redis-import structlog-from prometheus_client import Counter, Histogram--from posthog.dataclasses import frozen-from posthog.llm.gateway_client import GatewayNotConfiguredError-from posthog.ph_client import get_feature_flag_or_none-from posthog.redis import get_client--from products.ml_inference.backend.facade import api as decision_api-from products.ml_inference.backend.facade.contracts import (- MAX_QUESTIONS_PER_REQUEST,- ChoiceAnswer,- DecisionGatewayError,- DecisionQuestion,- DecisionRequest,- JsonValue,- NoulAnswer,-)-from products.ml_inference.backend.facade.enums import DecisionQuestionType-from products.replay_vision.backend.watch_feed import NOTABLE_MIN_SCORE, WatchFeedEntry, finish_feed--logger = structlog.get_logger(__name__)--WATCH_FEED_RANKER_FLAG = "vision-watch-feed-ranker"-RankerMode = Literal["weighted-score", "jev-shadow", "jev", "embedding"]-# The JevK5 build PostHog hosts on the AI gateway, which is the only backend the ml_inference-# facade reaches. The vendor (typesafe.ai) is not approved for customer data, and observation prose-# is customer data, so never point this at a vendor model id.-JEV_MODEL = "posthog/hogference/jevk5-fp8-0.2"-JEV_INPUT_USD_PER_MILLION = 0.042-# A chunk request carries up to WINDOW_CHUNK_SIZE questions over a multi-observation state, so it-# runs far longer than the single-question decisions Signals times out at 3s. Matches the gateway-# client's own default; the sweep is background work, so latency is cheap and a timeout loses a-# whole chunk's judgments.-JEV_TIMEOUT_SECONDS = 30.0-# Gateway refusals the batch itself can cause, so a retry fails the same way: a malformed request,-# a payload past the gateway's limit, unprocessable content. An allowlist, because the two mistakes-# cost differently: wrongly charging parks rows as judged-with-no-score for the cache entry's life,-# wrongly not charging re-buys one batch next sweep. Auth and routing refusals, rate limits, server-# errors, contract breaks reported as 200, and unknown statuses all retry free.-_BATCH_FAULT_STATUSES = frozenset({400, 413, 422})-# The gateway's rate limit. Every product's Jev calls share that limit, so a sweep that keeps-# sending after it would take capacity from features people are waiting on.-_RATE_LIMITED_STATUS = 429-# How far a viewed row drops on the 0-1 probability scale, the same intent as WATCH_SEEN_PENALTY in-# the weighted ranker: an unviewed peer with comparable evidence comes first, and a very strong seen-# row still holds its place above weak unseen rows.-JEV_SEEN_PENALTY = 0.3-# Below this probability a judged row carries no evidence: it falls to the same recency filler tier-# as an unjudged row, so its card never claims the model judged it worth watching, and a judged-low-# row cannot outrank a fresh observation the sweep has not seen yet. Tier membership uses the raw-# probability; the seen penalty only orders rows inside the evidence tier. JevK5 compresses the-# scale: it scores sessions with real friction near 0.3 and routine sessions near 0.1, so a higher-# bar empties the tier. Recalibrate from the judged event's watchability quantiles and top_scored-# sample, and bump the cache version below with any change here.-JEV_WATCHABLE_MIN = 0.3-# Observations per Jev request: the request carries the chunk as shared state and one question per-# observation, so a judgment is relative to its chunk, not to the whole window at once. Half the-# facade's per-request question cap: a request's prompt grows with its state, and a full 32-entry-# state was larger than the decision service admits in flight on one host, so while it ran that host-# refused every other product's requests.-WINDOW_CHUNK_SIZE = MAX_QUESTIONS_PER_REQUEST // 2-# About how many estimated prompt tokens one request may carry. A decision-service host admits-# 12,000 prompt tokens in flight and refuses callers past that, so a ~10k request fills most of a-# host by itself and the refusals land on whichever product calls next. Half the admission budget-# leaves room beside our request. Estimated at ~4 JSON characters per token; the judged event's-# input_tokens carries the measured sizes to calibrate against.-REQUEST_TOKEN_TARGET = 5_000---def _text_tokens(text: str) -> int:- """Estimated prompt tokens of a piece of request text."""- return max(1, len(text) // 4)---def _entry_tokens(entry: dict[str, Any]) -> int:- """Estimated prompt tokens one state entry adds to a request."""- return _text_tokens(json.dumps(entry, separators=(",", ":")))---def _pack_chunks(- entries: list[tuple[str, dict[str, Any]]], tokens_per_question: int-) -> list[tuple[list[tuple[str, dict[str, Any]]], int]]:- """Entries packed into request-sized chunks, each returned with its entries' token estimate: at- most WINDOW_CHUNK_SIZE questions and about REQUEST_TOKEN_TARGET estimated tokens each — counting- the question each entry brings, since its instructions ride along per question — so no request- can fill a decision-service host by itself however verbose the entries run. An entry over the- target alone still ships, alone."""- chunks: list[tuple[list[tuple[str, dict[str, Any]]], int]] = []- current: list[tuple[str, dict[str, Any]]] = []- current_tokens = 0- for entry_id, entry in entries:- tokens = _entry_tokens(entry) + tokens_per_question- if current and (len(current) >= WINDOW_CHUNK_SIZE or current_tokens + tokens > REQUEST_TOKEN_TARGET):- chunks.append((current, current_tokens))- current, current_tokens = [], 0- current.append((entry_id, entry))- current_tokens += tokens- if current:- chunks.append((current, current_tokens))- return chunks---# Per-entry prose caps, so a chunk state stays small and per-chunk latency predictable-# whatever the scanner's configured summary length.-_MAX_TITLE_CHARS = 300-_MAX_PROSE_CHARS = 1500-_MAX_TAGS = 20-# The judged set records sub-threshold rows without their scores, so a JEV_WATCHABLE_MIN change-# only reaches rows judged after it. Bump the version with the threshold: the sweep re-judges-# every window into a fresh cache, and the old keys lapse by TTL.-_WATCH_RANK_REDIS_PREFIX = "replay-vision:jev-watch-rank:v2:"-# Judgments are append-only per observation and the sweep prunes entries that leave the window, so-# a long lifetime is resilience, not staleness: the cache survives a day of failed sweeps before-# the feed falls back to the recency filler tier and coverage rebuilds at the judging cap per hour.-WATCH_RANK_TTL = timedelta(hours=25)--# User text stays in the request state; these instructions refer to it by observation id only, so-# session prose cannot become an instruction (the rule from posthog/llm/system_one.py). Each-# question is an independent probability, not a pick, and the instruction says so: without the-# "every answer may be no" sentence a relative phrasing grades the window on a curve, forcing-# winners out of an all-routine window and suppressing an all-failures one.-_WINDOW_INSTRUCTIONS = (- "The state holds recent AI scans of recorded product sessions, all from one scanner, keyed by "- "id. Judge the scan with id {index} on its own evidence: should a product team spend time "- "watching that session's recording in a 'What to watch' feed? "- "Sessions worth watching show user friction, failures, confusion, surprising behavior, or an "- "outcome unusual for this scanner. "- "Use the other scans in the state only as context for what is routine for this scanner; the "- "scans do not compete with each other. It is correct to answer no for every scan in the state "- "when nothing stands out, and to answer yes for many scans when many sessions show real "- "problems. "- "A session about a product whose subject matter is errors or debugging is not automatically "- "worth watching; only what the recorded user experienced counts."-)---class JevWatchReason(models.TextChoices):- """Why Jev rated a session worth watching, picked from a fixed list. The frontend maps each value to- the sentence on the row, so a wording change needs no re-judging."""-- VISIBLE_ERROR = "visible_error", "Error on screen"- SILENT_FAILURE = "silent_failure", "Action silently failed"- UNRESPONSIVE = "unresponsive", "Clicks went nowhere"- SLOW_OR_STUCK = "slow_or_stuck", "Slow or stuck"- BLOCKED = "blocked", "Blocked"- CANT_FIND = "cant_find", "Couldn't find it"- CONFUSED = "confused", "Confused"- WORKAROUND = "workaround", "Took a workaround"- ABANDONED = "abandoned", "Gave up"- CHURN_SIGNAL = "churn_signal", "Churn signal"- SUCCESS = "success", "Worked well"---# Jev picks this option when no reason fits. It never reaches the cache, so the row shows no reason.-_NO_WATCH_REASON = "nothing_notable"-_WATCH_REASON_CRITERIA: dict[str, str] = {- JevWatchReason.VISIBLE_ERROR.value: "An error message, toast, crash, or failed request is shown to the user.",- JevWatchReason.SILENT_FAILURE.value: (- "Something the user did did not take effect and nothing told them, such as a save that did not stick "- "or an action they had to repeat."- ),- JevWatchReason.UNRESPONSIVE.value: "Dead clicks or rage clicks: the user clicks something that does not respond.",- JevWatchReason.SLOW_OR_STUCK.value: "Long loading, spinners, timeouts, or a screen stuck in one state.",- JevWatchReason.BLOCKED.value: (- "A hard stop the user cannot get past: access denied, a limit reached, a 404 or dead end, or a feature "- "that is not available."- ),- JevWatchReason.CANT_FIND.value: (- "Repeated searching or navigating without finding what the user needs, or empty results."- ),- JevWatchReason.CONFUSED.value: (- "Hesitation, backtracking, trial and error, or misunderstanding how something works."- ),- JevWatchReason.WORKAROUND.value: (- "The user gave up on the intended path and reached the goal another way, such as SQL instead of the UI "- "or asking support."- ),- JevWatchReason.ABANDONED.value: "The user left a task, form, or setup unfinished.",- JevWatchReason.CHURN_SIGNAL.value: (- "Cancelling, downgrading, asking for a refund, deleting a project, or contacting support in frustration."- ),- JevWatchReason.SUCCESS.value: ("A smooth, notable success: the user used a feature as intended to reach a goal."),- _NO_WATCH_REASON: "A routine session. None of the other reasons fits.",-}-# Same rule as _WINDOW_INSTRUCTIONS: user text stays in the state, referred to by id only.-_REASON_INSTRUCTIONS = (- "The state holds recent AI scans of recorded product sessions, all from one scanner, keyed by "- "id. The session behind the scan with id {index} was judged worth watching. Pick the one reason "- "a product team would most want to watch it for. When several reasons apply, pick the most "- "severe one. Use the other scans in the state only as context. Judge only what the recorded "- "user experienced."-)--# What one question adds to a request beyond its state entry: the window instructions ride along-# per yes/no question, and the reason instructions plus the whole criteria list per CHOICE question.-# Per-request sizes grow with question count because of these, so packing and pacing count them.-_WINDOW_QUESTION_TOKENS = _text_tokens(_WINDOW_INSTRUCTIONS)-_REASON_QUESTION_TOKENS = _text_tokens(_REASON_INSTRUCTIONS) + _text_tokens(json.dumps(_WATCH_REASON_CRITERIA))--_CALLS = Counter(- "replay_vision_jev_watch_rank_calls",- "Jev watch rank chunk requests by outcome.",- ["outcome"],-)-_REQUESTS = Counter(- "replay_vision_jev_watch_rank_requests_total",- "Ranking requests by kind and HTTP status; non-HTTP failures use unknown",- ["kind", "status"],-)-_LATENCY = Histogram(- "replay_vision_jev_watch_rank_latency_seconds",- "Jev watch rank chunk request wall-clock latency.",- buckets=(0.25, 0.5, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0),-)-_INPUT_TOKENS = Counter(- "replay_vision_jev_watch_rank_input_tokens",- "Jev input tokens reported by the decision gateway.",-)-_ESTIMATED_COST = Counter(- "replay_vision_jev_watch_rank_estimated_cost_usd",- "Jev input-token cost at the gateway catalog price.",-)---def watch_feed_ranker_variant(team_id: int, team_uuid: UUID | str) -> str | None:- """The team's variant of the ranker flag, or None when the team takes no part in the experiment.-- Keyed on the project's uuid: both regions evaluate this one flag, and team ids repeat between- them. A region without the decision service has no sweep to fill the cache, so its teams take- no part in the Jev arms rather than land on an empty `jev` feed. The embedding arm needs no- decision service. Any flag failure also reads as None."""- value = get_feature_flag_or_none(- WATCH_FEED_RANKER_FLAG,- # Only a person-property condition reads the distinct id; a project-aggregated flag buckets- # and matches on the group below.- f"team-{team_id}",- groups={"project": str(team_uuid)},- group_properties={"project": {"id": team_id, "uuid": str(team_uuid)}},- send_feature_flag_events=False,- )- if not isinstance(value, str):- return None- if value in ("jev", "jev-shadow") and not decision_api.decisions_available_here():- return None- return value---def ranker_mode(variant: str | None) -> RankerMode:- """The ranker a flag variant selects. Any variant but the Jev and embedding arms is the default arm."""- if variant == "embedding":- return "embedding"- if variant == "jev-shadow":- return "jev-shadow"- if variant == "jev":- return "jev"- return "weighted-score"---def watch_feed_ranker(team_id: int, team_uuid: UUID | str) -> RankerMode:- """The team's arm of the watch feed ranker experiment. Any flag failure reads as the default- arm, so the sweep and the feed never fail on flag evaluation."""- return ranker_mode(watch_feed_ranker_variant(team_id, team_uuid))---@frozen-class WindowJudgment:- """Jev's judgment of one batch of observations: a probability per observation id (as a string)."""-- probabilities: dict[str, float]- # The JevWatchReason value Jev picked for each watchable row. A row without a fitting reason, or- # from a chunk whose reason request failed, is absent; the row still ranks on its probability.- reasons: dict[str, str]- failed_reason_chunks: int- # Rows with no prose to judge. The sweep records these as judged, so they settle into the- # filler tier once instead of being refetched every sweep; a failed chunk's rows are absent- # from `probabilities` and retry next sweep.- skipped_no_prose: tuple[str, ...]- # Rows from chunks that failed for a reason the batch itself caused (an invalid answer, or a- # gateway refusal in _BATCH_FAULT_STATUSES), so a retry will fail the same way. The sweep- # charges its retry budget only against these: an outage or a misconfigured gateway fails every- # chunk alike and must not park rows as judged-with-no-score.- batch_failed_ids: tuple[str, ...]- model: str | None- # Chunks the window actually sent (or tried to); chunks cut off by stop_after are not counted.- chunks: int- failed_chunks: int- # Failed chunks keyed by exception class name. The Prometheus counter carries the same- # breakdown, but prod workers do not ship their metrics into the product; the sweep's- # judged event does, so a failing sweep names its error without log or cluster access.- chunk_error_types: dict[str, int]- # The gateway rate-limited a request, so the window stopped there. The rows it did not reach- # retry free on the next sweep.- rate_limited: bool- # The window hit the caller's stop_after deadline and left chunks unsent; their rows retry- # on the next sweep.- stopped_early: bool- input_tokens: int- estimated_cost_usd: float- estimated_prompt_tokens: int = 0---def _is_rate_limited(error: Exception) -> bool:- return isinstance(error, DecisionGatewayError) and error.status_code == _RATE_LIMITED_STATUS---def _window_entry(row: dict[str, Any]) -> dict[str, Any] | None:- """The judgeable view of one observation row, or None when the row carries no prose to judge."""- result = row.get("scanner_result")- output = result.get("model_output") if isinstance(result, dict) else None- if not isinstance(output, dict):- return None- raw_tags = output.get("tags")- raw_freeform = output.get("tags_freeform")-- def clipped(value: Any, limit: int) -> Any:- return value[:limit] if isinstance(value, str) else value-- entry = {- key: value- for key, value in {- "scanner_type": output.get("scanner_type"),- "title": clipped(output.get("title"), _MAX_TITLE_CHARS),- "summary": clipped(output.get("summary"), _MAX_PROSE_CHARS),- "reasoning": clipped(output.get("reasoning"), _MAX_PROSE_CHARS),- "verdict": output.get("verdict"),- "score": output.get("score"),- "tags": [- *(raw_tags if isinstance(raw_tags, list) else []),- *(raw_freeform if isinstance(raw_freeform, list) else []),- ][:_MAX_TAGS],- "notability": output.get("notability"),- "signals_count": result.get("signals_count") if isinstance(result, dict) else None,- }.items()- if value not in (None, "", [])- }- if not any(isinstance(value := entry.get(key), str) and value.strip() for key in ("title", "summary", "reasoning")):- return None- return entry---def _decide(- team_id: int,- trace_id: str,- chunk: list[tuple[str, dict[str, Any]]],- context: list[dict[str, Any]],- questions: dict[str, DecisionQuestion],-) -> Any:- """One request over `chunk`, with `context` entries in the state as extra siblings. Context pads a- small chunk (a quiet hour adds only a few new rows) so its answers still see what routine looks- like for this scanner."""- entries = [entry for _, entry in chunk] + context- state: JsonValue = {"observations": {str(index): entry for index, entry in enumerate(entries)}}- return decision_api.decide_when_available(- DecisionRequest(- team_id=team_id,- state=state,- questions=questions,- model=JEV_MODEL,- ai_product="replay_vision",- trace_id=trace_id,- # The state holds recording-derived scan prose, which must stay out of the internal- # AI observability project — the same rule every other Replay Vision LLM call follows.- privacy_mode=True,- ),- timeout_seconds=JEV_TIMEOUT_SECONDS,- )---def _judge_chunk(- team_id: int,- trace_id: str,- chunk: list[tuple[str, dict[str, Any]]],- context: list[dict[str, Any]],-) -> tuple[dict[str, float], Any]:- """One yes/no question per observation in `chunk`: is the session worth watching."""- questions = {- f"watch_{index}": DecisionQuestion(- type=DecisionQuestionType.NOUL, instructions=_WINDOW_INSTRUCTIONS.format(index=index)- )- for index in range(len(chunk))- }- result = _decide(team_id, trace_id, chunk, context, questions)- probabilities: dict[str, float] = {}- for index, (observation_id, _) in enumerate(chunk):- answer = result.answers.get(f"watch_{index}")- if (- not isinstance(answer, NoulAnswer)- or not math.isfinite(answer.probability)- or not 0 <= answer.probability <= 1- ):- raise ValueError("Jev returned an invalid watchability probability")- probabilities[observation_id] = answer.probability- return probabilities, result---def _pick_watch_reasons(- team_id: int,- trace_id: str,- chunk: list[tuple[str, dict[str, Any]]],- context: list[dict[str, Any]],- watchable_ids: set[str],-) -> tuple[dict[str, str], Any]:- """One multiple-choice question per watchable observation in `chunk`: why it is worth watching.-- Asked only for watchable rows, because the feed shows a reason only on them. The state is the- same as the yes/no request's, so the pick sees the same context. An answer outside the list, or- no fitting reason, leaves the row without one instead of failing the chunk."""- indexes = [index for index, (observation_id, _) in enumerate(chunk) if observation_id in watchable_ids]- questions = {- f"reason_{index}": DecisionQuestion(- type=DecisionQuestionType.CHOICE,- instructions=_REASON_INSTRUCTIONS.format(index=index),- criteria=_WATCH_REASON_CRITERIA,- )- for index in indexes- }- result = _decide(team_id, trace_id, chunk, context, questions)- reasons: dict[str, str] = {}- for index in indexes:- answer = result.answers.get(f"reason_{index}")- if isinstance(answer, ChoiceAnswer) and answer.choice in JevWatchReason.values:- reasons[chunk[index][0]] = answer.choice- return reasons, result---def judge_scanner_window(- team_id: int,- scanner_id: UUID,- rows: list[dict[str, Any]],- context_rows: list[dict[str, Any]] | None = None,- pace: Callable[[int], None] | None = None,- stop_after: float | None = None,-) -> WindowJudgment:- """Ask Jev which of these observations are worth watching, one probability per row.-- `rows` and `context_rows` carry `id` and `scanner_result` (the shape the sweep loads). Context- rows enter the request state without questions, so a small batch of new rows is still judged- against the scanner's routine. Fail-soft per chunk: a failed chunk loses its rows' judgments- and counts as failed, and the other chunks still land, so one bad request never empties a- scanner's cache entry. `pace` is called with each request's estimated prompt tokens before it- is sent, so a caller can spread its requests at a steady token rate; it may block. Chunks are- not started past `stop_after` (a time.monotonic deadline): their rows stay unjudged and retry- on the next sweep, so a paced window cannot run away past the caller's time budget.- """- entries = [(str(row["id"]), entry) for row in rows if (entry := _window_entry(row)) is not None]- entry_ids = {entry_id for entry_id, _ in entries}- skipped_no_prose = tuple(str(row["id"]) for row in rows if str(row["id"]) not in entry_ids)- context = [entry for row in context_rows or [] if (entry := _window_entry(row)) is not None]- # Packed with the larger reason-question overhead: the reason pick resends the chunk's state- # with a criteria-bearing question per watchable row, so only a chunk sized for the all-watchable- # case keeps both requests under the target.- chunks = _pack_chunks(entries, max(_WINDOW_QUESTION_TOKENS, _REASON_QUESTION_TOKENS))- # One trace per window run, so a window's chunks group in AI observability without merging runs.- trace_id = str(uuid4())- probabilities: dict[str, float] = {}- reasons: dict[str, str] = {}- batch_failed_ids: list[str] = []- model: str | None = None- failed_chunks = 0- failed_reason_chunks = 0- chunk_error_types: dict[str, int] = {}- rate_limited = False- input_tokens = 0- estimated_cost = 0.0- estimated_prompt_tokens = 0-- def charge(result: Any) -> None:- nonlocal input_tokens, estimated_cost- input_tokens += result.input_tokens- chunk_cost = result.input_tokens * JEV_INPUT_USD_PER_MILLION / 1_000_000- estimated_cost += chunk_cost- _INPUT_TOKENS.inc(result.input_tokens)- _ESTIMATED_COST.inc(chunk_cost)-- context_with_tokens = [(entry, _entry_tokens(entry)) for entry in context]- chunks_started = 0- stopped_early = False-- def out_of_time() -> bool:- # Checked after each pace() too: the pacing sleep can carry the clock across the deadline,- # and a request started past it would overshoot the caller's budget by its whole timeout.- nonlocal stopped_early- if stop_after is not None and monotonic() > stop_after:- # Like rows cut off by the caller's time budget: unjudged, free to retry next sweep.- stopped_early = True- return stopped_early-- for chunk, packed_tokens in chunks:- if out_of_time():- break- # Context pads only the room the chunk leaves, by question count and by token budget, so a- # padded request stays no larger than a full one.- budget = REQUEST_TOKEN_TARGET - packed_tokens- chunk_context: list[dict[str, Any]] = []- context_tokens = 0- for context_entry, tokens in context_with_tokens:- if len(chunk) + len(chunk_context) >= WINDOW_CHUNK_SIZE or tokens > budget:- break- chunk_context.append(context_entry)- budget -= tokens- context_tokens += tokens- state_tokens = (- packed_tokens - len(chunk) * max(_WINDOW_QUESTION_TOKENS, _REASON_QUESTION_TOKENS) + context_tokens- )- if pace is not None:- pace(state_tokens + len(chunk) * _WINDOW_QUESTION_TOKENS)- if out_of_time():- break- chunks_started += 1- estimated_prompt_tokens += state_tokens + len(chunk) * _WINDOW_QUESTION_TOKENS- started = perf_counter()- try:- chunk_probabilities, result = _judge_chunk(team_id, trace_id, chunk, chunk_context)- except Exception as error:- _LATENCY.observe(perf_counter() - started)- status = str(error.status_code) if isinstance(error, DecisionGatewayError) else "unknown"- _REQUESTS.labels("judgment", status).inc()- error_type = type(error).__name__- _CALLS.labels(error_type).inc()- failed_chunks += 1- chunk_error_types[error_type] = chunk_error_types.get(error_type, 0) + 1- # An invalid answer or a batch-caused gateway refusal charges the retry budget. A- # missing gateway config raises a ValueError subclass and must not sneak in through- # the invalid-answer check: it fails every chunk alike, so its rows retry free, the- # same as an unreachable, disabled, rate-limited, or erroring gateway.- if (isinstance(error, ValueError) and not isinstance(error, GatewayNotConfiguredError)) or (- isinstance(error, DecisionGatewayError) and error.status_code in _BATCH_FAULT_STATUSES- ):- batch_failed_ids.extend(entry_id for entry_id, _ in chunk)- # The gateway error body can echo the state, which holds recording-derived prose, so- # only the error type leaves here.- logger.warning(- "Jev watch rank chunk failed",- team_id=team_id,- scanner_id=str(scanner_id),- error_type=error_type,- )- if _is_rate_limited(error):- rate_limited = True- break- continue- _LATENCY.observe(perf_counter() - started)- _CALLS.labels("ok").inc()- _REQUESTS.labels("judgment", "200").inc()- probabilities.update(chunk_probabilities)- model = result.model- charge(result)- watchable_ids = {oid for oid, probability in chunk_probabilities.items() if probability >= JEV_WATCHABLE_MIN}- if not watchable_ids:- continue- if pace is not None:- # The reason pick resends the same state, with one criteria-bearing question per- # watchable row — usually a few, so this toll is usually smaller than the judgment's.- pace(state_tokens + len(watchable_ids) * _REASON_QUESTION_TOKENS)- if out_of_time():- # The judgments above still land; only the reasons are left off, as on a failed pick.- break- started = perf_counter()- estimated_prompt_tokens += state_tokens + len(watchable_ids) * _REASON_QUESTION_TOKENS- try:- chunk_reasons, reason_result = _pick_watch_reasons(team_id, trace_id, chunk, chunk_context, watchable_ids)- except Exception as error:- # A reason only labels the row, so a failed pick keeps the chunk's probabilities and is- # not retried: the rows are judged, and the feed shows them without a reason.- _LATENCY.observe(perf_counter() - started)- _CALLS.labels(f"reason_{type(error).__name__}").inc()- status = str(error.status_code) if isinstance(error, DecisionGatewayError) else "unknown"- _REQUESTS.labels("reason", status).inc()- failed_reason_chunks += 1- logger.warning(- "Jev watch reason chunk failed",- team_id=team_id,- scanner_id=str(scanner_id),- error_type=type(error).__name__,- )- if _is_rate_limited(error):- rate_limited = True- break- continue- _LATENCY.observe(perf_counter() - started)- _CALLS.labels("reason_ok").inc()- _REQUESTS.labels("reason", "200").inc()- reasons.update(chunk_reasons)- charge(reason_result)- return WindowJudgment(- probabilities=probabilities,- reasons=reasons,- failed_reason_chunks=failed_reason_chunks,- skipped_no_prose=skipped_no_prose,- batch_failed_ids=tuple(batch_failed_ids),- model=model,- chunks=chunks_started,- failed_chunks=failed_chunks,- chunk_error_types=chunk_error_types,- rate_limited=rate_limited,- stopped_early=stopped_early,- input_tokens=input_tokens,- estimated_cost_usd=estimated_cost,- estimated_prompt_tokens=estimated_prompt_tokens,- )---# The watchable map (probability >= JEV_WATCHABLE_MIN, all the feed reads) lives apart from the-# judged-id set (everything the sweep has bought, read only by the sweep's skip logic): the feed-# loads every readable scanner's key on each request, so its keys must stay small however large a-# scanner's judged window grows.-def _watchable_key(team_id: int, scanner_id: UUID | str) -> str:- return f"{_WATCH_RANK_REDIS_PREFIX}watchable:{team_id}:{scanner_id}"---def _judged_key(team_id: int, scanner_id: UUID | str) -> str:- return f"{_WATCH_RANK_REDIS_PREFIX}judged:{team_id}:{scanner_id}"---# The sweep worker writes this cache and the feed API on the web fleet reads it, so it lives on-# the shared Redis, like the enqueue claims. The dedicated replay-vision Redis-# (REPLAY_VISION_REDIS_URL) is mounted only on the replay-vision temporal workers, so a key-# written there never reaches the feed.-def _watch_rank_client() -> redis.Redis:- return get_client()---def refresh_watch_ranks_ttl(team_id: int, scanner_id: UUID) -> None:- client = _watch_rank_client()- client.expire(_watchable_key(team_id, scanner_id), WATCH_RANK_TTL)- client.expire(_judged_key(team_id, scanner_id), WATCH_RANK_TTL)---def store_watch_ranks(- team_id: int,- scanner_id: UUID,- judged_ids: Collection[str],- watchable: dict[str, float],- attempts: dict[str, int],- model: str | None,- reasons: dict[str, str] | None = None,-) -> None:- client = _watch_rank_client()- client.setex(- _watchable_key(team_id, scanner_id),- WATCH_RANK_TTL,- json.dumps(- {- "model": model,- "judged_at": datetime.now(UTC).isoformat(),- "probabilities": watchable,- "reasons": {oid: reason for oid, reason in (reasons or {}).items() if oid in watchable},- }- ),- )- # The judged marker is written last on purpose: a partial failure then leaves scores without- # the marker, so the batch is re-bought next sweep. The reverse order would mark rows judged- # with their scores lost forever. (Not one MULTI: the two keys hash to different slots, so a- # transaction would break on a cluster.)- client.setex(- _judged_key(team_id, scanner_id),- WATCH_RANK_TTL,- json.dumps({"ids": sorted(judged_ids), "attempts": attempts}),- )---@frozen-class JudgedState:- """What the sweep already bought for one scanner: the judged observation ids, and how many- times each still-unjudged row's judgment has failed."""-- ids: set[str]- attempts: dict[str, int]---def load_judged_state(team_id: int, scanner_id: UUID) -> JudgedState:- """The sweep's judged set and attempt counts for this scanner. A missing key is empty state.-- A Redis read failure raises: an unreadable cache must not read as an empty one, or the sweep- re-buys the scanner's judgments and its next write replaces entries it never saw. A stored- value that cannot be parsed reads as empty instead, because rewriting it loses nothing.- """- value = _watch_rank_client().get(_judged_key(team_id, scanner_id))- if not value:- return JudgedState(ids=set(), attempts={})- try:- stored = json.loads(value)- except Exception:- logger.exception("Jev watch rank judged-set malformed", team_id=team_id)- return JudgedState(ids=set(), attempts={})- raw_ids = stored.get("ids")- raw_attempts = stored.get("attempts")- return JudgedState(- ids={str(judged_id) for judged_id in raw_ids} if isinstance(raw_ids, list) else set(),- attempts=(- {- str(observation_id): count- for observation_id, count in raw_attempts.items()- if isinstance(count, int) and not isinstance(count, bool)- }- if isinstance(raw_attempts, dict)- else {}- ),- )---@frozen-class WatchRanks:- """Cached Jev judgments for one or more scanners, keyed by observation id: the watchable- probabilities, and the JevWatchReason value for each watchable row that has one."""-- probabilities: dict[str, float]- reasons: dict[str, str]---def _parse_watchable(value: Any) -> WatchRanks:- if not value:- return WatchRanks(probabilities={}, reasons={})- try:- stored = json.loads(value)- except Exception:- logger.exception("Jev watch rank cache value malformed")- return WatchRanks(probabilities={}, reasons={})- stored_probabilities = stored.get("probabilities") if isinstance(stored, dict) else None- if not isinstance(stored_probabilities, dict):- return WatchRanks(probabilities={}, reasons={})- probabilities: dict[str, float] = {}- for observation_id, probability in stored_probabilities.items():- # Clamped like the weighted ranker clamps notability, because a cache can carry anything- # (and bool is an int subclass).- if isinstance(probability, int | float) and not isinstance(probability, bool):- probabilities[str(observation_id)] = min(1.0, max(0.0, float(probability)))- # Entries written before reasons shipped carry none, and a value outside the list never reaches- # the API, whose enum would reject it.- stored_reasons = stored.get("reasons")- reasons = (- {- str(observation_id): reason- for observation_id, reason in stored_reasons.items()- if str(observation_id) in probabilities and reason in JevWatchReason.values- }- if isinstance(stored_reasons, dict)- else {}- )- return WatchRanks(probabilities=probabilities, reasons=reasons)---def load_scanner_watch_ranks(team_id: int, scanner_id: UUID) -> WatchRanks:- """The sweep's read of one scanner's watchable map. Raises on a Redis read failure, because the- sweep merges what it loads back into the store, so writing over a map it never saw drops- entries. The feed reads through `load_watch_ranks`, which fails soft instead."""- return _parse_watchable(_watch_rank_client().get(_watchable_key(team_id, scanner_id)))---def load_watch_ranks(team_id: int, scanner_ids: list[UUID]) -> WatchRanks:- """The cached watchable probabilities and reasons for these scanners. Fail-soft: any malformed,- missing, or unreachable cache entry contributes nothing, so the Jev feed degrades to the recency- filler tier rather than failing the request."""- probabilities: dict[str, float] = {}- reasons: dict[str, str] = {}- if not scanner_ids:- return WatchRanks(probabilities=probabilities, reasons=reasons)- try:- values = _watch_rank_client().mget([_watchable_key(team_id, scanner_id) for scanner_id in scanner_ids])- for value in values:- ranks = _parse_watchable(value)- probabilities |= ranks.probabilities- reasons |= ranks.reasons- except Exception:- logger.exception("Jev watch rank cache read failed", team_id=team_id)- return WatchRanks(probabilities=probabilities, reasons=reasons)---def _scan_notability_reason(row: dict[str, Any]) -> str | None:- result = row.get("scanner_result")- output = result.get("model_output") if isinstance(result, dict) else None- if not isinstance(output, dict):- return None- # The scan writes a notability_reason on every session (including "nothing stands out"), so- # only carry it when the scan itself found the session notable — the same bar the weighted- # ranker applies. Jev can rate a session watchable relative to a dull window while the scan's- # own sentence says the opposite, and that sentence must not lead the card.- notability = output.get("notability")- if not isinstance(notability, int | float) or isinstance(notability, bool) or notability < NOTABLE_MIN_SCORE:- return None- reason = output.get("notability_reason")- return reason if isinstance(reason, str) and reason.strip() else None---def rank_watch_feed_by_jev(- rows: list[dict[str, Any]], probabilities: dict[str, float], reasons: dict[str, str] | None = None-) -> list[WatchFeedEntry]:- """Rank candidate rows (`id`, `scanner_id`, `created_at`, `scanner_result`, `feed_viewed`) on- Jev's cached watchability alone: highest probability first, one scanner held to a share of the- tier, viewed rows docked, newest as the tiebreak.-- Independent of `rank_watch_feed_candidates` on purpose: the flag picks a whole ranker, so this- arm measures Jev's judgment without any component of the weighted score mixed in. Rows without- evidence sort below every watchable row by recency, with the same filler reasons the weighted- ranker uses: rows the model rated below `JEV_WATCHABLE_MIN`, rows without a cached probability- (not yet swept, or the cache went cold), and rows outside the sweep's window. Filler is trimmed- to the same floor as the weighted feed, so a cold cache yields 3 newest clips, not a full page- of them.- """- watchable: list[tuple[float, Any, Any, WatchFeedEntry]] = []- filler_rows: list[dict[str, Any]] = []- for row in rows:- probability = probabilities.get(str(row["id"]))- viewed = bool(row.get("feed_viewed"))- if probability is not None and probability >= JEV_WATCHABLE_MIN:- reason: dict[str, Any] = {"kind": "jev_watchable", "jev_probability": probability}- # The scan's own sentence, so the card says why the session is worth watching instead- # of only that the model said so; the frontend prefers it over kind-derived copy.- if notability_reason := _scan_notability_reason(row):- reason["notability_reason"] = notability_reason- if watch_reason := (reasons or {}).get(str(row["id"])):- reason["watch_reason"] = watch_reason- entry = WatchFeedEntry(observation_id=row["id"], reason=reason)- watchable.append(- (probability - (JEV_SEEN_PENALTY if viewed else 0.0), row["created_at"], row.get("scanner_id"), entry)- )- else:- filler_rows.append(row)- return finish_feed(watchable, filler_rows)diff --git a/products/replay_vision/backend/learned_rules.py b/products/replay_vision/backend/learned_rules.pyindex 8c5b99be5399..4475109a582b 100644--- a/products/replay_vision/backend/learned_rules.py+++ b/products/replay_vision/backend/learned_rules.py@@ -60,7 +60,6 @@ MAX_CONTEXT_RATINGS_PER_SCANNER = 20_SCANNER_PROMPT_CHARS = 1200# Long enough for most answers whole: without the session, the scanner's own words are all a pattern can rest on._REASONING_CHARS = 2000-_NOTABILITY_CHARS = 300_FEEDBACK_CHARS = 600# A first run reads ratings this far back; older ones may describe a scanner that has changed since.FIRST_RUN_LOOKBACK = dt.timedelta(days=90)@@ -695,16 +694,12 @@ def _describe(rating: _Rating, scanner_version: int) -> str:def _output_details(output: dict) -> str:- """Confidence, notability and the moment the scanner picked, which a pattern across unexplained ratings- can turn on (for example, every rejected answer being low-confidence)."""+ """Confidence and the moment the scanner picked, which a pattern across unexplained ratings can turn on+ (for example, every rejected answer being low-confidence)."""parts: list[str] = []confidence = output.get("confidence")if isinstance(confidence, int | float):parts.append(f"confidence {confidence:.2f}")- notability = output.get("notability")- if isinstance(notability, int | float):- reason = " ".join(str(output.get("notability_reason") or "").split())[:_NOTABILITY_CHARS]- parts.append(f"notability {notability:.2f}" + (f" ({reason})" if reason else ""))key_moment_ms = output.get("key_moment_ms")if isinstance(key_moment_ms, int):parts.append(f"key moment at {key_moment_ms // 1000}s")diff --git a/products/replay_vision/backend/migrations/0137_retire_feed_rankers_schema.py b/products/replay_vision/backend/migrations/0137_retire_feed_rankers_schema.pynew file mode 100644index 000000000000..cbf425cf2d1b--- /dev/null+++ b/products/replay_vision/backend/migrations/0137_retire_feed_rankers_schema.py@@ -0,0 +1,48 @@+from django.db import migrations++from posthog.migration_helpers import untrack_field++# Databases created before the lens tables were renamed still carry these names; fresh databases already use the new ones.+_LEGACY_INDEX_NAMES = [+ ("replay_vision_replaylens_pkey", "replay_vision_replayscanner_pkey"),+ ("replay_vision_replaylens_team_id_6cbf8851", "replay_vision_replayscanner_team_id_5b48156f"),+ ("replay_vision_replaylens_created_by_id_d748ea54", "replay_vision_replayscanner_created_by_id_f7e76528"),+ ("replay_vision_replaylensobservation_pkey", "replay_vision_replayobservation_pkey"),+ (+ "replay_vision_replaylensob_triggered_by_user_id_55bd1882",+ "replay_vision_replayobservation_triggered_by_user_id_f913652f",+ ),+]+++class Migration(migrations.Migration):+ dependencies = [+ ("replay_vision", "0136_retire_search_suggestion_columns"),+ ]++ operations = [+ migrations.RunSQL(+ sql=[f'ALTER INDEX IF EXISTS "{old}" RENAME TO "{new}"' for old, new in _LEGACY_INDEX_NAMES],+ reverse_sql=migrations.RunSQL.noop,+ ),+ migrations.RenameIndex(+ model_name="replayobservationusage",+ new_name="rlou_org_observed_idx",+ old_name="replay_visi_organiz_a84ed4_idx",+ ),+ untrack_field(+ "replayscanner",+ "search_suggestions",+ "search_suggestions_watermark",+ "search_suggestions_generated_at",+ "search_last_viewed_at",+ ),+ untrack_field("replayobservationmedia", "rec_start_ms"),+ untrack_field(+ "teamreplayvisionconfig",+ "search_suggestions",+ "search_suggestions_sources",+ "search_suggestions_watermark",+ "search_suggestions_generated_at",+ ),+ ]diff --git a/products/replay_vision/backend/migrations/0138_drop_replayscanner_search_suggestion_columns.py b/products/replay_vision/backend/migrations/0138_drop_replayscanner_search_suggestion_columns.pynew file mode 100644index 000000000000..fc47d9483ba6--- /dev/null+++ b/products/replay_vision/backend/migrations/0138_drop_replayscanner_search_suggestion_columns.py@@ -0,0 +1,31 @@+from django.db import migrations++_COLUMNS = {+ "search_suggestions": "jsonb",+ "search_suggestions_watermark": "timestamptz",+ "search_suggestions_generated_at": "timestamptz",+ "search_last_viewed_at": "timestamptz",+}+++class Migration(migrations.Migration):+ # 0137 untracked the columns. Each table drops in its own migration, so one table's lock never waits on the other's.+ dependencies = [+ ("replay_vision", "0137_retire_feed_rankers_schema"),+ ]++ operations = [+ migrations.SeparateDatabaseAndState(+ state_operations=[],+ database_operations=[+ migrations.RunSQL(+ sql='ALTER TABLE "replay_vision_replayscanner" '+ + ", ".join(f'DROP COLUMN IF EXISTS "{column}"' for column in _COLUMNS),+ reverse_sql='ALTER TABLE "replay_vision_replayscanner" '+ + ", ".join(+ f'ADD COLUMN IF NOT EXISTS "{column}" {kind} NULL' for column, kind in _COLUMNS.items()+ ),+ ),+ ],+ ),+ ]diff --git a/products/replay_vision/backend/migrations/0139_drop_teamreplayvisionconfig_search_suggestion_columns.py b/products/replay_vision/backend/migrations/0139_drop_teamreplayvisionconfig_search_suggestion_columns.pynew file mode 100644index 000000000000..18df67ee95af--- /dev/null+++ b/products/replay_vision/backend/migrations/0139_drop_teamreplayvisionconfig_search_suggestion_columns.py@@ -0,0 +1,31 @@+from django.db import migrations++_COLUMNS = {+ "search_suggestions": "jsonb",+ "search_suggestions_sources": "jsonb",+ "search_suggestions_watermark": "timestamptz",+ "search_suggestions_generated_at": "timestamptz",+}+++class Migration(migrations.Migration):+ # 0137 untracked the columns. Each table drops in its own migration, so one table's lock never waits on the other's.+ dependencies = [+ ("replay_vision", "0138_drop_replayscanner_search_suggestion_columns"),+ ]++ operations = [+ migrations.SeparateDatabaseAndState(+ state_operations=[],+ database_operations=[+ migrations.RunSQL(+ sql='ALTER TABLE "replay_vision_teamreplayvisionconfig" '+ + ", ".join(f'DROP COLUMN IF EXISTS "{column}"' for column in _COLUMNS),+ reverse_sql='ALTER TABLE "replay_vision_teamreplayvisionconfig" '+ + ", ".join(+ f'ADD COLUMN IF NOT EXISTS "{column}" {kind} NULL' for column, kind in _COLUMNS.items()+ ),+ ),+ ],+ ),+ ]diff --git a/products/replay_vision/backend/migrations/0140_drop_replayobservationmedia_rec_start_ms.py b/products/replay_vision/backend/migrations/0140_drop_replayobservationmedia_rec_start_ms.pynew file mode 100644index 000000000000..8d6e40dcee78--- /dev/null+++ b/products/replay_vision/backend/migrations/0140_drop_replayobservationmedia_rec_start_ms.py@@ -0,0 +1,20 @@+from django.db import migrations+++class Migration(migrations.Migration):+ # 0137 untracked the column. Each table drops in its own migration, so one table's lock never waits on the other's.+ dependencies = [+ ("replay_vision", "0139_drop_teamreplayvisionconfig_search_suggestion_columns"),+ ]++ operations = [+ migrations.SeparateDatabaseAndState(+ state_operations=[],+ database_operations=[+ migrations.RunSQL(+ sql='ALTER TABLE "replay_vision_replayobservationmedia" DROP COLUMN IF EXISTS "rec_start_ms"',+ reverse_sql='ALTER TABLE "replay_vision_replayobservationmedia" ADD COLUMN IF NOT EXISTS "rec_start_ms" integer NULL',+ ),+ ],+ ),+ ]diff --git a/products/replay_vision/backend/migrations/max_migration.txt b/products/replay_vision/backend/migrations/max_migration.txtindex c3a6e3f2069c..795ba326782c 100644--- a/products/replay_vision/backend/migrations/max_migration.txt+++ b/products/replay_vision/backend/migrations/max_migration.txt@@ -1 +1 @@-0136_retire_search_suggestion_columns+0140_drop_replayobservationmedia_rec_start_msdiff --git a/products/replay_vision/backend/models/replay_observation_media.py b/products/replay_vision/backend/models/replay_observation_media.pyindex c6bd8f612e4f..15bf2b122388 100644--- a/products/replay_vision/backend/models/replay_observation_media.py+++ b/products/replay_vision/backend/models/replay_observation_media.py@@ -1,6 +1,5 @@from django.db import models-from posthog.migration_helpers import deprecate_fieldfrom posthog.models.scoping.root_mixin import TeamScopedRootMixinfrom posthog.models.utils import UUIDModel@@ -41,7 +40,6 @@ class ReplayObservationMedia(TeamScopedRootMixin, UUIDModel):# Analysis-video time, for seeking the player the person is looking at.video_start_ms = models.PositiveIntegerField()- rec_start_ms = deprecate_field(models.PositiveIntegerField(null=True, blank=True))created_at = models.DateTimeField(auto_now_add=True)diff --git a/products/replay_vision/backend/models/replay_observation_usage.py b/products/replay_vision/backend/models/replay_observation_usage.pyindex 293ef59536d6..ea9894d64333 100644--- a/products/replay_vision/backend/models/replay_observation_usage.py+++ b/products/replay_vision/backend/models/replay_observation_usage.py@@ -43,7 +43,7 @@ class ReplayObservationUsage(UUIDModel):class Meta:indexes = [- models.Index(fields=["organization", "observation_created_at"]),+ models.Index(fields=["organization", "observation_created_at"], name="rlou_org_observed_idx"),# Drives the org quota, which buckets receipts by write time so it agrees with billing.models.Index(fields=["organization", "created_at"], name="rlou_org_created_idx"),# Drives the daily per-team billing usage query, which buckets receipts by write time.diff --git a/products/replay_vision/backend/models/replay_scanner.py b/products/replay_vision/backend/models/replay_scanner.pyindex b83a9a77a8d9..58eb575c4fad 100644--- a/products/replay_vision/backend/models/replay_scanner.py+++ b/products/replay_vision/backend/models/replay_scanner.py@@ -7,7 +7,6 @@ from django.db import models, transactionfrom django.utils import timezonefrom django.utils.functional import Promise-from posthog.migration_helpers import deprecate_fieldfrom posthog.models.activity_logging.model_activity import ModelActivityMixinfrom posthog.models.tagged_items_relation import Taggablefrom posthog.models.utils import UUIDModel@@ -284,31 +283,6 @@ class ReplayScanner(Taggable, ModelActivityMixin, UUIDModel):help_text="When an estimate was last attempted, success or failure. Backs off the refresher on scanners whose estimate query keeps failing.",)- # Retired with the scheduled search suggestion refresher; the columns stay until a later drop.- search_suggestions = deprecate_field(- models.JSONField(- default=list,- null=True,- blank=True,- help_text="Example searches drawn from recent observations, shown on the Search tab's empty state.",- )- )- search_suggestions_watermark = deprecate_field(- models.DateTimeField(- null=True,- blank=True,- help_text="created_at of the newest observation the suggestions were drawn from.",- )- )- search_suggestions_generated_at = deprecate_field(models.DateTimeField(null=True, blank=True))- search_last_viewed_at = deprecate_field(- models.DateTimeField(- null=True,- blank=True,- help_text="When the Search tab last asked for this scanner's suggestions. Only viewed scanners refresh.",- )- )-# Written with the prompt by every path that sets one, see `prompt_questions`; inline scanners keep only a# built-in scan's question. Not version-tracked: it restates the prompt and changes nothing about how the scanner scans.prompt_question = models.TextField(diff --git a/products/replay_vision/backend/models/team_replay_vision_config.py b/products/replay_vision/backend/models/team_replay_vision_config.pyindex 3b3204f7de1a..a3be849ac411 100644--- a/products/replay_vision/backend/models/team_replay_vision_config.py+++ b/products/replay_vision/backend/models/team_replay_vision_config.py@@ -1,6 +1,5 @@from django.db import models-from posthog.migration_helpers import deprecate_fieldfrom posthog.models.team import Team@@ -11,33 +10,6 @@ class TeamReplayVisionConfig(models.Model):# migrating, stalling writes under traffic.team = models.OneToOneField(Team, on_delete=models.CASCADE, primary_key=True, db_constraint=False, related_name="+")- # Retired with the scheduled search suggestion refresher; the columns stay until a later drop.- search_suggestions = deprecate_field(- models.JSONField(- default=list,- null=True,- blank=True,- help_text="Example searches for the cross-scanner Search tab, drawn from findings across the team's scanners.",- )- )- search_suggestions_sources = deprecate_field(- models.JSONField(- default=list,- null=True,- blank=True,- help_text="Ids of the scanners whose observations fed `search_suggestions`. A viewer sees the phrases only "- "when they can read every one of them.",- )- )- search_suggestions_watermark = deprecate_field(- models.DateTimeField(- null=True,- blank=True,- help_text="Created time of the newest observation the stored phrases read.",- )- )- search_suggestions_generated_at = deprecate_field(models.DateTimeField(null=True, blank=True))-learned_rules_watermark = models.DateTimeField(null=True,blank=True,diff --git a/products/replay_vision/backend/observation_requests.py b/products/replay_vision/backend/observation_requests.pyindex ff5545ccccd5..a519f8dea951 100644--- a/products/replay_vision/backend/observation_requests.py+++ b/products/replay_vision/backend/observation_requests.py@@ -706,8 +706,8 @@ def _unanswered_reason(session: RequestSession) -> list[str]:return []-# The answer a workflow step gets, per scanner type. Internal fields (citation segments, idle periods,-# notability) would push a long session's answer over the step result budget and cut the chapters.+# The answer a workflow step gets, per scanner type. Internal fields (citation segments, idle periods)+# would push a long session's answer over the step result budget and cut the chapters.# Short fields come first: the size cap fills fields in order, so a long summary can't crowd them out._STEP_OUTPUT_FIELDS: dict[str, tuple[str, ...]] = {ScannerType.MONITOR: ("confidence", "verdict", "reasoning"),diff --git a/products/replay_vision/backend/scanner_dashboard.py b/products/replay_vision/backend/scanner_dashboard.pyindex 65764f3e2e78..47793b5700bd 100644--- a/products/replay_vision/backend/scanner_dashboard.py+++ b/products/replay_vision/backend/scanner_dashboard.py@@ -181,27 +181,24 @@ class _TileBuilder:def add_latest_sessions_tile(self) -> None:scanner_type = ScannerType(self.scanner.scanner_type)- name, description, order = "Latest matching sessions", "The newest observations", "timestamp DESC"detail = "properties.scanner_output_reasoning"- if scanner_type == ScannerType.CLASSIFIER:- detail = f"arrayStringConcat({_TAGS_EXPR}, ', ')"- elif scanner_type in (ScannerType.SUMMARIZER, ScannerType.EXPERIMENT):- name, description = "Most notable sessions", "The most notable observations"- order = "toFloat(properties.scanner_output_notability) DESC NULLS LAST"+ if scanner_type in (ScannerType.SUMMARIZER, ScannerType.EXPERIMENT):detail = "properties.scanner_output_title"+ elif scanner_type == ScannerType.CLASSIFIER:+ detail = f"arrayStringConcat({_TAGS_EXPR}, ', ')"score_column = ("toFloat(properties.scanner_output_score) AS score, " if scanner_type == ScannerType.SCORER else "")where = f" AND {_YES_SQL}" if self.is_monitor else ""self._sql_table(- name,- f"{description}, with a link to each recording.",+ "Latest matching sessions",+ "The newest observations, with a link to each recording.",f"""SELECT timestamp AS observed_at, {score_column}{detail} AS finding,properties.recording_distinct_id AS person, recordingButton(properties.session_id) AS recordingFROM eventsWHERE {_SCANNER_WHERE}{where}- ORDER BY {order}+ ORDER BY timestamp DESCLIMIT 50""",width=12,@@ -335,7 +332,6 @@ class _TileBuilder:)def _summarizer_tiles(self) -> None:- self._notability_distribution()self._trend("Average chapters per recording","A rough measure of how much happens in each session.",@@ -350,31 +346,6 @@ class _TileBuilder:breakdown={"breakdown": "experiment_variant", "breakdown_type": "event"},display="ActionsBar",)- self._trend(- "Average notability by variant",- "How notable the scanner found each variant's sessions over the date range. Higher means more unusual "- "behavior.",- series=[self._series("Notability", math="avg", math_property="scanner_output_notability")],- breakdown={"breakdown": "experiment_variant", "breakdown_type": "event"},- # One bar per variant: stacked weekly bars would add the variants' averages together.- display="ActionsBarValue",- )- self._notability_distribution()-- def _notability_distribution(self) -> None:- self._sql_chart(- "Notability distribution",- "How notable the scanner found each session, from 0 (routine) to 1 (unusual).",- f"""- SELECT round(toFloat(properties.scanner_output_notability), 1) AS notability, count() AS recordings- FROM events- WHERE {_SCANNER_WHERE} AND properties.scanner_output_notability IS NOT NULL- GROUP BY notability- ORDER BY notability- """,- x="notability",- y="recordings",- )# Helpers.diff --git a/products/replay_vision/backend/temporal/__init__.py b/products/replay_vision/backend/temporal/__init__.pyindex a7f226e6b9df..b2a6b1fed860 100644--- a/products/replay_vision/backend/temporal/__init__.py+++ b/products/replay_vision/backend/temporal/__init__.py@@ -60,10 +60,6 @@ from products.replay_vision.backend.temporal.gemini_cleanup_sweep import (ReplayVisionGeminiCleanupSweepWorkflow,sweep_gemini_files_activity,)-from products.replay_vision.backend.temporal.jev_watch_rank import (- ReplayVisionJevWatchRankWorkflow,- judge_watch_ranks_activity,-)from products.replay_vision.backend.temporal.learned_rules import RefreshLearnedRulesWorkflowfrom products.replay_vision.backend.temporal.media_workflow import ObservationMediaWorkflowfrom products.replay_vision.backend.temporal.read_meter import MeterScannerReadsWorkflow@@ -88,7 +84,6 @@ WORKFLOWS = [RefreshScannerEstimatesWorkflow,RefreshLearnedRulesWorkflow,ReplayVisionGeminiCleanupSweepWorkflow,- ReplayVisionJevWatchRankWorkflow,SweepScannerWorkflow,VisionAlertCheckWorkflow,]@@ -137,7 +132,6 @@ ACTIVITIES: list[Callable[..., Any]] = [start_launched_scanners_activity,complete_observation_requests_activity,sweep_gemini_files_activity,- judge_watch_ranks_activity,list_due_learned_rules_teams_activity,refresh_team_learned_rules_activity,snapshot_benchmark_labels_activity,diff --git a/products/replay_vision/backend/temporal/activities/observation_media.py b/products/replay_vision/backend/temporal/activities/observation_media.pyindex 792fd6278ab7..c9359de473bd 100644--- a/products/replay_vision/backend/temporal/activities/observation_media.py+++ b/products/replay_vision/backend/temporal/activities/observation_media.py@@ -234,7 +234,6 @@ async def prepare_observation_media_activity(inputs: ObservationMediaInputs) ->small_media_asset_id=small_media_asset.id,object_id=object_id,video_start_ms=int(video_time_s * 1000),- rec_start_ms=source.clock.video_s_to_session_ms(video_time_s) if source.clock else None,))diff --git a/products/replay_vision/backend/temporal/jev_watch_rank/__init__.py b/products/replay_vision/backend/temporal/jev_watch_rank/__init__.pydeleted file mode 100644index 67fe33619049..000000000000--- a/products/replay_vision/backend/temporal/jev_watch_rank/__init__.py+++ /dev/null@@ -1,9 +0,0 @@-from products.replay_vision.backend.temporal.jev_watch_rank.activities import judge_watch_ranks_activity-from products.replay_vision.backend.temporal.jev_watch_rank.schedule import create_replay_vision_jev_watch_rank_schedule-from products.replay_vision.backend.temporal.jev_watch_rank.workflow import ReplayVisionJevWatchRankWorkflow--__all__ = [- "ReplayVisionJevWatchRankWorkflow",- "create_replay_vision_jev_watch_rank_schedule",- "judge_watch_ranks_activity",-]diff --git a/products/replay_vision/backend/temporal/jev_watch_rank/activities.py b/products/replay_vision/backend/temporal/jev_watch_rank/activities.pydeleted file mode 100644index 87ed89d4715a..000000000000--- a/products/replay_vision/backend/temporal/jev_watch_rank/activities.py+++ /dev/null@@ -1,474 +0,0 @@-"""The hourly sweep that asks Jev to rank each enrolled scanner's recent window.--Runs outside both hot paths on purpose: the feed API is synchronous and must not make model calls,-and the scan pipeline must not carry a judgment that can only slow or fail a paid-for scan. Each-sweep judges only the rows without a cached probability (`judge_scanner_window`, with judged rows-padding the request as context) and merges the results into the Redis cache the feed reads-(`load_watch_ranks`), so coverage of a scanner's window accumulates at a bounded hourly cost-whatever the scanner's volume.--The gates run cheapest-first: region (one settings read), then the per-team flag, then the org's AI-data-processing consent, then per-team indexed observation queries. The observation table has no-cross-team index on (status, created_at), so the sweep never queries it without a team_id.-"""--import random-import asyncio-import threading-from collections import Counter-from collections.abc import Callable-from contextlib import suppress-from datetime import UTC, datetime-from statistics import fmean-from time import monotonic, sleep-from typing import Any-from uuid import UUID--import structlog-import posthoganalytics-from asgiref.sync import sync_to_async-from temporalio import activity--from posthog.dataclasses import frozen-from posthog.models.team import Team-from posthog.temporal.common.heartbeat import Heartbeater--from products.ml_inference.backend.facade import api as decision_api-from products.replay_vision.backend.consent import is_ai_data_processing_approved-from products.replay_vision.backend.jev_watch_feed import (- JEV_WATCHABLE_MIN,- WINDOW_CHUNK_SIZE,- judge_scanner_window,- load_judged_state,- load_scanner_watch_ranks,- refresh_watch_ranks_ttl,- store_watch_ranks,- watch_feed_ranker,-)-from products.replay_vision.backend.models.replay_observation import ObservationStatus, ReplayObservation-from products.replay_vision.backend.models.replay_scanner import ReplayScanner-from products.replay_vision.backend.temporal.decorators import track_activity-from products.replay_vision.backend.temporal.jev_watch_rank.constants import (- JUDGE_CONCURRENCY,- JUDGE_TOKENS_PER_SECOND,- MAX_JUDGE_ATTEMPTS,- MAX_JUDGED_PER_SCANNER,- MAX_SCANNERS_PER_SWEEP,- MAX_TEAMS_PER_SWEEP,- PINNED_TEAM_IDS,- RATE_LIMIT_GIVE_UP_AFTER,- RATE_LIMIT_PAUSE,- SWEEP_ACTIVITY_HEARTBEAT_TIMEOUT,- SWEEP_TIME_BUDGET,- WATCH_RANK_WINDOW,- WINDOW_SCAN_CAP,-)-from products.replay_vision.backend.temporal.jev_watch_rank.metrics import push_sweep_metrics-from products.replay_vision.backend.temporal.jev_watch_rank.types import (- JevWatchRankSweepInputs,- JevWatchRankSweepResult,-)-from products.replay_vision.backend.temporal.query_budget import bounded_queries--# The background heartbeat keeps a stalled query's attempt alive for the whole sweep, so each query gets its own-# heartbeat window instead.-_QUERY_BUDGET = SWEEP_ACTIVITY_HEARTBEAT_TIMEOUT--logger = structlog.get_logger(__name__)---def _teams_with_scanners() -> list[int]:- """Teams that own any scanner. The scanner table is small, so this is the cheap universe to- flag-check; the huge observation table is only queried per team below, where its indexes hold.-- Pinned teams go first. The rest are shuffled, so when a run stops at the time budget, the teams- it leaves for the next run change from run to run instead of always being the newest ones.- """- with bounded_queries(_QUERY_BUDGET, from_attempt_start=False):- team_ids = list(- ReplayScanner.all_origins.values_list("team_id", flat=True)- .distinct()- .order_by("team_id")[: MAX_TEAMS_PER_SWEEP + 1]- )- others = [team_id for team_id in team_ids if team_id not in PINNED_TEAM_IDS]- random.shuffle(others)- return [*PINNED_TEAM_IDS, *others]---def _team_uuids(team_ids: list[int]) -> dict[int, UUID]:- """The flag keys on the project's uuid, so the sweep looks them up in one query per run."""- with bounded_queries(_QUERY_BUDGET, from_attempt_start=False):- return dict(Team.objects.filter(id__in=team_ids).values_list("id", "uuid"))---def _team_scanner_ids(team_id: int, window_start: datetime) -> list[UUID]:- with bounded_queries(_QUERY_BUDGET, from_attempt_start=False):- return list(- ReplayObservation.objects.filter(- team_id=team_id, status=ObservationStatus.SUCCEEDED, created_at__gte=window_start- )- .values_list("scanner_id", flat=True)- .distinct()- )---def _scanner_window(team_id: int, scanner_id: UUID, window_start: datetime) -> dict[UUID, datetime]:- """Newest first, without prose: cheap enough to list the whole capped window every sweep, so the- sweep can tell which rows still lack a judgment and which cached entries left the window."""- with bounded_queries(_QUERY_BUDGET, from_attempt_start=False):- return dict(- ReplayObservation.objects.filter(- team_id=team_id,- scanner_id=scanner_id,- status=ObservationStatus.SUCCEEDED,- created_at__gte=window_start,- )- .order_by("-created_at")- .values_list("id", "created_at")[:WINDOW_SCAN_CAP]- )---def _rows_by_id(team_id: int, ids: list[UUID]) -> list[dict[str, Any]]:- with bounded_queries(_QUERY_BUDGET, from_attempt_start=False):- rows = list(ReplayObservation.objects.filter(team_id=team_id, id__in=ids).values("id", "scanner_result"))- by_id = {row["id"]: dict(row) for row in rows}- # `id__in` loses the caller's newest-first order.- return [by_id[row_id] for row_id in ids if row_id in by_id]--[email protected](name="replay_vision_judge_watch_ranks_activity")-@track_activity()-async def judge_watch_ranks_activity(inputs: JevWatchRankSweepInputs) -> JevWatchRankSweepResult:- """Judges every enrolled scanner's window. Per-scanner failures counted, never raised."""- # Continuous background heartbeats — a slow Jev fan-out otherwise outlives the heartbeat timeout.- async with Heartbeater(factor=4):- return await _judge_watch_ranks(inputs)---@frozen-class _ScannerOutcome:- """What one scanner's turn in the sweep did, summed into the sweep result."""-- judged: bool = False- skipped_unchanged: bool = False- # The scanner waited past the time budget and was left for the next run.- out_of_time: bool = False- # The organization revoked AI data-processing consent while the scanner waited for its turn.- consent_revoked: bool = False- # The gateway rate-limited the scanner's requests.- rate_limited: bool = False- # The run had been rate-limited RATE_LIMIT_GIVE_UP_AFTER turns in a row, so the scanner waits for- # the next run.- backed_off: bool = False- observations_judged: int = 0- observations_given_up: int = 0- cache_errors: int = 0- failed_chunks: int = 0- input_tokens: int = 0- estimated_cost_usd: float = 0.0- estimated_prompt_tokens: int = 0- failed_reason_chunks: int = 0- inspected: bool = False- window_capped: bool = False- pending_observations: int = 0- oldest_pending_age_seconds: float = 0.0---class _TokenPacer:- """Spreads the run's Jev requests across the hour at a steady token rate.-- Thread-safe, because judging runs in worker threads. wait() reserves the next free slot, sized- by the request's tokens, and sleeps until it; the first request in an idle run never waits."""-- def __init__(self, tokens_per_second: float) -> None:- self._seconds_per_token = 1.0 / tokens_per_second- self._lock = threading.Lock()- self._next_start = monotonic()- self.waited_seconds = 0.0-- def wait(self, tokens: int) -> None:- with self._lock:- now = monotonic()- start = max(now, self._next_start)- self._next_start = start + tokens * self._seconds_per_token- waiting = start - now- self.waited_seconds += waiting- if waiting > 0:- sleep(waiting)---async def _sweep_scanner(- team_id: int,- scanner_id: UUID,- mode: str,- window_start: datetime,- pace: Callable[[int], None],- stop_after: float,-) -> _ScannerOutcome:- window = await sync_to_async(_scanner_window)(team_id, scanner_id, window_start)- window_ids = list(window)- if not window_ids:- return _ScannerOutcome(inspected=True)- # Judgments accumulate: each sweep judges only the rows the judged set does not hold,- # newest first, and cache entries whose rows left the window are pruned. Coverage- # therefore grows across sweeps at MAX_JUDGED_PER_SCANNER per hour whatever the- # scanner's volume, and a fully judged window costs nothing.- try:- judged_state = await asyncio.to_thread(load_judged_state, team_id, scanner_id)- cached = await asyncio.to_thread(load_scanner_watch_ranks, team_id, scanner_id)- except Exception:- # A cache the sweep cannot read must not be judged over or written: without the- # judged set it re-buys rows, and the write would replace entries it never saw.- logger.exception(- "Jev watch rank cache unreadable, skipping scanner",- team_id=team_id,- scanner_id=str(scanner_id),- )- return _ScannerOutcome(cache_errors=1)- judged = judged_state.ids- unjudged_ids = [row_id for row_id in window_ids if str(row_id) not in judged][:MAX_JUDGED_PER_SCANNER]- if not unjudged_ids:- await asyncio.to_thread(refresh_watch_ranks_ttl, team_id, scanner_id)- return _ScannerOutcome(skipped_unchanged=True, inspected=True, window_capped=len(window) == WINDOW_SCAN_CAP)- judged_context_ids = [row_id for row_id in window_ids if str(row_id) in judged][:WINDOW_CHUNK_SIZE]- rows = await sync_to_async(_rows_by_id)(team_id, unjudged_ids)- context_rows = await sync_to_async(_rows_by_id)(team_id, judged_context_ids)- judgment = await asyncio.to_thread(judge_scanner_window, team_id, scanner_id, rows, context_rows, pace, stop_after)- # Sub-threshold judgments join only the judged set, keeping the watchable key — which- # the feed loads for every readable scanner on each request — small.- window_strs = {str(row_id) for row_id in window_ids}- watchable = {- **{oid: p for oid, p in cached.probabilities.items() if oid in window_strs},- **{oid: p for oid, p in judgment.probabilities.items() if p >= JEV_WATCHABLE_MIN},- }- reasons = {oid: reason for oid, reason in (cached.reasons | judgment.reasons).items() if oid in watchable}- # A row whose judgment failed through its own batch (an invalid answer, a gateway- # refusal the batch caused) retries on later sweeps, but only MAX_JUDGE_ATTEMPTS times: the- # newest-first pick would otherwise retry a deterministically failing batch every hour- # and starve older rows. An exhausted row is recorded as judged with no score, so it- # settles into the filler tier like a prose-less row. An outage charges nothing — its- # rows stay unjudged and retry free next sweep.- failed_ids = set(judgment.batch_failed_ids)- updated_attempts = {oid: judged_state.attempts.get(oid, 0) + 1 for oid in failed_ids}- exhausted = {oid for oid, count in updated_attempts.items() if count >= MAX_JUDGE_ATTEMPTS}- attempts = {- oid: count for oid, count in judged_state.attempts.items() if oid in window_strs and oid not in failed_ids- } | {oid: count for oid, count in updated_attempts.items() if count < MAX_JUDGE_ATTEMPTS}- all_judged = (judged & window_strs) | set(judgment.probabilities) | set(judgment.skipped_no_prose) | exhausted- cache_errors = 0- if all_judged or attempts:- try:- await asyncio.to_thread(- store_watch_ranks,- team_id,- scanner_id,- all_judged,- watchable,- attempts,- judgment.model,- reasons,- )- except Exception:- # The batch is re-bought next run, which beats one write failure ending the sweep.- logger.exception("Jev watch rank cache write failed", team_id=team_id, scanner_id=str(scanner_id))- cache_errors += 1- persisted_judged = all_judged if cache_errors == 0 else judged- pending = [created_at for row_id, created_at in window.items() if str(row_id) not in persisted_judged]- with suppress(Exception):- # Sub-threshold scores are cached nowhere, so this event is the only record of the- # score distribution — it is the data JEV_WATCHABLE_MIN is calibrated from.- scores = sorted(judgment.probabilities.values())- top_scored = [- {"id": oid, "p": round(probability, 3)}- for oid, probability in sorted(judgment.probabilities.items(), key=lambda entry: entry[1], reverse=True)[:5]- ]- posthoganalytics.capture(- event="replay_vision_jev_watch_rank_judged",- distinct_id=f"team-{team_id}",- properties={- "scanner_id": str(scanner_id),- "mode": mode,- "window_rows": len(rows),- "observations_judged": len(judgment.probabilities),- "mean_watchability": (fmean(judgment.probabilities.values()) if judgment.probabilities else None),- "watchable_count": sum(1 for probability in scores if probability >= JEV_WATCHABLE_MIN),- "watchability_p90": scores[int(0.9 * (len(scores) - 1))] if scores else None,- "watchability_max": scores[-1] if scores else None,- "top_scored": top_scored,- "chunks": judgment.chunks,- "stopped_early": judgment.stopped_early,- "failed_chunks": judgment.failed_chunks,- "chunk_error_types": judgment.chunk_error_types,- "watch_reasons": dict(Counter(judgment.reasons.values())),- "failed_reason_chunks": judgment.failed_reason_chunks,- "rate_limited": judgment.rate_limited,- "jev_model": judgment.model,- "input_tokens": judgment.input_tokens,- "estimated_cost_usd": judgment.estimated_cost_usd,- },- )- return _ScannerOutcome(- judged=True,- rate_limited=judgment.rate_limited,- # A window the deadline cut short surfaces as the time budget being hit, like a turn that- # never started.- out_of_time=judgment.stopped_early,- observations_judged=len(judgment.probabilities),- observations_given_up=len(exhausted),- cache_errors=cache_errors,- failed_chunks=judgment.failed_chunks,- input_tokens=judgment.input_tokens,- estimated_cost_usd=judgment.estimated_cost_usd,- estimated_prompt_tokens=judgment.estimated_prompt_tokens,- failed_reason_chunks=judgment.failed_reason_chunks,- inspected=True,- window_capped=len(window) == WINDOW_SCAN_CAP,- pending_observations=len(pending),- oldest_pending_age_seconds=max(0.0, (datetime.now(UTC) - min(pending)).total_seconds()) if pending else 0.0,- )---async def _judge_watch_ranks(inputs: JevWatchRankSweepInputs) -> JevWatchRankSweepResult:- if not decision_api.decisions_available_here():- result = JevWatchRankSweepResult(decisions_unavailable=True)- await asyncio.to_thread(push_sweep_metrics, result)- return result- started = monotonic()- deadline = started + SWEEP_TIME_BUDGET.total_seconds()- window_start = datetime.now(UTC) - WATCH_RANK_WINDOW- team_ids = await sync_to_async(_teams_with_scanners)()- team_uuids = await sync_to_async(_team_uuids)(team_ids[:MAX_TEAMS_PER_SWEEP])-- # Most of a scanner's turn waits on Jev, so scanners take turns JUDGE_CONCURRENCY at a time. The- # teams loop only flag-checks and lists scanners, so it runs ahead and queues them.- turns = asyncio.Semaphore(JUDGE_CONCURRENCY)- pacer = _TokenPacer(JUDGE_TOKENS_PER_SECOND)-- # Every product's Jev calls share the decision service's capacity. A rate-limited turn pauses new- # turns for RATE_LIMIT_PAUSE, so the run gives that capacity back instead of retrying into it. Only- # a run that stays rate-limited leaves its remaining scanners for the next run.- paused_until = 0.0- consecutive_rate_limited = 0-- async def take_turn(team_id: int, scanner_id: UUID, mode: str) -> _ScannerOutcome:- nonlocal paused_until, consecutive_rate_limited- async with turns:- pause = min(paused_until, deadline) - monotonic()- if pause > 0:- await asyncio.sleep(pause)- if monotonic() > deadline:- return _ScannerOutcome(out_of_time=True)- if consecutive_rate_limited >= RATE_LIMIT_GIVE_UP_AFTER:- return _ScannerOutcome(backed_off=True)- # A turn can wait most of the run after its team was queued, so consent is read again- # here: a revocation in between must stop the prose before it reaches the model.- if not await sync_to_async(is_ai_data_processing_approved)(team_id):- return _ScannerOutcome(consent_revoked=True)- # The deadline passes into the window too: the turn-level check cannot interrupt a- # scanner already judging, and a paced window of many chunks could otherwise run long- # past the budget toward the activity timeout.- outcome = await _sweep_scanner(team_id, scanner_id, mode, window_start, pacer.wait, deadline)- if outcome.rate_limited:- consecutive_rate_limited += 1- paused_until = monotonic() + RATE_LIMIT_PAUSE.total_seconds()- elif outcome.judged:- consecutive_rate_limited = 0- return outcome-- teams_enrolled = 0- teams_evaluated = 0- teams_without_consent = 0- scanners_seen = 0- hit_scanner_cap = False- hit_time_budget = False- queued: list[asyncio.Task[_ScannerOutcome]] = []- async with asyncio.TaskGroup() as scanner_turns:- for team_id in team_ids[:MAX_TEAMS_PER_SWEEP]:- if monotonic() > deadline:- hit_time_budget = True- break- team_uuid = team_uuids.get(team_id)- if team_uuid is None:- continue- mode = await asyncio.to_thread(watch_feed_ranker, team_id, team_uuid)- teams_evaluated += 1- if mode not in ("jev", "jev-shadow"):- continue- # The scan prose the sweep sends to the model derives from recordings, so a revoked consent- # stops the judgments even while the flag stays on.- if not await sync_to_async(is_ai_data_processing_approved)(team_id):- teams_without_consent += 1- continue- teams_enrolled += 1- scanner_ids = await sync_to_async(_team_scanner_ids)(team_id, window_start)- for scanner_id in scanner_ids:- if scanners_seen >= MAX_SCANNERS_PER_SWEEP:- hit_scanner_cap = True- break- scanners_seen += 1- queued.append(scanner_turns.create_task(take_turn(team_id, scanner_id, mode)))- if hit_scanner_cap:- break- outcomes = [turn.result() for turn in queued]-- result = JevWatchRankSweepResult(- teams_seen=min(len(team_ids), MAX_TEAMS_PER_SWEEP),- teams_evaluated=teams_evaluated,- teams_enrolled=teams_enrolled,- teams_without_consent=teams_without_consent,- scanners_consent_revoked=sum(outcome.consent_revoked for outcome in outcomes),- scanners_judged=sum(outcome.judged for outcome in outcomes),- scanners_skipped_unchanged=sum(outcome.skipped_unchanged for outcome in outcomes),- observations_judged=sum(outcome.observations_judged for outcome in outcomes),- observations_given_up=sum(outcome.observations_given_up for outcome in outcomes),- cache_errors=sum(outcome.cache_errors for outcome in outcomes),- failed_chunks=sum(outcome.failed_chunks for outcome in outcomes),- scanners_rate_limited=sum(outcome.rate_limited for outcome in outcomes),- scanners_backed_off=sum(outcome.backed_off for outcome in outcomes),- input_tokens=sum(outcome.input_tokens for outcome in outcomes),- estimated_cost_usd=sum(outcome.estimated_cost_usd for outcome in outcomes),- pacing_wait_seconds=round(pacer.waited_seconds, 1),- hit_team_cap=len(team_ids) > MAX_TEAMS_PER_SWEEP,- hit_scanner_cap=hit_scanner_cap,- hit_time_budget=hit_time_budget or any(outcome.out_of_time for outcome in outcomes),- estimated_prompt_tokens=sum(outcome.estimated_prompt_tokens for outcome in outcomes),- duration_seconds=monotonic() - started,- failed_reason_chunks=sum(outcome.failed_reason_chunks for outcome in outcomes),- scanners_seen=scanners_seen,- scanners_inspected=sum(outcome.inspected for outcome in outcomes),- scanners_window_capped=sum(outcome.window_capped for outcome in outcomes),- pending_observations=sum(outcome.pending_observations for outcome in outcomes),- oldest_pending_age_seconds=max((outcome.oldest_pending_age_seconds for outcome in outcomes), default=0.0),- backlog_complete=(- len(team_ids) <= MAX_TEAMS_PER_SWEEP- and not hit_scanner_cap- and not hit_time_budget- and all(outcome.inspected and not outcome.window_capped for outcome in outcomes)- ),- )- logger.info(- "replay_vision.jev_watch_rank.cycle_complete",- teams_seen=result.teams_seen,- teams_enrolled=result.teams_enrolled,- teams_without_consent=result.teams_without_consent,- scanners_consent_revoked=result.scanners_consent_revoked,- scanners_judged=result.scanners_judged,- scanners_skipped_unchanged=result.scanners_skipped_unchanged,- observations_judged=result.observations_judged,- observations_given_up=result.observations_given_up,- cache_errors=result.cache_errors,- failed_chunks=result.failed_chunks,- scanners_rate_limited=result.scanners_rate_limited,- scanners_backed_off=result.scanners_backed_off,- input_tokens=result.input_tokens,- pacing_wait_seconds=result.pacing_wait_seconds,- hit_team_cap=result.hit_team_cap,- hit_scanner_cap=result.hit_scanner_cap,- hit_time_budget=result.hit_time_budget,- estimated_prompt_tokens=result.estimated_prompt_tokens,- pending_observations=result.pending_observations,- oldest_pending_age_seconds=result.oldest_pending_age_seconds,- backlog_complete=result.backlog_complete,- )- await asyncio.to_thread(push_sweep_metrics, result)- return resultdiff --git a/products/replay_vision/backend/temporal/jev_watch_rank/constants.py b/products/replay_vision/backend/temporal/jev_watch_rank/constants.pydeleted file mode 100644index 5c038311adbc..000000000000--- a/products/replay_vision/backend/temporal/jev_watch_rank/constants.py+++ /dev/null@@ -1,58 +0,0 @@-from datetime import timedelta--SCHEDULE_ID = "replay-vision-jev-watch-rank-schedule"-WORKFLOW_ID = "replay-vision-jev-watch-rank"-WORKFLOW_NAME = "replay-vision-jev-watch-rank"-SCHEDULE_TYPE = "replay-vision-jev-watch-rank"--SCHEDULE_INTERVAL = timedelta(hours=1)--# Matches the feed's default window (`date_from=-7d`), so every row the default feed can show has-# been offered to Jev.-WATCH_RANK_WINDOW = timedelta(days=7)-# Ids listed per scanner per sweep. This bounds the cache size and the id query, and it is the-# coverage ceiling on a very-high-volume scanner: rows past it stay unjudged filler.-WINDOW_SCAN_CAP = 1000-# New rows judged per scanner per sweep. Judgments accumulate in the cache, so this caps the hourly-# Jev spend, not the coverage: a backlog drains at this rate across later sweeps, newest first.-MAX_JUDGED_PER_SCANNER = 100-# The newest-first pick would retry a deterministically failing batch every hour and starve older-# rows for the whole window. After this many batch-attributable failures (an invalid answer, a-# gateway refusal the batch caused) a row is recorded as judged with no score, so it-# settles into the recency filler tier like a prose-less row — for the life of the cache entry,-# since every sweep over an active scanner refreshes the TTL. Environmental failures (an outage,-# a rate limit, a misconfigured gateway) charge no attempt, so a bad hour parks nothing.-MAX_JUDGE_ATTEMPTS = 3-# Team 2 is PostHog's own team, swept first so the internal arm never waits behind the experiment.-PINNED_TEAM_IDS = (2,)-# Bound one sweep, so a misconfigured flag rollout cannot turn it into an unbounded flag-check or Jev-# fan-out. Sized above every team with a scanner and every scanner in a half rollout, so at-# experiment scale the time budget, not these counts, decides how far a run gets.-MAX_TEAMS_PER_SWEEP = 50_000-MAX_SCANNERS_PER_SWEEP = 20_000-# Scanners judged at once. A turn mostly waits on Jev, so this multiplies throughput. Eight at once-# kept hitting the gateway's rate limit, which every product's Jev calls share, so it stays low.-JUDGE_CONCURRENCY = 2-# Estimated prompt tokens per second one run (one region) may send, enforced across its concurrent-# turns. The concurrency cap bounds how many requests are in flight, not how fast they go out, so-# without this the hour's work lands on the decision service in the run's first minutes. About 30%-# of one L4 host's ~7,000 tokens/s, a peak the service owners asked us to stay under.-JUDGE_TOKENS_PER_SECOND = 2_000-# A rate-limited turn pauses new turns this long. The decision service frees capacity within-# seconds, so a pause lets the run continue without taking capacity from other products' calls.-RATE_LIMIT_PAUSE = timedelta(seconds=30)-# Rate-limited turns in a row, with no clean turn between, after which the rest of the run waits for-# the next sweep.-RATE_LIMIT_GIVE_UP_AFTER = 5-# Judging stops here even under the caps: the wall clock is the binding limit, not the counts, and-# paced runs are meant to use most of the hour. Capped under the shortest gap between one run and-# the next (hourly ticks carry up to 10 minutes of jitter, so a late run and an early next tick can-# sit 50 minutes apart). The activity and workflow timeouts below run past that gap, so a run that-# pathologically overshoots its budget can make the SKIP overlap policy drop one tick — accepted,-# because its rows just retry. Scanners cut off by the budget wait for the next hourly run, where-# already-judged rows cost nothing.-SWEEP_TIME_BUDGET = timedelta(minutes=45)--WORKFLOW_EXECUTION_TIMEOUT = timedelta(minutes=55)-SWEEP_ACTIVITY_TIMEOUT = timedelta(minutes=50)-SWEEP_ACTIVITY_HEARTBEAT_TIMEOUT = timedelta(minutes=2)diff --git a/products/replay_vision/backend/temporal/jev_watch_rank/metrics.py b/products/replay_vision/backend/temporal/jev_watch_rank/metrics.pydeleted file mode 100644index 5e8656b26c4f..000000000000--- a/products/replay_vision/backend/temporal/jev_watch_rank/metrics.py+++ /dev/null@@ -1,56 +0,0 @@-import time--from prometheus_client import Gauge--from posthog.metrics import pushed_metrics_registry--from products.replay_vision.backend.temporal.jev_watch_rank.constants import JUDGE_TOKENS_PER_SECOND, SWEEP_TIME_BUDGET-from products.replay_vision.backend.temporal.jev_watch_rank.types import JevWatchRankSweepResult---def push_sweep_metrics(result: JevWatchRankSweepResult) -> None:- # Each worker can run the singleton sweep, so replace one regional snapshot instead of retaining per-worker gauges.- with pushed_metrics_registry("replay_vision_jev_watch_rank_sweep") as registry:- values = {- "last_run_timestamp_seconds": (time.time(), "Unix time the last sweep returned"),- "duration_seconds": (result.duration_seconds, "Duration of the last sweep"),- "estimated_prompt_tokens": (- result.estimated_prompt_tokens,- "Estimated prompt tokens attempted in judgment and reason requests; excludes unsent pacing reservations",- ),- "token_budget": (- JUDGE_TOKENS_PER_SECOND * SWEEP_TIME_BUDGET.total_seconds(),- "Nominal estimated token envelope; request latency and backoff reduce the achievable work",- ),- "teams_evaluated": (result.teams_evaluated, "Teams whose ranking flag was evaluated in the last sweep"),- "teams_enrolled": (result.teams_enrolled, "Teams enrolled with AI consent in the last sweep"),- "teams_without_consent": (result.teams_without_consent, "Enrolled teams skipped without AI consent"),- "scanners_seen": (result.scanners_seen, "Scanner turns queued in the last sweep"),- "scanners_inspected": (result.scanners_inspected, "Scanner windows whose pending work was inspected"),- "scanners_window_capped": (result.scanners_window_capped, "Scanner windows at the row limit"),- "pending_observations": (- result.pending_observations,- "Unjudged rows remaining in inspected bounded windows; a lower bound unless backlog_complete is one",- ),- "oldest_pending_age_seconds": (- result.oldest_pending_age_seconds,- "Age of the oldest pending row in inspected bounded windows at inspection time",- ),- "backlog_complete": (result.backlog_complete, "One when every enrolled scanner window was fully inspected"),- "observations_judged": (result.observations_judged, "Observations judged in the last sweep"),- "observations_given_up": (- result.observations_given_up,- "Observations abandoned after repeated batch faults",- ),- "cache_errors": (result.cache_errors, "Cache failures in the last sweep"),- "failed_chunks": (result.failed_chunks, "Failed judgment requests in the last sweep"),- "failed_reason_chunks": (result.failed_reason_chunks, "Failed reason requests in the last sweep"),- "scanners_rate_limited": (result.scanners_rate_limited, "Scanner turns with a rate-limited request"),- "scanners_backed_off": (result.scanners_backed_off, "Scanner turns skipped after repeated rate limits"),- "hit_time_budget": (result.hit_time_budget, "One when the last sweep reached its time limit"),- "hit_team_cap": (result.hit_team_cap, "One when the last sweep reached its team limit"),- "hit_scanner_cap": (result.hit_scanner_cap, "One when the last sweep reached its scanner limit"),- "decisions_unavailable": (result.decisions_unavailable, "One when the decision service was unavailable"),- }- for name, (value, help_text) in values.items():- Gauge(f"replay_vision_jev_watch_rank_sweep_{name}", help_text, registry=registry).set(value)diff --git a/products/replay_vision/backend/temporal/jev_watch_rank/schedule.py b/products/replay_vision/backend/temporal/jev_watch_rank/schedule.pydeleted file mode 100644index 77a3c8714dc3..000000000000--- a/products/replay_vision/backend/temporal/jev_watch_rank/schedule.py+++ /dev/null@@ -1,34 +0,0 @@-from datetime import timedelta--from temporalio.client import Client-from temporalio.common import SearchAttributePair, TypedSearchAttributes--from posthog.temporal.common.search_attributes import POSTHOG_SCHEDULE_TYPE_KEY--from products.replay_vision.backend.temporal.jev_watch_rank.constants import (- SCHEDULE_ID,- SCHEDULE_INTERVAL,- SCHEDULE_TYPE,- WORKFLOW_EXECUTION_TIMEOUT,- WORKFLOW_ID,- WORKFLOW_NAME,-)-from products.replay_vision.backend.temporal.jev_watch_rank.types import JevWatchRankSweepInputs-from products.replay_vision.backend.temporal.schedule import upsert_interval_schedule---async def create_replay_vision_jev_watch_rank_schedule(client: Client) -> None:- await upsert_interval_schedule(- client,- schedule_id=SCHEDULE_ID,- workflow_name=WORKFLOW_NAME,- workflow_id=WORKFLOW_ID,- inputs=JevWatchRankSweepInputs(),- interval=SCHEDULE_INTERVAL,- offset=timedelta(minutes=2),- jitter=timedelta(minutes=10),- execution_timeout=WORKFLOW_EXECUTION_TIMEOUT,- search_attributes=TypedSearchAttributes(- search_attributes=[SearchAttributePair(key=POSTHOG_SCHEDULE_TYPE_KEY, value=SCHEDULE_TYPE)]- ),- )diff --git a/products/replay_vision/backend/temporal/jev_watch_rank/types.py b/products/replay_vision/backend/temporal/jev_watch_rank/types.pydeleted file mode 100644index 935a883d54ab..000000000000--- a/products/replay_vision/backend/temporal/jev_watch_rank/types.py+++ /dev/null@@ -1,49 +0,0 @@-from pydantic import BaseModel---class JevWatchRankSweepInputs(BaseModel, frozen=True):- pass---class JevWatchRankSweepResult(BaseModel, frozen=True):- # True when this region serves no decisions at all, in which case every other count is zero.- decisions_unavailable: bool = False- teams_seen: int = 0- teams_evaluated: int = 0- teams_enrolled: int = 0- # Enrolled in the flag, but the organization has not approved AI data processing.- teams_without_consent: int = 0- # Queued while the organization approved AI data processing, then skipped at their turn because- # it was revoked in between.- scanners_consent_revoked: int = 0- scanners_judged: int = 0- # Windows whose every row the judged set already holds, so no Jev call was spent.- scanners_skipped_unchanged: int = 0- observations_judged: int = 0- # Rows abandoned after MAX_JUDGE_ATTEMPTS failed judgments; they settle into the filler tier.- observations_given_up: int = 0- # Scanners skipped because their cache could not be read (judging over an unseen cache would- # overwrite it), plus writes that failed after judging.- cache_errors: int = 0- failed_chunks: int = 0- # Scanners whose requests the gateway rate-limited, and scanners left for the next run because- # the run had been rate-limited RATE_LIMIT_GIVE_UP_AFTER turns in a row.- scanners_rate_limited: int = 0- scanners_backed_off: int = 0- input_tokens: int = 0- # Seconds the run's requests spent waiting for their pacing slot, so a run that spends its- # budget paced rather than judging is visible without log access.- pacing_wait_seconds: float = 0.0- estimated_cost_usd: float = 0.0- hit_team_cap: bool = False- hit_scanner_cap: bool = False- hit_time_budget: bool = False- estimated_prompt_tokens: int = 0- duration_seconds: float = 0.0- failed_reason_chunks: int = 0- scanners_seen: int = 0- scanners_inspected: int = 0- scanners_window_capped: int = 0- pending_observations: int = 0- oldest_pending_age_seconds: float = 0.0- backlog_complete: bool = Falsediff --git a/products/replay_vision/backend/temporal/jev_watch_rank/workflow.py b/products/replay_vision/backend/temporal/jev_watch_rank/workflow.pydeleted file mode 100644index d243c4b8474e..000000000000--- a/products/replay_vision/backend/temporal/jev_watch_rank/workflow.py+++ /dev/null@@ -1,33 +0,0 @@-from typing import Any--from temporalio import workflow-from temporalio.common import RetryPolicy--from posthog.temporal.common.base import PostHogWorkflow--from products.replay_vision.backend.temporal.jev_watch_rank.constants import (- SWEEP_ACTIVITY_HEARTBEAT_TIMEOUT,- SWEEP_ACTIVITY_TIMEOUT,- WORKFLOW_NAME,-)-from products.replay_vision.backend.temporal.jev_watch_rank.types import JevWatchRankSweepInputs--with workflow.unsafe.imports_passed_through():- from products.replay_vision.backend.temporal.jev_watch_rank.activities import judge_watch_ranks_activity--[email protected](name=WORKFLOW_NAME)-class ReplayVisionJevWatchRankWorkflow(PostHogWorkflow):- inputs_cls = JevWatchRankSweepInputs- inputs_optional = True-- @workflow.run- async def run(self, inputs: JevWatchRankSweepInputs) -> dict[str, Any]:- result = await workflow.execute_activity(- judge_watch_ranks_activity,- inputs,- start_to_close_timeout=SWEEP_ACTIVITY_TIMEOUT,- heartbeat_timeout=SWEEP_ACTIVITY_HEARTBEAT_TIMEOUT,- retry_policy=RetryPolicy(maximum_attempts=1),- )- return result.model_dump()diff --git a/products/replay_vision/backend/temporal/media_types.py b/products/replay_vision/backend/temporal/media_types.pyindex c2e01684c962..fa6892b7f71e 100644--- a/products/replay_vision/backend/temporal/media_types.py+++ b/products/replay_vision/backend/temporal/media_types.py@@ -83,8 +83,6 @@ class PreparedFrame(BaseModel, frozen=True):small_media_asset_id: int | None = Noneobject_id: strvideo_start_ms: int- # Unused, kept so a worker on the previous release can still decode this payload.- rec_start_ms: int | None = Noneclass PrepareObservationMediaOutput(BaseModel, frozen=True):diff --git a/products/replay_vision/backend/temporal/pii_check.py b/products/replay_vision/backend/temporal/pii_check.pyindex e43b3311ba6d..e7cd07ec4837 100644--- a/products/replay_vision/backend/temporal/pii_check.py+++ b/products/replay_vision/backend/temporal/pii_check.py@@ -41,7 +41,7 @@ PII_FLAG_PROBABILITY = 0.4ASKS_FOR_IDENTITY_PROBABILITY = 0.5_JEV_TIMEOUT_SECONDS = 10.0_REWRITE_MODEL = "gemini-3.8-flash"-_TEXT_FIELDS = ("title", "summary", "reasoning", "notability_reason")+_TEXT_FIELDS = ("title", "summary", "reasoning")_NAME_QUESTION = DecisionQuestion(type=DecisionQuestionType.NOUL,@@ -113,7 +113,6 @@ class _Rewrite(BaseModel):title: str | None = Nonesummary: str | None = Nonereasoning: str | None = None- notability_reason: str | None = Nonedef answer_text(output: BaseModel) -> dict[str, str]:diff --git a/products/replay_vision/backend/temporal/scanners/base.py b/products/replay_vision/backend/temporal/scanners/base.pyindex f53aeb13f21a..f5caf43925e1 100644--- a/products/replay_vision/backend/temporal/scanners/base.py+++ b/products/replay_vision/backend/temporal/scanners/base.py@@ -179,26 +179,6 @@ def confidence_field() -> Any:return Field(ge=0, le=1, description=_CONFIDENCE_DESCRIPTION)-_NOTABILITY_REASON_DESCRIPTION = (- "One sentence a product team would read to decide whether to watch this session, naming the concrete moment "- "that makes it worth their time. Write it even when nothing stands out, saying so plainly."-)-_NOTABILITY_DESCRIPTION = (- "How much a product team would benefit from watching this session, 0.0 to 1.0 with one decimal. "- "Apply the notability calibration rules from the system prompt."-)---def notability_reason_field() -> Any:- """`notability_reason` field for LLM-response schemas. Declared before `notability` so the model names the- moment before scoring it.-- Optional on purpose: this rides every scan in the product, and a required field would turn a model that- skipped it into a failed, already-paid observation. Readers fall back when it is absent.- """- return Field(default=None, description=_NOTABILITY_REASON_DESCRIPTION)--_THUMBNAIL_DESCRIPTION = ("The moment to cut the thumbnail from, in whole seconds of video time counted from the start of the video ""file, the same scale you cite moments in, not the footer's `REC_T`."@@ -208,8 +188,8 @@ _THUMBNAIL_DESCRIPTION = (def thumbnail_field() -> Any:"""`thumbnail_t` field for LLM-response schemas, declared last so the pick never precedes the answer.- Optional for the same reason as `notability`: a skipped pick must not fail a paid-for scan. Readers fall back- to a cited or signal moment when absent.+ Optional on purpose: a required field would turn a model that skipped the pick into a failed, already-paid+ observation. Readers fall back to a cited or signal moment when absent."""return Field(default=None, ge=0, description=_THUMBNAIL_DESCRIPTION)@@ -229,16 +209,6 @@ def key_moment_field() -> Any:return Field(default=None, ge=0, description=_KEY_MOMENT_DESCRIPTION)-def notability_field() -> Any:- """`notability` field for LLM-response schemas.-- Judges the session on its own merits — friction, failure, confusion, surprise — rather than on the scanner's- question, so a scan whose own answer is a non-event can still flag a session worth watching. Optional for the- same reason as `notability_reason`.- """- return Field(default=None, ge=0, le=1, description=_NOTABILITY_DESCRIPTION)--@frozenclass EmbeddingDocument:"""One embedding row's identity and text: `rendering` names which field of the output it came from."""@@ -251,11 +221,6 @@ class BaseScannerOutput(BaseModel, frozen=True):"""Final output shape emitted as `$recording_observed` event properties (flattened with `scanner_output_*` keys)."""confidence: float = confidence_field()- # Optional because observations scanned before notability shipped have neither field; readers must treat- # `None` as "never judged" rather than "not notable", and fall back to their own heuristics. Uses the- # shared field so direct construction is bound to 0-1, not just the LLM-response step schemas.- notability: float | None = notability_field()- notability_reason: str | None = notability_reason_field()# Session-clock offset of `key_moment_t`, stamped after the scan. None when the model skipped the pick or# named a time past the video, and on observations scanned before key moments shipped.key_moment_ms: int | None = Field(default=None, ge=0)diff --git a/products/replay_vision/backend/temporal/scanners/classifier.py b/products/replay_vision/backend/temporal/scanners/classifier.pyindex 9aaae2db54a7..acbe65fd00fb 100644--- a/products/replay_vision/backend/temporal/scanners/classifier.py+++ b/products/replay_vision/backend/temporal/scanners/classifier.py@@ -14,8 +14,6 @@ from products.replay_vision.backend.temporal.scanners.base import (Segment,confidence_field,key_moment_field,- notability_field,- notability_reason_field,thumbnail_field,)@@ -98,8 +96,6 @@ class ClassifierScanner(BaseScanner, frozen=True):),),)- fields["notability_reason"] = (str | None, notability_reason_field())- fields["notability"] = (float | None, notability_field())fields["confidence"] = (float, confidence_field())fields["key_moment_t"] = (int | None, key_moment_field())fields["thumbnail_t"] = (int | None, thumbnail_field())diff --git a/products/replay_vision/backend/temporal/scanners/monitor.py b/products/replay_vision/backend/temporal/scanners/monitor.pyindex 7122063aeefb..e1705e8bdbe4 100644--- a/products/replay_vision/backend/temporal/scanners/monitor.py+++ b/products/replay_vision/backend/temporal/scanners/monitor.py@@ -11,8 +11,6 @@ from products.replay_vision.backend.temporal.scanners.base import (Segment,confidence_field,key_moment_field,- notability_field,- notability_reason_field,thumbnail_field,)@@ -34,8 +32,6 @@ class MonitorLlmResponse(BaseModel, frozen=True):reasoning: str = Field(description=_REASONING_DESCRIPTION)verdict: MonitorVerdict = Field(description=_VERDICT_DESCRIPTION)- notability_reason: str | None = notability_reason_field()- notability: float | None = notability_field()confidence: float = confidence_field()key_moment_t: int | None = key_moment_field()thumbnail_t: int | None = thumbnail_field()diff --git a/products/replay_vision/backend/temporal/scanners/prompts/_macros.jinja b/products/replay_vision/backend/temporal/scanners/prompts/_macros.jinjaindex 4bf255a36948..07c87dda86aa 100644--- a/products/replay_vision/backend/temporal/scanners/prompts/_macros.jinja+++ b/products/replay_vision/backend/temporal/scanners/prompts/_macros.jinja@@ -4,15 +4,6 @@Cite key moments inside `{{ field_name }}` as `(t <seconds>)`, following the Citations rules above.{%- endmacro -%}-{# Session-level notability, asked of every scanner type. Independent of the scanner's *answer* so a- non-event answer can still flag a session worth watching, but not of its *subject*: the area the- scanner watches is the team's declared interest, so trouble there counts for more than the same- trouble elsewhere. Calibration lives in the preamble's <notability_calibration> block (cached- once). Call as `{{ rate_notability() }}`. #}-{%- macro rate_notability() -%}-Separately from your answer above, judge the session itself: in `notability_reason` name the single moment a product team would most want to see (or say plainly that nothing stands out), then set `notability` per the notability calibration rules. Judge what happened to the person, not whether it matches your answer: a session can be worth watching when your answer is a non-event, and a routine session stays low when your answer is a hit.-{%- endmacro -%}-{# The moment the feed opens the recording at, asked in the core turn next to the poster frame. Call as `{{ mark_key_moment() }}`. #}{%- macro mark_key_moment() -%}Give `key_moment_t` in whole seconds of video time, the scale you cite in: where the deciding thing happens, the evidence your answer rests on most (the error, the click that failed, the step the user gave up on). A person opens the recording there to check your answer; it is often the right `thumbnail_t` too.diff --git a/products/replay_vision/backend/temporal/scanners/prompts/classifier_step.jinja b/products/replay_vision/backend/temporal/scanners/prompts/classifier_step.jinjaindex dd64b6c5fb0c..04cdf97eef57 100644--- a/products/replay_vision/backend/temporal/scanners/prompts/classifier_step.jinja+++ b/products/replay_vision/backend/temporal/scanners/prompts/classifier_step.jinja@@ -1,4 +1,4 @@-{% from "_macros.jinja" import cite_in, learned_preferences, mark_key_moment, pick_media, rate_notability %}Categorize this session according to: {{ user_prompt }}+{% from "_macros.jinja" import cite_in, learned_preferences, mark_key_moment, pick_media %}Categorize this session according to: {{ user_prompt }}{{ learned_preferences(scanner_rules) }}In `reasoning`, work through what the session shows that bears on this; then choose tags from this fixed vocabulary: [{{ vocabulary }}]. {% if multi_label %}Pick every tag that applies.{% else %}Pick exactly one tag.{% endif %} A tag is required even when the session never shows what the criterion describes, so in that case pick the least wrong one and follow the relevance rules: lead the `reasoning` with the fact that the session doesn't show it, and keep `confidence` low.{% if allow_freeform_tags %}@@ -8,8 +8,6 @@ Freeform tags this scanner has already used on other sessions: [{{ known_freefor{% endif %}{% endif %}{{ cite_in('reasoning') }}-{{ rate_notability() }}-{{ mark_key_moment() }}{{ pick_media() }}diff --git a/products/replay_vision/backend/temporal/scanners/prompts/experiment_step.jinja b/products/replay_vision/backend/temporal/scanners/prompts/experiment_step.jinjaindex cd85cc062fe7..0bd8a1abb7a5 100644--- a/products/replay_vision/backend/temporal/scanners/prompts/experiment_step.jinja+++ b/products/replay_vision/backend/temporal/scanners/prompts/experiment_step.jinja@@ -1,4 +1,4 @@-{% from "_macros.jinja" import cite_in, learned_preferences, pick_media, rate_notability %}Summarize this session as one exposed session of an A/B experiment.{% if experiment %}+{% from "_macros.jinja" import cite_in, learned_preferences, pick_media %}Summarize this session as one exposed session of an A/B experiment.{% if experiment %}The experiment under test:- Name: {{ experiment.name }}{% if experiment.description %}@@ -18,6 +18,4 @@ Additional context from the team operating the product being recorded:{{ cite_in('summary') }}-{{ rate_notability() }}-{{ pick_media() }}diff --git a/products/replay_vision/backend/temporal/scanners/prompts/monitor_step.jinja b/products/replay_vision/backend/temporal/scanners/prompts/monitor_step.jinjaindex 4d4c3f798f1c..59c2d5df8807 100644--- a/products/replay_vision/backend/temporal/scanners/prompts/monitor_step.jinja+++ b/products/replay_vision/backend/temporal/scanners/prompts/monitor_step.jinja@@ -1,11 +1,9 @@-{% from "_macros.jinja" import cite_in, learned_preferences, mark_key_moment, pick_media, rate_notability %}Decide whether the following condition occurred during the session: {{ user_prompt }}+{% from "_macros.jinja" import cite_in, learned_preferences, mark_key_moment, pick_media %}Decide whether the following condition occurred during the session: {{ user_prompt }}{{ learned_preferences(scanner_rules) }}In `reasoning`, weigh the evidence for and against, noting what the events confirmed or contradicted; then set `verdict` to {% if allow_inconclusive %}`yes` if there's direct visual or event-level evidence the condition occurred, `no` if there's direct evidence it did not, or `inconclusive` only when the session genuinely doesn't provide enough signal to decide.{% else %}`yes` only if there's direct visual or event-level evidence the condition occurred, otherwise `no` — never `inconclusive`.{% endif %} A `yes` needs at least one specific moment you checked with an events lookup; a plausible story the events do not support is not a `yes`. Never say you checked the events at a moment unless your lookups covered it.{{ cite_in('reasoning') }}-{{ rate_notability() }}-{{ mark_key_moment() }}{{ pick_media() }}diff --git a/products/replay_vision/backend/temporal/scanners/prompts/preamble.jinja b/products/replay_vision/backend/temporal/scanners/prompts/preamble.jinjaindex 9b2acd91f97d..10b25044d61a 100644--- a/products/replay_vision/backend/temporal/scanners/prompts/preamble.jinja+++ b/products/replay_vision/backend/temporal/scanners/prompts/preamble.jinja@@ -105,12 +105,6 @@ Write every letter as itself, including accented and non-English letters such asYou'll be asked to rate your own `confidence` (0 to 1) in an answer, on one scale. At most 0.3 when the session never shows what the criterion is about (see relevance). Around 0.5 when there is some evidence but it is ambiguous or points both ways. Up to 0.8 when the conclusion rests on a single moment with no corroboration. 0.9 or above only for an unambiguous conclusion backed by two or more independent signals. Masked or missing content lowers confidence within these bands; it never raises it.</confidence_calibration>-<notability_calibration>-You'll also be asked to rate a session's `notability` (0 to 1): how much a product team would benefit from watching it. Anchor it — 0.0-0.2 a routine session that went as intended; 0.3-0.5 minor hesitation or a small detour worth a glance; 0.6-0.8 real friction the person worked around, a confusing moment, or behaviour that contradicts how the product is meant to be used; 0.9-1.0 the person hit an error, got stuck, or abandoned what they came to do. Most sessions are routine, so most scores belong below 0.5 — reserve the top band for sessions where you can name the moment it went wrong.--Two things shift a session within those bands. Severity that repeats counts for more than the same trouble once: someone who abandons the same step three times belongs a band above someone who hesitates at it once. And where the trouble happened counts: difficulty inside the area your criterion targets is worth more to this team than the same difficulty elsewhere in the session, because that area is what they asked to watch. When the two pull apart, the bands win: an error or a dead end anywhere still reaches the top band, and the area your criterion targets only decides how high within a band the session goes.-</notability_calibration>-<session_metadata>Session-level context (fields are omitted when unknown):{% for key, value in session_metadata.items() %}- {{ key }}: {{ value }}diff --git a/products/replay_vision/backend/temporal/scanners/prompts/scorer_step.jinja b/products/replay_vision/backend/temporal/scanners/prompts/scorer_step.jinjaindex d8a451c9488d..bff91722863d 100644--- a/products/replay_vision/backend/temporal/scanners/prompts/scorer_step.jinja+++ b/products/replay_vision/backend/temporal/scanners/prompts/scorer_step.jinja@@ -1,11 +1,9 @@-{% from "_macros.jinja" import cite_in, learned_preferences, mark_key_moment, pick_media, rate_notability %}Rate this session against the following criterion: {{ user_prompt }}+{% from "_macros.jinja" import cite_in, learned_preferences, mark_key_moment, pick_media %}Rate this session against the following criterion: {{ user_prompt }}{{ learned_preferences(scanner_rules) }}In `reasoning`, work through what you observed in the session that bears on this criterion, noting what the events confirmed or contradicted; then set `score` on {% if scale_label %}the '{{ scale_label }}' scale{% else %}the scale{% endif %} from {{ scale_min }} to {{ scale_max }}. A score toward either end of the scale needs moments you checked with an events lookup behind it, not just how the session looks. A score is required even when the session never shows what the criterion describes, so in that case follow the relevance rules and stay away from both ends of the scale: an extreme score reads as a real finding, and there's nothing here to find.{{ cite_in('reasoning') }}-{{ rate_notability() }}-{{ mark_key_moment() }}{{ pick_media() }}diff --git a/products/replay_vision/backend/temporal/scanners/prompts/summarizer_summary_step.jinja b/products/replay_vision/backend/temporal/scanners/prompts/summarizer_summary_step.jinjaindex a93cca1a7cba..e252f76d1539 100644--- a/products/replay_vision/backend/temporal/scanners/prompts/summarizer_summary_step.jinja+++ b/products/replay_vision/backend/temporal/scanners/prompts/summarizer_summary_step.jinja@@ -1,4 +1,4 @@-{% from "_macros.jinja" import cite_in, learned_preferences, mark_key_moment, pick_media, rate_notability %}Summarize this session.{% if user_prompt %}+{% from "_macros.jinja" import cite_in, learned_preferences, mark_key_moment, pick_media %}Summarize this session.{% if user_prompt %}Additional context from the team operating the product being recorded:{{ user_prompt }}{% endif %}@@ -9,8 +9,6 @@ Additional context from the team operating the product being recorded:Then break the whole recording down into `chapters`: the distinct parts of what the user did, in order, from the start of the video to its end. A new chapter starts where the user moves on to something else: a new page, a new task, a change of approach, or a different section or feature of the same page. Aim for about {{ chapter_target }} chapters when the session has that many distinct parts, and never split where nothing changes on screen. Keep each chapter at least about 20 seconds long, except a notable moment. A long stretch on one page still splits by what the user reads or does there, and no chapter should cover several minutes of varied activity with a generic heading like "Browses page". Each chapter gives the video second it starts at, a heading of a few words, and the frame that best shows it. The chapters describe the recording as it happened, even when the context above asks the summary to focus on one area.-{{ rate_notability() }}-{{ mark_key_moment() }}{{ pick_media() }}diff --git a/products/replay_vision/backend/temporal/scanners/scorer.py b/products/replay_vision/backend/temporal/scanners/scorer.pyindex 7e5b04d6b8c2..7e5899e5af5a 100644--- a/products/replay_vision/backend/temporal/scanners/scorer.py+++ b/products/replay_vision/backend/temporal/scanners/scorer.py@@ -11,8 +11,6 @@ from products.replay_vision.backend.temporal.scanners.base import (Segment,confidence_field,key_moment_field,- notability_field,- notability_reason_field,thumbnail_field,)@@ -67,8 +65,6 @@ class ScorerScanner(BaseScanner, frozen=True):Field(description=_REASONING_DESCRIPTION),),score=(float, Field(ge=self.scale.min, le=self.scale.max, description=score_description)),- notability_reason=(str | None, notability_reason_field()),- notability=(float | None, notability_field()),confidence=(float, confidence_field()),key_moment_t=(int | None, key_moment_field()),thumbnail_t=(int | None, thumbnail_field()),diff --git a/products/replay_vision/backend/temporal/scanners/summarizer.py b/products/replay_vision/backend/temporal/scanners/summarizer.pyindex 892012ee646f..fea8e0e9b232 100644--- a/products/replay_vision/backend/temporal/scanners/summarizer.py+++ b/products/replay_vision/backend/temporal/scanners/summarizer.py@@ -13,8 +13,6 @@ from products.replay_vision.backend.temporal.scanners.base import (Segment,confidence_field,key_moment_field,- notability_field,- notability_reason_field,strip_citation_markers,thumbnail_field,)@@ -127,8 +125,6 @@ class SummarizerSummaryResponse(BaseModel, frozen=True):"burst of rage clicks) its own short chapter so it can be jumped to directly."),)- notability_reason: str | None = notability_reason_field()- notability: float | None = notability_field()confidence: float = confidence_field()key_moment_t: int | None = key_moment_field()thumbnail_t: int | None = thumbnail_field()diff --git a/products/replay_vision/backend/tests/test_api.py b/products/replay_vision/backend/tests/test_api.pyindex 14dc3454f4e9..75a41f5b9c29 100644--- a/products/replay_vision/backend/tests/test_api.py+++ b/products/replay_vision/backend/tests/test_api.py@@ -36,7 +36,6 @@ from posthog.session_recordings.queries.test.session_replay_sql import produce_rfrom products.access_control.backend.models.access_control import AccessControlfrom products.experiments.backend.models.experiment import Experimentfrom products.replay_vision.backend.api.scanners import (- WATCH_FEED_CANDIDATE_CAP,WATCH_FEED_PICKS_BUCKET_S,ReplayScannerSerializer,WatchFeedQuerySerializer,@@ -46,7 +45,6 @@ from products.replay_vision.backend.billing import observation_credits_for_modelfrom products.replay_vision.backend.built_in_scans import BUILT_IN_SCANS, BuiltInScanKeyfrom products.replay_vision.backend.enqueue_claims import _scanner_key, _team_key, pending_enqueue_claims_for_teamfrom products.replay_vision.backend.experiment_launch import start_scanners_of_launched_experiments-from products.replay_vision.backend.jev_watch_feed import store_watch_ranksfrom products.replay_vision.backend.models.replay_observation import (ObservationStatus,ObservationTrigger,@@ -4818,534 +4816,63 @@ class TestWatchFeedAPI(_VisionAPITestCase):ReplayObservation.objects.filter(pk=obs.pk).update(created_at=timezone.now() - timedelta(minutes=minutes_ago))return obs- def _monitor_result(self, verdict: str, signals: int = 0) -> dict[str, Any]:+ def _monitor_result(self, verdict: str) -> dict[str, Any]:return {"model_output": {"scanner_type": "monitor", "verdict": verdict, "reasoning": "r", "confidence": 0.9},- "signals_count": signals,- }-- def test_ranks_signal_then_hit_then_friction_and_drops_the_rest(self) -> None:- scanner = self._create_scanner(name="m")- summarizer = self._create_scanner(- name="s", scanner_type=ScannerType.SUMMARIZER, scanner_config={"prompt": "p", "length": "short"}- )- plain_new = self._succeeded_observation(scanner, "plain-new", 1, self._monitor_result("no"))- hit = self._succeeded_observation(scanner, "hit", 10, self._monitor_result("yes"))- signal = self._succeeded_observation(scanner, "signal", 20, self._monitor_result("no", signals=2))- plain_old = self._succeeded_observation(scanner, "plain-old", 30, self._monitor_result("no"))- # Friction prose outranks a fresher happy-path row, despite being older.- self._succeeded_observation(- scanner,- "friction-sess",- 25,- {- "model_output": {- "scanner_type": "summarizer",- "title": "Checkout gone wrong",- "summary": "The user hit an error at checkout and retried payment twice.",- "confidence": 0.9,- },- "signals_count": 0,- },- )- assert summarizer-- resp = self.client.get(self.feed_url)- self.assertEqual(resp.status_code, 200, resp.json())- items = resp.json()["results"]- # Three findings reach the floor, so the two rows that carry nothing are left out entirely.- self.assertEqual(- [item["observation"]["session_id"] for item in items],- ["signal", "hit", "friction-sess"],- )- self.assertEqual(items[0]["reason"], {"kind": "signal_emitted", "signals_count": 2})- self.assertEqual(items[1]["reason"], {"kind": "verdict_yes"})- self.assertEqual(items[2]["reason"], {"kind": "friction"})- assert plain_new and hit and signal and plain_old-- def test_signal_reason_carries_the_persisted_problem_types(self) -> None:- # The card counts what kinds of issue a signal row carries, so the reason must surface one entry- # per signal in order, repeats kept — not a deduplicated set.- scanner = self._create_scanner(name="m")- self._succeeded_observation(- scanner,- "signal",- 10,- {- "model_output": {"scanner_type": "monitor", "verdict": "no", "reasoning": "r", "confidence": 0.9},- "signals_count": 3,- "signal_problem_types": ["bug", "bug", "crash"],- },- )- reason = self.client.get(self.feed_url).json()["results"][0]["reason"]- self.assertEqual(reason["kind"], "signal_emitted")- self.assertEqual(reason["problem_types"], ["bug", "bug", "crash"])-- def test_signal_reason_names_each_signal(self) -> None:- # A count told a reader nothing, so the row carries the scan's own headline per finding. The- # confidence rides the row too, but it ranks the row rather than saying anything to a reader.- scanner = self._create_scanner(name="m")- self._succeeded_observation(- scanner,- "signal",- 10,- {- "model_output": {"scanner_type": "monitor", "verdict": "no", "reasoning": "r", "confidence": 0.9},- "signals_count": 2,- "signal_problem_types": ["bug", "ux_friction"],- "signal_summaries": [- {"problem_type": "bug", "headline": "Checkout button does nothing", "confidence": 0.9},- {"problem_type": "ux_friction", "headline": "Search results load twice", "confidence": 0.6},- ],- },- )- reason = self.client.get(self.feed_url).json()["results"][0]["reason"]- self.assertEqual(reason["kind"], "signal_emitted")- self.assertEqual(- reason["signals"],- [- {"problem_type": "bug", "headline": "Checkout button does nothing"},- {"problem_type": "ux_friction", "headline": "Search results load twice"},- ],- )-- def test_signal_reason_omits_the_signals_on_rows_scanned_before_headlines(self) -> None:- # Those rows carry a count and the types alone, and the card falls back to counting them.- scanner = self._create_scanner(name="m")- self._succeeded_observation(scanner, "signal", 10, self._monitor_result("no", signals=2))- reason = self.client.get(self.feed_url).json()["results"][0]["reason"]- self.assertEqual(reason["kind"], "signal_emitted")- self.assertNotIn("signals", reason)-- def _friction_result(self) -> dict[str, Any]:- return {- "model_output": {- "scanner_type": "summarizer",- "title": "Checkout gone wrong",- "summary": "The user hit an error at checkout and retried payment twice.",- "confidence": 0.9,- },"signals_count": 0,}- def test_signal_rows_never_hold_more_than_their_share_of_the_findings(self) -> None:- # Signals corroborate across sessions, so a scanner raising them on every session used to crowd the- # other findings out of the feed. They are held against the other findings rather than against- # padding, because padding no longer survives the trim.- scanner = self._create_scanner(name="m")- summarizer = self._create_scanner(- name="s", scanner_type=ScannerType.SUMMARIZER, scanner_config={"prompt": "p", "length": "short"}- )- for i in range(6):- self._succeeded_observation(scanner, f"signal-{i}", 10 + i, self._monitor_result("no", signals=1))- self._succeeded_observation(summarizer, f"friction-{i}", 40 + i, self._friction_result())-- items = self.client.get(f"{self.feed_url}?limit=50").json()["results"]- self.assertEqual(len(items), 12) # every finding is returned; the cap reorders, it never drops- kinds = [item["reason"]["kind"] for item in items]- self.assertEqual(kinds[0], "signal_emitted") # the best row leads whatever it is- # Six signal rows cannot all fit under the share, so the last of them trail the feed. The share- # governs everything up to there, which is the part a reader with a normal limit ever sees.- interleaved = len(kinds) - kinds[::-1].index("friction")- self.assertEqual(set(kinds[interleaved:]), {"signal_emitted"})- self.assertEqual(kinds[:5].count("signal_emitted"), 2) # the share, exactly, over the first five- # The lead is exempt, so the bound is the share or one card, whichever is larger.- for length in range(2, interleaved + 1):- prefix = kinds[:length]- allowed = max(1, int(length * 0.4))- self.assertLessEqual(prefix.count("signal_emitted"), allowed, f"prefix of {length}: {prefix}")-- def test_clips_that_carry_no_finding_are_dropped_once_the_feed_has_three(self) -> None:- # The feed used to fill `limit` with whatever was newest, so a window with three findings returned- # those three under seventeen clips that said only "new since you last looked".- scanner = self._create_scanner(name="m")- summarizer = self._create_scanner(- name="s", scanner_type=ScannerType.SUMMARIZER, scanner_config={"prompt": "p", "length": "short"}- )- for i in range(3):- self._succeeded_observation(summarizer, f"friction-{i}", 10 + i, self._friction_result())- for i in range(9):- self._succeeded_observation(scanner, f"plain-{i}", 40 + i, self._monitor_result("no"))-- items = self.client.get(f"{self.feed_url}?limit=50").json()["results"]- self.assertEqual(- [item["observation"]["session_id"] for item in items], ["friction-0", "friction-1", "friction-2"]- )-- def test_every_finding_is_returned_however_many_there_are(self) -> None:- # The floor pads a thin feed; it never caps a rich one.- summarizer = self._create_scanner(- name="s", scanner_type=ScannerType.SUMMARIZER, scanner_config={"prompt": "p", "length": "short"}- )- for i in range(10):- self._succeeded_observation(summarizer, f"friction-{i}", 10 + i, self._friction_result())-- items = self.client.get(f"{self.feed_url}?limit=50").json()["results"]- self.assertEqual(len(items), 10)- self.assertEqual({item["reason"]["kind"] for item in items}, {"friction"})-- def test_a_thin_feed_is_padded_to_the_floor_rather_than_to_the_limit(self) -> None:- # One finding alone reads as a broken feed, and twenty reads as noise, so the padding stops at three.- scanner = self._create_scanner(name="m")- summarizer = self._create_scanner(- name="s", scanner_type=ScannerType.SUMMARIZER, scanner_config={"prompt": "p", "length": "short"}+ def _ranked_observation(+ self,+ scanner: ReplayScanner,+ session_id: str,+ minutes_ago: int,+ category: str = "routine",+ contrast: float = -0.1,+ group: str | None = None,+ ) -> ReplayObservation:+ obs = self._succeeded_observation(scanner, session_id, minutes_ago, self._monitor_result("no"))+ ReplayObservation.objects.filter(id=obs.id).update(+ watch_rank={"category": category, "contrast": contrast, "group": group, "anchor_version": ANCHOR_VERSION})- self._succeeded_observation(summarizer, "friction-0", 5, self._friction_result())- for i in range(9):- self._succeeded_observation(scanner, f"plain-{i}", 40 + i, self._monitor_result("no"))-- items = self.client.get(f"{self.feed_url}?limit=50").json()["results"]- self.assertEqual([item["reason"]["kind"] for item in items], ["friction", "unviewed_recent", "unviewed_recent"])-- def test_a_window_with_no_findings_returns_the_floor_rather_than_nothing(self) -> None:- # A feed that empties whenever nothing is wrong reads as broken, so the newest clips still answer.- scanner = self._create_scanner(name="m")- for i in range(9):- self._succeeded_observation(scanner, f"plain-{i}", 10 + i, self._monitor_result("no"))-- items = self.client.get(f"{self.feed_url}?limit=50").json()["results"]- self.assertEqual([item["observation"]["session_id"] for item in items], ["plain-0", "plain-1", "plain-2"])+ return obs- def test_a_window_of_only_signals_still_returns_a_full_feed(self) -> None:- # With nothing to interleave, the held rows fill the tail rather than vanishing from the feed.+ @parameterized.expand(+ [+ # A feed that empties whenever nothing is wrong reads as broken, so the newest clips still answer.+ ("no_findings", 0, ["unviewed_recent"] * 3),+ ("enough_findings", 4, ["embedding_pick"] * 4),+ ]+ )+ def test_filler_brings_a_thin_feed_up_to_the_floor_and_never_pads_a_full_one(+ self, _name: str, findings: int, expected_kinds: list[str]+ ) -> None:scanner = self._create_scanner(name="m")- for i in range(5):- self._succeeded_observation(scanner, f"signal-{i}", 10 + i, self._monitor_result("no", signals=1))-- items = self.client.get(f"{self.feed_url}?limit=50").json()["results"]- self.assertEqual(- [item["observation"]["session_id"] for item in items],- ["signal-0", "signal-1", "signal-2", "signal-3", "signal-4"],- )-- def test_a_wide_outlier_outranks_and_outlabels_a_weak_signal(self) -> None:- # The reason names the strongest evidence now. A row used to read `signal_emitted` whenever it- # carried a signal, even where a far wider outlier was the thing worth watching.- scorer = self._create_scanner(- name="sc", scanner_type=ScannerType.SCORER, scanner_config={"prompt": "p", "scale_max": 10}- )-- def _scored(session_id: str, minutes_ago: int, score: float, summaries: list[dict] | None = None) -> None:- self._succeeded_observation(- scorer,- session_id,- minutes_ago,- {- "model_output": {"scanner_type": "scorer", "score": score, "reasoning": "r", "confidence": 0.9},- "signals_count": len(summaries or []),- "signal_problem_types": [entry["problem_type"] for entry in summaries or []],- "signal_summaries": summaries or [],- },+ # Of 20 ranked rows the top four clear the percentile cut, so only the failures among them are picks.+ for i in range(20):+ self._ranked_observation(+ scanner, f"row-{i}", 10 + i, "failure" if i < findings else "routine", 0.9 - i / 100)- for i in range(6):- _scored(f"usual-{i}", 30 + i, 5.0 + (i % 2) * 0.1)- _scored("outlier", 20, 10.0)- _scored(- "weak-signal",- 10,- 5.0,- [{"problem_type": "bug", "headline": "Tooltip sits behind the header", "confidence": 0.45}],- )-items = self.client.get(f"{self.feed_url}?limit=50").json()["results"]- self.assertEqual(items[0]["observation"]["session_id"], "outlier")- self.assertEqual(items[0]["reason"]["kind"], "outlier_score")-- def test_minority_verdict_is_the_hit_regardless_of_prompt_polarity(self) -> None:- # "Was the experience good?" answers yes almost always, so its rare "no" is the notable- # one; a majority "yes" must not rank as a hit just for being yes.- scanner = self._create_scanner(name="good-experience")- for i in range(6):- self._succeeded_observation(scanner, f"good-{i}", 10 + i, self._monitor_result("yes"))- self._succeeded_observation(scanner, "bad-one", 1, self._monitor_result("no"))-- resp = self.client.get(f"{self.feed_url}?limit=50")- items = resp.json()["results"]- self.assertEqual(items[0]["observation"]["session_id"], "bad-one")- self.assertEqual(items[0]["reason"]["kind"], "unusual_verdict")- self.assertEqual(items[0]["reason"]["verdict"], "no")- reasons = {item["observation"]["session_id"]: item["reason"]["kind"] for item in items}- self.assertEqual(reasons["good-0"], "unviewed_recent")-- def test_high_notability_outranks_a_weak_hit_but_still_beats_friction_and_carries_its_sentence(self) -> None:- # The blend lets the scan's own judgment surface: a high notability outranks a bare verdict-yes- # hit (weak, no baseline), still beats keyword friction, and its sentence reaches the card.- scanner = self._create_scanner(name="m")- self._succeeded_observation(- scanner,- "notable-sess",- 20,- {- "model_output": {- "scanner_type": "monitor",- "verdict": "no",- "reasoning": "r",- "confidence": 0.9,- "notability": 0.8,- "notability_reason": "Tried the same export three times and never saw an error.",- },- "signals_count": 0,- },- )- self._succeeded_observation(- scanner,- "friction-sess",- 10,- {- "model_output": {- "scanner_type": "summarizer",- "title": "t",- "summary": "The user hit an error and retried.",- "confidence": 0.9,- },- "signals_count": 0,- },- )- self._succeeded_observation(scanner, "intent-hit", 30, self._monitor_result("yes"))-- resp = self.client.get(self.feed_url)- items = resp.json()["results"]- self.assertEqual(- [item["observation"]["session_id"] for item in items],- ["notable-sess", "intent-hit", "friction-sess"],- )- self.assertEqual(items[0]["reason"]["kind"], "notable")- self.assertEqual(- items[0]["reason"]["notability_reason"], "Tried the same export three times and never saw an error."- )-- def test_notability_reason_stays_off_rows_that_did_not_rank_on_notability(self) -> None:- # The scan writes a notability_reason on every session, so a routine or signal row carries one- # too; it must not override the copy those rows earned from their own reason kind.- scanner = self._create_scanner(name="m")- self._succeeded_observation(- scanner,- "routine",- 10,- {- "model_output": {- "scanner_type": "monitor",- "verdict": "no",- "reasoning": "r",- "confidence": 0.9,- "notability": 0.1,- "notability_reason": "Nothing stands out in this session.",- },- "signals_count": 0,- },- )- # Notability above the notable threshold, so a candidate-level gate would wrongly attach the- # sentence; the signal still won the row, so its own copy must survive.- self._succeeded_observation(- scanner,- "signal",- 20,- {- "model_output": {- "scanner_type": "monitor",- "verdict": "no",- "reasoning": "r",- "confidence": 0.9,- "notability": 0.8,- "notability_reason": "Tried the export three times.",- },- "signals_count": 2,- },- )-- reasons = {- item["observation"]["session_id"]: item["reason"]- for item in self.client.get(self.feed_url).json()["results"]- }- self.assertEqual(reasons["signal"]["kind"], "signal_emitted")- self.assertNotIn("notability_reason", reasons["signal"])- self.assertEqual(reasons["routine"]["kind"], "unviewed_recent")- self.assertNotIn("notability_reason", reasons["routine"])-- def test_notability_breaks_ties_between_rows_in_the_same_tier(self) -> None:- # Two unviewed plain rows share a tier, so the scan's notability score orders them.- scanner = self._create_scanner(name="m")- self._succeeded_observation(- scanner,- "lower",- 10,- {- "model_output": {- "scanner_type": "monitor",- "verdict": "no",- "reasoning": "r",- "confidence": 0.9,- "notability": 0.2,- },- "signals_count": 0,- },- )- self._succeeded_observation(- scanner,- "higher",- 20,- {- "model_output": {- "scanner_type": "monitor",- "verdict": "no",- "reasoning": "r",- "confidence": 0.9,- "notability": 0.5,- },- "signals_count": 0,- },- )-- sessions = [item["observation"]["session_id"] for item in self.client.get(self.feed_url).json()["results"]]- self.assertEqual(sessions, ["higher", "lower"])-- def test_stored_notability_is_clamped_and_booleans_are_ignored(self) -> None:- # ge/le only bind the LLM response; a stored row can carry anything, and bool is an int subclass.- scanner = self._create_scanner(name="m")- self._succeeded_observation(- scanner,- "out-of-range",- 10,- {- "model_output": {- "scanner_type": "monitor",- "verdict": "no",- "reasoning": "r",- "confidence": 0.9,- "notability": 5.0,- },- "signals_count": 0,- },- )- self._succeeded_observation(- scanner,- "boolean",- 20,- {- "model_output": {- "scanner_type": "monitor",- "verdict": "no",- "reasoning": "r",- "confidence": 0.9,- "notability": True,- },- "signals_count": 0,- },- )-- reasons = {- item["observation"]["session_id"]: item["reason"]- for item in self.client.get(self.feed_url).json()["results"]- }- self.assertEqual(reasons["out-of-range"]["kind"], "notable")- self.assertEqual(reasons["out-of-range"]["notability"], 1.0)- self.assertEqual(reasons["boolean"]["kind"], "unviewed_recent")-- def test_no_verdict_monitor_negation_is_not_friction(self) -> None:- # "Did they struggle? No" reasoning restates the question; keyword matching must not- # read the negation as a friction hit.- scanner = self._create_scanner(name="m")- self._succeeded_observation(- scanner,- "calm",- 1,- {- "model_output": {- "scanner_type": "monitor",- "verdict": "no",- "reasoning": "The user did not struggle and saw no errors.",- "confidence": 0.9,- },- "signals_count": 0,- },- )- resp = self.client.get(self.feed_url)- self.assertEqual(resp.json()["results"][0]["reason"], {"kind": "unviewed_recent"})-- def test_the_flag_switches_the_whole_ranker_between_arms(self) -> None:- # The two rankers are independent: a cached Jev probability must not move the weighted-score- # feed (weighted-score and jev-shadow arms), and the jev arm must rank on the cached- # probabilities alone, with unjudged and judged-low rows in the recency filler tier.- scanner = self._create_scanner(name="m")- jev_high = self._succeeded_observation(scanner, "jev-high", 40, self._monitor_result("no"))- self._succeeded_observation(scanner, "signal", 20, self._monitor_result("no", signals=2))- jev_low = self._succeeded_observation(scanner, "jev-low", 10, self._monitor_result("yes"))- store_watch_ranks(- self.team.id, scanner.id, {str(jev_high.id), str(jev_low.id)}, {str(jev_high.id): 0.95}, {}, "jevk5-fp8-0.2"- )-- ranker = "products.replay_vision.backend.api.scanners.watch_feed_ranker_variant"- for mode in ("control", "jev-shadow", None):- with patch(ranker, return_value=mode):- resp = self.client.get(self.feed_url)- items = resp.json()["results"]- # The shadow arm ranks on the weighted score, so the response names that ranker too, but- # it still reports its own variant, which the client records as the experiment exposure.- self.assertEqual(resp.json()["ranker"], "weighted-score", mode)- self.assertEqual(resp.json()["ranker_variant"], mode)- # The signal and the verdict hit lead as today; the cached 0.95 moves nothing.- self.assertEqual(- [item["observation"]["session_id"] for item in items],- ["signal", "jev-low", "jev-high"],- mode,- )- self.assertEqual(items[0]["reason"]["kind"], "signal_emitted", mode)- self.assertEqual(items[2]["reason"], {"kind": "unviewed_recent"}, mode)-- with patch(ranker, return_value="jev"):- resp = self.client.get(self.feed_url)- items = resp.json()["results"]- self.assertEqual(resp.json()["ranker"], "jev")- # jev-high carries evidence; the 0.2 row and the unjudged row fall to the filler tier by- # recency, so neither claims the model judged it worth watching.- self.assertEqual(- [item["observation"]["session_id"] for item in items],- ["jev-high", "jev-low", "signal"],- )- self.assertEqual(items[0]["reason"], {"kind": "jev_watchable", "jev_probability": 0.95})- self.assertEqual(items[1]["reason"], {"kind": "unviewed_recent"})- self.assertEqual(items[2]["reason"], {"kind": "unviewed_recent"})-- # The embedding arm ranks on the rank the scan wrote, once the scanner has enough ranked rows.- for index in range(10):- filler = self._succeeded_observation(scanner, f"routine-{index}", 50 + index, self._monitor_result("no"))- ReplayObservation.objects.filter(id=filler.id).update(- watch_rank={- "category": "routine",- "contrast": -0.1,- "group": None,- "anchor_version": ANCHOR_VERSION,- }- )- ReplayObservation.objects.filter(id=jev_low.id).update(- watch_rank={"category": "failure", "contrast": 0.2, "group": None, "anchor_version": ANCHOR_VERSION}- )- with patch(ranker, return_value="embedding"):- resp = self.client.get(self.feed_url)- items = resp.json()["results"]- self.assertEqual(resp.json()["ranker"], "embedding")- self.assertEqual(items[0]["observation"]["session_id"], "jev-low")+ self.assertEqual([item["reason"]["kind"] for item in items], expected_kinds)self.assertEqual(- items[0]["reason"],- {"kind": "embedding_pick", "watch_category": "failure", "similar_count": 0, "affected_people": 1},+ [item["observation"]["session_id"] for item in items], [f"row-{i}" for i in range(len(expected_kinds))])def test_pages_do_not_shift_when_the_reader_opens_a_card_mid_scroll(self) -> None:- scanner = self._create_scanner(- name="s", scanner_type=ScannerType.SUMMARIZER, scanner_config={"prompt": "p", "length": "short"}- )- for i in range(6):- self._succeeded_observation(scanner, f"hit-{i}", 10 + i, self._friction_result())+ scanner = self._create_scanner(name="m")+ for i in range(30):+ self._ranked_observation(scanner, f"hit-{i}", 10 + i, "failure" if i < 6 else "routine", 0.9 - i / 100)first = self.client.get(f"{self.feed_url}?limit=3").json()assert first["has_more"] is True# Opening the top card would dock it below page two, and a card arriving would push everything down.top_id = first["results"][0]["observation"]["id"]assert self.client.post(f"{self.observations_url(str(scanner.id))}{top_id}/viewed/").status_code == 204- self._succeeded_observation(scanner, "late-hit", 0, self._friction_result())+ self._ranked_observation(scanner, "late-hit", 0, "failure", 1.0)# A scan already running at page one, which completes before page two, would also shift it.- self._succeeded_observation(scanner, "finished-late", 12, self._friction_result())+ self._ranked_observation(scanner, "finished-late", 12, "failure", 1.0)second = self.client.get(self.feed_url, {"limit": 3, "offset": 3, "as_of": first["as_of"]}).json()sessions = [item["observation"]["session_id"] for item in first["results"] + second["results"]]@@ -5376,37 +4903,18 @@ class TestWatchFeedAPI(_VisionAPITestCase):response = self.client.get(self.feed_url, {"limit": "20", "as_of": as_of.isoformat()}).json()return [item["observation"]["session_id"] for item in response["results"]]- with patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value="embedding"):- assert "row-0" in sessions(later)- assert "row-0" not in sessions(earlier)+ assert "row-0" in sessions(later)+ assert "row-0" not in sessions(earlier)- def test_embedding_picks_reach_past_the_recency_slice_and_count_hidden_duplicates(self) -> None:+ def test_an_embedding_pick_counts_its_hidden_duplicates(self) -> None:scanner = self._create_scanner(name="m")-- def ranked(session_id: str, minutes_ago: int, category: str, contrast: float, group: str | None = None):- obs = self._succeeded_observation(scanner, session_id, minutes_ago, self._monitor_result("no"))- ReplayObservation.objects.filter(id=obs.id).update(- watch_rank={- "category": category,- "contrast": contrast,- "group": group,- "anchor_version": ANCHOR_VERSION,- }- )- return obs-- oldest = ranked("oldest-failure", 500, "failure", 0.9)+ oldest = self._ranked_observation(scanner, "oldest-failure", 500, "failure", 0.9)# A copy of the pick that scores below the cut still makes the group bigger.- ranked("copy", 400, "failure", -0.5, group=str(oldest.id))+ self._ranked_observation(scanner, "copy", 400, "failure", -0.5, group=str(oldest.id))for index in range(10):- ranked(f"routine-{index}", index, "routine", -0.1)+ self._ranked_observation(scanner, f"routine-{index}", index)- # The candidate slice holds two rows per scanner, so the oldest row only surfaces from the whole window.- with (- patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value="embedding"),- patch("products.replay_vision.backend.api.scanners.WATCH_FEED_PER_SCANNER_CAP", 2),- ):- items = self.client.get(self.feed_url).json()["results"]+ items = self.client.get(self.feed_url).json()["results"]picks = [item for item in items if item["reason"]["kind"] == "embedding_pick"]self.assertEqual([item["observation"]["session_id"] for item in picks], ["oldest-failure"])self.assertEqual(picks[0]["reason"]["similar_count"], 1)@@ -5414,28 +4922,14 @@ class TestWatchFeedAPI(_VisionAPITestCase):def test_a_thin_embedding_feed_pads_by_group_and_never_pads_with_a_copy_of_a_pick(self) -> None:scanner = self._create_scanner(name="m")-- def ranked(session_id: str, minutes_ago: int, category: str, contrast: float, group: str | None = None):- obs = self._succeeded_observation(scanner, session_id, minutes_ago, self._monitor_result("no"))- ReplayObservation.objects.filter(id=obs.id).update(- watch_rank={- "category": category,- "contrast": contrast,- "group": group,- "anchor_version": ANCHOR_VERSION,- }- )- return obs-- root = ranked("pick", 500, "failure", 0.9)+ root = self._ranked_observation(scanner, "pick", 500, "failure", 0.9)# Two copies of the one finding: one above the cut, one newest of all and below it.- ranked("copy-above", 400, "failure", 0.8, group=str(root.id))- ranked("copy-newest", 0, "failure", -0.9, group=str(root.id))+ self._ranked_observation(scanner, "copy-above", 400, "failure", 0.8, group=str(root.id))+ self._ranked_observation(scanner, "copy-newest", 0, "failure", -0.9, group=str(root.id))for index in range(10):- ranked(f"routine-{index}", 10 + index, "routine", -0.1)+ self._ranked_observation(scanner, f"routine-{index}", 10 + index)- with patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value="embedding"):- items = self.client.get(self.feed_url).json()["results"]+ items = self.client.get(self.feed_url).json()["results"]sessions = [item["observation"]["session_id"] for item in items]self.assertEqual(sessions[0], "pick")self.assertEqual(len(sessions), 3)@@ -5457,10 +4951,7 @@ class TestWatchFeedAPI(_VisionAPITestCase):})- with (- patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value="embedding"),- patch("products.replay_vision.backend.api.scanners.WATCH_FEED_CANDIDATE_CAP", 2),- ):+ with patch("products.replay_vision.backend.api.scanners.WATCH_FEED_CANDIDATE_CAP", 2):items = self.client.get(self.feed_url).json()["results"]picked = {item["observation"]["session_id"] for item in items if item["reason"]["kind"] == "embedding_pick"}self.assertEqual(picked, {"busy-0", "quiet-0"})@@ -5489,7 +4980,6 @@ class TestWatchFeedAPI(_VisionAPITestCase):root_id = root_id or str(obs.id)with (- patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value="embedding"),patch("products.replay_vision.backend.api.scanners.WATCH_FEED_CANDIDATE_CAP", cap),patch("products.replay_vision.backend.watch_rank.WATCH_PICK_GROUP_CANDIDATES", group_candidates),):@@ -5530,10 +5020,7 @@ class TestWatchFeedAPI(_VisionAPITestCase):load = uuid.uuid4()def load_picks(**params: str) -> tuple[list[str], str]:- with patch(- "products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value="embedding"- ):- body = self.client.get(self.feed_url, params).json()+ body = self.client.get(self.feed_url, params).json()picks = [item["observation"]["session_id"]for item in body["results"]@@ -5558,286 +5045,6 @@ class TestWatchFeedAPI(_VisionAPITestCase):assert load_picks(as_of=as_of)[0] == ["s-0", "s-1"]assert load_picks()[0] == ["s-1", "s-0"]- def test_the_jev_arm_surfaces_a_watchable_row_the_recency_slice_cut_off(self) -> None:- # The candidate query keeps only each scanner's newest rows, so on a high-volume scanner a- # watchable row from days ago never reaches the weighted feed. The jev arm must fetch it- # back from the cache and rank it first.- scanner = self._create_scanner(name="m")- interesting_result = self._monitor_result("yes")- interesting_result["model_output"]["notability"] = 0.9- interesting_result["model_output"]["notability_reason"] = "The user paid twice for one order."- old_interesting = self._succeeded_observation(scanner, "old-interesting", 60 * 24 * 2, interesting_result)- for index in range(100):- self._succeeded_observation(scanner, f"routine-{index}", index + 1, self._monitor_result("no"))- store_watch_ranks(- self.team.id,- scanner.id,- {str(old_interesting.id)},- {str(old_interesting.id): 0.9},- {},- "jevk5-fp8-0.2",- {str(old_interesting.id): "visible_error"},- )-- ranker = "products.replay_vision.backend.api.scanners.watch_feed_ranker_variant"- with patch(ranker, return_value="control"):- resp = self.client.get(self.feed_url)- self.assertNotIn("old-interesting", [item["observation"]["session_id"] for item in resp.json()["results"]])-- with patch(ranker, return_value="jev"):- resp = self.client.get(self.feed_url)- items = resp.json()["results"]- self.assertEqual(items[0]["observation"]["session_id"], "old-interesting")- # The scan found the session notable, so its own sentence reaches the card's reason, next to- # the reason Jev picked from its fixed list.- self.assertEqual(- items[0]["reason"],- {- "kind": "jev_watchable",- "jev_probability": 0.9,- "notability_reason": "The user paid twice for one order.",- "watch_reason": "visible_error",- },- )- # The 100 routine rows are filler: they pad the one finding only to the feed's floor.- self.assertEqual(len(items), 3)-- def test_the_jev_bypass_filters_before_it_caps(self) -> None:- # The cache knows nothing about the request's filters: cached watchable ids that match no- # candidate row must not use up the bypass and push out a lower-probability row that does.- scanner = self._create_scanner(name="m")- interesting = self._monitor_result("yes")- interesting["model_output"]["reasoning"] = "the checkout flow needle"- old_interesting = self._succeeded_observation(scanner, "old-interesting", 60 * 24 * 2, interesting)- for index in range(100):- routine = self._monitor_result("no")- routine["model_output"]["reasoning"] = "the checkout flow needle"- self._succeeded_observation(scanner, f"routine-{index}", index + 1, routine)- # More cached watchable ids than the old pre-filter slice kept, all rated above the one row- # that matches the search and none of them a real observation.- watchable = {str(uuid.uuid4()): 0.9 for _ in range(WATCH_FEED_CANDIDATE_CAP)}- watchable[str(old_interesting.id)] = 0.6- store_watch_ranks(self.team.id, scanner.id, set(watchable), watchable, {}, "jevk5-fp8-0.2")-- ranker = "products.replay_vision.backend.api.scanners.watch_feed_ranker_variant"- with patch(ranker, return_value="jev"):- resp = self.client.get(f"{self.feed_url}?search=needle")- items = resp.json()["results"]- self.assertEqual(items[0]["observation"]["session_id"], "old-interesting")- self.assertEqual(items[0]["reason"], {"kind": "jev_watchable", "jev_probability": 0.6})- self.assertEqual(len(items), 3)-- def test_viewed_orders_within_tiers_but_never_sinks_a_signal_below_plain_rows(self) -> None:- # Seen-state is a within-tier order, not a top-level one: a signal the reader saw yesterday- # still outranks unviewed routine rows, while among plain rows unviewed comes first.- scanner = self._create_scanner(name="m")- viewed_signal = self._succeeded_observation(scanner, "viewed-signal", 1, self._monitor_result("no", signals=3))- viewed_plain = self._succeeded_observation(scanner, "viewed-plain", 10, self._monitor_result("no"))- self._succeeded_observation(scanner, "fresh-plain", 30, self._monitor_result("no"))- for viewed in (viewed_signal, viewed_plain):- self.assertEqual(- self.client.post(f"{self.observations_url(str(scanner.id))}{viewed.id}/viewed/").status_code, 204- )-- resp = self.client.get(self.feed_url)- items = resp.json()["results"]- self.assertEqual(- [item["observation"]["session_id"] for item in items],- ["viewed-signal", "fresh-plain", "viewed-plain"],- )- self.assertEqual(items[0]["reason"]["kind"], "signal_emitted")- self.assertTrue(items[0]["observation"]["viewed"])-- def test_outlier_score_and_rare_tag_reasons(self) -> None:- scorer = self._create_scanner(- name="s", scanner_type=ScannerType.SCORER, scanner_config={"prompt": "p", "scale": {"min": 0, "max": 10}}- )- for i in range(5):- self._succeeded_observation(- scorer,- f"score-{i}",- 10 + i,- {- "model_output": {"scanner_type": "scorer", "score": 5.0, "reasoning": "r", "confidence": 0.9},- "signals_count": 0,- },- )- self._succeeded_observation(- scorer,- "score-outlier",- 1,- {- "model_output": {"scanner_type": "scorer", "score": 9.5, "reasoning": "r", "confidence": 0.9},- "signals_count": 0,- },- )- classifier = self._create_scanner(- name="c",- scanner_type=ScannerType.CLASSIFIER,- scanner_config={"prompt": "p", "tags": ["common", "rare"], "multi_label": True},- )- for i in range(9):- self._succeeded_observation(- classifier,- f"tag-{i}",- 10 + i,- {- "model_output": {- "scanner_type": "classifier",- "tags": ["common"],- "reasoning": "r",- "confidence": 0.9,- },- "signals_count": 0,- },- )- self._succeeded_observation(- classifier,- "tag-rare",- 1,- {- "model_output": {- "scanner_type": "classifier",- "tags": ["common", "rare"],- "reasoning": "r",- "confidence": 0.9,- },- "signals_count": 0,- },- )-- summarizer = self._create_scanner(- name="sum-novel",- scanner_type=ScannerType.SUMMARIZER,- scanner_config={"prompt": "p", "length": "short"},- )- for i in range(5):- self._succeeded_observation(- summarizer,- f"sum-{i}",- 10 + i,- {- "model_output": {- "scanner_type": "summarizer",- "title": "Browsed the pricing page",- "summary": "The user browsed the pricing page and compared the available plans.",- "confidence": 0.9,- },- "signals_count": 0,- },- )- self._succeeded_observation(- summarizer,- "sum-novel-sess",- 1,- {- "model_output": {- "scanner_type": "summarizer",- "title": "Deleted every project",- "summary": "Removed all workspaces then immediately churned out.",- "confidence": 0.9,- },- "signals_count": 0,- },- )-- resp = self.client.get(f"{self.feed_url}?limit=50")- reasons = {item["observation"]["session_id"]: item["reason"] for item in resp.json()["results"]}- self.assertEqual(reasons["score-outlier"]["kind"], "outlier_score")- self.assertEqual(reasons["score-outlier"]["score"], 9.5)- self.assertEqual(reasons["tag-rare"]["kind"], "rare_tag")- self.assertEqual(reasons["tag-rare"]["tag"], "rare")- self.assertEqual(reasons["sum-novel-sess"]["kind"], "novel_summary")- # Near-identical summaries are the baseline, not novel, and a row at the scanner's own average is- # not an outlier. Neither earns a finding, so with three findings already in the feed both are gone.- self.assertNotIn("sum-0", reasons)- self.assertNotIn("score-0", reasons)-- def test_singleton_tag_is_rare_even_in_a_thin_window(self) -> None:- # A pure share cutoff makes rarity impossible below 10 rows and then fire on every singleton- # at exactly 10 — the feed would change character as a scanner accumulates data.- classifier = self._create_scanner(- name="c-thin",- scanner_type=ScannerType.CLASSIFIER,- scanner_config={"prompt": "p", "tags": ["common", "rare"], "multi_label": True},- )- for i in range(4):- self._succeeded_observation(- classifier,- f"thin-{i}",- 10 + i,- {- "model_output": {- "scanner_type": "classifier",- "tags": ["common"],- "reasoning": "r",- "confidence": 0.9,- },- "signals_count": 0,- },- )- self._succeeded_observation(- classifier,- "thin-rare",- 1,- {- "model_output": {- "scanner_type": "classifier",- "tags": ["common", "rare"],- "reasoning": "r",- "confidence": 0.9,- },- "signals_count": 0,- },- )- resp = self.client.get(f"{self.feed_url}?limit=50")- reasons = {item["observation"]["session_id"]: item["reason"] for item in resp.json()["results"]}- self.assertEqual(reasons["thin-rare"]["kind"], "rare_tag")- self.assertEqual(reasons["thin-rare"]["tag"], "rare")-- def test_rare_tag_suppressed_when_the_vocabulary_never_repeats(self) -> None:- # Freeform tagging that never repeats makes every tag a singleton; rarity means nothing- # there, and without the guard every row would rank as a top-tier hit.- classifier = self._create_scanner(- name="c-chaos",- scanner_type=ScannerType.CLASSIFIER,- scanner_config={"prompt": "p", "tags": [], "multi_label": True, "allow_freeform_tags": True},- )- for i in range(5):- self._succeeded_observation(- classifier,- f"chaos-{i}",- 10 + i,- {- "model_output": {- "scanner_type": "classifier",- "tags": [],- "tags_freeform": [f"unique_{i}"],- "reasoning": "r",- "confidence": 0.9,- },- "signals_count": 0,- },- )- resp = self.client.get(f"{self.feed_url}?limit=50")- kinds = {item["reason"]["kind"] for item in resp.json()["results"]}- self.assertEqual(kinds, {"unviewed_recent"})-- def test_per_scanner_cap_keeps_quiet_scanners_in_the_candidate_slice(self) -> None:- # Without the per-scanner cap a busy scanner fills the global slice and quiet scanners lose- # both feed slots and their own baselines.- busy = self._create_scanner(name="busy")- quiet = self._create_scanner(name="quiet")- for i in range(4):- self._succeeded_observation(busy, f"busy-{i}", i, self._monitor_result("no"))- self._succeeded_observation(quiet, "quiet-sess", 30, self._monitor_result("no"))-- with patch("products.replay_vision.backend.api.scanners.WATCH_FEED_PER_SCANNER_CAP", 2):- resp = self.client.get(f"{self.feed_url}?limit=50")- sessions = [item["observation"]["session_id"] for item in resp.json()["results"]]- self.assertIn("quiet-sess", sessions)- self.assertEqual(len([s for s in sessions if s.startswith("busy-")]), 2)-def test_window_type_filter_and_limit(self) -> None:monitor = self._create_scanner(name="m")summarizer = self._create_scanner(@@ -5860,10 +5067,9 @@ class TestWatchFeedAPI(_VisionAPITestCase):sessions = [item["observation"]["session_id"] for item in resp.json()["results"]]self.assertEqual(sessions, ["summary", "in-window"])- # The wider window admits the old row, whose verdict-yes hit outranks plain recency.resp = self.client.get(f"{self.feed_url}?date_from=-30d&scanner_type=monitor")sessions = [item["observation"]["session_id"] for item in resp.json()["results"]]- self.assertEqual(sessions, ["too-old", "in-window"])+ self.assertEqual(sessions, ["in-window", "too-old"])resp = self.client.get(f"{self.feed_url}?limit=1")self.assertEqual(len(resp.json()["results"]), 1)@@ -5871,8 +5077,12 @@ class TestWatchFeedAPI(_VisionAPITestCase):def test_scanner_ids_filter_restricts_between_readable_scanners(self) -> None:wanted = self._create_scanner(name="wanted")other = self._create_scanner(name="other")- self._succeeded_observation(wanted, "wanted-sess", 1, self._monitor_result("no"))- self._succeeded_observation(other, "other-sess", 2, self._monitor_result("no"))+ self._succeeded_observation(wanted, "wanted-sess", 30, self._monitor_result("no"))+ # The other scanner holds a pick, so the filter has to reach the pick query as well as the filler.+ for index in range(10):+ self._ranked_observation(+ other, f"other-{index}", index, "failure" if index == 0 else "routine", 0.9 - index / 100+ )resp = self.client.get(f"{self.feed_url}?scanner_ids={wanted.id}")self.assertEqual(@@ -5892,18 +5102,6 @@ class TestWatchFeedAPI(_VisionAPITestCase):["inside"],)- def test_global_candidate_cap_keeps_the_newest_rows(self) -> None:- scanner = self._create_scanner(name="m")- for i in range(3):- self._succeeded_observation(scanner, f"sess-{i}", i + 1, self._monitor_result("no"))-- with patch("products.replay_vision.backend.api.scanners.WATCH_FEED_CANDIDATE_CAP", 2):- resp = self.client.get(f"{self.feed_url}?limit=50")- self.assertEqual(- [item["observation"]["session_id"] for item in resp.json()["results"]],- ["sess-0", "sess-1"],- )-def test_window_wider_than_the_maximum_is_rejected(self) -> None:resp = self.client.get(f"{self.feed_url}?date_from=-365d")self.assertEqual(resp.status_code, 400, resp.json())@@ -5929,37 +5127,6 @@ class TestWatchFeedAPI(_VisionAPITestCase):self.assertEqual(allowed.status_code, 200, allowed.json())self.assertEqual(allowed.json()["results"][0]["observation"]["session_id"], "scoped-sess")- def test_search_matches_scan_prose_and_scanner_name_across_the_whole_window(self) -> None:- # Search runs before ranking, so a match that the ranking would never have surfaced still- # comes back — that is the whole point of the box on a capped feed.- checkout = self._create_scanner(name="Checkout watcher")- other = self._create_scanner(name="Inbox watcher")- self._succeeded_observation(- checkout,- "coupon-sess",- 30,- {- "model_output": {- "scanner_type": "summarizer",- "title": "Coupon rejected",- "summary": "The coupon field rejected a valid code.",- "confidence": 0.9,- },- "signals_count": 0,- },- )- self._succeeded_observation(other, "inbox-sess", 1, self._monitor_result("no"))-- resp = self.client.get(f"{self.feed_url}?search=coupon")- self.assertEqual([i["observation"]["session_id"] for i in resp.json()["results"]], ["coupon-sess"])-- # The scanner's own name matches too, so typing an area name works.- resp = self.client.get(f"{self.feed_url}?search=inbox")- self.assertEqual([i["observation"]["session_id"] for i in resp.json()["results"]], ["inbox-sess"])-- resp = self.client.get(f"{self.feed_url}?search=nothingmatchesthis")- self.assertEqual(resp.json()["results"], [])-def test_tag_filter_follows_current_scanner_tags_not_the_snapshot(self) -> None:# Snapshots freeze config at scan time and never carried tags, so a retag has to take effect# on existing observations immediately.@@ -5992,16 +5159,12 @@ class TestWatchFeedAPI(_VisionAPITestCase):self.assertEqual(resp.status_code, 200, resp.json())self.assertEqual(resp.json()["results"][0]["reason"]["kind"], "unviewed_recent")- @parameterized.expand([("weighted_score", "control"), ("embedding", "embedding")])- def test_a_scope_with_no_readable_scanner_returns_an_empty_feed(self, _name: str, variant: str) -> None:+ def test_a_scope_with_no_readable_scanner_returns_an_empty_feed(self) -> None:hidden = self._create_scanner(name="hidden")self._succeeded_observation(hidden, "hidden-sess", 1, self._monitor_result("no"))- with (- patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value=variant),- patch(- "products.access_control.backend.facade.user_access_control.UserAccessControl.filter_queryset_by_access_level",- side_effect=lambda qs, **_: qs.none() if qs.model is ReplayScanner else qs,- ),+ with patch(+ "products.access_control.backend.facade.user_access_control.UserAccessControl.filter_queryset_by_access_level",+ side_effect=lambda qs, **_: qs.none() if qs.model is ReplayScanner else qs,):resp = self.client.get(self.feed_url)self.assertEqual(resp.status_code, 200, resp.json())@@ -6038,13 +5201,10 @@ class TestWatchFeedAPI(_VisionAPITestCase):self._succeeded_observation(scanner, "open-sess", 3, self._monitor_result("no"))self._succeeded_observation(scanner, "open-sess-2", 4, self._monitor_result("no"))- # The newest rows fill the first candidate page and are all restricted, so the feed must read on.- with (- patch(- "products.access_control.backend.facade.user_access_control.UserAccessControl.filter_queryset_by_access_level",- side_effect=lambda qs, **_: qs.exclude(pk=experiment.pk) if qs.model is Experiment else qs,- ),- patch("products.replay_vision.backend.api.scanners.WATCH_FEED_CANDIDATE_CAP", 2),+ # The restricted rows are the newest, so the access gate has to run before the filler's limit.+ with patch(+ "products.access_control.backend.facade.user_access_control.UserAccessControl.filter_queryset_by_access_level",+ side_effect=lambda qs, **_: qs.exclude(pk=experiment.pk) if qs.model is Experiment else qs,):resp = self.client.get(self.feed_url)self.assertEqual(@@ -6066,41 +5226,37 @@ class TestWatchFeedAPI(_VisionAPITestCase):)scanner.experiment_targeting = Nonescanner.save()- ranker = "products.replay_vision.backend.api.scanners.watch_feed_ranker_variant"-- with patch(ranker, return_value="embedding"):- first = self.client.get(self.feed_url, {"limit": 1}).json()- self.assertEqual(len(first["results"]), 1)- # The second page reads picks cached by the first, so access has to be checked again on the page itself.- with patch(- "products.access_control.backend.facade.user_access_control.UserAccessControl.filter_queryset_by_access_level",- side_effect=lambda qs, **_: qs.exclude(pk=experiment.pk) if qs.model is Experiment else qs,- ):- second = self.client.get(self.feed_url, {"limit": 5, "offset": 1, "as_of": first["as_of"]}).json()++ first = self.client.get(self.feed_url, {"limit": 1}).json()+ self.assertEqual(len(first["results"]), 1)+ # The second page reads picks cached by the first, so access has to be checked again on the page itself.+ with patch(+ "products.access_control.backend.facade.user_access_control.UserAccessControl.filter_queryset_by_access_level",+ side_effect=lambda qs, **_: qs.exclude(pk=experiment.pk) if qs.model is Experiment else qs,+ ):+ second = self.client.get(self.feed_url, {"limit": 5, "offset": 1, "as_of": first["as_of"]}).json()self.assertEqual(second["results"], [])- @parameterized.expand([("weighted_score", "control"), ("embedding_reload_in_one_bucket", "embedding")])- def test_not_useful_hides_a_row_for_that_user_only_until_undone(self, _name: str, variant: str) -> None:- with patch("products.replay_vision.backend.api.scanners.watch_feed_ranker_variant", return_value=variant):- scanner = self._create_scanner(name="m")- dismissed = self._succeeded_observation(scanner, "dismissed-sess", 1, self._monitor_result("yes"))- self._succeeded_observation(scanner, "kept-sess", 2, self._monitor_result("yes"))- dismissal_url = f"/api/projects/{self.team.id}/vision/observations/{dismissed.id}/feed_dismissal/"+ def test_not_useful_hides_a_row_for_that_user_only_until_undone(self) -> None:+ scanner = self._create_scanner(name="m")+ dismissed = self._succeeded_observation(scanner, "dismissed-sess", 1, self._monitor_result("yes"))+ self._succeeded_observation(scanner, "kept-sess", 2, self._monitor_result("yes"))+ dismissal_url = f"/api/projects/{self.team.id}/vision/observations/{dismissed.id}/feed_dismissal/"- def feed_sessions() -> list[str]:- return [item["observation"]["session_id"] for item in self.client.get(self.feed_url).json()["results"]]+ def feed_sessions() -> list[str]:+ return [item["observation"]["session_id"] for item in self.client.get(self.feed_url).json()["results"]]- self.assertEqual(self.client.post(dismissal_url, {"reason": "nonsense"}).status_code, 400)- self.assertEqual(self.client.post(dismissal_url, {"reason": "already_known"}).status_code, 204)- self.assertEqual(feed_sessions(), ["kept-sess"])+ self.assertEqual(self.client.post(dismissal_url, {"reason": "nonsense"}).status_code, 400)+ self.assertEqual(self.client.post(dismissal_url, {"reason": "already_known"}).status_code, 204)+ self.assertEqual(feed_sessions(), ["kept-sess"])- other = User.objects.create_and_join(self.organization, "[email protected]", None)- self.client.force_login(other)- self.assertEqual(sorted(feed_sessions()), ["dismissed-sess", "kept-sess"])+ other = User.objects.create_and_join(self.organization, "[email protected]", None)+ self.client.force_login(other)+ self.assertEqual(sorted(feed_sessions()), ["dismissed-sess", "kept-sess"])- self.client.force_login(self.user)- self.assertEqual(self.client.delete(dismissal_url).status_code, 204)- self.assertEqual(sorted(feed_sessions()), ["dismissed-sess", "kept-sess"])+ self.client.force_login(self.user)+ self.assertEqual(self.client.delete(dismissal_url).status_code, 204)+ self.assertEqual(sorted(feed_sessions()), ["dismissed-sess", "kept-sess"])def test_requires_session_recording_read(self) -> None:with patch(@@ -6112,14 +5268,26 @@ class TestWatchFeedAPI(_VisionAPITestCase):def test_query_count_stays_flat_as_data_grows(self) -> None:scanner = self._create_scanner(name="m")- self._succeeded_observation(scanner, "s-0", 1, self._monitor_result("no"))+ # Enough rows to fill the feed's floor, so both samples skip the extra query a thin feed makes for filler.+ for i in range(3):+ self._succeeded_observation(scanner, f"s-0-{i}", i + 1, self._monitor_result("no"))self.client.get(self.feed_url) # warm request-scoped caches- with CaptureQueriesContext(connection) as one:+ # Both measured loads rank afresh, since a load in the same bucket would reuse the picks cached before it.+ real_get = cache.get+ uncached_picks = patch.object(+ cache,+ "get",+ side_effect=lambda key, *args, **kwargs: (+ None if key.startswith("replay_vision:watch_picks:") else real_get(key, *args, **kwargs)+ ),+ )+ with uncached_picks, CaptureQueriesContext(connection) as one:self.assertEqual(self.client.get(self.feed_url).status_code, 200)for i in range(1, 6):other = self._create_scanner(name=f"m{i}")- self._succeeded_observation(other, f"s-{i}", i, self._monitor_result("yes", signals=i % 2))- with CaptureQueriesContext(connection) as six:+ self._succeeded_observation(other, f"s-{i}", i, self._monitor_result("yes"))+ self.client.get(self.feed_url)+ with uncached_picks, CaptureQueriesContext(connection) as six:self.assertEqual(self.client.get(self.feed_url).status_code, 200)self.assertEqual(len(one.captured_queries), len(six.captured_queries))diff --git a/products/replay_vision/backend/tests/test_facade.py b/products/replay_vision/backend/tests/test_facade.pyindex 6ab3d21663fd..04903fde76a6 100644--- a/products/replay_vision/backend/tests/test_facade.py+++ b/products/replay_vision/backend/tests/test_facade.py@@ -186,7 +186,6 @@ class TestStartWorkflowObservationRequest(APIBaseTest):"reasoning": "r","confidence": 0.9,"reasoning_segments": [{"text": "r", "t": 12}] * 200,- "notability_reason": "n" * 2000,},{"confidence": 0.9, "verdict": "yes", "reasoning": "r"},[],diff --git a/products/replay_vision/backend/tests/test_jev_watch_feed.py b/products/replay_vision/backend/tests/test_jev_watch_feed.pydeleted file mode 100644index da33e45ccf20..000000000000--- a/products/replay_vision/backend/tests/test_jev_watch_feed.py+++ /dev/null@@ -1,966 +0,0 @@-import itertools-import threading-from datetime import UTC, datetime, timedelta-from time import monotonic-from typing import Any-from uuid import uuid4--import pytest-from posthog.test.base import BaseTest-from unittest.mock import patch--from django.test import SimpleTestCase-from django.utils import timezone--from asgiref.sync import async_to_sync-from parameterized import parameterized--from posthog.llm.gateway_client import GatewayNotConfiguredError-from posthog.redis import get_client--from products.ml_inference.backend.facade.contracts import (- ChoiceAnswer,- DecisionGatewayError,- DecisionRequest,- DecisionResult,- DecisionsDisabledError,- NoulAnswer,-)-from products.ml_inference.backend.facade.enums import DecisionQuestionType-from products.replay_vision.backend.jev_watch_feed import (- _REASON_QUESTION_TOKENS,- _WINDOW_QUESTION_TOKENS,- REQUEST_TOKEN_TARGET,- WINDOW_CHUNK_SIZE,- _entry_tokens,- judge_scanner_window,- load_judged_state,- load_scanner_watch_ranks,- load_watch_ranks,- rank_watch_feed_by_jev,- store_watch_ranks,- watch_feed_ranker,- watch_feed_ranker_variant,-)-from products.replay_vision.backend.models.replay_observation import (- ObservationStatus,- ObservationTrigger,- ReplayObservation,-)-from products.replay_vision.backend.models.replay_scanner import ReplayScanner, ScannerModel, ScannerType-from products.replay_vision.backend.temporal.jev_watch_rank.activities import (- _judge_watch_ranks,- _teams_with_scanners,- _TokenPacer,-)-from products.replay_vision.backend.temporal.jev_watch_rank.constants import (- MAX_JUDGE_ATTEMPTS,- RATE_LIMIT_GIVE_UP_AFTER,-)-from products.replay_vision.backend.temporal.jev_watch_rank.types import JevWatchRankSweepInputs-from products.replay_vision.backend.tests.helpers import snapshot_for as _snapshot_for--_FLAG = "products.replay_vision.backend.jev_watch_feed.get_feature_flag_or_none"-_API = "products.replay_vision.backend.jev_watch_feed.decision_api"-_REGION = f"{_API}.decisions_available_here"---def _answer_every_question(probability: float, reason: str = "visible_error"):- """A decide_when_available side effect that answers each yes/no question with `probability` and- each multiple-choice question with `reason`."""-- def _decide(request: DecisionRequest, **kwargs: Any) -> DecisionResult:- return DecisionResult(- model="jevk5-fp8-0.2",- answers={- name: (- ChoiceAnswer(choice=reason, confidence=0.8, probabilities={reason: 0.8})- if question.type == DecisionQuestionType.CHOICE- else NoulAnswer(probability=probability)- )- for name, question in request.questions.items()- },- input_tokens=120,- )-- return _decide---def _prose_row(observation_id: object, summary: str) -> dict[str, Any]:- return {- "id": observation_id,- "scanner_result": {- "model_output": {"scanner_type": "summarizer", "title": "t", "summary": summary, "confidence": 0.9},- "signals_count": 0,- },- }---class TestWatchFeedRankerFlag(SimpleTestCase):- @parameterized.expand(- [- ("flag_off", None, "weighted-score"),- ("boolean_flag", True, "weighted-score"),- ("unknown_variant", "something-else", "weighted-score"),- ("shadow", "jev-shadow", "jev-shadow"),- ("graduated", "jev", "jev"),- ("embedding", "embedding", "embedding"),- ]- )- def test_only_known_variants_leave_the_default_arm(self, _name: str, flag_value: object, expected: str) -> None:- with patch(_FLAG, return_value=flag_value), patch(_REGION, return_value=True):- assert watch_feed_ranker(1, "project-uuid") == expected-- def test_the_flag_keys_on_the_project_uuid_and_keeps_jev_arms_out_of_regions_without_jev(self) -> None:- # Both regions evaluate this one flag and team ids repeat between them, so only the uuid is unique.- with patch(_FLAG, return_value="jev") as flag, patch(_REGION, return_value=True):- assert watch_feed_ranker_variant(2, "project-uuid") == "jev"- assert flag.call_args.kwargs["groups"] == {"project": "project-uuid"}- # A region with no sweep to fill the cache must not land a team on an empty jev feed.- with patch(_FLAG, return_value="jev"), patch(_REGION, return_value=False):- assert watch_feed_ranker_variant(2, "project-uuid") is None- assert watch_feed_ranker(2, "project-uuid") == "weighted-score"- # The embedding arm needs no decision service.- with patch(_FLAG, return_value="embedding"), patch(_REGION, return_value=False):- assert watch_feed_ranker(2, "project-uuid") == "embedding"---class TestJudgeScannerWindow(SimpleTestCase):- def test_a_window_larger_than_one_chunk_is_judged_across_requests(self) -> None:- rows = [_prose_row(uuid4(), f"summary {index}") for index in range(WINDOW_CHUNK_SIZE + 6)]- with patch(_API) as api:- api.decide_when_available.side_effect = _answer_every_question(0.7)- judgment = judge_scanner_window(1, uuid4(), rows)- # Each chunk is one yes/no request, then one reason request for its watchable rows. How many- # rows fit a chunk follows from the token target, so derive the geometry instead of pinning it.- assert judgment.chunks >= 2- assert api.decide_when_available.call_count == 2 * judgment.chunks- assert judgment.failed_chunks == 0- assert len(judgment.probabilities) == len(rows)- assert judgment.probabilities[str(rows[0]["id"])] == 0.7- assert judgment.input_tokens == 120 * api.decide_when_available.call_count- first_request = api.decide_when_available.call_args_list[0].args[0]- # The chunk travels as shared state with one question per observation, so each judgment is- # relative to its siblings.- assert 1 < len(first_request.state["observations"]) < len(rows)- assert len(first_request.questions) == len(first_request.state["observations"])- assert "0" in first_request.questions["watch_0"].instructions- # The state carries recording-derived prose, which must stay out of the internal project.- assert first_request.privacy_mode is True-- def test_a_failed_chunk_loses_only_its_own_rows(self) -> None:- rows = [_prose_row(uuid4(), f"summary {index}") for index in range(WINDOW_CHUNK_SIZE + 6)]- answer = _answer_every_question(0.7)- attempts: list[int] = []-- def _first_chunk_fails(request: DecisionRequest, **kwargs: Any) -> DecisionResult:- attempts.append(1)- if len(attempts) == 1:- raise DecisionsDisabledError(1)- return answer(request)-- with patch(_API) as api:- api.decide_when_available.side_effect = _first_chunk_fails- judgment = judge_scanner_window(1, uuid4(), rows)- assert judgment.failed_chunks == 1- first_request = api.decide_when_available.call_args_list[0].args[0]- assert len(judgment.probabilities) == len(rows) - len(first_request.state["observations"])- # An unavailable decision service is not the batch's fault, so no retry budget is charged.- assert judgment.batch_failed_ids == ()- assert judgment.chunk_error_types == {"DecisionsDisabledError": 1}-- def test_an_invalid_probability_fails_the_whole_chunk(self) -> None:- rows = [_prose_row(uuid4(), "summary")]- with patch(_API) as api:- api.decide_when_available.side_effect = _answer_every_question(7.0)- judgment = judge_scanner_window(1, uuid4(), rows)- assert judgment.probabilities == {}- assert judgment.failed_chunks == 1- # An invalid answer is the batch's own fault, so its rows are charged a retry attempt.- assert judgment.batch_failed_ids == (str(rows[0]["id"]),)- assert judgment.chunk_error_types == {"ValueError": 1}-- @parameterized.expand(- [- ("bad_request", 400, True),- ("unprocessable", 422, True),- ("unauthorized", 401, False),- ("not_found", 404, False),- ("rate_limit", 429, False),- ("server_error", 503, False),- ]- )- def test_only_a_batch_attributable_gateway_refusal_charges_the_retry_budget(- self, _name: str, status_code: int, charged: bool- ) -> None:- rows = [_prose_row(uuid4(), "summary")]- with patch(_API) as api:- api.decide_when_available.side_effect = DecisionGatewayError(status_code, "detail")- judgment = judge_scanner_window(1, uuid4(), rows)- assert judgment.failed_chunks == 1- assert judgment.batch_failed_ids == ((str(rows[0]["id"]),) if charged else ())-- def test_a_misconfigured_gateway_charges_no_retry_budget(self) -> None:- # GatewayNotConfiguredError subclasses ValueError, so the invalid-answer check must not- # catch it: a broken gateway config would otherwise park the newest rows within hours.- rows = [_prose_row(uuid4(), "summary")]- with patch(_API) as api:- api.decide_when_available.side_effect = GatewayNotConfiguredError("no gateway configured")- judgment = judge_scanner_window(1, uuid4(), rows)- assert judgment.failed_chunks == 1- assert judgment.batch_failed_ids == ()- assert judgment.chunk_error_types == {"GatewayNotConfiguredError": 1}-- def test_rows_without_prose_are_reported_instead_of_sent(self) -> None:- no_output, no_result = uuid4(), uuid4()- rows = [- _prose_row(uuid4(), "summary"),- {"id": no_output, "scanner_result": {"model_output": {}, "signals_count": 0}},- {"id": no_result, "scanner_result": None},- ]- with patch(_API) as api:- api.decide_when_available.side_effect = _answer_every_question(0.5)- judgment = judge_scanner_window(1, uuid4(), rows)- assert len(judgment.probabilities) == 1- assert set(judgment.skipped_no_prose) == {str(no_output), str(no_result)}-- def test_context_rows_enter_the_state_without_questions(self) -> None:- rows = [_prose_row(uuid4(), f"new {index}") for index in range(3)]- context_rows = [_prose_row(uuid4(), f"old {index}") for index in range(40)]- with patch(_API) as api:- api.decide_when_available.side_effect = _answer_every_question(0.5)- judgment = judge_scanner_window(1, uuid4(), rows, context_rows)- request = api.decide_when_available.call_args.args[0]- # A quiet hour's small batch is still judged against the scanner's routine, and the state- # never exceeds one chunk's size.- assert len(request.questions) == 3- assert len(request.state["observations"]) == WINDOW_CHUNK_SIZE- assert len(judgment.probabilities) == 3-- def test_long_prose_is_clipped_before_it_enters_the_request(self) -> None:- rows = [_prose_row(uuid4(), "x" * 100_000)]- with patch(_API) as api:- api.decide_when_available.side_effect = _answer_every_question(0.5)- judge_scanner_window(1, uuid4(), rows)- request = api.decide_when_available.call_args.args[0]- assert len(request.state["observations"]["0"]["summary"]) == 1500-- @parameterized.expand(- [- ("picked", "visible_error", None, {"watchable": "visible_error"}),- ("no_reason_fits", "nothing_notable", None, {}),- ("off_the_list", "made_up", None, {}),- ("request_fails", "visible_error", DecisionGatewayError(503, "detail"), {}),- ]- )- def test_only_watchable_rows_get_a_reason_and_a_failed_pick_keeps_the_scores(- self, _name: str, choice: str, reason_error: Exception | None, expected: dict[str, str]- ) -> None:- rows = [_prose_row("watchable", "The user hit an error."), _prose_row("routine", "The user read docs.")]- answer_scores = _answer_every_question(0.7, choice)-- def _decide(request: DecisionRequest, **kwargs: Any) -> DecisionResult:- if any(question.type == DecisionQuestionType.CHOICE for question in request.questions.values()):- if reason_error:- raise reason_error- return answer_scores(request)- return DecisionResult(- model="jevk5-fp8-0.2",- answers={"watch_0": NoulAnswer(probability=0.7), "watch_1": NoulAnswer(probability=0.1)},- input_tokens=120,- )-- with patch(_API) as api:- api.decide_when_available.side_effect = _decide- judgment = judge_scanner_window(1, uuid4(), rows)- reason_request = api.decide_when_available.call_args_list[1].args[0]- # The feed shows a reason only on watchable rows, so only they are asked.- assert list(reason_request.questions) == ["reason_0"]- assert judgment.reasons == expected- # A reason only labels the row: a failed pick must not cost the chunk its scores.- assert judgment.probabilities == {"watchable": 0.7, "routine": 0.1}- assert judgment.failed_chunks == 0- assert judgment.failed_reason_chunks == (1 if reason_error else 0)-- def test_a_rate_limit_stops_the_window_and_charges_no_retry_budget(self) -> None:- # The rate limit is shared with every product's Jev calls, so the window must not keep- # sending after it. The rows it did not reach retry free on the next sweep.- rows = [_prose_row(uuid4(), f"summary {index}") for index in range(WINDOW_CHUNK_SIZE + 6)]- with patch(_API) as api:- api.decide_when_available.side_effect = DecisionGatewayError(429, "rate limited")- judgment = judge_scanner_window(1, uuid4(), rows)- assert api.decide_when_available.call_count == 1- assert judgment.rate_limited- assert judgment.batch_failed_ids == ()-- def test_requests_pack_to_the_token_target(self) -> None:- # A full-count chunk of verbose entries must not become a host-filling request. Every row is- # watchable here, the worst case: the reason request resends the state with a criteria-bearing- # question per row, so it, not the judgment request, is the largest request a chunk produces.- rows = [_prose_row(uuid4(), f"entry {index} " + "x" * 1400) for index in range(WINDOW_CHUNK_SIZE)]- with patch(_API) as api:- api.decide_when_available.side_effect = _answer_every_question(0.7)- judgment = judge_scanner_window(1, uuid4(), rows)- assert judgment.failed_chunks == 0- assert len(judgment.probabilities) == WINDOW_CHUNK_SIZE- requests = [call.args[0] for call in api.decide_when_available.call_args_list]- # Count-only chunking would fit all of these into one judgment request.- assert len(requests) > 2- for request in requests:- # state["observations"] is a dict keyed by index, so size the entries, not the keys.- entry_tokens = sum(_entry_tokens(entry) for entry in request.state["observations"].values())- question_tokens = sum(- _REASON_QUESTION_TOKENS if question_id.startswith("reason_") else _WINDOW_QUESTION_TOKENS- for question_id in request.questions- )- assert entry_tokens + question_tokens <= REQUEST_TOKEN_TARGET-- def test_each_request_pays_the_pacing_toll_before_it_sends(self) -> None:- rows = [_prose_row(uuid4(), f"summary {index}") for index in range(3)]- paid: list[int] = []- with patch(_API) as api:- api.decide_when_available.side_effect = _answer_every_question(0.7)- judgment = judge_scanner_window(1, uuid4(), rows, pace=paid.append)- # One judgment request plus one reason request (the rows are watchable), each paced.- assert api.decide_when_available.call_count == 2- assert len(paid) == 2- assert all(tokens > 0 for tokens in paid)- assert judgment.estimated_prompt_tokens == sum(paid)-- def test_a_window_starts_no_chunk_past_its_stop_time(self) -> None:- # Rows past the deadline stay unjudged and retry next sweep, charging no retry budget.- rows = [_prose_row(uuid4(), "The user hit an error.") for _ in range(2)]- with patch(_API) as api:- judgment = judge_scanner_window(1, uuid4(), rows, stop_after=monotonic() - 1)- api.decide_when_available.assert_not_called()- assert judgment.probabilities == {}- assert judgment.batch_failed_ids == ()- # The cut-off is visible: no chunk counts as attempted, and the window says it stopped early.- assert judgment.stopped_early- assert judgment.chunks == 0- assert judgment.estimated_prompt_tokens == 0-- def test_an_empty_window_makes_no_request(self) -> None:- with patch(_API) as api:- judgment = judge_scanner_window(1, uuid4(), [])- api.decide_when_available.assert_not_called()- assert judgment.probabilities == {}---def _feed_row(- observation_id: str,- minutes_ago: int,- *,- viewed: bool = False,- scanner: str = "scanner-a",- notability: str | None = None,- notability_score: float = 0.9,-) -> dict[str, Any]:- row: dict[str, Any] = {- "id": observation_id,- "scanner_id": scanner,- "created_at": datetime(2026, 9, 25, 12, 0, tzinfo=UTC) - timedelta(minutes=minutes_ago),- "feed_viewed": viewed,- }- if notability is not None:- row["scanner_result"] = {"model_output": {"notability_reason": notability, "notability": notability_score}}- return row---class TestTokenPacer(SimpleTestCase):- # The clock and sleep are patched at the activities module, so these assert the reservation- # math deterministically instead of timing real sleeps (which flake on a loaded CI machine).- _MOD = "products.replay_vision.backend.temporal.jev_watch_rank.activities"-- def test_a_request_waits_for_the_slot_the_one_before_it_reserved(self) -> None:- with (- patch(f"{self._MOD}.monotonic", side_effect=[100.0, 100.0, 100.05]),- patch(f"{self._MOD}.sleep") as slept,- ):- pacer = _TokenPacer(tokens_per_second=10_000)- pacer.wait(2_000) # sends at once and reserves until 100.2- pacer.wait(1)- slept.assert_called_once()- assert slept.call_args.args[0] == pytest.approx(0.15)- assert pacer.waited_seconds == pytest.approx(0.15)-- def test_the_first_request_of_an_idle_run_never_waits(self) -> None:- with (- patch(f"{self._MOD}.monotonic", side_effect=[100.0, 200.0]),- patch(f"{self._MOD}.sleep") as slept,- ):- pacer = _TokenPacer(tokens_per_second=1) # any backlog at this rate would show instantly- pacer.wait(10)- slept.assert_not_called()- assert pacer.waited_seconds == 0.0---class TestRankWatchFeedByJev(SimpleTestCase):- def test_watchable_rows_rank_by_probability_and_filler_pads_only_to_the_floor(self) -> None:- # A judged-low row falls to the same recency filler tier as an unjudged one, so a stale low- # judgment never outranks a fresh observation the sweep has not seen yet, and its card never- # claims the model judged it worth watching. Filler then pads the findings only up to the- # feed's floor, newest first, so the older filler rows drop.- ranked = rank_watch_feed_by_jev(- [- _feed_row("old-unjudged", 50),- _feed_row("judged-low", 40),- _feed_row("high", 30),- _feed_row("higher", 20, notability="The user paid twice for one order."),- _feed_row("new-unjudged", 10),- ],- {"judged-low": 0.2, "high": 0.7, "higher": 0.9},- # A cached reason reaches only a watchable row's card.- {"higher": "churn_signal", "judged-low": "blocked"},- )- assert [entry.observation_id for entry in ranked] == ["higher", "high", "new-unjudged"]- # The scan's own sentence rides along so the card can explain the pick; a row without one- # carries only the kind and probability.- assert ranked[0].reason == {- "kind": "jev_watchable",- "jev_probability": 0.9,- "notability_reason": "The user paid twice for one order.",- "watch_reason": "churn_signal",- }- assert ranked[1].reason == {"kind": "jev_watchable", "jev_probability": 0.7}- assert ranked[2].reason == {"kind": "unviewed_recent"}-- def test_a_cold_cache_returns_the_filler_floor_not_a_full_page(self) -> None:- # With no cached judgments every row is filler: the feed shows the same 3-row floor as the- # weighted ranker, unviewed rows first and newest first, instead of a whole page of "new- # since you last looked" cards.- rows = [_feed_row(f"row-{index}", index, viewed=index == 0) for index in range(20)]- ranked = rank_watch_feed_by_jev(rows, {})- assert [entry.observation_id for entry in ranked] == ["row-1", "row-2", "row-3"]-- def test_one_scanner_cannot_flood_the_top_of_the_evidence_tier(self) -> None:- # One incident's near-identical sessions must leave room for other scanners' findings, and- # the overflow trails the tier instead of dropping out of it.- rows = [_feed_row(f"flood-{index}", 10 + index, scanner="flooding") for index in range(6)]- rows += [_feed_row("other-0", 30, scanner="quiet"), _feed_row("other-1", 31, scanner="quiet")]- probabilities = {f"flood-{index}": 0.96 - index / 100 for index in range(6)} | {- "other-0": 0.7,- "other-1": 0.69,- }- ranked = rank_watch_feed_by_jev(rows, probabilities)- assert [entry.observation_id for entry in ranked] == [- "flood-0",- "flood-1",- "flood-2",- "other-0",- "other-1",- "flood-3",- "flood-4",- "flood-5",- ]-- def test_a_viewed_row_is_docked_inside_the_watchable_tier_only(self) -> None:- ranked = rank_watch_feed_by_jev(- [- # The scan itself found nothing notable, so its sentence must not lead a card Jev- # rated watchable relative to a dull window.- _feed_row("viewed-strong", 30, viewed=True, notability="Nothing stands out.", notability_score=0.2),- _feed_row("unviewed-mid", 20),- # Raw probability decides the tier, so the dock cannot push a watchable row into filler.- _feed_row("viewed-borderline", 15, viewed=True),- _feed_row("viewed-filler", 5, viewed=True),- _feed_row("unviewed-filler", 1),- ],- {"viewed-strong": 0.95, "unviewed-mid": 0.6, "viewed-borderline": 0.55},- )- # 0.95 - 0.3 dock = 0.65 still beats 0.6; 0.55 - 0.3 = 0.25 stays watchable, ordered last.- # Three findings meet the floor, so no filler row survives the trim.- assert [entry.observation_id for entry in ranked] == [- "viewed-strong",- "unviewed-mid",- "viewed-borderline",- ]- assert ranked[0].reason == {"kind": "jev_watchable", "jev_probability": 0.95}- assert ranked[2].reason == {"kind": "jev_watchable", "jev_probability": 0.55}---class TestWatchRankCache(SimpleTestCase):- def test_stored_ranks_round_trip_and_malformed_values_are_dropped_or_clamped(self) -> None:- team_id = 990_001- scanner_id, other_scanner_id, missing_scanner_id = uuid4(), uuid4(), uuid4()- store_watch_ranks(- team_id,- scanner_id,- {"obs-a", "obs-b", "obs-low"},- {"obs-a": 0.9, "obs-b": 7.0},- {"obs-f": 2},- "jevk5",- # A sub-threshold row's reason is never stored, and an off-list value never loads.- {"obs-a": "visible_error", "obs-b": "made_up", "obs-low": "blocked"},- )- store_watch_ranks(team_id, other_scanner_id, {"obs-c"}, {"obs-c": 0.4}, {}, "jevk5", {"obs-c": "confused"})- loaded = load_watch_ranks(team_id, [scanner_id, other_scanner_id, missing_scanner_id])- assert loaded.probabilities == {"obs-a": 0.9, "obs-b": 1.0, "obs-c": 0.4}- assert loaded.reasons == {"obs-a": "visible_error", "obs-c": "confused"}- # The feed's per-request keys carry only the watchable map; the judged set remembers every- # row the sweep has bought, including the sub-threshold ones the feed never needs, plus the- # attempt counts for rows whose judgment keeps failing.- judged_state = load_judged_state(team_id, scanner_id)- assert judged_state.ids == {"obs-a", "obs-b", "obs-low"}- assert judged_state.attempts == {"obs-f": 2}- assert load_judged_state(team_id, missing_scanner_id).ids == set()- # Another team's cache never leaks in.- assert load_watch_ranks(team_id + 1, [scanner_id]).probabilities == {}-- def test_a_partial_store_failure_leaves_rows_rejudgeable(self) -> None:- # The judged marker is written last: when the watchable write lands and the judged write- # fails, the next sweep re-buys the batch. The reverse order would mark the rows judged- # with their scores lost forever.- team_id = 990_002- scanner_id = uuid4()- inner = get_client()-- class _JudgedWritesFail:- def setex(self, key: str, ttl: Any, value: str) -> None:- if ":judged:" in key:- raise ConnectionError("redis down")- inner.setex(key, ttl, value)-- with patch("products.replay_vision.backend.jev_watch_feed.get_client", return_value=_JudgedWritesFail()):- with self.assertRaises(ConnectionError):- store_watch_ranks(team_id, scanner_id, {"obs-a"}, {"obs-a": 0.9}, {}, "jevk5")- assert load_watch_ranks(team_id, [scanner_id]).probabilities == {"obs-a": 0.9}- assert load_judged_state(team_id, scanner_id).ids == set()-- def test_a_read_failure_raises_for_the_sweep_and_stays_soft_for_the_feed(self) -> None:- # An unreachable cache must not read as an empty one: the sweep would re-buy the scanner's- # judgments and its write would replace entries it never saw. The feed only degrades.- scanner_id = uuid4()- with patch(- "products.replay_vision.backend.jev_watch_feed.get_client",- side_effect=ConnectionError("redis down"),- ):- with self.assertRaises(ConnectionError):- load_judged_state(1, scanner_id)- with self.assertRaises(ConnectionError):- load_scanner_watch_ranks(1, scanner_id)- assert load_watch_ranks(1, [scanner_id]).probabilities == {}---class TestJevWatchRankSweep(BaseTest):- def _succeeded_observation(self, scanner: ReplayScanner, session_id: str, summary: str) -> ReplayObservation:- return ReplayObservation.objects.create(- scanner=scanner,- session_id=session_id,- scanner_snapshot=_snapshot_for(scanner),- triggered_by=ObservationTrigger.SCHEDULE,- status=ObservationStatus.SUCCEEDED,- completed_at=timezone.now(),- scanner_result={- "model_output": {"scanner_type": "summarizer", "title": "t", "summary": summary, "confidence": 0.9},- "signals_count": 0,- },- )-- def test_the_sweep_gates_judges_and_skips_unchanged_windows(self) -> None:- self.organization.is_ai_data_processing_approved = True- self.organization.save()- scanner = ReplayScanner.objects.create(- team=self.team,- name="s",- scanner_type=ScannerType.SUMMARIZER,- scanner_config={"prompt": "p", "length": "short"},- model=ScannerModel.GEMINI_3_8_FLASH,- )- first = self._succeeded_observation(scanner, "s1", "The user hit an error at checkout.")- second = self._succeeded_observation(scanner, "s2", "The user skimmed the pricing page.")-- activities = "products.replay_vision.backend.temporal.jev_watch_rank.activities"- flag = f"{activities}.watch_feed_ranker"- region = f"{activities}.decision_api.decisions_available_here"-- # A team on the default or embedding arm costs no Jev calls and gets no cache entry.- for arm in ("weighted-score", "embedding"):- with patch(flag, return_value=arm), patch(region, return_value=True), patch(_API) as api:- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- api.decide_when_available.assert_not_called()- assert result.teams_enrolled == 0- assert load_watch_ranks(self.team.id, [scanner.id]).probabilities == {}-- with (- patch(flag, side_effect=self._flag_arm("jev-shadow")),- patch(region, return_value=False),- patch(_API) as api,- ):- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- # A region without the decision service does no work at all.- api.decide_when_available.assert_not_called()- assert result.decisions_unavailable-- self.organization.is_ai_data_processing_approved = False- self.organization.save()- with (- patch(flag, side_effect=self._flag_arm("jev-shadow")),- patch(region, return_value=True),- patch(_API) as api,- ):- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- # An enrolled team without AI data-processing consent sends nothing to the model.- api.decide_when_available.assert_not_called()- assert result.teams_without_consent == 1- self.organization.is_ai_data_processing_approved = True- self.organization.save()-- with (- patch(flag, side_effect=self._flag_arm("jev-shadow")),- patch(region, return_value=True),- patch(_API) as api,- patch("posthoganalytics.capture") as captured,- ):- api.decide_when_available.side_effect = _answer_every_question(0.7)- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- assert result.teams_enrolled == 1- assert result.scanners_judged == 1- assert result.observations_judged == 2- assert result.estimated_prompt_tokens > 0- assert result.pending_observations == 0- assert result.oldest_pending_age_seconds == 0- assert result.backlog_complete- assert load_watch_ranks(self.team.id, [scanner.id]).probabilities == {str(first.id): 0.7, str(second.id): 0.7}- # Sub-threshold scores are cached nowhere, so the judged event must carry the score sample- # the threshold is calibrated from.- judged = [- call for call in captured.call_args_list if call.kwargs["event"] == "replay_vision_jev_watch_rank_judged"- ]- payload = judged[0].kwargs["properties"]- assert payload["watchable_count"] == 2- assert payload["watchability_max"] == 0.7- assert sorted(entry["id"] for entry in payload["top_scored"]) == sorted([str(first.id), str(second.id)])- assert all(entry["p"] == 0.7 for entry in payload["top_scored"])- assert payload["watch_reasons"] == {"visible_error": 2}-- with (- patch(flag, side_effect=self._flag_arm("jev-shadow")),- patch(region, return_value=True),- patch(_API) as api,- ):- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- # Every row is judged already, so the second run keeps the cache without a Jev call.- api.decide_when_available.assert_not_called()- assert result.scanners_skipped_unchanged == 1- assert result.estimated_prompt_tokens == 0- assert result.backlog_complete- assert load_watch_ranks(self.team.id, [scanner.id]).probabilities == {str(first.id): 0.7, str(second.id): 0.7}-- third = self._succeeded_observation(scanner, "s3", "The user deleted the whole workspace.")- with (- patch(flag, side_effect=self._flag_arm("jev-shadow")),- patch(region, return_value=True),- patch(_API) as api,- patch("posthoganalytics.capture"),- ):- api.decide_when_available.side_effect = _answer_every_question(0.9, "churn_signal")- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- # Only the new row is judged, with the already-judged rows in the state as context, and the- # earlier judgments survive the merge.- request = api.decide_when_available.call_args.args[0]- assert len(request.questions) == 1- assert len(request.state["observations"]) == 3- assert result.observations_judged == 1- ranks = load_watch_ranks(self.team.id, [scanner.id])- assert ranks.probabilities == {str(first.id): 0.7, str(second.id): 0.7, str(third.id): 0.9}- # The earlier reasons survive the merge too, so the feed keeps saying why.- assert ranks.reasons == {- str(first.id): "visible_error",- str(second.id): "visible_error",- str(third.id): "churn_signal",- }-- def test_an_unreadable_cache_skips_the_scanner_without_judging_or_writing(self) -> None:- # A transient Redis failure must not read as an empty cache: judging over it would re-buy- # rows, and the write would replace the scanner's entries with only the current batch.- self.organization.is_ai_data_processing_approved = True- self.organization.save()- scanner = ReplayScanner.objects.create(- team=self.team,- name="s",- scanner_type=ScannerType.SUMMARIZER,- scanner_config={"prompt": "p", "length": "short"},- model=ScannerModel.GEMINI_3_8_FLASH,- )- self._succeeded_observation(scanner, "s1", "The user hit an error at checkout.")- store_watch_ranks(self.team.id, scanner.id, {"judged-earlier"}, {"judged-earlier": 0.9}, {}, "jevk5")-- activities = "products.replay_vision.backend.temporal.jev_watch_rank.activities"- with (- patch(f"{activities}.watch_feed_ranker", side_effect=self._flag_arm("jev-shadow")),- patch(f"{activities}.decision_api.decisions_available_here", return_value=True),- patch(f"{activities}.load_judged_state", side_effect=ConnectionError("redis down")),- patch(_API) as api,- ):- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- api.decide_when_available.assert_not_called()- assert result.cache_errors == 1- assert not result.backlog_complete- assert result.scanners_judged == 0- assert load_watch_ranks(self.team.id, [scanner.id]).probabilities == {"judged-earlier": 0.9}-- @parameterized.expand([(2, False), (3, True)])- def test_pending_work_includes_rows_beyond_the_judging_limit(self, window_cap: int, complete: bool) -> None:- self.organization.is_ai_data_processing_approved = True- self.organization.save()- scanner = ReplayScanner.objects.create(- team=self.team,- name="example",- scanner_type=ScannerType.SUMMARIZER,- scanner_config={"prompt": "Example workflow", "length": "short"},- model=ScannerModel.GEMINI_3_8_FLASH,- )- older = self._succeeded_observation(scanner, "old", "An example user opens a menu.")- newer = self._succeeded_observation(scanner, "new", "An example user closes a menu.")- now = timezone.now()- ReplayObservation.objects.filter(id=older.id).update(created_at=now - timedelta(hours=2))- ReplayObservation.objects.filter(id=newer.id).update(created_at=now - timedelta(minutes=30))- activities = "products.replay_vision.backend.temporal.jev_watch_rank.activities"- with (- patch(f"{activities}.watch_feed_ranker", side_effect=self._flag_arm("jev-shadow")),- patch(f"{activities}.decision_api.decisions_available_here", return_value=True),- patch(f"{activities}.MAX_JUDGED_PER_SCANNER", 1),- patch(f"{activities}.WINDOW_SCAN_CAP", window_cap),- patch(f"{activities}.datetime") as clock,- patch("posthog.metrics.push_to_gateway") as push,- self.settings(PROM_PUSHGATEWAY_ADDRESS="http://pushgateway.example.com"),- patch(_API) as api,- ):- clock.now.return_value = now- api.decide_when_available.side_effect = _answer_every_question(0.1)- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- assert result.observations_judged == 1- assert result.pending_observations == 1- assert result.oldest_pending_age_seconds == 7200- assert result.backlog_complete == complete- registry = push.call_args.kwargs["registry"]- assert registry.get_sample_value("replay_vision_jev_watch_rank_sweep_pending_observations") == 1- assert registry.get_sample_value("replay_vision_jev_watch_rank_sweep_backlog_complete") == int(complete)-- def test_scanners_are_judged_concurrently(self) -> None:- # One scanner at a time cannot cover a half rollout inside the time budget. The barrier only- # opens while both scanners' Jev calls are in flight, so a sequential sweep fails every chunk.- self.organization.is_ai_data_processing_approved = True- self.organization.save()- for name in ("a", "b"):- scanner = ReplayScanner.objects.create(- team=self.team,- name=name,- scanner_type=ScannerType.SUMMARIZER,- scanner_config={"prompt": "p", "length": "short"},- model=ScannerModel.GEMINI_3_8_FLASH,- )- self._succeeded_observation(scanner, f"session-{name}", "The user hit an error at checkout.")- barrier = threading.Barrier(2, timeout=10)- answer = _answer_every_question(0.7)-- def _together(request: DecisionRequest, **kwargs: Any) -> DecisionResult:- barrier.wait()- return answer(request)-- activities = "products.replay_vision.backend.temporal.jev_watch_rank.activities"- with (- patch(f"{activities}.watch_feed_ranker", side_effect=self._flag_arm("jev-shadow")),- patch(f"{activities}.decision_api.decisions_available_here", return_value=True),- patch(_API) as api,- patch("posthoganalytics.capture"),- ):- api.decide_when_available.side_effect = _together- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- assert result.scanners_judged == 2- assert result.observations_judged == 2- assert result.failed_chunks == 0-- @parameterized.expand(- [- # A run that keeps sending into the shared rate limit takes capacity from other features.- ("every_call_rate_limited", 1, True),- # Occasional 429s clear within seconds, so a pause must not cost the run its remaining scanners.- ("every_other_call_rate_limited", 2, False),- ]- )- def test_a_rate_limited_run_pauses_and_gives_up_only_when_the_limit_persists(- self, _name: str, rate_limit_every: int, gives_up: bool- ) -> None:- self.organization.is_ai_data_processing_approved = True- self.organization.save()- # Enough scanners that some are still queued when the run would give up.- scanner_count = 2 * RATE_LIMIT_GIVE_UP_AFTER + 4- for index in range(scanner_count):- scanner = ReplayScanner.objects.create(- team=self.team,- name=f"s{index}",- scanner_type=ScannerType.SUMMARIZER,- scanner_config={"prompt": "p", "length": "short"},- model=ScannerModel.GEMINI_3_8_FLASH,- )- self._succeeded_observation(scanner, f"session-{index}", "The user hit an error at checkout.")- answer = _answer_every_question(0.1)- calls = itertools.count()- lock = threading.Lock()-- def _decide(request: DecisionRequest, **kwargs: Any) -> DecisionResult:- with lock:- call = next(calls)- if call % rate_limit_every == 0:- raise DecisionGatewayError(429, "rate limited")- return answer(request)-- activities = "products.replay_vision.backend.temporal.jev_watch_rank.activities"- with (- patch(f"{activities}.watch_feed_ranker", side_effect=self._flag_arm("jev-shadow")),- patch(f"{activities}.decision_api.decisions_available_here", return_value=True),- patch(f"{activities}.RATE_LIMIT_PAUSE", timedelta(0)),- patch(_API) as api,- patch("posthoganalytics.capture"),- ):- api.decide_when_available.side_effect = _decide- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- if gives_up:- assert RATE_LIMIT_GIVE_UP_AFTER <= result.scanners_rate_limited < scanner_count- assert result.scanners_backed_off == scanner_count - result.scanners_rate_limited- assert not result.backlog_complete- else:- assert result.scanners_rate_limited >= 3- assert result.scanners_backed_off == 0- assert result.scanners_judged == scanner_count- assert result.backlog_complete- assert result.pending_observations == result.scanners_rate_limited-- def test_a_rate_limited_turn_delays_the_next_request(self) -> None:- # The pause is what gives the shared capacity back, so the request after a 429 must wait it out.- self.organization.is_ai_data_processing_approved = True- self.organization.save()- for index in range(2):- scanner = ReplayScanner.objects.create(- team=self.team,- name=f"s{index}",- scanner_type=ScannerType.SUMMARIZER,- scanner_config={"prompt": "p", "length": "short"},- model=ScannerModel.GEMINI_3_8_FLASH,- )- self._succeeded_observation(scanner, f"session-{index}", "The user hit an error at checkout.")- pause = timedelta(seconds=0.3)- answer = _answer_every_question(0.1)- call_times: list[float] = []-- def _decide(request: DecisionRequest, **kwargs: Any) -> DecisionResult:- call_times.append(monotonic())- if len(call_times) == 1:- raise DecisionGatewayError(429, "rate limited")- return answer(request)-- activities = "products.replay_vision.backend.temporal.jev_watch_rank.activities"- with (- patch(f"{activities}.watch_feed_ranker", side_effect=self._flag_arm("jev-shadow")),- patch(f"{activities}.decision_api.decisions_available_here", return_value=True),- patch(f"{activities}.RATE_LIMIT_PAUSE", pause),- # One turn at a time, so the second request can only start after the first turn ends.- patch(f"{activities}.JUDGE_CONCURRENCY", 1),- patch(_API) as api,- patch("posthoganalytics.capture"),- ):- api.decide_when_available.side_effect = _decide- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- assert result.scanners_rate_limited == 1- assert result.scanners_judged == 2- assert call_times[1] - call_times[0] >= pause.total_seconds()-- def test_a_consent_revoked_while_a_scanner_waits_sends_nothing(self) -> None:- # Scanners are queued when their team passes the consent check but judged at their turn,- # which can come most of the run later. A revocation in between must stop the prose from- # reaching the model.- self.organization.is_ai_data_processing_approved = True- self.organization.save()- scanner = ReplayScanner.objects.create(- team=self.team,- name="s",- scanner_type=ScannerType.SUMMARIZER,- scanner_config={"prompt": "p", "length": "short"},- model=ScannerModel.GEMINI_3_8_FLASH,- )- self._succeeded_observation(scanner, "s1", "The user hit an error at checkout.")-- activities = "products.replay_vision.backend.temporal.jev_watch_rank.activities"- with (- patch(f"{activities}.watch_feed_ranker", side_effect=self._flag_arm("jev-shadow")),- patch(f"{activities}.decision_api.decisions_available_here", return_value=True),- # Approved when the team is queued, revoked by the time its scanner's turn comes.- patch(f"{activities}.is_ai_data_processing_approved", side_effect=[True, False]),- patch(_API) as api,- patch("posthoganalytics.capture"),- ):- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- api.decide_when_available.assert_not_called()- assert result.teams_enrolled == 1- assert result.scanners_consent_revoked == 1- assert result.scanners_judged == 0-- def _flag_arm(self, arm: str):- """The experiment arm for self.team only, so the pinned team 2 stays on the default arm."""- return lambda team_id, _team_uuid: arm if team_id == self.team.id else "weighted-score"-- def test_posthogs_own_team_is_always_swept_first(self) -> None:- # While the experiment is internal-only, team 2 must never fall past the team cap,- # whatever order the other teams come back in.- assert _teams_with_scanners()[0] == 2-- def test_a_batch_that_keeps_failing_is_abandoned_after_max_attempts(self) -> None:- # A deterministically failing batch must not stay newest-unjudged forever, re-bought every- # hour while older rows starve. After MAX_JUDGE_ATTEMPTS batch-attributable failures it- # joins the judged set with no score and the sweep moves on. An outage sweep charges no- # attempt, so a bad hour cannot park the rows.- self.organization.is_ai_data_processing_approved = True- self.organization.save()- scanner = ReplayScanner.objects.create(- team=self.team,- name="s",- scanner_type=ScannerType.SUMMARIZER,- scanner_config={"prompt": "p", "length": "short"},- model=ScannerModel.GEMINI_3_8_FLASH,- )- self._succeeded_observation(scanner, "s1", "The user hit an error at checkout.")-- activities = "products.replay_vision.backend.temporal.jev_watch_rank.activities"- # One outage sweep first: it fails the chunk but burns none of the batch's retry budget,- # so the batch still takes the full MAX_JUDGE_ATTEMPTS failing sweeps below to give up.- with (- patch(f"{activities}.watch_feed_ranker", side_effect=self._flag_arm("jev-shadow")),- patch(f"{activities}.decision_api.decisions_available_here", return_value=True),- patch(_API) as api,- patch("posthoganalytics.capture") as captured,- ):- api.decide_when_available.side_effect = DecisionsDisabledError(self.team.id)- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- assert result.failed_chunks == 1- assert result.observations_given_up == 0- # The judged event names the failure class, so a broken sweep is diagnosable from the- # product alone — worker logs and Prometheus never leave the cluster.- judged = [- call for call in captured.call_args_list if call.kwargs["event"] == "replay_vision_jev_watch_rank_judged"- ]- assert judged[0].kwargs["properties"]["chunk_error_types"] == {"DecisionsDisabledError": 1}-- for attempt in range(1, MAX_JUDGE_ATTEMPTS + 1):- with (- patch(f"{activities}.watch_feed_ranker", side_effect=self._flag_arm("jev-shadow")),- patch(f"{activities}.decision_api.decisions_available_here", return_value=True),- patch(_API) as api,- patch("posthoganalytics.capture"),- ):- api.decide_when_available.side_effect = _answer_every_question(7.0)- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- assert result.failed_chunks == 1, attempt- assert result.observations_given_up == (1 if attempt == MAX_JUDGE_ATTEMPTS else 0), attempt-- with (- patch(f"{activities}.watch_feed_ranker", side_effect=self._flag_arm("jev-shadow")),- patch(f"{activities}.decision_api.decisions_available_here", return_value=True),- patch(_API) as api,- ):- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- # The abandoned row is judged with no score: no more Jev spend, filler tier in the feed.- api.decide_when_available.assert_not_called()- assert result.scanners_skipped_unchanged == 1- assert load_watch_ranks(self.team.id, [scanner.id]).probabilities == {}diff --git a/products/replay_vision/backend/tests/test_scanner_dashboard.py b/products/replay_vision/backend/tests/test_scanner_dashboard.pyindex e080f2607e0c..048512585702 100644--- a/products/replay_vision/backend/tests/test_scanner_dashboard.py+++ b/products/replay_vision/backend/tests/test_scanner_dashboard.py@@ -39,12 +39,10 @@ _OUTPUT_BY_TYPE: dict[ScannerType, dict[str, Any]] = {ScannerType.SCORER: {"scanner_output_score": 2.0, "scanner_output_reasoning": "Slow progress."},ScannerType.SUMMARIZER: {"scanner_output_title": "Set up a dashboard",- "scanner_output_notability": 0.4,"scanner_output_chapter_count": 3,},ScannerType.EXPERIMENT: {"scanner_output_title": "Tried the new pricing page",- "scanner_output_notability": 0.6,"scanner_output_chapter_count": 2,"experiment_variant": "test",},diff --git a/products/replay_vision/backend/tests/test_scanners.py b/products/replay_vision/backend/tests/test_scanners.pyindex 452b8371794e..8b7df2f4be47 100644--- a/products/replay_vision/backend/tests/test_scanners.py+++ b/products/replay_vision/backend/tests/test_scanners.py@@ -882,14 +882,17 @@ class TestSummaryEmbeddingText:class TestToEventProperties:def test_flattens_with_scanner_output_prefix(self) -> None:- # Notability and the key moment ride onto the event too, so they are queryable in insights alongside the verdict.- out = MonitorOutput(- verdict="yes",- reasoning="found it",- confidence=0.9,- notability=0.8,- notability_reason="the export failed twice",- key_moment_ms=42_000,+ # The key moment rides onto the event too, so it is queryable in insights alongside the verdict. Stored+ # answers from before notability was retired still carry its keys, and must load without putting them back.+ out = MonitorOutput.model_validate(+ {+ "verdict": "yes",+ "reasoning": "found it",+ "confidence": 0.9,+ "notability": 0.8,+ "notability_reason": "the export failed twice",+ "key_moment_ms": 42_000,+ })props = out.to_event_properties()assert props == {@@ -897,18 +900,9 @@ class TestToEventProperties:"scanner_output_reasoning": "found it","scanner_output_reasoning_segments": [],"scanner_output_confidence": 0.9,- "scanner_output_notability": 0.8,- "scanner_output_notability_reason": "the export failed twice","scanner_output_key_moment_ms": 42_000,}- def test_unjudged_notability_flattens_as_null_not_zero(self) -> None:- # A scan that skipped notability must not read as "not notable" downstream.- out = MonitorOutput(verdict="yes", reasoning="found it", confidence=0.9)- props = out.to_event_properties()- assert props["scanner_output_notability"] is None- assert props["scanner_output_notability_reason"] is None-def test_summary_chapters_flatten_to_a_count(self) -> None:out = SummarizerOutput(title="t",diff --git a/products/replay_vision/backend/tests/test_watch_rank.py b/products/replay_vision/backend/tests/test_watch_rank.pyindex b85803c322a1..73844ccb8881 100644--- a/products/replay_vision/backend/tests/test_watch_rank.py+++ b/products/replay_vision/backend/tests/test_watch_rank.py@@ -15,7 +15,13 @@ from products.replay_vision.backend import watch_rank_backfillfrom products.replay_vision.backend.models.replay_observation import ReplayObservationfrom products.replay_vision.backend.tests.helpers import axis_anchorsfrom products.replay_vision.backend.watch_feed import SESSION_SPACING-from products.replay_vision.backend.watch_rank import ANCHOR_VERSION, ANCHORS, classify, rank_watch_feed_by_embedding+from products.replay_vision.backend.watch_rank import (+ ANCHOR_VERSION,+ ANCHORS,+ WATCH_PICK_MIN_SCANNER_ROWS,+ classify,+ rank_watch_feed_by_embedding,+)from products.replay_vision.backend.watch_rank_backfill import _rank_scanner, _stored_vectors_NOW = datetime(2026, 10, 23, 12, 0, tzinfo=UTC)@@ -53,6 +59,21 @@ def _row(}+def _with_percentiles(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:+ """Each row's contrast percentile within its scanner, as the pick query attaches it."""+ by_scanner: dict[str, list[dict[str, Any]]] = {}+ for row in rows:+ if row["watch_rank"] and row["watch_rank"].get("anchor_version") == ANCHOR_VERSION:+ by_scanner.setdefault(row["scanner_id"], []).append(row)+ for scanner_rows in by_scanner.values():+ if len(scanner_rows) < WATCH_PICK_MIN_SCANNER_ROWS:+ continue+ scanner_rows.sort(key=lambda row: row["watch_rank"]["contrast"])+ for position, row in enumerate(scanner_rows):+ row.setdefault("watch_percentile", (position + 0.5) / len(scanner_rows))+ return rows++class TestClassify(SimpleTestCase):@parameterized.expand([@@ -88,7 +109,7 @@ class TestRankWatchFeedByEmbedding(SimpleTestCase):for row in rows:row["watch_percentile"] = {"one-off": 0.99, "widespread": 0.85}.get(str(row["id"]))row["watch_group_people"] = 6 if row["id"] == "widespread" else 1- ranked = rank_watch_feed_by_embedding(rows)+ ranked = rank_watch_feed_by_embedding(_with_percentiles(rows))assert [entry.observation_id for entry in ranked][:2] == ["widespread", "one-off"]assert ranked[0].reason["affected_people"] == 6@@ -107,7 +128,7 @@ class TestRankWatchFeedByEmbedding(SimpleTestCase):_row("b-top", 0.50, scanner="scanner-b", minutes_ago=1),]- ranked = rank_watch_feed_by_embedding(rows)+ ranked = rank_watch_feed_by_embedding(_with_percentiles(rows))assert ranked[0].observation_id == "dup-copy"assert ranked[0].reason == {@@ -129,7 +150,7 @@ class TestRankWatchFeedByEmbedding(SimpleTestCase):*[_row(f"a-{index}", 0.01 * index, minutes_ago=index + 3) for index in range(7)],]- ranked = rank_watch_feed_by_embedding(rows)+ ranked = rank_watch_feed_by_embedding(_with_percentiles(rows))picks = {str(entry.observation_id): entry.reason for entry in ranked if entry.reason["kind"] == "embedding_pick"@@ -148,7 +169,7 @@ class TestRankWatchFeedByEmbedding(SimpleTestCase):_row("c-top", 0.9, scanner="scanner-c", minutes_ago=2),]- ranked = [str(entry.observation_id) for entry in rank_watch_feed_by_embedding(rows)]+ ranked = [str(entry.observation_id) for entry in rank_watch_feed_by_embedding(_with_percentiles(rows))]assert ranked[0] == "a-top"assert ranked.index("b-top") - ranked.index("a-top") >= SESSION_SPACING@@ -157,7 +178,7 @@ class TestRankWatchFeedByEmbedding(SimpleTestCase):def test_rows_without_a_rank_fall_back_to_the_filler_floor(self) -> None:rows = [_row(f"row-{index}", None, minutes_ago=index, viewed=index == 0) for index in range(20)]- ranked = rank_watch_feed_by_embedding(rows)+ ranked = rank_watch_feed_by_embedding(_with_percentiles(rows))assert [entry.observation_id for entry in ranked] == ["row-1", "row-2", "row-3"]diff --git a/products/replay_vision/backend/watch_feed.py b/products/replay_vision/backend/watch_feed.pyindex 192f539177e1..227a9cc02760 100644--- a/products/replay_vision/backend/watch_feed.py+++ b/products/replay_vision/backend/watch_feed.py@@ -1,412 +1,35 @@-"""Ranking for the What to watch feed: which succeeded observations in a window are worth a look.+"""Ordering for the Highlights feed: a cap on any one scanner's share, spacing between one session's cards, and+the floor of filler rows a quiet window returns."""-Deterministic v1, no model calls. Baselines come from the candidate rows themselves (the scanner's-own window), so "outlier" and "rare" mean unusual for that scanner lately, not against all history.-"""--import reimport math-from collections import deque-from datetime import datetime-from statistics import fmean, stdevfrom typing import Anyfrom uuid import UUIDfrom posthog.dataclasses import frozen-# Below this many rows a scanner's window baseline is noise, so the scanner gets no type hit.-MIN_BASELINE_ROWS = 5-OUTLIER_STDDEVS = 1.5-RARE_TAG_MAX_SHARE = 0.1-# When more than this share of a scanner's tagged rows carry a "rare" tag, the vocabulary isn't-# discriminating (e.g. freeform tags that never repeat) and rarity means nothing — suppress the rule.-RARE_TAG_MAX_HIT_SHARE = 0.5-# The scan's own judgment of how much a team would benefit from watching the session. At or above this it-# ranks as notable on its own, regardless of what the scanner was asked.-NOTABLE_MIN_SCORE = 0.6-# A monitor verdict carried by at most this share of the scanner's window rows is the unusual-# answer, whatever the prompt's polarity ("did they struggle?" vs "was the experience good?").-UNUSUAL_VERDICT_MAX_SHARE = 0.35-# A summary whose best token overlap with the scanner's other summaries stays at or below this is-# treated as an unusual session. Conservative on purpose: prose varies, so only clear outliers hit.-NOVEL_SUMMARY_MAX_SIMILARITY = 0.15-# Jaccard divides by the union, so a very short summary reads as "novel" against any longer one-# regardless of content. Below this many tokens the comparison measures length, not meaning.-NOVEL_SUMMARY_MIN_TOKENS = 8-# Compare against at most this many of the scanner's newest summaries. Bounds the novelty pass to-# O(rows x this) — all-pairs would be quadratic in the scanner's window, which an authenticated-# viewer could lean on by hammering the broad feed.-NOVEL_SUMMARY_MAX_SIBLINGS = 40--# Component weights for the blended watchability score. The feed is a portfolio of the window's most-# watchable rows, not signals first and everything after, so the signal weight sits below the hit weight: a-# wide outlier or a rare verdict outranks even the most confident lone signal. Retune here.-WATCH_WEIGHT_SIGNAL = 0.8-WATCH_WEIGHT_HIT = 0.9-WATCH_WEIGHT_NOTABILITY = 0.7-WATCH_WEIGHT_FRICTION = 0.3-# A row the reader already opened keeps its evidence but drops by this much, so an unviewed peer with-# comparable evidence comes first while a strong seen row (a signal, a wide outlier) still holds its-# place above routine unseen rows.-WATCH_SEEN_PENALTY = 0.5-# A no-baseline "yes" verdict is the weak conventional something-happened phrasing, so it enters the-# blend below a verdict the scanner's own window shows to be rare.-VERDICT_YES_STRENGTH = 0.5-# Signal strength is the row's most confident finding, and emission floors confidence at 0.4, so a live value-# runs 0.4-1.0. A row scanned before summaries shipped carries a count and no confidence, and takes the-# midpoint of that range: it neither outranks the rows around it nor sinks below them.-SIGNAL_LEGACY_STRENGTH = 0.7-# The share of the feed signal rows may hold, above the leading row and while other rows remain to place.-# Signals corroborate across sessions, so a window full of them says the scanner is working, not that a-# reader should watch all of them.-WATCH_FEED_MAX_SIGNAL_SHARE = 0.4-# Reason kinds that say nothing about the session: the fall-through for a row that scored on no source at-# all. They are ordering, not evidence, so the feed carries them only to reach the floor below.-FILLER_REASON_KINDS = frozenset({"unviewed_recent", "recent"})-# How short the feed may get before filler rows pad it. A feed of 3 real findings beats one of 3 findings-# and 17 "new since you last looked", and a feed that empties the moment nothing is wrong reads as broken.+# Filler pads a quiet feed to this many cards, so a window with nothing wrong still shows the newest clips.WATCH_FEED_MIN_ITEMS = 3-# The score deviation, in stddevs, at which an outlier counts full strength. The hit fires at-# OUTLIER_STDDEVS; its strength climbs from there to 1.0 by this many stddevs, so a wider outlier ranks-# above a marginal one instead of both reading as the same boolean hit.-OUTLIER_FULL_STDDEVS = 3.0--_TOKEN_RE = re.compile(r"[a-z0-9]+")--# Friction language in the model's own prose about the session. Stems, so "retried"/"retrying",-# "failed"/"failure", "confused"/"confusing" all hit. Known weakness, not fixable with keywords: a team-# whose product is about errors (error tracking, debugging) matches on happy sessions too, which is why-# friction ranks below seen-state and never reaches the top tiers. Widen with care.-_FRICTION_RE = re.compile(- r"\b(error|errors|fail(?:ed|ing|ure|s)?|stuck|struggl\w*|confus\w*|frustrat\w*|abandon\w*|"- r"rage.?click\w*|dead.?end\w*|broken|crash\w*|bug|bugs|retr(?:y|ied|ies|ying)|time(?:d|s)?.?out|"- r"gave up|give up|churn\w*|unable|couldn.?t|blocked|glitch\w*|freez\w*|froze|unresponsive)\b",- re.IGNORECASE,-)@frozenclass WatchFeedEntry:observation_id: UUIDreason: dict[str, Any]- # Set only by rankers that space a recording's cards apart; None opts the entry out.+ # None opts the entry out of session spacing.session_id: str | None = None-@frozen-class _Candidate:- observation_id: UUID- scanner_id: UUID- created_at: datetime- viewed: bool- signals_count: int- signal_problem_types: tuple[str, ...]- # (problem_type, headline, confidence) per emitted signal. Empty on rows scanned before this shipped.- signal_summaries: tuple[tuple[str, str, float], ...]- scanner_type: str | None- verdict: str | None- score: float | None- tags: tuple[str, ...]- summary_tokens: frozenset[str]- friction: bool- notability: float | None- notability_reason: str | None---def _parse_candidate(row: dict[str, Any]) -> _Candidate:- """Read the ranking features off a raw row, tolerating malformed `scanner_result` — a bad row- ranks by recency only rather than failing the whole feed."""- result = row.get("scanner_result")- output = result.get("model_output") if isinstance(result, dict) else None- if not isinstance(output, dict):- output = {}- signals_count = result.get("signals_count") if isinstance(result, dict) else 0- raw_problem_types = result.get("signal_problem_types") if isinstance(result, dict) else None- signal_problem_types = (- tuple(pt for pt in raw_problem_types if isinstance(pt, str)) if isinstance(raw_problem_types, list) else ()- )- raw_summaries = result.get("signal_summaries") if isinstance(result, dict) else None- signal_summaries = (- tuple(- (entry["problem_type"], entry["headline"], float(entry["confidence"]))- for entry in raw_summaries- if isinstance(entry, dict)- and isinstance(entry.get("problem_type"), str)- # A blank headline would leave the card's sentence trailing off, so the row counts instead.- and isinstance(entry.get("headline"), str)- and entry["headline"].strip()- and isinstance(entry.get("confidence"), int | float)- and not isinstance(entry.get("confidence"), bool)- )- if isinstance(raw_summaries, list)- else ()- )- score = output.get("score")- raw_tags = output.get("tags")- raw_freeform = output.get("tags_freeform")- tags = [- *(raw_tags if isinstance(raw_tags, list) else []),- *(raw_freeform if isinstance(raw_freeform, list) else []),- ]- title = output.get("title")- summary = output.get("summary")- reasoning = output.get("reasoning")- summary_text = " ".join(part for part in (title, summary) if isinstance(part, str))- prose = " ".join(part for part in (title, summary, reasoning) if isinstance(part, str))- notability = output.get("notability")- notability_reason = output.get("notability_reason")- scanner_type = output.get("scanner_type") if isinstance(output.get("scanner_type"), str) else None- verdict = output.get("verdict") if isinstance(output.get("verdict"), str) else None- # A no-verdict monitor's reasoning restates its question in the negative ("did not struggle"),- # so keyword matching there reads negations as friction; the scan already judged it a non-event.- friction_eligible = not (scanner_type == "monitor" and verdict == "no")- return _Candidate(- observation_id=row["id"],- scanner_id=row["scanner_id"],- created_at=row["created_at"],- viewed=bool(row.get("feed_viewed")),- signals_count=signals_count if isinstance(signals_count, int) else 0,- signal_problem_types=signal_problem_types,- signal_summaries=signal_summaries,- scanner_type=scanner_type,- verdict=verdict,- score=float(score) if isinstance(score, int | float) else None,- tags=tuple(tag for tag in tags if isinstance(tag, str)),- summary_tokens=frozenset(_TOKEN_RE.findall(summary_text.lower())),- friction=friction_eligible and bool(_FRICTION_RE.search(" ".join([prose, *tags]))),- # The LLM-response schema bounds this to 0-1, but a stored row (or a bool, since bool is an int- # subclass) can carry anything, so clamp defensively — an out-of-range value would outrank its tier.- notability=(- min(1.0, max(0.0, float(notability)))- if isinstance(notability, int | float) and not isinstance(notability, bool)- else None- ),- notability_reason=notability_reason if isinstance(notability_reason, str) and notability_reason else None,- )---@frozen-class _ScannerBaseline:- row_count: int- score_mean: float | None- score_stddev: float | None- tag_shares: dict[str, float]- rare_tags: frozenset[str]- verdict_shares: dict[str, float]---def _baselines(candidates: list[_Candidate]) -> dict[UUID, _ScannerBaseline]:- by_scanner: dict[UUID, list[_Candidate]] = {}- for candidate in candidates:- by_scanner.setdefault(candidate.scanner_id, []).append(candidate)- baselines: dict[UUID, _ScannerBaseline] = {}- for scanner_id, rows in by_scanner.items():- scores = [row.score for row in rows if row.score is not None]- tag_counts: dict[str, int] = {}- verdict_counts: dict[str, int] = {}- tagged_rows = [row for row in rows if row.tags]- for row in rows:- for tag in set(row.tags):- tag_counts[tag] = tag_counts.get(tag, 0) + 1- if row.verdict is not None:- verdict_counts[row.verdict] = verdict_counts.get(row.verdict, 0) + 1- verdict_total = sum(verdict_counts.values())- # Shares are over rows that carry tags at all, so untagged/malformed rows don't make every- # real tag look rarer than it is (verdict_shares already divides by verdict_total for the- # same reason). A singleton always counts as rare: a pure share cutoff makes rarity- # impossible below 1/RARE_TAG_MAX_SHARE rows and then fire on every singleton at exactly- # that size, so the feed would change character as a scanner accumulates data.- tag_shares = {tag: count / len(tagged_rows) for tag, count in tag_counts.items()} if tagged_rows else {}- rare_tags = (- frozenset(tag for tag, count in tag_counts.items() if count == 1 or tag_shares[tag] <= RARE_TAG_MAX_SHARE)- if len(tagged_rows) >= MIN_BASELINE_ROWS- else frozenset()- )- if rare_tags:- rare_rows = sum(1 for row in tagged_rows if rare_tags & set(row.tags))- if rare_rows / len(tagged_rows) > RARE_TAG_MAX_HIT_SHARE:- rare_tags = frozenset()- baselines[scanner_id] = _ScannerBaseline(- row_count=len(rows),- score_mean=fmean(scores) if len(scores) >= MIN_BASELINE_ROWS else None,- score_stddev=stdev(scores) if len(scores) >= MIN_BASELINE_ROWS else None,- tag_shares=tag_shares,- rare_tags=rare_tags,- verdict_shares=(- {verdict: count / verdict_total for verdict, count in verdict_counts.items()}- if verdict_total >= MIN_BASELINE_ROWS- else {}- ),- )- return baselines---def _max_summary_similarity(candidate: _Candidate, siblings: list[_Candidate]) -> float | None:- """Best token-set Jaccard similarity of this summary against the scanner's newest other summaries- (at most NOVEL_SUMMARY_MAX_SIBLINGS — siblings arrive newest-first from the candidate query).- None when the window is too thin to say what "usual" reads like."""- others = [- row.summary_tokens for row in siblings if row.observation_id != candidate.observation_id and row.summary_tokens- ][:NOVEL_SUMMARY_MAX_SIBLINGS]- if len(candidate.summary_tokens) < NOVEL_SUMMARY_MIN_TOKENS or len(others) < MIN_BASELINE_ROWS - 1:- return None- return max(len(candidate.summary_tokens & other) / len(candidate.summary_tokens | other) for other in others)---def _type_hit(- candidate: _Candidate, baseline: _ScannerBaseline, siblings: list[_Candidate]-) -> tuple[dict[str, Any], float] | None:- """The type-specific reason this row stands out for its scanner and how strongly, or None.-- The strength is in (0, 1]: a wide score outlier, a very rare verdict or tag, or a summary unlike- its siblings scores near 1; a marginal hit scores near the floor of its rule. It feeds the blended- score only — the returned reason dict is unchanged, so the card copy stays the same.- """- if candidate.scanner_type == "monitor" and candidate.verdict in ("yes", "no"):- # With a baseline, the notable answer is the scanner's own minority verdict — prompt- # polarity doesn't matter ("did they struggle?" vs "was the experience good?"). Without- # one, fall back to yes, the conventional something-happened phrasing.- if baseline.verdict_shares:- share = baseline.verdict_shares.get(candidate.verdict, 0.0)- if share <= UNUSUAL_VERDICT_MAX_SHARE:- reason = {"kind": "unusual_verdict", "verdict": candidate.verdict, "verdict_share": round(share, 3)}- return reason, 1.0 - share- elif candidate.verdict == "yes":- return {"kind": "verdict_yes"}, VERDICT_YES_STRENGTH- if candidate.scanner_type in ("summarizer", "experiment"):- similarity = _max_summary_similarity(candidate, siblings)- if similarity is not None and similarity <= NOVEL_SUMMARY_MAX_SIMILARITY:- return {"kind": "novel_summary"}, 1.0 - similarity- if (- candidate.scanner_type == "scorer"- and candidate.score is not None- and baseline.score_mean is not None- and baseline.score_stddev is not None- # A flat window (every score identical) makes any deviation infinite sigmas; skip it.- and baseline.score_stddev > 0- and abs(candidate.score - baseline.score_mean) >= OUTLIER_STDDEVS * baseline.score_stddev- ):- sigma = abs(candidate.score - baseline.score_mean) / baseline.score_stddev- reason = {"kind": "outlier_score", "score": candidate.score, "window_mean": round(baseline.score_mean, 2)}- return reason, min(1.0, sigma / OUTLIER_FULL_STDDEVS)- if candidate.scanner_type == "classifier" and baseline.rare_tags:- rare = [(baseline.tag_shares[tag], tag) for tag in set(candidate.tags) if tag in baseline.rare_tags]- if rare:- share, tag = min(rare)- return {"kind": "rare_tag", "tag": tag, "tag_share": round(share, 3)}, 1.0 - share- return None---def _signal_strength(candidate: _Candidate) -> float:- """How strongly the row's signals argue for watching it: the confidence of its most confident finding.- A flat 1.0 for any signal made a marginal finding rank level with an unmistakable one."""- if candidate.signal_summaries:- return max(confidence for _, _, confidence in candidate.signal_summaries)- return SIGNAL_LEGACY_STRENGTH if candidate.signals_count > 0 else 0.0---def _contributions(candidate: _Candidate, hit_strength: float) -> dict[str, float]:- """The weighted evidence this row carries, one entry per source. `_watchability` sums these and the- displayed reason names the largest of them, so the order and the copy read the same numbers."""- return {- "signal": WATCH_WEIGHT_SIGNAL * _signal_strength(candidate),- "hit": WATCH_WEIGHT_HIT * hit_strength,- "notability": WATCH_WEIGHT_NOTABILITY * (candidate.notability if candidate.notability is not None else 0.0),- "friction": WATCH_WEIGHT_FRICTION * (1.0 if candidate.friction else 0.0),- }---def _watchability(candidate: _Candidate, hit_strength: float) -> float:- """Blend the row's signal, type hit, notability, and friction into one score, then dock a row the- reader already opened. Every source contributes at once, so a row wins on the sum of its evidence- rather than on a single dominant flag."""- score = sum(_contributions(candidate, hit_strength).values())- if candidate.viewed:- score -= WATCH_SEEN_PENALTY- return score---def _strongest_reason_part(candidate: _Candidate, contributions: dict[str, float]) -> str | None:- """Which source the card should name, or None when the row carries no evidence at all.-- The largest weighted part wins. Ties break signal, hit, notability, friction — the order the old- fixed ladder used — so a row with two equal parts reads the way it always did.- """- parts = dict(contributions)- # Notability below the tier threshold still adds to the score, but the card must not say the scan judged- # the session worth watching when it did not. The old ladder held this rule; the label holds it now.- if candidate.notability is None or candidate.notability < NOTABLE_MIN_SCORE:- parts["notability"] = 0.0- winner = max(("signal", "hit", "notability", "friction"), key=lambda part: parts[part])- return winner if parts[winner] > 0 else None---def _cap_signal_share(ranked: list[WatchFeedEntry]) -> list[WatchFeedEntry]:- """Hold signal cards to WATCH_FEED_MAX_SIGNAL_SHARE of the feed as it is placed.-- The share is checked per place rather than over the whole list, because the view slices the head of this- list: a feed of 5 obeys the same share as a feed of 50. Two rows are exempt by design. The best row- always leads, whatever it is. And a window with more signal rows than the share can hold runs out of- other rows to interleave, so the rest trail the feed rather than disappear from it — reordering the feed- is the job here, shortening it is not.-- Because `_trim_filler` runs first, the rows a signal is held against are other findings, not padding.- So a window whose only findings are signals returns a feed of signals: the share keeps signals from- crowding out an outlier or a rare verdict, and was never meant to dilute them with clips that carry- nothing.- """- placed: list[WatchFeedEntry] = []- # A deque, because the candidate cap is 1000 and every admitted row pops from the front.- held: deque[WatchFeedEntry] = deque()- signals_placed = 0- for entry in ranked:- if entry.reason["kind"] == "signal_emitted":- held.append(entry)- else:- placed.append(entry)- while held and (not placed or (signals_placed + 1) / (len(placed) + 1) <= WATCH_FEED_MAX_SIGNAL_SHARE):- placed.append(held.popleft())- signals_placed += 1- return placed + list(held)---def _trim_filler(ranked: list[WatchFeedEntry]) -> list[WatchFeedEntry]:- """Drop the no-evidence rows once WATCH_FEED_MIN_ITEMS real findings are in the feed.-- `limit` is a ceiling the feed had been filling: a quiet window returned 20 cards of which 17 said- "New since you last looked", which buries the few that meant something. Findings are kept whatever- their count; filler only makes up the difference to the floor, so a window with nothing to say- returns 3 newest clips rather than 20 or an empty state.- """- findings = [entry for entry in ranked if entry.reason["kind"] not in FILLER_REASON_KINDS]- if len(findings) >= WATCH_FEED_MIN_ITEMS:- return findings- # Walk rather than concatenate, because a row the reader already opened is docked half a point and can- # sort below an unviewed filler row — findings are not always a prefix of the list.- budget = WATCH_FEED_MIN_ITEMS - len(findings)- kept: list[WatchFeedEntry] = []- for entry in ranked:- if entry.reason["kind"] not in FILLER_REASON_KINDS:- kept.append(entry)- elif budget > 0:- kept.append(entry)- budget -= 1- return kept---# One incident can put many near-identical watchable sessions on one scanner, and a pure score-# sort would fill the top of the feed with them. Hold each scanner to this share of the evidence-# tier as it is placed (same intent as WATCH_FEED_MAX_SIGNAL_SHARE in the weighted ranker), with a-# floor so a short feed is not over-constrained; the overflow trails the tier instead of leaving it.+# One incident can put many near-identical sessions on one scanner, so each scanner holds at most this share of the+# picks, above a floor that keeps a short feed from being over-constrained.MAX_SCANNER_SHARE = 0.4_SCANNER_SPREAD_FLOOR = 3def spread_scanners(ordered: list[tuple[Any, WatchFeedEntry]]) -> list[WatchFeedEntry]:- """Hold each scanner to MAX_SCANNER_SHARE of the evidence tier as it is placed.+ """Hold each scanner to MAX_SCANNER_SHARE of the picks as it is placed.Checked per place rather than over the whole list because the view slices the head: a feed of 5- obeys the same share as a feed of 50. Overflow trails the tier in probability order — reordering+ obeys the same share as a feed of 50. Overflow trails the picks in score order — reorderingfor breadth is the job here, shortening the feed is not."""placed: list[WatchFeedEntry] = []@@ -459,73 +82,5 @@ def finish_feed(entry = WatchFeedEntry(observation_id=row["id"], reason={"kind": "recent" if viewed else "unviewed_recent"})filler.append((not viewed, row["created_at"], entry))filler.sort(key=lambda item: (item[0], item[1]), reverse=True)- return _trim_filler(- space_sessions(spread_scanners([(scanner_id, entry) for _, _, scanner_id, entry in ordered]))- + [entry for *_, entry in filler]- )---def rank_watch_feed_candidates(rows: list[dict[str, Any]]) -> list[WatchFeedEntry]:- """Rank candidate rows (`id`, `scanner_id`, `created_at`, `scanner_result`, `feed_viewed`) most- watchable first by a blended score, then newest.-- The score adds four sources at once — emitted signals, a type-specific hit, the scan's own- notability judgment, and prose that reads as friction — each weighted, so a row wins on the sum of- its evidence. A type hit plus a high notability can outrank a routine signal, rather than every- signal outranking everything else. A row the reader already opened keeps its evidence but is docked,- so an unviewed peer comes first while a strong seen row still holds its place above routine unseen- rows.-- The displayed reason names the largest weighted part of that same score, so the card explains the- position it earned. It used to name a signal whenever the row carried one, which labelled a row as- `signal_emitted` even when a wide outlier was the stronger evidence and made the feed read as signals- with a few other things in it.-- Rows that scored on nothing are then trimmed to a floor of WATCH_FEED_MIN_ITEMS, and the signal rows- that survive are interleaved down to WATCH_FEED_MAX_SIGNAL_SHARE of the result; see `_trim_filler` and- `_cap_signal_share`. The trim runs first on purpose: trimming afterwards would strip the very rows the- cap interleaved signals against, so a feed that satisfied the share when built would breach it on the- way out.- """- candidates = [_parse_candidate(row) for row in rows]- baselines = _baselines(candidates)- by_scanner: dict[UUID, list[_Candidate]] = {}- for candidate in candidates:- by_scanner.setdefault(candidate.scanner_id, []).append(candidate)- scored: list[tuple[tuple[float, datetime], WatchFeedEntry]] = []- for candidate in candidates:- hit_result = _type_hit(candidate, baselines[candidate.scanner_id], by_scanner[candidate.scanner_id])- hit = hit_result[0] if hit_result is not None else None- hit_strength = hit_result[1] if hit_result is not None else 0.0- contributions = _contributions(candidate, hit_strength)- strongest = _strongest_reason_part(candidate, contributions)- if strongest == "signal":- reason: dict[str, Any] = {"kind": "signal_emitted", "signals_count": candidate.signals_count}- if candidate.signal_problem_types:- reason["problem_types"] = list(candidate.signal_problem_types)- if candidate.signal_summaries:- # The card names the findings from this; the confidence stays behind, because it ranks the- # row rather than telling a reader anything they can act on.- reason["signals"] = [- {"problem_type": problem_type, "headline": headline}- for problem_type, headline, _ in candidate.signal_summaries- ]- elif strongest == "hit" and hit is not None:- reason = hit- elif strongest == "notability":- reason = {"kind": "notable", "notability": candidate.notability}- elif strongest == "friction":- reason = {"kind": "friction"}- elif not candidate.viewed:- reason = {"kind": "unviewed_recent"}- else:- reason = {"kind": "recent"}- # The scan writes a notability_reason on every session (including "nothing stands out"), so only- # carry it when notability actually won the row — a signal or type hit can coexist with a high- # score, and gating on `notable` alone would attach the sentence over the copy that row earned.- if candidate.notability_reason and reason["kind"] == "notable":- reason["notability_reason"] = candidate.notability_reason- sort_key = (_watchability(candidate, hit_strength), candidate.created_at)- scored.append((sort_key, WatchFeedEntry(observation_id=candidate.observation_id, reason=reason)))- scored.sort(key=lambda item: item[0], reverse=True)- return _cap_signal_share(_trim_filler([entry for _, entry in scored]))+ placed = space_sessions(spread_scanners([(scanner_id, entry) for _, _, scanner_id, entry in ordered]))+ return placed + [entry for *_, entry in filler][: max(0, WATCH_FEED_MIN_ITEMS - len(placed))]diff --git a/products/replay_vision/backend/watch_feed_impressions.py b/products/replay_vision/backend/watch_feed_impressions.pyindex 43cc9eb1b2dd..c0428061982c 100644--- a/products/replay_vision/backend/watch_feed_impressions.py+++ b/products/replay_vision/backend/watch_feed_impressions.py@@ -1,4 +1,4 @@-"""Per-user counts of how often What to watch showed each observation."""+"""Per-user counts of how often the Highlights feed showed each observation."""from collections.abc import Sequencefrom datetime import datetimediff --git a/products/replay_vision/backend/watch_rank.py b/products/replay_vision/backend/watch_rank.pyindex ff5fd3ff4884..194cdbc7f3c2 100644--- a/products/replay_vision/backend/watch_rank.py+++ b/products/replay_vision/backend/watch_rank.py@@ -1,4 +1,4 @@-"""What to watch ranking from observation embeddings.+"""Highlights feed ranking from observation embeddings.A consumer of the embedding worker's results ranks each observation once, from the vector the scanalready embeds for search, so the feed needs no model call and no sweep. Each vector is compared with fixed anchor sentences: the@@ -109,6 +109,8 @@ WATCH_PICK_PEOPLE_BOOST_CAP = 2.0WATCH_PICK_GROUP_CANDIDATES = 3# Group members older than the feed's longest window before the oldest pick can never fall inside a requested window.WATCH_FEED_MEMBER_LOOKBACK = timedelta(days=90)+# How much wider the second pick fetch reads when the first is all copies of a few feed groups.+WATCH_PICK_REFILL_FACTOR = 10_ANCHOR_CACHE_TTL_S = 7 * 24 * 3600@@ -257,22 +259,6 @@ async def rank_observation(request: RankRequest) -> Literal["ranked", "already_rreturn "ranked"-def _scanner_percentiles(rows: list[dict[str, Any]]) -> dict[str, float]:- """Each ranked row's score as a percentile among its scanner's ranked rows in the window."""- by_scanner: dict[Any, list[tuple[float, str]]] = {}- for row in rows:- if rank := current_rank(row):- by_scanner.setdefault(row["scanner_id"], []).append((float(rank["contrast"]), str(row["id"])))- percentiles: dict[str, float] = {}- for scored in by_scanner.values():- if len(scored) < WATCH_PICK_MIN_SCANNER_ROWS:- continue- scored.sort()- for position, (_, observation_id) in enumerate(scored):- percentiles[observation_id] = (position + 0.5) / len(scored)- return percentiles--def people_boost(people: int) -> float:"""How much a pick's score grows with the number of people its feed group reached."""return min(WATCH_PICK_PEOPLE_BOOST_CAP, 1 + WATCH_PICK_PEOPLE_BOOST * math.log(max(people, 1)))@@ -296,7 +282,7 @@ def _cap_feed_groups(rows: list[dict[str, Any]], cap: int) -> dict[Any, dict[strdef _pick_window_rows(- ranked: QuerySet[ReplayObservation], current_ranks: QuerySet[ReplayObservation], limit: int | None+ ranked: QuerySet[ReplayObservation], current_ranks: QuerySet[ReplayObservation], limit: int) -> list[dict[str, Any]]:"""Each scanner's top-fifth issue rows in pick order: every scanner's best first, then every second-best.@@ -378,12 +364,15 @@ def select_embedding_picks(watch_category=KeyTextTransform("category", "watch_rank"),contrast_type=jsonb_typeof(KeyTransform("contrast", "watch_rank")),).filter(contrast_type="number")- # Bounds the fetch, since a client-side cursor buffers every row; only a window full of copies needs the rest.+ # Bounds the fetch, since a client-side cursor buffers every row. A window full of copies reads a wider slice+ # once, still bounded, and a feed that stays short past it shows fewer picks rather than reading every row.fetch_limit = cap * (WATCH_PICK_GROUP_CANDIDATES + 1)rows = _pick_window_rows(ranked, current_ranks, fetch_limit)window_values = _cap_feed_groups(rows, cap)if len(window_values) < cap and len(rows) == fetch_limit:- window_values = _cap_feed_groups(_pick_window_rows(ranked, current_ranks, None), cap)+ window_values = _cap_feed_groups(+ _pick_window_rows(ranked, current_ranks, fetch_limit * WATCH_PICK_REFILL_FACTOR), cap+ )if not window_values:return []# Counted over readable rows only, so a group never reveals how many restricted sessions it holds. Members@@ -467,15 +456,10 @@ def rank_watch_feed_by_embedding(rows: list[dict[str, Any]]) -> list[WatchFeedEnA pick is a row in its scanner's top fifth by score whose category is an issue. Rows of one feed groupcollapse to their best-scoring row, which carries the size of the group. Picks sort by weightedpercentile, decayed for each earlier impression, viewed rows docked, one scanner held to a share- of the picks, and one session's picks spaced apart. Every other row is filler, trimmed to the same- floor as the other rankers.+ of the picks, and one session's picks spaced apart. Every other row is filler, kept only to reach the+ feed's floor."""- # Rows from `select_embedding_picks` carry their whole-window percentile; a plain candidate slice does not.- percentiles = (- {str(row["id"]): row["watch_percentile"] for row in rows if row.get("watch_percentile") is not None}- if any("watch_percentile" in row for row in rows)- else _scanner_percentiles(rows)- )+ percentiles = {str(row["id"]): row["watch_percentile"] for row in rows if row.get("watch_percentile") is not None}best_in_group: dict[str, tuple[float, dict[str, Any]]] = {}group_members: dict[str, list[str]] = {}for row in rows:diff --git a/products/replay_vision/backend/watch_themes.py b/products/replay_vision/backend/watch_themes.pyindex 18d409524bd0..a2af67333375 100644--- a/products/replay_vision/backend/watch_themes.py+++ b/products/replay_vision/backend/watch_themes.py@@ -64,6 +64,8 @@ THEME_NAME_SAMPLES = 8THEME_NAMING_DAILY_CAP = 100MAX_THEME_LABEL_WORDS = 10_THEME_MODEL = "gemini-3.5-flash-lite"+# A label is a few words, so a request that runs this long is stuck rather than slow.+_NAMING_REQUEST_TIMEOUT_MS = 20_000_SAMPLE_CHARS = 400_IDENTIFIER = re.compile(r"https?://|www\.|@")@@ -285,7 +287,11 @@ async def name_theme(team_id: int, theme_id: uuid.UUID, now: datetime) -> str |scanner_prompt=neutralize_markup((prompt or "")[:600]),samples=neutralize_markup("\n".join(f"- {sample}" for sample in samples)),),- config=types.GenerateContentConfig(response_mime_type="application/json", response_schema=_ThemeName),+ config=types.GenerateContentConfig(+ response_mime_type="application/json",+ response_schema=_ThemeName,+ http_options=types.HttpOptions(timeout=_NAMING_REQUEST_TIMEOUT_MS),+ ),posthog_distinct_id=replay_vision_distinct_id(team_id),posthog_trace_id=str(uuid.uuid4()),posthog_properties={"$ai_span_name": "name_search_theme", "team_id": team_id},diff --git a/products/replay_vision/frontend/generated/api.schemas.ts b/products/replay_vision/frontend/generated/api.schemas.tsindex ebfa6ba10f82..6c615569d148 100644--- a/products/replay_vision/frontend/generated/api.schemas.ts+++ b/products/replay_vision/frontend/generated/api.schemas.ts@@ -871,7 +871,7 @@ export const FeedDismissalReasonEnumApi = {} as const/**- * Why the calling user marked an observation not useful in What to watch.+ * Why the calling user marked an observation not useful in the Highlights feed.*/export interface ObservationFeedDismissalApi {/** Optional reason the observation was not useful. Omit or send null when the user gave none.@@ -2912,15 +2912,6 @@ export interface ScannerStatsResponseApi {}/**- * * `signal_emitted` - Signal Emitted- * * `unusual_verdict` - Unusual Verdict- * * `notable` - Notable- * * `verdict_yes` - Verdict Yes- * * `outlier_score` - Outlier Score- * * `rare_tag` - Rare Tag- * * `novel_summary` - Novel Summary- * * `friction` - Friction- * * `jev_watchable` - Jev Watchable* * `embedding_pick` - Embedding Pick* * `unviewed_recent` - Unviewed Recent* * `recent` - Recent@@ -2928,59 +2919,11 @@ export interface ScannerStatsResponseApi {export type WatchFeedReasonEnumApi = (typeof WatchFeedReasonEnumApi)[keyof typeof WatchFeedReasonEnumApi]export const WatchFeedReasonEnumApi = {- SignalEmitted: 'signal_emitted',- UnusualVerdict: 'unusual_verdict',- Notable: 'notable',- VerdictYes: 'verdict_yes',- OutlierScore: 'outlier_score',- RareTag: 'rare_tag',- NovelSummary: 'novel_summary',- Friction: 'friction',- JevWatchable: 'jev_watchable',EmbeddingPick: 'embedding_pick',UnviewedRecent: 'unviewed_recent',Recent: 'recent',} as const-/**- * One signal an observation raised, named rather than counted.- */-export interface WatchFeedSignalApi {- /** Issue type: `bug`, `crash`, `design_flaw`, or `ux_friction`. */- problem_type: string- /** The finding in a few words, written by the scan. The full description lives on the signal itself. */- headline: string-}--/**- * * `visible_error` - Error on screen- * * `silent_failure` - Action silently failed- * * `unresponsive` - Clicks went nowhere- * * `slow_or_stuck` - Slow or stuck- * * `blocked` - Blocked- * * `cant_find` - Couldn't find it- * * `confused` - Confused- * * `workaround` - Took a workaround- * * `abandoned` - Gave up- * * `churn_signal` - Churn signal- * * `success` - Worked well- */-export type JevWatchReasonEnumApi = (typeof JevWatchReasonEnumApi)[keyof typeof JevWatchReasonEnumApi]--export const JevWatchReasonEnumApi = {- VisibleError: 'visible_error',- SilentFailure: 'silent_failure',- Unresponsive: 'unresponsive',- SlowOrStuck: 'slow_or_stuck',- Blocked: 'blocked',- CantFind: 'cant_find',- Confused: 'confused',- Workaround: 'workaround',- Abandoned: 'abandoned',- ChurnSignal: 'churn_signal',- Success: 'success',-} as const-/*** * `failure` - Failure* * `struggle` - Struggle@@ -2998,89 +2941,12 @@ export const WatchFeedCategoryEnumApi = {* Machine-readable reason an observation made the feed; the frontend renders the copy.*/export interface WatchFeedReasonApi {- /** Highest-priority rule the observation satisfied: `signal_emitted` (it pushed a signal), `unusual_verdict` (a monitor answer that is the minority for that scanner this window), `verdict_yes` (a monitor hit, when the window is too thin to know which answer is unusual), `outlier_score` (far from the scanner's window average), `rare_tag` (a tag uncommon for the scanner this window), `novel_summary` (a summary that reads unlike the scanner's other sessions this window), `notable` (the scan itself judged the session worth watching), `friction` (the scan describes errors, retries, or dead ends), `jev_watchable` (the decision model judged the session worth watching; teams on the Jev ranker experiment only), `embedding_pick` (the session reads like a failure, struggle, or revenue moment and ranks in its scanner's top fifth; teams on the embedding ranker only), `unviewed_recent` (new to you), `recent` (nothing special, newest available).+ /** Why the observation is in the feed: `embedding_pick` (the session reads like a failure, struggle, or revenue moment and ranks in its scanner's top fifth), `unviewed_recent` (new to you), `recent` (nothing special, newest available).*- * * `signal_emitted` - Signal Emitted- * * `unusual_verdict` - Unusual Verdict- * * `notable` - Notable- * * `verdict_yes` - Verdict Yes- * * `outlier_score` - Outlier Score- * * `rare_tag` - Rare Tag- * * `novel_summary` - Novel Summary- * * `friction` - Friction- * * `jev_watchable` - Jev Watchable* * `embedding_pick` - Embedding Pick* * `unviewed_recent` - Unviewed Recent* * `recent` - Recent */kind: WatchFeedReasonEnumApi- /**- * Signals this observation emitted, for `signal_emitted`.- * @nullable- */- signals_count?: number | null- /** Issue type of each emitted signal (`bug`, `crash`, `design_flaw`, `ux_friction`), one entry per signal in the order raised, for `signal_emitted`. Absent on signals scanned before this shipped. */- problem_types?: string[]- /** Each emitted signal in the order raised, for `signal_emitted`. Carries what the card needs to name the findings instead of counting them. Absent on sessions scanned before this shipped, which carry `problem_types` alone. */- signals?: WatchFeedSignalApi[]- /**- * The monitor's answer, for `unusual_verdict`.- * @nullable- */- verdict?: string | null- /**- * Share (0-1) of the scanner's window observations with this answer, for `unusual_verdict`.- * @nullable- */- verdict_share?: number | null- /**- * The scan's own 0-1 judgment of how much a team would benefit from watching, for `notable`.- * @nullable- */- notability?: number | null- /**- * The decision model's 0-1 judgment that the session is worth watching, for `jev_watchable`.- * @nullable- */- jev_probability?: number | null- /**- * The scan's own sentence naming why the session is worth watching. Present on the `notable` and `jev_watchable` reason kinds when the scan itself found the session notable, and preferred over copy derived from the reason kind. Absent on observations scanned before notability shipped.- * @nullable- */- notability_reason?: string | null- /** Why the decision model rated the session worth watching, picked from a fixed list, for `jev_watchable`. Absent when no reason on the list fits, or on sessions judged before reasons shipped.- *- * * `visible_error` - Error on screen- * * `silent_failure` - Action silently failed- * * `unresponsive` - Clicks went nowhere- * * `slow_or_stuck` - Slow or stuck- * * `blocked` - Blocked- * * `cant_find` - Couldn't find it- * * `confused` - Confused- * * `workaround` - Took a workaround- * * `abandoned` - Gave up- * * `churn_signal` - Churn signal- * * `success` - Worked well */- watch_reason?: JevWatchReasonEnumApi- /**- * The observation's score, for `outlier_score`.- * @nullable- */- score?: number | null- /**- * The scanner's mean score in the window, for `outlier_score`.- * @nullable- */- window_mean?: number | null- /**- * The rare tag that ranked the observation, for `rare_tag`.- * @nullable- */- tag?: string | null- /**- * Share (0-1) of the scanner's window observations carrying `tag`, for `rare_tag`.- * @nullable- */- tag_share?: number | null/** What the session reads like, for `embedding_pick`.** * `failure` - Failure@@ -3093,7 +2959,7 @@ export interface WatchFeedReasonApi {*/similar_count?: number | null/**- * Distinct people across the session's duplicate group in the window, for `embedding_pick`. A finding that reached more people ranks higher.+ * Distinct people across the session's feed group in the window, for `embedding_pick`. A finding that reached more people ranks higher.* @nullable*/affected_people?: number | null@@ -3109,19 +2975,6 @@ export interface WatchFeedItemApi {reason: WatchFeedReasonApi}-/**- * * `weighted-score` - Weighted Score- * * `jev` - Jev- * * `embedding` - Embedding- */-export type WatchFeedRankerEnumApi = (typeof WatchFeedRankerEnumApi)[keyof typeof WatchFeedRankerEnumApi]--export const WatchFeedRankerEnumApi = {- WeightedScore: 'weighted-score',- Jev: 'jev',- Embedding: 'embedding',-} as const-/*** Response of GET /vision/scanners/watch_feed/.*/@@ -3132,17 +2985,6 @@ export interface WatchFeedResponseApi {has_more: boolean/** The time this feed ranks as of. Send it back with later pages. */as_of: string- /** Which ranker ordered this feed: `jev` ranks on the decision model's cached judgments, `embedding` on the rank the watch rank consumer wrote from the observation's embedding, `weighted-score` on the deterministic blend. The arm is decided server-side per team, so clients read it from here rather than evaluating the flag themselves.- *- * * `weighted-score` - Weighted Score- * * `jev` - Jev- * * `embedding` - Embedding */- ranker: WatchFeedRankerEnumApi- /**- * The team's variant of the `vision-watch-feed-ranker` experiment flag (`control`, `jev-shadow`, `jev`, `embedding`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from control. Clients report it on the feed-viewed event as `$feature/vision-watch-feed-ranker`, which is the exposure the experiment counts.- * @nullable- */- ranker_variant: string | null}/**@@ -3152,7 +2994,7 @@ export interface WatchFeedImpressionsApi {/** The client's id for the feed load that showed these cards. One load counts each card once. */feed_load_id: string/**- * Observations the calling user saw in What to watch, at most 100. Ids the user cannot read are ignored.+ * Observations the calling user saw in the Highlights feed, at most 100. Ids the user cannot read are ignored.* @minItems 1* @maxItems 100*/@@ -3791,11 +3633,6 @@ export type VisionScannersWatchFeedRetrieveParams = {* @minLength 1*/scanner_type?: VisionScannersWatchFeedRetrieveScannerType- /**- * Case-insensitive text to match against the scan's own words (title, summary, reasoning, and the notability sentence) and the scanner's name. Applied before ranking, so it searches the whole window rather than the items that would have surfaced without it.- * @minLength 1- */- search?: string/*** Comma-separated scanner tags to restrict the feed to. A team with many scanners uses these to follow one area without naming every scanner in it.* @minLength 1diff --git a/products/replay_vision/frontend/generated/api.ts b/products/replay_vision/frontend/generated/api.tsindex 96348470b61d..9b1dbd05a58f 100644--- a/products/replay_vision/frontend/generated/api.ts+++ b/products/replay_vision/frontend/generated/api.ts@@ -385,7 +385,7 @@ export const getVisionObservationsFeedDismissalCreateUrl = (projectId: string, i}/**- * Mark this observation not useful for the calling user, with an optional reason. What to watch stops showing it to that user. Sending it again replaces the reason.+ * Mark this observation not useful for the calling user, with an optional reason. The Highlights feed stops showing it to that user. Sending it again replaces the reason.*/export const visionObservationsFeedDismissalCreate = async (projectId: string,@@ -406,7 +406,7 @@ export const getVisionObservationsFeedDismissalDestroyUrl = (projectId: string,}/**- * Undo the calling user's not-useful mark, so What to watch can show the observation again.+ * Undo the calling user's not-useful mark, so the Highlights feed can show the observation again.*/export const visionObservationsFeedDismissalDestroy = async (projectId: string,@@ -1226,7 +1226,7 @@ export const getVisionScannersObservationsFeedDismissalCreateUrl = (}/**- * Mark this observation not useful for the calling user, with an optional reason. What to watch stops showing it to that user. Sending it again replaces the reason.+ * Mark this observation not useful for the calling user, with an optional reason. The Highlights feed stops showing it to that user. Sending it again replaces the reason.*/export const visionScannersObservationsFeedDismissalCreate = async (projectId: string,@@ -1252,7 +1252,7 @@ export const getVisionScannersObservationsFeedDismissalDestroyUrl = (}/**- * Undo the calling user's not-useful mark, so What to watch can show the observation again.+ * Undo the calling user's not-useful mark, so the Highlights feed can show the observation again.*/export const visionScannersObservationsFeedDismissalDestroy = async (projectId: string,@@ -1660,7 +1660,7 @@ export const getVisionScannersWatchFeedRetrieveUrl = (}/**- * Succeeded observations in the window worth watching, ranked — feeds the What to watch tab.+ * Succeeded observations in the window worth watching, ranked for the Highlights tab.*/export const visionScannersWatchFeedRetrieve = async (projectId: string,@@ -1678,7 +1678,7 @@ export const getVisionScannersWatchFeedImpressionsCreateUrl = (projectId: string}/**- * Record that the calling user saw these cards in What to watch, so the feed ranks them lower next time.+ * Record that the calling user saw these cards in the Highlights feed, so the feed ranks them lower next time.*/export const visionScannersWatchFeedImpressionsCreate = async (projectId: string,diff --git a/products/replay_vision/frontend/generated/api.zod.ts b/products/replay_vision/frontend/generated/api.zod.tsindex 8d4dba3c547c..5bd4a185378a 100644--- a/products/replay_vision/frontend/generated/api.zod.ts+++ b/products/replay_vision/frontend/generated/api.zod.ts@@ -446,7 +446,7 @@ export const VisionAlertsDestinationsDeleteCreateBody = /* @__PURE__ */ zod.obje})/**- * Mark this observation not useful for the calling user, with an optional reason. What to watch stops showing it to that user. Sending it again replaces the reason.+ * Mark this observation not useful for the calling user, with an optional reason. The Highlights feed stops showing it to that user. Sending it again replaces the reason.*/export const VisionObservationsFeedDismissalCreateBody = /* @__PURE__ */ zod.object({@@ -464,7 +464,7 @@ export const VisionObservationsFeedDismissalCreateBody = /* @__PURE__ */ zod'Optional reason the observation was not useful. Omit or send null when the user gave none.\n\n\* `not_a_real_problem` - Not a real problem\n\* `already_known` - Already known\n\* `not_my_area` - Not my area\n\* `other` - Other'),})- .describe('Why the calling user marked an observation not useful in What to watch.')+ .describe('Why the calling user marked an observation not useful in the Highlights feed.')/*** Set or update the observation's shared label: whether the scanner scored the session correctly, plus optional feedback on what it got wrong. One label per observation, shared across the team; these labels feed prompt improvement. Requires editor access to the scanner.@@ -1037,7 +1037,7 @@ export const VisionScannersBackfillsEstimateCreateBody = /* @__PURE__ */ zod.obj})/**- * Mark this observation not useful for the calling user, with an optional reason. What to watch stops showing it to that user. Sending it again replaces the reason.+ * Mark this observation not useful for the calling user, with an optional reason. The Highlights feed stops showing it to that user. Sending it again replaces the reason.*/export const VisionScannersObservationsFeedDismissalCreateBody = /* @__PURE__ */ zod.object({@@ -1055,7 +1055,7 @@ export const VisionScannersObservationsFeedDismissalCreateBody = /* @__PURE__ */'Optional reason the observation was not useful. Omit or send null when the user gave none.\n\n\* `not_a_real_problem` - Not a real problem\n\* `already_known` - Already known\n\* `not_my_area` - Not my area\n\* `other` - Other'),})- .describe('Why the calling user marked an observation not useful in What to watch.')+ .describe('Why the calling user marked an observation not useful in the Highlights feed.')/*** Set or update the observation's shared label: whether the scanner scored the session correctly, plus optional feedback on what it got wrong. One label per observation, shared across the team; these labels feed prompt improvement. Requires editor access to the scanner.@@ -1542,7 +1542,7 @@ export const VisionScannersInlineScanCreateBody = /* @__PURE__ */ zod)/**- * Record that the calling user saw these cards in What to watch, so the feed ranks them lower next time.+ * Record that the calling user saw these cards in the Highlights feed, so the feed ranks them lower next time.*/export const visionScannersWatchFeedImpressionsCreateBodyObservationIdsMax = 100@@ -1556,7 +1556,7 @@ export const VisionScannersWatchFeedImpressionsCreateBody = /* @__PURE__ */ zod.min(1).max(visionScannersWatchFeedImpressionsCreateBodyObservationIdsMax).describe(- 'Observations the calling user saw in What to watch, at most 100. Ids the user cannot read are ignored.'+ 'Observations the calling user saw in the Highlights feed, at most 100. Ids the user cannot read are ignored.'),}).describe('Body of POST \/vision\/scanners\/watch_feed_impressions\/.')diff --git a/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.stories.tsx b/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.stories.tsxindex 3abde05469ab..03cb2b612ed8 100644--- a/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.stories.tsx+++ b/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.stories.tsx@@ -916,7 +916,7 @@ const meta: Meta = {'/api/projects/:team_id/vision/scanners/': scanners,'/api/projects/:team_id/vision/scanners/stats/': scannerStats,'/api/projects/:team_id/vision/scanners/creators/': { creators: [alice, bob] },- // One card per reason kind. Only the first carries a key moment, so the rest show no time on the tile.+ // Two picks, then each filler kind. Only the first carries a key moment, so the rest show no time on the tile.'/api/projects/:team_id/vision/scanners/watch_feed/': {results: [{@@ -950,7 +950,7 @@ const meta: Meta = {},viewed: false,}),- reason: { kind: 'signal_emitted', signals_count: 2 },+ reason: { kind: 'embedding_pick', watch_category: 'failure', similar_count: 0 },},{observation: observation({@@ -982,7 +982,7 @@ const meta: Meta = {},viewed: false,}),- reason: { kind: 'outlier_score', score: 9.5, window_mean: 5.1 },+ reason: { kind: 'embedding_pick', watch_category: 'revenue', similar_count: 0 },},{observation: observation({@@ -1131,107 +1131,7 @@ export const UsageTab: StoryObj = {parameters: { pageUrl: `${urls.replayVision()}?tab=usage` },}-// The home page lands on the What to watch feed. The viewer created one scanner, so the feed offers the scope tabs.-export const HomeWatchFeed: StoryObj = {- decorators: [- mswDecorator({- get: {- '/api/projects/:team_id/vision/scanners/': {- ...scanners,- results: scanners.results.map((item, index) =>- index === 0- ? { ...item, created_by: { ...MOCK_DEFAULT_BASIC_USER, id: MOCK_DEFAULT_USER.id } }- : item- ),- },- },- }),- ],-}--// The jev ranker arm: the same rows as the default arm, plus the reason Jev picked each finding. The-// first row leads with the scan's notability sentence, the second falls back to the derived headline,-// and the filler row reads muted with no finding claim or reason.-export const HomeWatchFeedJevArm: StoryObj = {- decorators: [- mswDecorator({- get: {- '/api/projects/:team_id/vision/scanners/watch_feed/': {- ranker: 'jev',- results: [- {- observation: observation({- id: '00000000-0000-0000-0000-0000000000e1',- prompt_question: 'Did the user hesitate at checkout?',- scanner_id: scanners.results[0].id,- scanner_snapshot: {- name: 'Confused checkout',- scanner_type: 'monitor',- scanner_version: 1,- model: 'gemini-3.8-flash',- provider: 'google',- emits_signals: true,- scanner_config: { prompt: 'Did the user hesitate at checkout?' },- },- scanner_result: {- model_output: {- scanner_type: 'monitor',- verdict: 'yes',- confidence: 0.92,- reasoning: 'Retried the payment form twice before completing.',- key_moment_ms: 154000,- },- signals_count: 0,- },- viewed: false,- }),- reason: {- kind: 'jev_watchable',- jev_probability: 0.91,- notability_reason:- 'The card form rejected a valid card three times before the user abandoned the checkout.',- watch_reason: 'visible_error',- },- },- {- observation: observation({- id: '00000000-0000-0000-0000-0000000000e2',- prompt_question: 'How strong is the buying intent in this session?',- scanner_id: scanners.results[3].id,- scanner_snapshot: {- name: 'Intent score',- scanner_type: 'scorer',- scanner_version: 1,- model: 'gemini-3.8-flash',- provider: 'google',- emits_signals: false,- scanner_config: { prompt: 'Score this session.', scale: { min: 0, max: 10 } },- },- scanner_result: {- model_output: {- scanner_type: 'scorer',- score: 9.5,- confidence: 0.88,- reasoning: 'Compared plans, opened billing, invited a teammate.',- },- signals_count: 0,- },- viewed: true,- }),- reason: { kind: 'jev_watchable', jev_probability: 0.48, watch_reason: 'success' },- },- {- observation: observation({ id: '00000000-0000-0000-0000-0000000000e3' }),- reason: { kind: 'unviewed_recent' },- },- ],- },- },- }),- ],-}--// The other sessions in the first embedding card's duplicate group, as the scanner's observation list returns them.+// The other sessions in the first embedding card's feed group, as the scanner's observation list returns them.const checkoutGroupMembers = [{email: '[email protected]',@@ -1283,11 +1183,20 @@ const checkoutGroupMembers = [}))-// The embedding ranker arm: each pick names its category, and the first stands for seven similar sessions.-export const HomeWatchFeedEmbeddingArm: StoryObj = {+// The home page lands on the Highlights feed. Each pick names its category, and the first stands for seven similar+// sessions. The viewer created one scanner, so the feed offers the scope tabs.+export const HomeWatchFeed: StoryObj = {decorators: [mswDecorator({get: {+ '/api/projects/:team_id/vision/scanners/': {+ ...scanners,+ results: scanners.results.map((item, index) =>+ index === 0+ ? { ...item, created_by: { ...MOCK_DEFAULT_BASIC_USER, id: MOCK_DEFAULT_USER.id } }+ : item+ ),+ },'/api/projects/:team_id/vision/scanners/:id/observations/': {// The group holds seven other sessions, so the preview of five ends with a link to the rest.count: 7,@@ -1296,7 +1205,6 @@ export const HomeWatchFeedEmbeddingArm: StoryObj = {results: checkoutGroupMembers,},'/api/projects/:team_id/vision/scanners/watch_feed/': {- ranker: 'embedding',has_more: false,results: [{@@ -1355,7 +1263,6 @@ export const HomeWatchFeedEmbeddingArm: StoryObj = {},}),],- parameters: { featureFlags: { [FEATURE_FLAGS.VISION_WATCH_FEED_RANKER]: 'embedding' } },}// A quiet window: nothing scored on any source, so the feed pads to three newest clips and says sodiff --git a/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.tsx b/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.tsxindex ee1e21718af5..289255fa9910 100644--- a/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.tsx+++ b/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.tsx@@ -324,11 +324,7 @@ export function ReplayScannersScene(): JSX.Element {tabs={[{key: 'watch',- // The new name ships with the embedding ranker, so it stays off the arms it does not describe.- label:- featureFlags[FEATURE_FLAGS.VISION_WATCH_FEED_RANKER] === 'embedding'- ? 'Highlights'- : 'What to watch',+ label: 'Highlights',content: <></>,},{ key: 'scanners', label: 'Scanners', content: <></> },diff --git a/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.test.ts b/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.test.tsindex 8689fedee27c..e15e98f45d9f 100644--- a/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.test.ts+++ b/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.test.ts@@ -4,7 +4,6 @@ import {watchFeedRowTitle,observationKeyMomentMs,watchCardHeadline,- watchFeedRowWhy,watchReasonCopy,watchStartSeconds,} from './WatchFeedCard'@@ -12,119 +11,6 @@ import {describe('WatchFeedCard helpers', () => {describe('watchReasonCopy', () => {it.each<{ name: string; reason: WatchFeedReasonApi; expected: string }>([- {- name: 'counts signals on a session scanned before headlines shipped',- reason: { kind: 'signal_emitted', signals_count: 3 } as WatchFeedReasonApi,- expected: 'The scanner raised 3 signals from this session.',- },- {- name: 'names a single signal problem type',- reason: { kind: 'signal_emitted', signals_count: 1, problem_types: ['bug'] } as WatchFeedReasonApi,- expected: 'The scanner raised a bug signal from this session.',- },- {- name: 'names and counts one repeated problem type',- reason: {- kind: 'signal_emitted',- signals_count: 3,- problem_types: ['bug', 'bug', 'bug'],- } as WatchFeedReasonApi,- expected: 'The scanner raised 3 bug signals from this session.',- },- {- name: 'breaks mixed problem types down by count',- reason: {- kind: 'signal_emitted',- signals_count: 2,- problem_types: ['bug', 'ux_friction'],- } as WatchFeedReasonApi,- expected: 'The scanner raised 2 signals from this session: 1 bug signal, 1 UX friction signal.',- },- {- name: 'pluralizes and orders each type in a mixed breakdown',- reason: {- kind: 'signal_emitted',- signals_count: 3,- problem_types: ['bug', 'bug', 'ux_friction'],- } as WatchFeedReasonApi,- expected: 'The scanner raised 3 signals from this session: 2 bug signals, 1 UX friction signal.',- },- {- name: 'names the finding behind a single signal',- reason: {- kind: 'signal_emitted',- signals_count: 1,- problem_types: ['bug'],- signals: [{ problem_type: 'bug', headline: 'Checkout button does nothing' }],- } as WatchFeedReasonApi,- expected: 'The scanner raised a bug signal from this session: Checkout button does nothing.',- },- {- name: 'lists the findings when one type raised several',- reason: {- kind: 'signal_emitted',- signals_count: 2,- problem_types: ['bug', 'bug'],- signals: [- { problem_type: 'bug', headline: 'Checkout button does nothing' },- { problem_type: 'bug', headline: 'Card form rejects a valid card' },- ],- } as WatchFeedReasonApi,- expected:- 'The scanner raised 2 bug signals from this session: Checkout button does nothing, and Card form rejects a valid card.',- },- {- name: 'groups the findings under each type in a mixed breakdown',- reason: {- kind: 'signal_emitted',- signals_count: 3,- problem_types: ['bug', 'bug', 'ux_friction'],- signals: [- { problem_type: 'bug', headline: 'Checkout button does nothing' },- { problem_type: 'bug', headline: 'Card form rejects a valid card' },- { problem_type: 'ux_friction', headline: 'Search results load twice' },- ],- } as WatchFeedReasonApi,- expected:- 'The scanner raised 3 signals from this session: 2 bug (Checkout button does nothing, Card form rejects a valid card), and 1 UX friction (Search results load twice).',- },- {- name: 'counts the findings it has no room to name',- reason: {- kind: 'signal_emitted',- signals_count: 5,- problem_types: ['bug', 'bug', 'bug', 'bug', 'bug'],- signals: [- { problem_type: 'bug', headline: 'Checkout button does nothing' },- { problem_type: 'bug', headline: 'Card form rejects a valid card' },- { problem_type: 'bug', headline: 'Order total shows zero' },- { problem_type: 'bug', headline: 'Address lookup returns nothing' },- { problem_type: 'bug', headline: 'Receipt page is blank' },- ],- } as WatchFeedReasonApi,- expected:- 'The scanner raised 5 bug signals from this session: Checkout button does nothing, Card form rejects a valid card, Order total shows zero, and 2 more.',- },- {- name: 'rounds the score and window average',- reason: { kind: 'outlier_score', score: 9.53846, window_mean: 5.1428 } as WatchFeedReasonApi,- expected: "Scored 9.54, far from this scanner's recent average of 5.14.",- },- {- name: 'falls back when an outlier_score arrives without its numbers',- reason: { kind: 'outlier_score' } as WatchFeedReasonApi,- expected: "Scored far from this scanner's recent average.",- },- {- name: 'falls back when an unusual_verdict arrives without a verdict',- reason: { kind: 'unusual_verdict' } as WatchFeedReasonApi,- expected: 'The scanner gave a rare answer for this window.',- },- {- name: 'falls back when a rare_tag arrives without a tag',- reason: { kind: 'rare_tag' } as WatchFeedReasonApi,- expected: 'Tagged something uncommon for this scanner lately.',- },{name: 'names the category of an embedding pick with no similar sessions',reason: { kind: 'embedding_pick', watch_category: 'failure', similar_count: 0 } as WatchFeedReasonApi,@@ -155,37 +41,6 @@ describe('WatchFeedCard helpers', () => {})})- describe('watchFeedRowWhy', () => {- it.each<{ name: string; reason: WatchFeedReasonApi; expected: string }>([- {- name: "leads with Jev's pick when it gave one",- reason: {- kind: 'jev_watchable',- jev_probability: 0.9,- watch_reason: 'churn_signal',- } as WatchFeedReasonApi,- expected: 'The user showed signs of churn or downgrading in this session.',- },- {- name: "explains a weighted-score row from its reason kind, never repeating the title's sentence",- reason: {- kind: 'outlier_score',- score: 9.5,- window_mean: 5,- notability_reason: 'The card form rejected a valid card three times.',- } as WatchFeedReasonApi,- expected: "Scored 9.5, far from this scanner's recent average of 5.",- },- {- name: 'falls back to the kind copy for a reason this build does not know',- reason: { kind: 'jev_watchable', watch_reason: 'some_future_reason' } as unknown as WatchFeedReasonApi,- expected: 'The decision model judged this session worth watching.',- },- ])('$name', ({ reason, expected }) => {- expect(watchFeedRowWhy(reason)).toBe(expected)- })- })-const observation = (scannerType: string | undefined, output: Record<string, unknown>): ReplayObservationApi =>({scanner_snapshot: scannerType ? { scanner_type: scannerType } : undefined,@@ -215,22 +70,9 @@ describe('WatchFeedCard helpers', () => {})describe('watchFeedRowSentence', () => {- it("prefers the scan's notability sentence over the derived headline", () => {+ it('derives the headline from the scan prose', () => {const sentence = watchFeedRowSentence(- observation('monitor', { reasoning: 'Retried the form twice. The submit then failed.' }),- {- kind: 'jev_watchable',- jev_probability: 0.9,- notability_reason: 'The card form rejected a valid card three times.',- } as WatchFeedReasonApi- )- expect(sentence).toBe('The card form rejected a valid card three times.')- })-- it('derives the headline when the reason carries no notability sentence', () => {- const sentence = watchFeedRowSentence(- observation('monitor', { reasoning: 'Retried the form twice. The submit then failed.' }),- { kind: 'jev_watchable', jev_probability: 0.9 } as WatchFeedReasonApi+ observation('monitor', { reasoning: 'Retried the form twice. The submit then failed.' }))expect(sentence).toBe('Retried the form twice.')})@@ -240,39 +82,32 @@ describe('WatchFeedCard helpers', () => {scanner_snapshot: { name: 'Confused checkout', scanner_type: 'monitor' },scanner_result: { model_output: {} },} as unknown as ReplayObservationApi- expect(watchFeedRowSentence(bare, { kind: 'unviewed_recent' } as WatchFeedReasonApi)).toBe(- 'Confused checkout'- )+ expect(watchFeedRowSentence(bare)).toBe('Confused checkout')})})describe('watchFeedRowTitle', () => {- const notable = {- kind: 'jev_watchable',- jev_probability: 0.9,- notability_reason: 'The card form rejected a valid card three times.',- } as WatchFeedReasonApiit.each<{ name: string; scannerType: string; output: Record<string, unknown>; expected: string }>([{- name: "leads with a summarizer's authored title over the scan's sentence",+ name: "leads with a summarizer's authored title",scannerType: 'summarizer',output: { title: 'Checkout card rejected', summary: 'Tried the card three times. Left.' },expected: 'Checkout card rejected',},{- name: 'falls back to the card sentence for an untitled summarizer',+ name: 'derives the headline for an untitled summarizer',scannerType: 'summarizer',output: { title: '', summary: 'Tried the card three times. Left.' },- expected: 'The card form rejected a valid card three times.',+ expected: 'Tried the card three times.',},{- name: 'leads other scan types with the card sentence',+ name: 'leads other scan types with the derived headline',scannerType: 'monitor',output: { reasoning: 'Retried the form twice. The submit then failed.' },- expected: 'The card form rejected a valid card three times.',+ expected: 'Retried the form twice.',},])('$name', ({ scannerType, output, expected }) => {- expect(watchFeedRowTitle(observation(scannerType, output), notable)).toBe(expected)+ expect(watchFeedRowTitle(observation(scannerType, output))).toBe(expected)})})diff --git a/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.tsx b/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.tsxindex 0f0bee1f53d8..70ca14b595ec 100644--- a/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.tsx+++ b/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.tsx@@ -15,7 +15,6 @@ import { ObservationThumbnail } from '../../components/ObservationThumbnail'import { scannerTypeIcon } from '../../components/ScannerTypeBadge'import { UnviewedObservationTag } from '../../components/UnviewedObservationTag'import type {- JevWatchReasonEnumApi,ReplayObservationApi,WatchFeedCategoryEnumApi,WatchFeedItemApi,@@ -34,114 +33,15 @@ import { WatchFeedGroupList, WatchFeedGroupToggle } from './WatchFeedGroup'* feed, so a feed made up entirely of them means the window turned up no findings at all. */export const FILLER_REASON_KINDS = new Set(['unviewed_recent', 'recent'])-const roundScore = (value: number): number => Math.round(value * 100) / 100--const PROBLEM_TYPE_LABELS: Record<string, string> = {- bug: 'bug',- crash: 'crash',- design_flaw: 'design flaw',- ux_friction: 'UX friction',-}--const problemTypeLabel = (problemType: string): string =>- PROBLEM_TYPE_LABELS[problemType] ?? problemType.replace(/_/g, ' ')--// A card lists the findings on one line, so it names the first few and counts the rest.-const MAX_SHOWN_HEADLINES = 3--const joinWithAnd = (parts: string[]): string =>- parts.length > 1 ? `${parts.slice(0, -1).join(', ')}, and ${parts[parts.length - 1]}` : parts[0]-const WATCH_CATEGORY_COPY: Record<WatchFeedCategoryEnumApi, string> = {failure: 'Something failed in this session.',struggle: 'The user struggled in this session.',revenue: 'This session touches billing or a plan limit.',}-/** What the reason kind says about the session. Never the scan's notability sentence: the row's title- * already leads with it. */+/** What the reason kind says about the session. */export function watchReasonCopy(reason: WatchFeedReasonApi): string {switch (reason.kind) {- case 'signal_emitted': {- const total = reason.signals_count ?? 0- const signals = reason.signals ?? []- const problemTypes =- signals.length > 0 ? signals.map((signal) => signal.problem_type) : (reason.problem_types ?? [])- if (problemTypes.length === 0) {- return total > 1- ? `The scanner raised ${total} signals from this session.`- : 'The scanner raised a signal from this session.'- }- // Count each problem type, keeping the order the scan first raised them.- const order: string[] = []- const countByType = new Map<string, number>()- for (const problemType of problemTypes) {- if (!countByType.has(problemType)) {- order.push(problemType)- }- countByType.set(problemType, (countByType.get(problemType) ?? 0) + 1)- }- // Sessions scanned before headlines shipped carry the types alone, so those cards still count.- if (signals.length === 0) {- if (order.length === 1) {- const label = problemTypeLabel(order[0])- return total > 1- ? `The scanner raised ${total} ${label} signals from this session.`- : `The scanner raised a ${label} signal from this session.`- }- const breakdown = order- .map((problemType) => {- const n = countByType.get(problemType) ?? 0- return `${n} ${problemTypeLabel(problemType)} signal${n === 1 ? '' : 's'}`- })- .join(', ')- return `The scanner raised ${total} signals from this session: ${breakdown}.`- }- const shown = signals.slice(0, MAX_SHOWN_HEADLINES)- // Against the count, not the named list: the backend drops a finding whose headline came back- // blank, so the card can hold fewer names than the session raised signals.- const hidden = Math.max(total, signals.length) - shown.length- if (order.length === 1) {- const label = problemTypeLabel(order[0])- const named = shown.map((signal) => signal.headline)- const listed = joinWithAnd(hidden > 0 ? [...named, `${hidden} more`] : named)- return total > 1- ? `The scanner raised ${total} ${label} signals from this session: ${listed}.`- : `The scanner raised a ${label} signal from this session: ${listed}.`- }- const groups = order.map((problemType) => {- const n = countByType.get(problemType) ?? 0- const named = shown- .filter((signal) => signal.problem_type === problemType)- .map((signal) => signal.headline)- const label = `${n} ${problemTypeLabel(problemType)}`- return named.length > 0 ? `${label} (${named.join(', ')})` : label- })- const listed = joinWithAnd(hidden > 0 ? [...groups, `${hidden} more`] : groups)- return `The scanner raised ${total} signals from this session: ${listed}.`- }- case 'unusual_verdict':- return reason.verdict- ? `The scanner answered ${reason.verdict}, which is rare for it in this window.`- : 'The scanner gave a rare answer for this window.'- case 'verdict_yes':- return 'The scanner answered yes for this session.'- case 'outlier_score':- return reason.score != null && reason.window_mean != null- ? `Scored ${roundScore(reason.score)}, far from this scanner's recent average of ${roundScore(reason.window_mean)}.`- : "Scored far from this scanner's recent average."- case 'rare_tag':- return reason.tag- ? `Tagged "${reason.tag}", which is uncommon for this scanner lately.`- : 'Tagged something uncommon for this scanner lately.'- case 'novel_summary':- return "Reads unlike this scanner's other sessions in this window."- case 'notable':- return 'The scanner judged this session worth watching.'- case 'friction':- return 'The session shows signs of friction, like errors, retries, or dead ends.'- case 'jev_watchable':- return 'The decision model judged this session worth watching.'case 'embedding_pick': {const category = reason.watch_category? WATCH_CATEGORY_COPY[reason.watch_category]@@ -265,15 +165,8 @@ function WatchCardPerson({)}-/**- * The one sentence a row leads with when the scan wrote no short title. The scan's notability- * sentence names the finding, so it outranks prose derived from the result; the derived headline- * covers scans from before notability shipped, and the scanner's name covers scans with no prose at all.- */-export function watchFeedRowSentence(observation: ReplayObservationApi, reason: WatchFeedReasonApi): string {- if (reason.notability_reason) {- return reason.notability_reason- }+/** The one sentence a row leads with when the scan wrote no short title, or the scanner's name for scans with no prose at all. */+export function watchFeedRowSentence(observation: ReplayObservationApi): string {const headline = watchCardHeadline(observation)return headline?.title ?? scannerLabel(observation)}@@ -340,33 +233,11 @@ function WatchFeedScannerLink({)}-/** The sentence that says why a row was picked, for each reason Jev can pick from its fixed list. */-const JEV_WATCH_REASON_COPY: Record<JevWatchReasonEnumApi, string> = {- visible_error: 'The user saw an error message or a failed action in this session.',- silent_failure: "Something the user did didn't take effect, and nothing told them.",- unresponsive: "The user clicked on something that didn't respond.",- slow_or_stuck: 'The page was slow to load or got stuck while the user waited.',- blocked: "The user hit a dead end they couldn't get past.",- cant_find: "The user searched for something and couldn't find it.",- confused: 'The user seemed unsure how something worked and went back and forth.',- workaround: 'The user gave up on the usual path and found another way to finish.',- abandoned: 'The user started a task and left without finishing it.',- churn_signal: 'The user showed signs of churn or downgrading in this session.',- success: 'The user reached their goal smoothly, a good example of the product working.',-}--/** Why a row was picked: Jev's pick when it gave one, otherwise what the reason kind says, so both- * ranker arms explain each row. */-export function watchFeedRowWhy(reason: WatchFeedReasonApi): string {- // A value this build does not know (the backend list grew first) falls back to the kind's copy.- return (reason.watch_reason && JEV_WATCH_REASON_COPY[reason.watch_reason]) || watchReasonCopy(reason)-}-/*** The title of a row. A summarizer's authored title is already short, so it leads. Other scans* lead with watchFeedRowSentence.*/-export function watchFeedRowTitle(observation: ReplayObservationApi, reason: WatchFeedReasonApi): string {+export function watchFeedRowTitle(observation: ReplayObservationApi): string {const result = readResult(observation)const scannerType =(observation.scanner_snapshot?.scanner_type as ScannerType | undefined) ??@@ -374,14 +245,13 @@ export function watchFeedRowTitle(observation: ReplayObservationApi, reason: Watif (scannerType === 'summarizer' && typeof result?.title === 'string' && result.title) {return result.title}- return watchFeedRowSentence(observation, reason)+ return watchFeedRowSentence(observation)}/*** One feed row, laid out as a post: who and where on top, then the title and the reason it was- * picked, then the key-moment poster and a button that starts the player there. Every ranker arm- * renders it, so the experiment compares rankings, not layouts. The whole card opens the- * observation page.+ * picked, then the key-moment poster and a button that starts the player there. The whole card+ * opens the observation page.*/export function WatchFeedRow({ item, position }: WatchFeedRowProps): JSX.Element {const { observation, reason } = item@@ -397,7 +267,7 @@ export function WatchFeedRow({ item, position }: WatchFeedRowProps): JSX.Elementconst person = observation.recording_subject_email || observation.distinct_idconst keyMomentMs = observationKeyMomentMs(observation)const observationUrl = watchFeedObservationUrl(observation)- const title = watchFeedRowTitle(observation, reason)+ const title = watchFeedRowTitle(observation)const filler = FILLER_REASON_KINDS.has(reason.kind)const similarCount = reason.kind === 'embedding_pick' ? (reason.similar_count ?? 0) : 0const groupProps = {@@ -457,7 +327,7 @@ export function WatchFeedRow({ item, position }: WatchFeedRowProps): JSX.Element</Tooltip><span className="line-clamp-3"><span className="sr-only">Why this recording: </span>- {watchFeedRowWhy(reason)}+ {watchReasonCopy(reason)}</span></p></div>diff --git a/products/replay_vision/frontend/replay_scanners/components/WatchFeedGroup.tsx b/products/replay_vision/frontend/replay_scanners/components/WatchFeedGroup.tsxindex bba80f95f440..0dc21dec314d 100644--- a/products/replay_vision/frontend/replay_scanners/components/WatchFeedGroup.tsx+++ b/products/replay_vision/frontend/replay_scanners/components/WatchFeedGroup.tsx@@ -81,7 +81,7 @@ function WatchFeedGroupMember({)}-/** Opens the other sessions in a card's duplicate group under the card, so a reader can see who else hit it. */+/** Opens the other sessions in a card's feed group under the card, so a reader can see who else hit it. */export function WatchFeedGroupToggle({ similarCount, cardContext, ...logicProps }: WatchFeedGroupProps): JSX.Element {const { expanded } = useValues(watchFeedGroupLogic(logicProps))const { toggleExpanded } = useActions(watchFeedGroupLogic(logicProps))diff --git a/products/replay_vision/frontend/replay_scanners/components/WatchPickPreview.tsx b/products/replay_vision/frontend/replay_scanners/components/WatchPickPreview.tsxindex 2f3ec08605f6..ca56a3b5d999 100644--- a/products/replay_vision/frontend/replay_scanners/components/WatchPickPreview.tsx+++ b/products/replay_vision/frontend/replay_scanners/components/WatchPickPreview.tsx@@ -6,7 +6,7 @@ import { ScannerTypeBadge } from '../../components/ScannerTypeBadge'import type { WatchFeedItemApi } from '../../generated/api.schemas'import { citedTextToPlainText } from '../../utils/citations'import { watchPickSummary } from '../watchPickSummary'-import { watchFeedRowWhy } from './WatchFeedCard'+import { watchReasonCopy } from './WatchFeedCard'export function WatchPickPreview({ item, className }: { item: WatchFeedItemApi; className?: string }): JSX.Element {const { observation, reason } = item@@ -22,7 +22,7 @@ export function WatchPickPreview({ item, className }: { item: WatchFeedItemApi;<span>·</span><TZLabel time={observation.created_at} showPopover={false} noStyles className="whitespace-nowrap" /></span>- <span className="text-xs leading-snug">{citedTextToPlainText(watchFeedRowWhy(reason), undefined)}</span>+ <span className="text-xs leading-snug">{citedTextToPlainText(watchReasonCopy(reason), undefined)}</span></div>)}diff --git a/products/replay_vision/frontend/replay_scanners/components/WatchPickRow.tsx b/products/replay_vision/frontend/replay_scanners/components/WatchPickRow.tsxindex 23486fa7bbcb..f0147be68fc5 100644--- a/products/replay_vision/frontend/replay_scanners/components/WatchPickRow.tsx+++ b/products/replay_vision/frontend/replay_scanners/components/WatchPickRow.tsx@@ -12,7 +12,7 @@ import { citedTextToPlainText } from '../../utils/citations'import { CARD_SEEN_DWELL_MS } from '../watchFeedTelemetry'import { watchPicksLogic } from '../watchPicksLogic'import { watchPickSummary } from '../watchPickSummary'-import { watchFeedRowWhy } from './WatchFeedCard'+import { watchReasonCopy } from './WatchFeedCard'import { WatchPickPreview } from './WatchPickPreview'import { WatchPickRankBadge } from './WatchPickRankBadge'@@ -65,7 +65,7 @@ export function WatchPickRow({ item, position, isActive, rank }: WatchPickRowPro/></span><span className="truncate text-xs font-normal text-secondary">- {citedTextToPlainText(watchFeedRowWhy(reason), undefined)}+ {citedTextToPlainText(watchReasonCopy(reason), undefined)}</span></span></span>diff --git a/products/replay_vision/frontend/replay_scanners/scannerScout.ts b/products/replay_vision/frontend/replay_scanners/scannerScout.tsindex bdee47d2dc6e..c700e456dd06 100644--- a/products/replay_vision/frontend/replay_scanners/scannerScout.ts+++ b/products/replay_vision/frontend/replay_scanners/scannerScout.ts@@ -348,7 +348,7 @@ function rootCauseTemplate(scannerId: string, scannerName: string, lens: RootCauheading: 'Replay Vision root cause',role: `You explain why sessions end up in one of this Replay Vision scanner's outcomes. Your bucket is ${lens.bucket}. You find the causes behind it, mainly from what the scanner saw in the recordings.`,window: 'Your window is the last 30 days, whatever ran before, because a cause needs more sessions than one week collects. The previous run tells you which causes you already reported.',- reads: `1. The reasoning, which is your primary evidence. \`execute-sql\` over \`$recording_observed\` for the bucket in the window (\`${lens.bucketFilter}\`): \`properties.session_id\`, \`properties.scanner_output_reasoning\`, and \`properties.scanner_output_key_moment_ms\`, ordered by \`properties.scanner_output_notability\` descending. Read up to about 150. The scanner wrote this reasoning from the recording itself, so it says what was on screen. Group it into causes you name yourself (a modal over the submit button, a spinner that never resolves, an error message below the fold) and count the distinct sessions behind each. Never group on the raw text.+ reads: `1. The reasoning, which is your primary evidence. \`execute-sql\` over \`$recording_observed\` for the bucket in the window (\`${lens.bucketFilter}\`): \`properties.session_id\`, \`properties.scanner_output_reasoning\`, and \`properties.scanner_output_key_moment_ms\`, ordered by \`timestamp\` descending. Read up to about 150. The scanner wrote this reasoning from the recording itself, so it says what was on screen. Group it into causes you name yourself (a modal over the submit button, a spinner that never resolves, an error message below the fold) and count the distinct sessions behind each. Never group on the raw text.2. The contrast. Read a similar sample of reasoning from ${lens.contrast}. A cause that shows up there just as often does not explain the bucket: drop it, or say that it is everywhere.3. The moments. For each cause you keep, \`vision-scanners-observations-list\` (scanner_id \`${scannerId}\`, \`date_from=-30d\`, ${lens.listFilter}) for its clearest sessions, and take the chip timestamps in \`reasoning_segments\` to link the moment it happens.4. Event data, only to corroborate. \`execute-sql\` over \`events\` joined on \`$session_id\` for the bucket sessions that show a cause can confirm or narrow it: their \`$exception\` types, \`$pathname\`, \`$browser\`, \`$device_type\`, app version, or person properties. A pattern that only the events show, with no counterpart in the reasoning, is not yours to report. This scout explains what the recordings show; it does not rebuild a session summary from analytics events.`,diff --git a/products/replay_vision/frontend/replay_scanners/watchFeedGroupLogic.ts b/products/replay_vision/frontend/replay_scanners/watchFeedGroupLogic.tsindex 690f6aaec352..c00748f44e11 100644--- a/products/replay_vision/frontend/replay_scanners/watchFeedGroupLogic.ts+++ b/products/replay_vision/frontend/replay_scanners/watchFeedGroupLogic.ts@@ -7,7 +7,7 @@ import { visionScannersObservationsList } from '../generated/api'import type { ReplayObservationApi } from '../generated/api.schemas'export interface WatchFeedGroupLogicProps {- /** The card's observation. The backend resolves its duplicate group from it. */+ /** The card's observation. The backend resolves its feed group from it. */observationId: stringscannerId: stringdateFrom: string@@ -57,7 +57,7 @@ export type watchFeedGroupLogicType = MakeLogicType</** How many of the group's other sessions the card lists before it links to the full list. */export const WATCH_GROUP_PREVIEW_SIZE = 5-/** The other sessions in a feed card's duplicate group, loaded the first time the card expands. */+/** The other sessions in a feed card's feed group, loaded the first time the card expands. */export const watchFeedGroupLogic = kea<watchFeedGroupLogicType>([path((key) => ['products', 'replay_vision', 'frontend', 'replay_scanners', 'watchFeedGroupLogic', key]),props({} as WatchFeedGroupLogicProps),diff --git a/products/replay_vision/frontend/replay_scanners/watchFeedLogic.test.ts b/products/replay_vision/frontend/replay_scanners/watchFeedLogic.test.tsindex a208c9e2bdff..dc88ccc178cd 100644--- a/products/replay_vision/frontend/replay_scanners/watchFeedLogic.test.ts+++ b/products/replay_vision/frontend/replay_scanners/watchFeedLogic.test.ts@@ -1,6 +1,5 @@import { waitFor } from '@testing-library/react'import { expectLogic } from 'kea-test-utils'-import posthog from 'posthog-js'import { lemonToast } from 'lib/lemon-ui/LemonToast'import { userLogic } from 'scenes/userLogic'@@ -32,7 +31,7 @@ describe('watchFeedLogic', () => {beforeEach(() => {// The scope choice persists, so one test's choice must not leak into the next.localStorage.clear()- feedSpy = jest.fn(() => [200, { results: [item('o1', 'signal_emitted'), item('o2', 'recent')] }])+ feedSpy = jest.fn(() => [200, { results: [item('o1', 'embedding_pick'), item('o2', 'recent')] }])useMocks({get: {'/api/projects/:team/vision/scanners/watch_feed/': feedSpy,@@ -48,21 +47,11 @@ describe('watchFeedLogic', () => {})it('loads on mount and reloads with params when filters change', async () => {- const captureSpy = jest.spyOn(posthog, 'capture').mockImplementation()- const lastFeedViewed = (): Record<string, any> | null | undefined =>- captureSpy.mock.calls.filter(([name]) => name === 'replay_vision_watch_feed_viewed').at(-1)?.[1]logic.mount()await expectLogic(logic).toDispatchActions(['loadFeed', 'loadFeedSuccess']).toFinishAllListeners()expect(logic.values.feedItems).toHaveLength(2)expect(new URL(feedSpy.mock.calls[0][0].request.url).searchParams.get('date_from')).toBe('-7d')- // A response without a ranker (an older API) reads as the default arm, outside the experiment.- expect(logic.values.feedRanker).toBe('weighted-score')- expect(lastFeedViewed()).not.toHaveProperty('$feature/vision-watch-feed-ranker')- feedSpy.mockImplementation(() => [- 200,- { results: [item('o1', 'jev_watchable')], ranker: 'jev', ranker_variant: 'jev' },- ])const owned = [{ id: 'scanner-a', created_by: userLogic.values.user } as unknown as ReplayScannerApi]const lastCreatedBy = (): string | null =>new URL(feedSpy.mock.calls.at(-1)[0].request.url).searchParams.get('created_by')@@ -96,10 +85,6 @@ describe('watchFeedLogic', () => {.toFinishAllListeners()expect(feedSpy.mock.calls).toHaveLength(requestsBeforeSwitch + 1)expect(lastCreatedBy()).toBeNull()- // The response names the ranker that ordered the feed, and the feed-viewed event reports the- // project's variant, which is the exposure the experiment counts.- expect(logic.values.feedRanker).toBe('jev')- expect(lastFeedViewed()).toMatchObject({ ranker: 'jev', '$feature/vision-watch-feed-ranker': 'jev' })await expectLogic(logic, () => {logic.actions.setDateFrom('-30d')diff --git a/products/replay_vision/frontend/replay_scanners/watchFeedLogic.ts b/products/replay_vision/frontend/replay_scanners/watchFeedLogic.tsindex de380de16a2f..d0eb9d5438ce 100644--- a/products/replay_vision/frontend/replay_scanners/watchFeedLogic.ts+++ b/products/replay_vision/frontend/replay_scanners/watchFeedLogic.ts@@ -2,7 +2,6 @@ import { MakeLogicType, actions, afterMount, connect, kea, listeners, path, reduimport { loaders } from 'kea-loaders'import { subscriptions } from 'kea-subscriptions'-import { FEATURE_FLAGS } from 'lib/constants'import posthog from 'lib/posthog-typed'import { getCurrentTeamIdOrNone } from 'lib/utils/getAppContext'import { teamLogic } from 'scenes/teamLogic'@@ -10,9 +9,8 @@ import { userLogic } from 'scenes/userLogic'import type { UserType } from '../../../../frontend/src/types'import { visionScannersWatchFeedRetrieve } from '../generated/api'-import type { ReplayScannerApi, VisionQuotaApi, WatchFeedResponseApi } from '../generated/api.schemas'+import type { ReplayScannerApi, VisionQuotaApi } from '../generated/api.schemas'import type { VisionScannersWatchFeedRetrieveParams, WatchFeedItemApi } from '../generated/api.schemas'-import type { WatchFeedRankerEnumApi } from '../generated/api.schemas'import type { FeedDismissalReasonEnumApi } from '../generated/api.schemas'import { visionQuotaLogic } from '../logics/visionQuotaLogic'import { visionScannersListLogic } from '../logics/visionScannersListLogic'@@ -44,8 +42,6 @@ export interface watchFeedLogicValues {feedItemsLoading: booleanfeedLoad: WatchFeedLoad | nullfeedMoreFailed: boolean- feedRanker: WatchFeedRanker- feedRankerVariant: string | nullfeedReloading: booleanfeedServed: numberhasOwnScanners: boolean | null@@ -132,13 +128,6 @@ export interface watchFeedLogicActions {hasMore: booleanserved: number}- setFeedRanker: (- ranker: WatchFeedRanker,- variant: string | null- ) => {- ranker: WatchFeedRankerEnumApi- variant: string | null- }setOnlyMyScanners: (onlyMyScanners: boolean) => {onlyMyScanners: boolean}@@ -196,10 +185,6 @@ export function feedWindowPhrase(dateFrom: string): string {export const FEED_PAGE_SIZE = 20-/** Which ranker ordered the loaded feed. Server-decided per team (the flag evaluates against the- * team, not the viewer), so the client reads it from the response instead of checking the flag. */-export type WatchFeedRanker = WatchFeedResponseApi['ranker']-function feedParams(values: watchFeedLogicValues): VisionScannersWatchFeedRetrieveParams {const params: VisionScannersWatchFeedRetrieveParams = { date_from: values.dateFrom, limit: FEED_PAGE_SIZE }if (values.scopedToMine && values.user) {@@ -236,7 +221,6 @@ export const watchFeedLogic = kea<watchFeedLogicType>([setDateFrom: (dateFrom: string) => ({ dateFrom }),setOnlyMyScanners: (onlyMyScanners: boolean) => ({ onlyMyScanners }),loadFeed: true,- setFeedRanker: (ranker: WatchFeedRanker, variant: string | null) => ({ ranker, variant }),setFeedLoad: (load: WatchFeedLoad) => ({ load }),extendFeedLoad: (load: WatchFeedLoad) => ({ load }),setFeedPaging: (hasMore: boolean, asOf: string | null, served: number) => ({ hasMore, asOf, served }),@@ -265,8 +249,7 @@ export const watchFeedLogic = kea<watchFeedLogicType>([const response = await visionScannersWatchFeedRetrieve(String(teamId), feedParams(values))// Drop out-of-order responses — the most recent filter change owns the feed.breakpoint()- actions.setFeedRanker(response.ranker ?? 'weighted-score', response.ranker_variant ?? null)- actions.setFeedLoad(newWatchFeedLoad(response.ranker, response.results))+ actions.setFeedLoad(newWatchFeedLoad(response.results))actions.setFeedPaging(response.has_more ?? false, response.as_of ?? null, response.results.length)return response.results},@@ -390,19 +373,6 @@ export const watchFeedLogic = kea<watchFeedLogicType>([loadFeedFailure: () => false,},],- // Set from each response, so the card layout always matches the ranking it shows.- feedRanker: [- 'weighted-score' as WatchFeedRanker,- {- setFeedRanker: (_, { ranker }) => ranker,- },- ],- feedRankerVariant: [- null as string | null,- {- setFeedRanker: (_, { variant }) => variant,- },- ],}),listeners(({ actions, values, cache }) => ({@@ -450,12 +420,6 @@ export const watchFeedLogic = kea<watchFeedLogicType>([clip_count: items.length,scanner_count: new Set(items.map((item) => item.observation.scanner_id)).size,reason_kind_counts: countByReasonKind,- ranker: values.feedRanker,- // The ranker experiment counts this event as its exposure, so it carries the project's variant.- ...(values.feedRankerVariant- ? { [`$feature/${FEATURE_FLAGS.VISION_WATCH_FEED_RANKER}`]: values.feedRankerVariant }- : {}),- signal_share: items.length > 0 ? (countByReasonKind.signal_emitted ?? 0) / items.length : 0,// Which dead end the reader hit, so the empty screens can be counted and ranked// against each other instead of only showing up as a clip count of zero....(items.length === 0 ? { empty_reason: values.emptyReason ?? 'unresolved' } : {}),diff --git a/products/replay_vision/frontend/replay_scanners/watchFeedTelemetry.ts b/products/replay_vision/frontend/replay_scanners/watchFeedTelemetry.tsindex 6a625cf69d76..138aa0651c11 100644--- a/products/replay_vision/frontend/replay_scanners/watchFeedTelemetry.ts+++ b/products/replay_vision/frontend/replay_scanners/watchFeedTelemetry.ts@@ -9,15 +9,13 @@ export const CARD_SEEN_DWELL_MS = 1000/** One feed fetch. Every card event carries its id, so a click, dismissal, or impression joins back to the ranked list it came from. */export interface WatchFeedLoad {feedLoadId: string- ranker: string// Each card's place as served, so a dismissal that shortens the list does not shift later cards' positions.positions: Record<string, number>}-export function newWatchFeedLoad(ranker: string | null | undefined, items: WatchFeedItemApi[]): WatchFeedLoad {+export function newWatchFeedLoad(items: WatchFeedItemApi[]): WatchFeedLoad {return {feedLoadId: uuid(),- ranker: ranker ?? 'weighted-score',positions: Object.fromEntries(items.map((item, index) => [item.observation.id, index])),}}@@ -39,7 +37,6 @@ function servedPosition(load: WatchFeedLoad | null, item: WatchFeedItemApi, posiexport function servedListProperties(load: WatchFeedLoad | null, items: WatchFeedItemApi[]): Record<string, unknown> {return {feed_load_id: load?.feedLoadId ?? null,- ranker: load?.ranker ?? null,observation_ids: items.map((item) => item.observation.id),reason_kinds: items.map((item) => item.reason.kind),}@@ -55,7 +52,6 @@ export function cardProperties(const createdAt = Date.parse(observation.created_at)return {feed_load_id: load?.feedLoadId ?? null,- ranker: load?.ranker ?? null,surface,observation_id: observation.id,scanner_id: observation.scanner_id,@@ -106,7 +102,6 @@ export class WatchFeedImpressions {}posthog.capture('replay_vision_watch_feed_impressions', {feed_load_id: this.load?.feedLoadId ?? null,- ranker: this.load?.ranker ?? null,surface: this.surface,count: this.seen.size,impressions: Array.from(this.seen, ([observation_id, card]) => ({ observation_id, ...card })),diff --git a/products/replay_vision/frontend/replay_scanners/watchPicksLogic.test.ts b/products/replay_vision/frontend/replay_scanners/watchPicksLogic.test.tsindex 7e2310b4f3bb..40171f4d7a75 100644--- a/products/replay_vision/frontend/replay_scanners/watchPicksLogic.test.ts+++ b/products/replay_vision/frontend/replay_scanners/watchPicksLogic.test.ts@@ -24,7 +24,7 @@ describe('watchPicksLogic', () => {viewed: false,scanner_result: keyMomentMs === undefined ? null : { model_output: { key_moment_ms: keyMomentMs } },},- reason: { kind: 'signal_emitted' },+ reason: { kind: 'embedding_pick' },})beforeEach(() => {diff --git a/products/replay_vision/frontend/replay_scanners/watchPicksLogic.ts b/products/replay_vision/frontend/replay_scanners/watchPicksLogic.tsindex 72cf69a35039..22b5800d6894 100644--- a/products/replay_vision/frontend/replay_scanners/watchPicksLogic.ts+++ b/products/replay_vision/frontend/replay_scanners/watchPicksLogic.ts@@ -187,7 +187,7 @@ export const watchPicksLogic: LogicWrapper<watchPicksLogicType> = kea<watchPickslimit: PICKS_LIMIT,})breakpoint()- actions.setPicksLoad(newWatchFeedLoad(response.ranker, response.results))+ actions.setPicksLoad(newWatchFeedLoad(response.results))return response.results},},diff --git a/products/replay_vision/mcp/tools.yaml b/products/replay_vision/mcp/tools.yamlindex 44e162d25c0f..badfd2e27111 100644--- a/products/replay_vision/mcp/tools.yaml+++ b/products/replay_vision/mcp/tools.yaml@@ -1035,8 +1035,8 @@ tools:Rank succeeded replay vision observations in a window by how worth watching their recordings are, acrossevery scanner you can read. Each item pairs the `observation` (with its `session_id`) and the `reason` itmade the feed. `date_from` defaults to `-7d` (ISO 8601 or relative); the window spans at most 90 days.- Narrow with `scanner_ids`, `scanner_type`, scanner `tags`, or a `search` over the scan's own words. `limit`- is at most 50. Use it to answer "which recordings should I watch this week?".+ Narrow with `scanner_ids`, `scanner_type`, or scanner `tags`. `limit` is at most 50. Use it to answer "which+ recordings should I watch this week?".agent_note: |Point the person at one or two recordings with a one-line reason each, rather than listing the whole feed.response:diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.jsonindex abc7256d47f7..7692610d6466 100644--- a/services/mcp/schema/generated-tool-definitions.json+++ b/services/mcp/schema/generated-tool-definitions.json@@ -15946,7 +15946,7 @@}},"vision-scanners-watch-feed": {- "description": "Rank succeeded replay vision observations in a window by how worth watching their recordings are, across every scanner you can read. Each item pairs the `observation` (with its `session_id`) and the `reason` it made the feed. `date_from` defaults to `-7d` (ISO 8601 or relative); the window spans at most 90 days. Narrow with `scanner_ids`, `scanner_type`, scanner `tags`, or a `search` over the scan's own words. `limit` is at most 50. Use it to answer \"which recordings should I watch this week?\".",+ "description": "Rank succeeded replay vision observations in a window by how worth watching their recordings are, across every scanner you can read. Each item pairs the `observation` (with its `session_id`) and the `reason` it made the feed. `date_from` defaults to `-7d` (ISO 8601 or relative); the window spans at most 90 days. Narrow with `scanner_ids`, `scanner_type`, or scanner `tags`. `limit` is at most 50. Use it to answer \"which recordings should I watch this week?\".","category": "Replay vision","feature": "replay_vision","summary": "Get ranked recordings to watch",diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.jsonindex 84be830a0379..04143f590430 100644--- a/services/mcp/schema/tool-definitions-all.json+++ b/services/mcp/schema/tool-definitions-all.json@@ -16722,7 +16722,7 @@}},"vision-scanners-watch-feed": {- "description": "Rank succeeded replay vision observations in a window by how worth watching their recordings are, across every scanner you can read. Each item pairs the `observation` (with its `session_id`) and the `reason` it made the feed. `date_from` defaults to `-7d` (ISO 8601 or relative); the window spans at most 90 days. Narrow with `scanner_ids`, `scanner_type`, scanner `tags`, or a `search` over the scan's own words. `limit` is at most 50. Use it to answer \"which recordings should I watch this week?\".",+ "description": "Rank succeeded replay vision observations in a window by how worth watching their recordings are, across every scanner you can read. Each item pairs the `observation` (with its `session_id`) and the `reason` it made the feed. `date_from` defaults to `-7d` (ISO 8601 or relative); the window spans at most 90 days. Narrow with `scanner_ids`, `scanner_type`, or scanner `tags`. `limit` is at most 50. Use it to answer \"which recordings should I watch this week?\".","category": "Replay vision","feature": "replay_vision","summary": "Get ranked recordings to watch",diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex b38bc19f3b26..11d057716249 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -66061,36 +66061,6 @@ export namespace Schemas {recipe_snapshot: IterationTrailWithRecipeRecipeSnapshot;}- /**- * * `visible_error` - Error on screen- * * `silent_failure` - Action silently failed- * * `unresponsive` - Clicks went nowhere- * * `slow_or_stuck` - Slow or stuck- * * `blocked` - Blocked- * * `cant_find` - Couldn't find it- * * `confused` - Confused- * * `workaround` - Took a workaround- * * `abandoned` - Gave up- * * `churn_signal` - Churn signal- * * `success` - Worked well- */- export type JevWatchReasonEnum = typeof JevWatchReasonEnum[keyof typeof JevWatchReasonEnum];--- export const JevWatchReasonEnum = {- VisibleError: 'visible_error',- SilentFailure: 'silent_failure',- Unresponsive: 'unresponsive',- SlowOrStuck: 'slow_or_stuck',- Blocked: 'blocked',- CantFind: 'cant_find',- Confused: 'confused',- Workaround: 'workaround',- Abandoned: 'abandoned',- ChurnSignal: 'churn_signal',- Success: 'success',- } as const;-export interface JiraIssueSignalExtra {key: string;url: string | null;@@ -74192,7 +74162,7 @@ export namespace Schemas {}/**- * Why the calling user marked an observation not useful in What to watch.+ * Why the calling user marked an observation not useful in the Highlights feed.*/export interface ObservationFeedDismissal {/** Optional reason the observation was not useful. Omit or send null when the user gave none.@@ -122986,7 +122956,7 @@ export namespace Schemas {/** The client's id for the feed load that showed these cards. One load counts each card once. */feed_load_id: string;/**- * Observations the calling user saw in What to watch, at most 100. Ids the user cannot read are ignored.+ * Observations the calling user saw in the Highlights feed, at most 100. Ids the user cannot read are ignored.* @minItems 1* @maxItems 100*/@@ -122994,15 +122964,6 @@ export namespace Schemas {}/**- * * `signal_emitted` - Signal Emitted- * * `unusual_verdict` - Unusual Verdict- * * `notable` - Notable- * * `verdict_yes` - Verdict Yes- * * `outlier_score` - Outlier Score- * * `rare_tag` - Rare Tag- * * `novel_summary` - Novel Summary- * * `friction` - Friction- * * `jev_watchable` - Jev Watchable* * `embedding_pick` - Embedding Pick* * `unviewed_recent` - Unviewed Recent* * `recent` - Recent@@ -123011,117 +122972,21 @@ export namespace Schemas {export const WatchFeedReasonEnum = {- SignalEmitted: 'signal_emitted',- UnusualVerdict: 'unusual_verdict',- Notable: 'notable',- VerdictYes: 'verdict_yes',- OutlierScore: 'outlier_score',- RareTag: 'rare_tag',- NovelSummary: 'novel_summary',- Friction: 'friction',- JevWatchable: 'jev_watchable',EmbeddingPick: 'embedding_pick',UnviewedRecent: 'unviewed_recent',Recent: 'recent',} as const;- /**- * One signal an observation raised, named rather than counted.- */- export interface WatchFeedSignal {- /** Issue type: `bug`, `crash`, `design_flaw`, or `ux_friction`. */- problem_type: string;- /** The finding in a few words, written by the scan. The full description lives on the signal itself. */- headline: string;- }-/*** Machine-readable reason an observation made the feed; the frontend renders the copy.*/export interface WatchFeedReason {- /** Highest-priority rule the observation satisfied: `signal_emitted` (it pushed a signal), `unusual_verdict` (a monitor answer that is the minority for that scanner this window), `verdict_yes` (a monitor hit, when the window is too thin to know which answer is unusual), `outlier_score` (far from the scanner's window average), `rare_tag` (a tag uncommon for the scanner this window), `novel_summary` (a summary that reads unlike the scanner's other sessions this window), `notable` (the scan itself judged the session worth watching), `friction` (the scan describes errors, retries, or dead ends), `jev_watchable` (the decision model judged the session worth watching; teams on the Jev ranker experiment only), `embedding_pick` (the session reads like a failure, struggle, or revenue moment and ranks in its scanner's top fifth; teams on the embedding ranker only), `unviewed_recent` (new to you), `recent` (nothing special, newest available).- *- * * `signal_emitted` - Signal Emitted- * * `unusual_verdict` - Unusual Verdict- * * `notable` - Notable- * * `verdict_yes` - Verdict Yes- * * `outlier_score` - Outlier Score- * * `rare_tag` - Rare Tag- * * `novel_summary` - Novel Summary- * * `friction` - Friction- * * `jev_watchable` - Jev Watchable+ /** Why the observation is in the feed: `embedding_pick` (the session reads like a failure, struggle, or revenue moment and ranks in its scanner's top fifth), `unviewed_recent` (new to you), `recent` (nothing special, newest available).+ ** * `embedding_pick` - Embedding Pick* * `unviewed_recent` - Unviewed Recent* * `recent` - Recent */kind: WatchFeedReasonEnum;- /**- * Signals this observation emitted, for `signal_emitted`.- * @nullable- */- signals_count?: number | null;- /** Issue type of each emitted signal (`bug`, `crash`, `design_flaw`, `ux_friction`), one entry per signal in the order raised, for `signal_emitted`. Absent on signals scanned before this shipped. */- problem_types?: string[];- /** Each emitted signal in the order raised, for `signal_emitted`. Carries what the card needs to name the findings instead of counting them. Absent on sessions scanned before this shipped, which carry `problem_types` alone. */- signals?: WatchFeedSignal[];- /**- * The monitor's answer, for `unusual_verdict`.- * @nullable- */- verdict?: string | null;- /**- * Share (0-1) of the scanner's window observations with this answer, for `unusual_verdict`.- * @nullable- */- verdict_share?: number | null;- /**- * The scan's own 0-1 judgment of how much a team would benefit from watching, for `notable`.- * @nullable- */- notability?: number | null;- /**- * The decision model's 0-1 judgment that the session is worth watching, for `jev_watchable`.- * @nullable- */- jev_probability?: number | null;- /**- * The scan's own sentence naming why the session is worth watching. Present on the `notable` and `jev_watchable` reason kinds when the scan itself found the session notable, and preferred over copy derived from the reason kind. Absent on observations scanned before notability shipped.- * @nullable- */- notability_reason?: string | null;- /** Why the decision model rated the session worth watching, picked from a fixed list, for `jev_watchable`. Absent when no reason on the list fits, or on sessions judged before reasons shipped.- *- * * `visible_error` - Error on screen- * * `silent_failure` - Action silently failed- * * `unresponsive` - Clicks went nowhere- * * `slow_or_stuck` - Slow or stuck- * * `blocked` - Blocked- * * `cant_find` - Couldn't find it- * * `confused` - Confused- * * `workaround` - Took a workaround- * * `abandoned` - Gave up- * * `churn_signal` - Churn signal- * * `success` - Worked well */- watch_reason?: JevWatchReasonEnum;- /**- * The observation's score, for `outlier_score`.- * @nullable- */- score?: number | null;- /**- * The scanner's mean score in the window, for `outlier_score`.- * @nullable- */- window_mean?: number | null;- /**- * The rare tag that ranked the observation, for `rare_tag`.- * @nullable- */- tag?: string | null;- /**- * Share (0-1) of the scanner's window observations carrying `tag`, for `rare_tag`.- * @nullable- */- tag_share?: number | null;/** What the session reads like, for `embedding_pick`.** * `failure` - Failure@@ -123134,7 +122999,7 @@ export namespace Schemas {*/similar_count?: number | null;/**- * Distinct people across the session's duplicate group in the window, for `embedding_pick`. A finding that reached more people ranks higher.+ * Distinct people across the session's feed group in the window, for `embedding_pick`. A finding that reached more people ranks higher.* @nullable*/affected_people?: number | null;@@ -123150,20 +123015,6 @@ export namespace Schemas {reason: WatchFeedReason;}- /**- * * `weighted-score` - Weighted Score- * * `jev` - Jev- * * `embedding` - Embedding- */- export type WatchFeedRankerEnum = typeof WatchFeedRankerEnum[keyof typeof WatchFeedRankerEnum];--- export const WatchFeedRankerEnum = {- WeightedScore: 'weighted-score',- Jev: 'jev',- Embedding: 'embedding',- } as const;-/*** Response of GET /vision/scanners/watch_feed/.*/@@ -123174,17 +123025,6 @@ export namespace Schemas {has_more: boolean;/** The time this feed ranks as of. Send it back with later pages. */as_of: string;- /** Which ranker ordered this feed: `jev` ranks on the decision model's cached judgments, `embedding` on the rank the watch rank consumer wrote from the observation's embedding, `weighted-score` on the deterministic blend. The arm is decided server-side per team, so clients read it from here rather than evaluating the flag themselves.- *- * * `weighted-score` - Weighted Score- * * `jev` - Jev- * * `embedding` - Embedding */- ranker: WatchFeedRankerEnum;- /**- * The team's variant of the `vision-watch-feed-ranker` experiment flag (`control`, `jev-shadow`, `jev`, `embedding`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from control. Clients report it on the feed-viewed event as `$feature/vision-watch-feed-ranker`, which is the exposure the experiment counts.- * @nullable- */- ranker_variant: string | null;}export interface WebAnalyticsBotCondition {@@ -143000,11 +142840,6 @@ export namespace Schemas {* @minLength 1*/scanner_type?: VisionScannersWatchFeedRetrieveScannerType;- /**- * Case-insensitive text to match against the scan's own words (title, summary, reasoning, and the notability sentence) and the scanner's name. Applied before ranking, so it searches the whole window rather than the items that would have surfaced without it.- * @minLength 1- */- search?: string;/*** Comma-separated scanner tags to restrict the feed to. A team with many scanners uses these to follow one area without naming every scanner in it.* @minLength 1diff --git a/services/mcp/src/generated/replay_vision/api.ts b/services/mcp/src/generated/replay_vision/api.tsindex bddfb7b97f44..01cc06d8fb7f 100644--- a/services/mcp/src/generated/replay_vision/api.ts+++ b/services/mcp/src/generated/replay_vision/api.ts@@ -2323,7 +2323,7 @@ export const VisionScannersStatsRetrieveParams = () => zod.object({})/**- * Succeeded observations in the window worth watching, ranked — feeds the What to watch tab.+ * Succeeded observations in the window worth watching, ranked for the Highlights tab.*/export const VisionScannersWatchFeedRetrieveParams = () => zod.object({project_id: zod@@ -2393,13 +2393,6 @@ export const VisionScannersWatchFeedRetrieveQueryParams = () => zod.object({.describe('Restrict the feed to observations from scanners of this type.\n\n\* `monitor` - Monitor\n\* `classifier` - Classifier\n\* `scorer` - Scorer\n\* `summarizer` - Summarizer\n\* `experiment` - Experiment'),- search: zod- .string()- .min(1)- .optional()- .describe(- "Case-insensitive text to match against the scan's own words (title, summary, reasoning, and the notability sentence) and the scanner's name. Applied before ranking, so it searches the whole window rather than the items that would have surfaced without it."- ),tags: zod.string().min(1)diff --git a/services/mcp/src/tools/generated/replay_vision.ts b/services/mcp/src/tools/generated/replay_vision.tsindex 89fe33bc3575..1bb42628dc83 100644--- a/services/mcp/src/tools/generated/replay_vision.ts+++ b/services/mcp/src/tools/generated/replay_vision.ts@@ -1664,7 +1664,6 @@ const visionScannersWatchFeed = (): ToolBase<offset: params.offset,scanner_ids: params.scanner_ids,scanner_type: params.scanner_type,- search: params.search,tags: params.tags,},})diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/vision-scanners-watch-feed.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/vision-scanners-watch-feed.jsonindex 95256ddcece3..b276d2848e77 100644--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/vision-scanners-watch-feed.json+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/vision-scanners-watch-feed.json@@ -47,11 +47,6 @@"enum": ["monitor", "classifier", "scorer", "summarizer", "experiment"],"type": "string"},- "search": {- "description": "Case-insensitive text to match against the scan's own words (title, summary, reasoning, and the notability sentence) and the scanner's name. Applied before ranking, so it searches the whole window rather than the items that would have surfaced without it.",- "minLength": 1,- "type": "string"- },"tags": {"description": "Comma-separated scanner tags to restrict the feed to. A team with many scanners uses these to follow one area without naming every scanner in it.","minLength": 1,