:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-vision): rank what to watch from observation embeddings (#122748)
Tue Haulund77c5f52cfec9
diff --git a/bin/mprocs.yaml b/bin/mprocs.yamlindex 6cbcb453f342..403d094d1030 100755--- a/bin/mprocs.yaml+++ b/bin/mprocs.yaml@@ -313,6 +313,16 @@ procs:layer: Product servicestech: Rust+ replay-vision-watch-rank:+ shell: |-+ bin/wait-for-docker && \+ python manage.py run_replay_vision_watch_rank_consumer --health-port 8092 --metrics-port 8093+ capability: embedding_service+ ready_pattern: 'replay_vision.watch_rank_consumer.started'+ groups:+ layer: Product services+ tech: Python+feature-flags:shell: |-bin/wait-for-docker && \diff --git a/products/replay_vision/backend/api/scanners.py b/products/replay_vision/backend/api/scanners.pyindex b4455a9f2cf7..713fc56577dd 100644--- a/products/replay_vision/backend/api/scanners.py+++ b/products/replay_vision/backend/api/scanners.py@@ -188,6 +188,7 @@ from products.replay_vision.backend.temporal.constants import VISION_SIGNALS_SOUfrom products.replay_vision.backend.temporal.metrics import record_estimate_outcome, record_scanner_limit_reachedfrom products.replay_vision.backend.temporal.read_meter_types import current_sweep_throttle_factorfrom products.replay_vision.backend.watch_feed import rank_watch_feed_candidates+from products.replay_vision.backend.watch_rank import rank_watch_feed_by_embeddingfrom products.signals.backend.facade.api import get_outcomes_for_signal_source_slice# Date is set by the schedule at trigger time, not by the user — strip on save.@@ -1644,6 +1645,18 @@ WATCH_FEED_PER_SCANNER_CAP = 100WATCH_FEED_BYPASS_CHUNK = 200+class WatchFeedRanker(models.TextChoices):+ WEIGHTED_SCORE = "weighted-score"+ JEV = "jev"+ EMBEDDING = "embedding"+++class WatchFeedCategory(models.TextChoices):+ FAILURE = "failure"+ STRUGGLE = "struggle"+ REVENUE = "revenue"++class WatchFeedReason(models.TextChoices):SIGNAL_EMITTED = "signal_emitted"UNUSUAL_VERDICT = "unusual_verdict"@@ -1654,6 +1667,7 @@ class WatchFeedReason(models.TextChoices):NOVEL_SUMMARY = "novel_summary"FRICTION = "friction"JEV_WATCHABLE = "jev_watchable"+ EMBEDDING_PICK = "embedding_pick"UNVIEWED_RECENT = "unviewed_recent"RECENT = "recent"@@ -1762,6 +1776,8 @@ class WatchFeedReasonSerializer(serializers.Serializer):"session worth watching), `friction` (the scan describes errors, retries, or dead ends), ""`jev_watchable` (the decision model judged the session worth watching; teams on the ""Jev ranker experiment only), "+ "`embedding_pick` (the session reads like a failure, struggle, or revenue moment and ranks in "+ "its scanner's top fifth; teams on the embedding ranker only), ""`unviewed_recent` (new to you), `recent` (nothing special, newest available)."),)@@ -1834,6 +1850,19 @@ class WatchFeedReasonSerializer(serializers.Serializer):allow_null=True,help_text="Share (0-1) of the scanner's window observations carrying `tag`, for `rare_tag`.",)+ watch_category = serializers.ChoiceField(+ choices=WatchFeedCategory.choices,+ required=False,+ help_text="What the session reads like, for `embedding_pick`.",+ )+ similar_count = serializers.IntegerField(+ required=False,+ allow_null=True,+ help_text=(+ "Other sessions from the same scanner in the window that describe nearly the same thing, for "+ "`embedding_pick`. The feed shows one session per group."+ ),+ )class WatchFeedItemSerializer(serializers.Serializer):@@ -1856,9 +1885,10 @@ class WatchFeedResponseSerializer(serializers.Serializer):),)ranker = serializers.ChoiceField(- choices=["weighted-score", "jev"],+ choices=WatchFeedRanker.choices,help_text=("Which ranker ordered this feed: `jev` ranks on the decision model's cached judgments, "+ "`embedding` on the rank the watch rank consumer wrote from the observation's embedding, ""`weighted-score` on the deterministic blend. The arm is decided server-side per team, so ""clients read it from here rather than evaluating the flag themselves."),@@ -1867,7 +1897,7 @@ class WatchFeedResponseSerializer(serializers.Serializer):allow_null=True,help_text=("The team's variant of the `vision-watch-feed-ranker` experiment flag (`control`, `jev-shadow`, "- "`jev`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from "+ "`jev`, `embedding`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from ""control. Clients report it on the feed-viewed event as `$feature/vision-watch-feed-ranker`, ""which is the exposure the experiment counts."),@@ -2611,7 +2641,7 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi),).filter(scanner_recency_rank__lte=WATCH_FEED_PER_SCANNER_CAP)- .values("id", "scanner_id", "created_at", "scanner_result", "feed_viewed")+ .values("id", "scanner_id", "session_id", "created_at", "scanner_result", "watch_rank", "feed_viewed").order_by("-created_at", "-id")[:WATCH_FEED_CANDIDATE_CAP])# The flag selects one of two independent rankers; nothing is blended between them. Shadow@@ -2620,8 +2650,11 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi# ranker that ordered it, so the shadow arm reads as weighted-score to the client, and the# flag variant separately, which the client reports as the experiment's exposure.variant = watch_feed_ranker_variant(self.team_id, self.team.uuid)- ranker = "jev" if ranker_mode(variant) == "jev" else "weighted-score"- if ranker == "jev":+ mode = ranker_mode(variant)+ ranker = mode if mode in ("jev", "embedding") else "weighted-score"+ if ranker == "embedding":+ ranked = rank_watch_feed_by_embedding(candidate_rows)[: params["limit"]]+ elif ranker == "jev":ranks = load_watch_ranks(self.team_id, allowed_ids)probabilities = ranks.probabilitiesjev_rows = list(candidate_rows)diff --git a/products/replay_vision/backend/embeddings.py b/products/replay_vision/backend/embeddings.pyindex ecdf6b741621..905e3679370b 100644--- a/products/replay_vision/backend/embeddings.py+++ b/products/replay_vision/backend/embeddings.py@@ -4,8 +4,30 @@ A search only finds rows written with the same (model, product, document_type) timport from here rather than defining their own copies."""+from typing import Any+from posthog.schema import EmbeddingModelNameOBSERVATION_EMBEDDING_MODEL = EmbeddingModelName.TEXT_EMBEDDING_3_LARGE_3072EMBEDDING_PRODUCT = "replay-vision"EMBEDDING_DOCUMENT_TYPE = "replay-observation"++# The physical table, read directly because the HogQL table pushes only the team filter down to storage.+OBSERVATION_EMBEDDINGS_TABLE = (+ f"distributed_posthog_document_embeddings_{OBSERVATION_EMBEDDING_MODEL.value.replace('-', '_')}"+)++# One team's observation embeddings from the given scanners; bind its values with `observation_scope_params`.+OBSERVATION_SCOPE_PREWHERE = """team_id = %(team_id)s+ AND product = %(product)s+ AND document_type = %(document_type)s+ AND JSONExtractString(metadata, 'scanner_id') IN %(scanner_ids)s"""+++def observation_scope_params(team_id: int, scanner_ids: list[str]) -> dict[str, Any]:+ return {+ "team_id": team_id,+ "product": EMBEDDING_PRODUCT,+ "document_type": EMBEDDING_DOCUMENT_TYPE,+ "scanner_ids": scanner_ids,+ }diff --git a/products/replay_vision/backend/jev_watch_feed.py b/products/replay_vision/backend/jev_watch_feed.pyindex f6567a7b3e61..206dd34f5947 100644--- a/products/replay_vision/backend/jev_watch_feed.py+++ b/products/replay_vision/backend/jev_watch_feed.py@@ -51,12 +51,12 @@ from products.ml_inference.backend.facade.contracts import (NoulAnswer,)from products.ml_inference.backend.facade.enums import DecisionQuestionType-from products.replay_vision.backend.watch_feed import NOTABLE_MIN_SCORE, WatchFeedEntry, _trim_filler+from products.replay_vision.backend.watch_feed import NOTABLE_MIN_SCORE, WatchFeedEntry, finish_feedlogger = structlog.get_logger(__name__)WATCH_FEED_RANKER_FLAG = "vision-watch-feed-ranker"-RankerMode = Literal["weighted-score", "jev-shadow", "jev"]+RankerMode = Literal["weighted-score", "jev-shadow", "jev", "embedding"]# The JevK5 build PostHog hosts on the AI gateway, which is the only backend the ml_inference# facade reaches. The vendor (typesafe.ai) is not approved for customer data, and observation prose# is customer data, so never point this at a vendor model id.@@ -88,12 +88,6 @@ JEV_SEEN_PENALTY = 0.3# bar empties the tier. Recalibrate from the judged event's watchability quantiles and top_scored# sample, and bump the cache version below with any change here.JEV_WATCHABLE_MIN = 0.3-# One incident can put many near-identical watchable sessions on one scanner, and a pure probability-# sort would fill the top of the feed with them. Hold each scanner to this share of the evidence-# tier as it is placed (same intent as WATCH_FEED_MAX_SIGNAL_SHARE in the weighted ranker), with a-# floor so a short feed is not over-constrained; the overflow trails the tier instead of leaving it.-JEV_MAX_SCANNER_SHARE = 0.4-_SCANNER_SPREAD_FLOOR = 3# Observations per Jev request: the request carries the chunk as shared state and one question per# observation, so a judgment is relative to its chunk, not to the whole window at once. Half the# facade's per-request question cap: a request's prompt grows with its state, and a full 32-entry@@ -268,9 +262,8 @@ def watch_feed_ranker_variant(team_id: int, team_uuid: UUID | str) -> str | NoneKeyed on the project's uuid: both regions evaluate this one flag, and team ids repeat betweenthem. A region without the decision service has no sweep to fill the cache, so its teams take- no part rather than land on an empty `jev` feed. Any flag failure also reads as None."""- if not decision_api.decisions_available_here():- return None+ no part in the Jev arms rather than land on an empty `jev` feed. The embedding arm needs no+ decision service. Any flag failure also reads as None."""value = get_feature_flag_or_none(WATCH_FEED_RANKER_FLAG,# Only a person-property condition reads the distinct id; a project-aggregated flag buckets@@ -280,11 +273,17 @@ def watch_feed_ranker_variant(team_id: int, team_uuid: UUID | str) -> str | Nonegroup_properties={"project": {"id": team_id, "uuid": str(team_uuid)}},send_feature_flag_events=False,)- return value if isinstance(value, str) else None+ if not isinstance(value, str):+ return None+ if value in ("jev", "jev-shadow") and not decision_api.decisions_available_here():+ return None+ return valuedef ranker_mode(variant: str | None) -> RankerMode:- """The ranker a flag variant selects. Any variant but the two Jev arms is the default arm."""+ """The ranker a flag variant selects. Any variant but the Jev and embedding arms is the default arm."""+ if variant == "embedding":+ return "embedding"if variant == "jev-shadow":return "jev-shadow"if variant == "jev":@@ -818,26 +817,6 @@ def _scan_notability_reason(row: dict[str, Any]) -> str | None:return reason if isinstance(reason, str) and reason.strip() else None-def _spread_scanners(ordered: list[tuple[Any, WatchFeedEntry]]) -> list[WatchFeedEntry]:- """Hold each scanner to JEV_MAX_SCANNER_SHARE of the evidence tier as it is placed.-- Checked per place rather than over the whole list because the view slices the head: a feed of 5- obeys the same share as a feed of 50. Overflow trails the tier in probability order — reordering- for breadth is the job here, shortening the feed is not.- """- placed: list[WatchFeedEntry] = []- deferred: list[WatchFeedEntry] = []- counts: dict[Any, int] = {}- for scanner_id, entry in ordered:- allowed = max(_SCANNER_SPREAD_FLOOR, math.ceil(JEV_MAX_SCANNER_SHARE * (len(placed) + 1)))- if counts.get(scanner_id, 0) >= allowed:- deferred.append(entry)- else:- counts[scanner_id] = counts.get(scanner_id, 0) + 1- placed.append(entry)- return placed + deferred--def rank_watch_feed_by_jev(rows: list[dict[str, Any]], probabilities: dict[str, float], reasons: dict[str, str] | None = None) -> list[WatchFeedEntry]:@@ -854,7 +833,7 @@ def rank_watch_feed_by_jev(of them."""watchable: list[tuple[float, Any, Any, WatchFeedEntry]] = []- filler: list[tuple[bool, Any, WatchFeedEntry]] = []+ filler_rows: list[dict[str, Any]] = []for row in rows:probability = probabilities.get(str(row["id"]))viewed = bool(row.get("feed_viewed"))@@ -871,11 +850,5 @@ def rank_watch_feed_by_jev((probability - (JEV_SEEN_PENALTY if viewed else 0.0), row["created_at"], row.get("scanner_id"), entry))else:- entry = WatchFeedEntry(observation_id=row["id"], reason={"kind": "recent" if viewed else "unviewed_recent"})- filler.append((not viewed, row["created_at"], entry))- watchable.sort(key=lambda item: (item[0], item[1]), reverse=True)- filler.sort(key=lambda item: (item[0], item[1]), reverse=True)- return _trim_filler(- _spread_scanners([(scanner_id, entry) for _, _, scanner_id, entry in watchable])- + [entry for *_, entry in filler]- )+ filler_rows.append(row)+ return finish_feed(watchable, filler_rows)diff --git a/products/replay_vision/backend/management/commands/backfill_replay_vision_watch_ranks.py b/products/replay_vision/backend/management/commands/backfill_replay_vision_watch_ranks.pynew file mode 100644index 000000000000..378d91a69469--- /dev/null+++ b/products/replay_vision/backend/management/commands/backfill_replay_vision_watch_ranks.py@@ -0,0 +1,33 @@+from datetime import timedelta+from typing import Any++from django.core.management.base import BaseCommand, CommandParser+from django.utils import timezone++from products.replay_vision.backend.watch_rank import DUPLICATE_LOOKBACK+from products.replay_vision.backend.watch_rank_backfill import backfill_team_watch_ranks, teams_with_observations+++class Command(BaseCommand):+ help = "Rank recent Replay Vision observations for What to watch from the embeddings search already stored"++ def add_arguments(self, parser: CommandParser) -> None:+ parser.add_argument("--team-id", type=int, action="append", help="Only this team; repeat for more")+ parser.add_argument("--days", type=int, default=DUPLICATE_LOOKBACK.days, help="How far back to rank, in days")+ parser.add_argument("--dry-run", action="store_true", help="Count what would be ranked, write nothing")++ def handle(self, *args: Any, **options: Any) -> None:+ since = timezone.now() - timedelta(days=options["days"])+ team_ids = options["team_id"] or teams_with_observations(since)+ verb = "Would rank" if options["dry_run"] else "Ranked"+ for team_id in team_ids:+ try:+ result = backfill_team_watch_ranks(team_id, since, dry_run=bool(options["dry_run"]))+ except Exception as e:+ # One team without AI data processing, or a slow query, must not stop the rest.+ self.stderr.write(f"Team {team_id}: skipped ({e.__class__.__name__}: {e})")+ continue+ self.stdout.write(+ f"Team {team_id}: {verb} {result.ranked} of {result.observations} observations, "+ f"{result.without_embedding} had no stored embedding."+ )diff --git a/products/replay_vision/backend/management/commands/run_replay_vision_watch_rank_consumer.py b/products/replay_vision/backend/management/commands/run_replay_vision_watch_rank_consumer.pynew file mode 100644index 000000000000..dfb72cd6813e--- /dev/null+++ b/products/replay_vision/backend/management/commands/run_replay_vision_watch_rank_consumer.py@@ -0,0 +1,38 @@+import asyncio+from typing import Any++from django.core.management.base import BaseCommand, CommandParser++from prometheus_client import start_http_server++from posthog.temporal.common.health_server import HealthCheckServer+from posthog.temporal.common.liveness_tracker import LivenessTracker++from products.replay_vision.backend.watch_rank_consumer import WatchRankConsumer+++class Command(BaseCommand):+ help = "Rank Replay Vision observations for What to watch from the embedding worker's results topic"++ def add_arguments(self, parser: CommandParser) -> None:+ parser.add_argument("--health-port", type=int, default=8090, help="Port for /healthz and /readyz")+ parser.add_argument(+ "--health-max-idle-seconds",+ type=float,+ default=120.0,+ help="Seconds without a consume loop heartbeat before the health checks fail",+ )+ parser.add_argument("--metrics-port", type=int, default=8001, help="Port for Prometheus metrics")++ def handle(self, *args: Any, **options: Any) -> None:+ start_http_server(options["metrics_port"])+ asyncio.run(self._run(options["health_port"], options["health_max_idle_seconds"]))++ async def _run(self, health_port: int, max_idle_seconds: float) -> None:+ liveness = LivenessTracker()+ health = HealthCheckServer(health_port, liveness, max_idle_seconds=max_idle_seconds)+ await health.start()+ try:+ await WatchRankConsumer(liveness).run()+ finally:+ await health.stop()diff --git a/products/replay_vision/backend/migrations/0126_replayobservation_watch_rank.py b/products/replay_vision/backend/migrations/0126_replayobservation_watch_rank.pynew file mode 100644index 000000000000..82cd0240f0bc--- /dev/null+++ b/products/replay_vision/backend/migrations/0126_replayobservation_watch_rank.py@@ -0,0 +1,21 @@+# Generated by Django 5.2.18 on 2026-10-23 14:34++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("replay_vision", "0125_replayobservationfeeddismissal"),+ ]++ operations = [+ migrations.AddField(+ model_name="replayobservation",+ name="watch_rank",+ field=models.JSONField(+ blank=True,+ help_text="What to watch ranking derived from the observation's embedding; see `watch_rank.WatchRank`.",+ null=True,+ ),+ ),+ ]diff --git a/products/replay_vision/backend/migrations/max_migration.txt b/products/replay_vision/backend/migrations/max_migration.txtindex 094e746900bc..52d3ffafe31f 100644--- a/products/replay_vision/backend/migrations/max_migration.txt+++ b/products/replay_vision/backend/migrations/max_migration.txt@@ -1 +1 @@-0125_replayobservationfeeddismissal+0126_replayobservation_watch_rankdiff --git a/products/replay_vision/backend/models/replay_observation.py b/products/replay_vision/backend/models/replay_observation.pyindex ee66a413b9af..5601a4757a11 100644--- a/products/replay_vision/backend/models/replay_observation.py+++ b/products/replay_vision/backend/models/replay_observation.py@@ -98,6 +98,11 @@ class ReplayObservation(UUIDModel):default=dict,help_text="Result data persisted on success (model output, signals count); see `temporal.types.ScannerResult`.",)+ watch_rank = models.JSONField(+ null=True,+ blank=True,+ help_text="What to watch ranking derived from the observation's embedding; see `watch_rank.WatchRank`.",+ )created_task_id = models.UUIDField(null=True,blank=True,diff --git a/products/replay_vision/backend/search.py b/products/replay_vision/backend/search.pyindex 9459f758027d..99f2c51ddc1a 100644--- a/products/replay_vision/backend/search.py+++ b/products/replay_vision/backend/search.py@@ -31,9 +31,10 @@ from posthog.utils import relative_date_parse_with_delta_mappingfrom products.access_control.backend.facade.user_access_control import UserAccessControlfrom products.replay_vision.backend.embeddings import (- EMBEDDING_DOCUMENT_TYPE,- EMBEDDING_PRODUCT,OBSERVATION_EMBEDDING_MODEL,+ OBSERVATION_EMBEDDINGS_TABLE,+ OBSERVATION_SCOPE_PREWHERE,+ observation_scope_params,)from products.replay_vision.backend.models.replay_observation import (ObservationStatus,@@ -78,11 +79,6 @@ _QUERY_VECTOR_EXECUTOR = ThreadPoolExecutor(max_workers=8, thread_name_prefix="r# Warming gets its own pool so a burst of page views never queues a live search behind it._WARM_VECTOR_EXECUTOR = ThreadPoolExecutor(max_workers=2, thread_name_prefix="replay-vision-query-vector-warm")-_EMBEDDINGS_TABLE = f"distributed_posthog_document_embeddings_{OBSERVATION_EMBEDDING_MODEL.value.replace('-', '_')}"-_SCOPE_PREWHERE = """team_id = %(team_id)s- AND product = %(product)s- AND document_type = %(document_type)s- AND JSONExtractString(metadata, 'scanner_id') IN %(scanner_ids)s"""_CANDIDATE_QUERY_TYPE = "replay_vision_search_candidates"_RANK_QUERY_TYPE = "replay_vision_search_rank"@@ -214,7 +210,7 @@ def candidate_cutoff(team: Team, scanner_ids: list[str], filters: ObservationSeaSELECT minOrNull(timestamp)FROM (SELECT timestamp- FROM {_EMBEDDINGS_TABLE}+ FROM {OBSERVATION_EMBEDDINGS_TABLE}PREWHERE {scope}ORDER BY timestamp DESCLIMIT %(candidate_cap)s@@ -230,17 +226,13 @@ def candidate_cutoff(team: Team, scanner_ids: list[str], filters: ObservationSeadef _scope_params(team: Team, scanner_ids: list[str]) -> dict[str, Any]:- return {- "team_id": team.id,- "product": EMBEDDING_PRODUCT,- "document_type": EMBEDDING_DOCUMENT_TYPE,- "scanner_ids": scanner_ids,- "candidate_cap": _MAX_CANDIDATE_ROWS,- }+ return {**observation_scope_params(team.id, scanner_ids), "candidate_cap": _MAX_CANDIDATE_ROWS}def _scope_clause(filters: ObservationSearchFilters, params: dict[str, Any]) -> str:- return _SCOPE_PREWHERE + "".join(f"\n AND {clause}" for clause in filters.where_clauses(params))+ return OBSERVATION_SCOPE_PREWHERE + "".join(+ f"\n AND {clause}" for clause in filters.where_clauses(params)+ )def rank_observations(@@ -287,7 +279,7 @@ def rank_observations(document_id,cosineDistance(embedding, %(embedding)s) AS row_distance,substring(content, 1, %(snippet_chars)s) AS snippet- FROM {_EMBEDDINGS_TABLE}+ FROM {OBSERVATION_EMBEDDINGS_TABLE}PREWHERE {scope}AND timestamp >= %(cutoff)sWHERE row_distance <= %(max_distance)sdiff --git a/products/replay_vision/backend/temporal/activities/embed_observation.py b/products/replay_vision/backend/temporal/activities/embed_observation.pyindex 5ecad96cffb4..ec2fdcbaa2ce 100644--- a/products/replay_vision/backend/temporal/activities/embed_observation.py+++ b/products/replay_vision/backend/temporal/activities/embed_observation.py@@ -6,6 +6,8 @@ via `metadata.scanner_id` for later natural-language search. monitor/classifier/document under `rendering="summary"`. Empty content is skipped. Failures surface to the workflow, whichlogs and swallows them — the observation is already marked succeeded.+The worker's results topic feeds What to watch ranking, see `watch_rank_consumer`.+The structured result (monitor `verdict`, scorer `score`, classifier `tags`) is stamped into `metadata` sosearch can filter by exact outcome inside the same ClickHouse ranking query, with no second-pass lookup."""diff --git a/products/replay_vision/backend/temporal/jev_watch_rank/activities.py b/products/replay_vision/backend/temporal/jev_watch_rank/activities.pyindex c220d7f6f4f0..87ed89d4715a 100644--- a/products/replay_vision/backend/temporal/jev_watch_rank/activities.py+++ b/products/replay_vision/backend/temporal/jev_watch_rank/activities.py@@ -392,7 +392,7 @@ async def _judge_watch_ranks(inputs: JevWatchRankSweepInputs) -> JevWatchRankSwecontinuemode = await asyncio.to_thread(watch_feed_ranker, team_id, team_uuid)teams_evaluated += 1- if mode == "weighted-score":+ if mode not in ("jev", "jev-shadow"):continue# The scan prose the sweep sends to the model derives from recordings, so a revoked consent# stops the judgments even while the flag stays on.diff --git a/products/replay_vision/backend/tests/test_api.py b/products/replay_vision/backend/tests/test_api.pyindex 1945dd1c4959..5ffa3ef63a49 100644--- a/products/replay_vision/backend/tests/test_api.py+++ b/products/replay_vision/backend/tests/test_api.py@@ -76,6 +76,7 @@ from products.replay_vision.backend.tests.helpers import (seed_scanner_spend,snapshot_for as _snapshot_for,)+from products.replay_vision.backend.watch_rank import ANCHOR_VERSIONfrom products.signals.backend.facade.api import (SignalSourceSliceOutcomes,SignalSourceSlicePullRequest,@@ -5108,7 +5109,7 @@ class TestWatchFeedAPI(_VisionAPITestCase):resp = self.client.get(self.feed_url)self.assertEqual(resp.json()["results"][0]["reason"], {"kind": "unviewed_recent"})- def test_the_flag_switches_the_whole_ranker_between_weighted_score_and_jev(self) -> None:+ def test_the_flag_switches_the_whole_ranker_between_arms(self) -> None:# The two rankers are independent: a cached Jev probability must not move the weighted-score# feed (weighted-score and jev-shadow arms), and the jev arm must rank on the cached# probabilities alone, with unjudged and judged-low rows in the recency filler tier.@@ -5152,6 +5153,29 @@ class TestWatchFeedAPI(_VisionAPITestCase):self.assertEqual(items[1]["reason"], {"kind": "unviewed_recent"})self.assertEqual(items[2]["reason"], {"kind": "unviewed_recent"})+ # The embedding arm ranks on the rank the scan wrote, once the scanner has enough ranked rows.+ for index in range(10):+ filler = self._succeeded_observation(scanner, f"routine-{index}", 50 + index, self._monitor_result("no"))+ ReplayObservation.objects.filter(id=filler.id).update(+ watch_rank={+ "category": "routine",+ "contrast": -0.1,+ "duplicate_of": None,+ "anchor_version": ANCHOR_VERSION,+ }+ )+ ReplayObservation.objects.filter(id=jev_low.id).update(+ watch_rank={"category": "failure", "contrast": 0.2, "duplicate_of": None, "anchor_version": ANCHOR_VERSION}+ )+ with patch(ranker, return_value="embedding"):+ resp = self.client.get(self.feed_url)+ items = resp.json()["results"]+ self.assertEqual(resp.json()["ranker"], "embedding")+ self.assertEqual(items[0]["observation"]["session_id"], "jev-low")+ self.assertEqual(+ items[0]["reason"], {"kind": "embedding_pick", "watch_category": "failure", "similar_count": 0}+ )+def test_the_jev_arm_surfaces_a_watchable_row_the_recency_slice_cut_off(self) -> None:# The candidate query keeps only each scanner's newest rows, so on a high-volume scanner a# watchable row from days ago never reaches the weighted feed. The jev arm must fetch itdiff --git a/products/replay_vision/backend/tests/test_jev_watch_feed.py b/products/replay_vision/backend/tests/test_jev_watch_feed.pyindex a0d34d987936..da33e45ccf20 100644--- a/products/replay_vision/backend/tests/test_jev_watch_feed.py+++ b/products/replay_vision/backend/tests/test_jev_watch_feed.py@@ -104,22 +104,25 @@ class TestWatchFeedRankerFlag(SimpleTestCase):("unknown_variant", "something-else", "weighted-score"),("shadow", "jev-shadow", "jev-shadow"),("graduated", "jev", "jev"),+ ("embedding", "embedding", "embedding"),])def test_only_known_variants_leave_the_default_arm(self, _name: str, flag_value: object, expected: str) -> None:with patch(_FLAG, return_value=flag_value), patch(_REGION, return_value=True):assert watch_feed_ranker(1, "project-uuid") == expected- def test_the_flag_keys_on_the_project_uuid_and_skips_regions_without_jev(self) -> None:+ def test_the_flag_keys_on_the_project_uuid_and_keeps_jev_arms_out_of_regions_without_jev(self) -> None:# Both regions evaluate this one flag and team ids repeat between them, so only the uuid is unique.with patch(_FLAG, return_value="jev") as flag, patch(_REGION, return_value=True):assert watch_feed_ranker_variant(2, "project-uuid") == "jev"assert flag.call_args.kwargs["groups"] == {"project": "project-uuid"}# A region with no sweep to fill the cache must not land a team on an empty jev feed.- with patch(_FLAG, return_value="jev") as flag, patch(_REGION, return_value=False):+ with patch(_FLAG, return_value="jev"), patch(_REGION, return_value=False):assert watch_feed_ranker_variant(2, "project-uuid") is Noneassert watch_feed_ranker(2, "project-uuid") == "weighted-score"- flag.assert_not_called()+ # The embedding arm needs no decision service.+ with patch(_FLAG, return_value="embedding"), patch(_REGION, return_value=False):+ assert watch_feed_ranker(2, "project-uuid") == "embedding"class TestJudgeScannerWindow(SimpleTestCase):@@ -571,12 +574,13 @@ class TestJevWatchRankSweep(BaseTest):flag = f"{activities}.watch_feed_ranker"region = f"{activities}.decision_api.decisions_available_here"- with patch(flag, return_value="weighted-score"), patch(region, return_value=True), patch(_API) as api:- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())- # A team on the default arm costs no Jev calls and gets no cache entry.- api.decide_when_available.assert_not_called()- assert result.teams_enrolled == 0- assert load_watch_ranks(self.team.id, [scanner.id]).probabilities == {}+ # A team on the default or embedding arm costs no Jev calls and gets no cache entry.+ for arm in ("weighted-score", "embedding"):+ with patch(flag, return_value=arm), patch(region, return_value=True), patch(_API) as api:+ result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())+ api.decide_when_available.assert_not_called()+ assert result.teams_enrolled == 0+ assert load_watch_ranks(self.team.id, [scanner.id]).probabilities == {}with (patch(flag, side_effect=self._flag_arm("jev-shadow")),diff --git a/products/replay_vision/backend/tests/test_watch_rank.py b/products/replay_vision/backend/tests/test_watch_rank.pynew file mode 100644index 000000000000..d1381e8c23ba--- /dev/null+++ b/products/replay_vision/backend/tests/test_watch_rank.py@@ -0,0 +1,192 @@+from datetime import UTC, datetime, timedelta+from typing import Any++from django.test import SimpleTestCase++import numpy as np+from parameterized import parameterized++from products.replay_vision.backend.watch_feed import SESSION_SPACING+from products.replay_vision.backend.watch_rank import (+ ANCHOR_VERSION,+ WatchCategory,+ classify,+ duplicate_root,+ rank_watch_feed_by_embedding,+)+from products.replay_vision.backend.watch_rank_backfill import _rank_scanner, _RecentVectors++_NOW = datetime(2026, 10, 23, 12, 0, tzinfo=UTC)+++def _row(+ observation_id: str,+ contrast: float | None,+ *,+ category: str = "failure",+ scanner: str = "scanner-a",+ duplicate_of: str | None = None,+ anchor_version: int = ANCHOR_VERSION,+ minutes_ago: int = 0,+ viewed: bool = False,+ session: str | None = None,+) -> dict[str, Any]:+ rank = (+ None+ if contrast is None+ else {+ "category": category,+ "contrast": contrast,+ "duplicate_of": duplicate_of,+ "anchor_version": anchor_version,+ }+ )+ return {+ "id": observation_id,+ "scanner_id": scanner,+ "session_id": session or f"session-{observation_id}",+ "created_at": _NOW - timedelta(minutes=minutes_ago),+ "watch_rank": rank,+ "feed_viewed": viewed,+ }+++class TestClassify(SimpleTestCase):+ @parameterized.expand(+ [+ ("closest_to_an_issue", [1.0, 0.1, 0.0], "failure", True),+ ("closest_to_routine", [0.1, 0.0, 1.0], "routine", False),+ ]+ )+ def test_routine_wins_over_the_closest_issue_when_it_is_closer(+ self, _name: str, vector: list[float], category: str, positive: bool+ ) -> None:+ anchors: dict[WatchCategory, np.ndarray] = {+ "failure": np.array([[1.0, 0.0, 0.0]]),+ "struggle": np.array([[0.0, 1.0, 0.0]]),+ "revenue": np.array([[0.0, -1.0, 0.0]]),+ "routine": np.array([[0.0, 0.0, 1.0]]),+ }+ got_category, contrast = classify(vector, anchors)+ assert got_category == category+ assert (contrast > 0) is positive+++class TestDuplicateRoot(SimpleTestCase):+ @parameterized.expand(+ [+ ("unranked_smaller_neighbour_is_the_root", "obs-b", "obs-a", None, "obs-a"),+ ("unranked_larger_neighbour_defers_to_this_one", "obs-a", "obs-b", None, None),+ ("ranked_neighbour_passes_on_its_root", "obs-a", "obs-b", {"duplicate_of": "obs-0"}, "obs-0"),+ ("ranked_root_neighbour_is_the_root", "obs-a", "obs-b", {"duplicate_of": None}, "obs-b"),+ ]+ )+ def test_two_concurrent_copies_agree_on_one_root(+ self,+ _name: str,+ observation_id: str,+ neighbour_id: str,+ neighbour_rank: dict[str, Any] | None,+ expected: str | None,+ ) -> None:+ assert duplicate_root(observation_id, neighbour_id, neighbour_rank) == expected+++class TestRankWatchFeedByEmbedding(SimpleTestCase):+ def test_picks_the_scanner_top_issues_once_per_duplicate_group(self) -> None:+ rows = [+ # scanner-a has enough ranked rows for a percentile; its top two are one duplicate group.+ _row("dup-root", 0.30),+ _row("dup-copy", 0.32, duplicate_of="dup-root", minutes_ago=50),+ _row("dup-low", -0.01, duplicate_of="dup-root"),+ # A copy that points at another copy, as when its root was ranked after it, still joins the group.+ _row("dup-chain", 0.02, duplicate_of="dup-copy", minutes_ago=3),+ _row("routine-top", 0.40, category="routine", minutes_ago=50),+ *[_row(f"a-{index}", 0.01 * index, minutes_ago=index + 1) for index in range(10)],+ # A rank from older anchors does not compare with current scores.+ _row("stale", 0.90, anchor_version=ANCHOR_VERSION - 1),+ # scanner-b is too thin for a percentile, so its top row is only filler.+ _row("b-top", 0.50, scanner="scanner-b", minutes_ago=1),+ ]++ ranked = rank_watch_feed_by_embedding(rows)++ assert ranked[0].observation_id == "dup-copy"+ assert ranked[0].reason == {"kind": "embedding_pick", "watch_category": "failure", "similar_count": 3}+ picks = [entry.observation_id for entry in ranked if entry.reason["kind"] == "embedding_pick"]+ assert picks == ["dup-copy"]+ # Collapsed copies, even those below the cutoff, must not come back as filler.+ assert {"dup-root", "dup-low", "dup-chain"}.isdisjoint(entry.observation_id for entry in ranked)++ def test_a_viewed_pick_still_stands_for_its_group_over_a_copy_below_the_cutoff(self) -> None:+ rows = [+ _row("seen-top", 0.9, viewed=True),+ _row("other-top", 0.8, minutes_ago=1),+ _row("mid-copy", 0.7, duplicate_of="seen-top", minutes_ago=2),+ *[_row(f"a-{index}", 0.01 * index, minutes_ago=index + 3) for index in range(7)],+ ]++ ranked = rank_watch_feed_by_embedding(rows)++ picks = {+ str(entry.observation_id): entry.reason for entry in ranked if entry.reason["kind"] == "embedding_pick"+ }+ assert picks["seen-top"]["similar_count"] == 1+ assert "mid-copy" not in {entry.observation_id for entry in ranked}++ def test_two_scanners_picking_one_session_land_apart_without_dropping_a_pick(self) -> None:+ rows = [+ _row(f"{scanner}-{index}", index / 20, scanner=scanner, minutes_ago=index)+ for scanner in ("scanner-a", "scanner-b", "scanner-c")+ for index in range(9)+ ] + [+ _row("a-top", 0.9, scanner="scanner-a", session="shared"),+ _row("b-top", 0.9, scanner="scanner-b", session="shared", minutes_ago=1),+ _row("c-top", 0.9, scanner="scanner-c", minutes_ago=2),+ ]++ ranked = [str(entry.observation_id) for entry in rank_watch_feed_by_embedding(rows)]++ assert ranked[0] == "a-top"+ assert ranked.index("b-top") - ranked.index("a-top") >= SESSION_SPACING+ assert {"a-top", "b-top", "c-top", "scanner-a-8", "scanner-b-8", "scanner-c-8"} <= set(ranked)++ def test_rows_without_a_rank_fall_back_to_the_filler_floor(self) -> None:+ rows = [_row(f"row-{index}", None, minutes_ago=index, viewed=index == 0) for index in range(20)]++ ranked = rank_watch_feed_by_embedding(rows)++ assert [entry.observation_id for entry in ranked] == ["row-1", "row-2", "row-3"]+++class TestBackfillRankScanner(SimpleTestCase):+ def test_walks_oldest_first_so_a_copy_joins_the_group_of_an_already_ranked_row(self) -> None:+ anchors: dict[WatchCategory, np.ndarray] = {+ "failure": np.array([[1.0, 0.0, 0.0]]),+ "struggle": np.array([[0.0, 1.0, 0.0]]),+ "revenue": np.array([[0.0, -1.0, 0.0]]),+ "routine": np.array([[0.0, 0.0, 1.0]]),+ }+ error = np.array([1.0, 0.0, 0.0])+ current = {"category": "failure", "contrast": 1.0, "duplicate_of": "root", "anchor_version": ANCHOR_VERSION}+ rows: list[dict[str, Any]] = [+ {"id": "ranked", "watch_rank": current},+ {"id": "no-vector", "watch_rank": None},+ {"id": "copy", "watch_rank": None},+ {"id": "stale", "watch_rank": {**current, "anchor_version": ANCHOR_VERSION - 1}},+ {"id": "calm", "watch_rank": None},+ ]+ vectors = {"ranked": error, "copy": error, "stale": error, "calm": np.array([0.0, 0.0, 1.0])}++ # Two batches that share the recent vectors, as the backfill runs them.+ recent = _RecentVectors()+ written = {+ **_rank_scanner(rows[:2], vectors, anchors, recent),+ **_rank_scanner(rows[2:], vectors, anchors, recent),+ }++ assert set(written) == {"copy", "stale", "calm"}+ assert written["copy"]["duplicate_of"] == "root"+ assert written["stale"]["duplicate_of"] == "root"+ assert written["calm"]["category"] == "routine"+ assert written["calm"]["duplicate_of"] is Nonediff --git a/products/replay_vision/backend/tests/test_watch_rank_consumer.py b/products/replay_vision/backend/tests/test_watch_rank_consumer.pynew file mode 100644index 000000000000..a45aa5ca6804--- /dev/null+++ b/products/replay_vision/backend/tests/test_watch_rank_consumer.py@@ -0,0 +1,196 @@+import json+import math+import time+from typing import Any++from posthog.test.base import BaseTest+from unittest.mock import AsyncMock, MagicMock, patch++from django.test import SimpleTestCase++import numpy as np+from asgiref.sync import async_to_sync+from confluent_kafka import KafkaException+from parameterized import parameterized++from products.replay_vision.backend.embeddings import (+ EMBEDDING_DOCUMENT_TYPE,+ EMBEDDING_PRODUCT,+ OBSERVATION_EMBEDDING_MODEL,+)+from products.replay_vision.backend.models.replay_observation import ObservationTrigger, ReplayObservation+from products.replay_vision.backend.models.replay_scanner import ReplayScanner, ScannerModel, ScannerType+from products.replay_vision.backend.tests.helpers import snapshot_for+from products.replay_vision.backend.watch_rank import ANCHOR_VERSION, RankRequest+from products.replay_vision.backend.watch_rank_consumer import WatchRankConsumer, parse_rank_request++_WATCH_RANK = "products.replay_vision.backend.watch_rank"+_CURRENT_RANK = {"category": "struggle", "contrast": 0.2, "duplicate_of": None, "anchor_version": ANCHOR_VERSION}+_CONSUMER = "products.replay_vision.backend.watch_rank_consumer"+++def _result_message(+ *,+ team_id: int = 7,+ observation_id: str = "obs-1",+ scanner_id: str | None = "scanner-1",+ product: str = EMBEDDING_PRODUCT,+ outcome: str = "success",+ vector: list[float] | None = None,+) -> bytes:+ result: dict[str, Any] = {"model": OBSERVATION_EMBEDDING_MODEL.value, "outcome": outcome}+ if outcome == "success":+ result["embedding"] = [1.0, 0.1, 0.0] if vector is None else vector+ else:+ result["error"] = "provider refused"+ return json.dumps(+ {+ "team_id": team_id,+ "product": product,+ "document_type": EMBEDDING_DOCUMENT_TYPE,+ "rendering": "reasoning",+ "document_id": observation_id,+ "timestamp": "2026-10-24T12:00:00Z",+ "content": "the payment form errored",+ "models": [OBSERVATION_EMBEDDING_MODEL.value],+ "metadata": {} if scanner_id is None else {"scanner_id": scanner_id},+ "results": [result],+ },+ separators=(",", ":"),+ ).encode()+++class TestParseRankRequest(SimpleTestCase):+ def test_reads_an_observation_result(self) -> None:+ assert parse_rank_request(_result_message()) == RankRequest(+ team_id=7, scanner_id="scanner-1", observation_id="obs-1", vector=[1.0, 0.1, 0.0]+ )++ @parameterized.expand(+ [+ ("another_product", _result_message(product="error-tracking")),+ ("failed_embedding", _result_message(outcome="failure")),+ ("no_scanner", _result_message(scanner_id=None)),+ ("zero_vector", _result_message(vector=[0.0, 0.0, 0.0])),+ ("not_json", b'{"document_type":"replay-observation"'),+ ]+ )+ def test_skips_what_it_cannot_rank(self, _name: str, value: bytes) -> None:+ assert parse_rank_request(value) is None+++class TestWatchRankConsumer(BaseTest):+ _ANCHORS = {+ "failure": np.array([[1.0, 0.0, 0.0]]),+ "struggle": np.array([[0.0, 1.0, 0.0]]),+ "revenue": np.array([[0.0, -1.0, 0.0]]),+ "routine": np.array([[0.0, 0.0, 1.0]]),+ }++ def _observation(self, scanner: ReplayScanner, session_id: str, **overrides: Any) -> ReplayObservation:+ return ReplayObservation.objects.create(+ scanner=scanner,+ team=self.team,+ session_id=session_id,+ triggered_by=ObservationTrigger.ON_DEMAND,+ scanner_snapshot=snapshot_for(scanner),+ **overrides,+ )++ @parameterized.expand(+ [+ (+ "duplicate_points_at_the_group_root",+ None,+ None,+ "ranked",+ {"category": "failure", "duplicate_of": "root-id"},+ ),+ ("ranking_failure_leaves_no_rank", None, RuntimeError("clickhouse down"), "failed", None),+ ("redelivery_keeps_the_current_rank", _CURRENT_RANK, None, "already_ranked", _CURRENT_RANK),+ ("deleted_observation_is_skipped", None, None, "missing", None),+ ]+ )+ def test_ranks_the_observation_in_a_result(+ self,+ _name: str,+ existing_rank: dict[str, Any] | None,+ nearest_error: Exception | None,+ outcome: str,+ expected_rank: dict[str, Any] | None,+ ) -> None:+ scanner = ReplayScanner.objects.create(+ team=self.team,+ name="s",+ scanner_type=ScannerType.MONITOR,+ scanner_config={"prompt": "p"},+ model=ScannerModel.GEMINI_3_8_FLASH,+ )+ neighbour = self._observation(scanner, "sess-earlier", watch_rank={"duplicate_of": "root-id"})+ observation = self._observation(scanner, "sess-new", watch_rank=existing_rank)+ nearest = MagicMock(side_effect=nearest_error, return_value=(str(neighbour.id), 0.05))+ message = _result_message(team_id=self.team.id, observation_id=str(observation.id), scanner_id=str(scanner.id))+ if outcome == "missing":+ observation.delete()++ with (+ patch(f"{_WATCH_RANK}.anchor_vectors", AsyncMock(return_value=self._ANCHORS)),+ patch(f"{_WATCH_RANK}.nearest_earlier_observation", nearest),+ patch(f"{_CONSUMER}._RETRY_WINDOW_S", 0),+ # It would close the connection that holds the test transaction.+ patch(f"{_CONSUMER}.close_old_connections"),+ ):+ assert async_to_sync(WatchRankConsumer(MagicMock()).handle)(message, 0.0, math.inf) == outcome++ if outcome == "missing":+ return+ observation.refresh_from_db()+ if expected_rank is None:+ assert observation.watch_rank is None+ else:+ assert observation.watch_rank is not None+ assert expected_rank.items() <= observation.watch_rank.items()++ @parameterized.expand(+ [+ # A revoked partition stops the batch, and the rest of it is read again rather than skipped.+ ("revoked_partition", KafkaException("partition revoked"), "ranked", [0], 1),+ # A message still failing at the end of the batch budget goes back to Kafka unstored.+ ("deferred_retry", None, "deferred", [], 0),+ ]+ )+ def test_unfinished_messages_are_read_again(+ self, _name: str, store_error: Exception | None, outcome: str, stored: list[int], rewind_from: int+ ) -> None:+ messages = [+ MagicMock(+ **{+ "error.return_value": None,+ "timestamp.return_value": (1, 0),+ "value.return_value": b"{}",+ "topic.return_value": "results",+ "partition.return_value": 0,+ "offset.return_value": offset,+ }+ )+ for offset in range(3)+ ]+ consumer = MagicMock(**{"store_offsets.side_effect": store_error})++ with patch.object(WatchRankConsumer, "handle", AsyncMock(return_value=outcome)):+ async_to_sync(WatchRankConsumer(MagicMock()).process_batch)(consumer, messages)++ assert [c.kwargs["message"] for c in consumer.store_offsets.call_args_list] == [messages[i] for i in stored]+ (seek,) = consumer.seek.call_args_list+ assert seek.args[0].offset == rewind_from++ def test_a_deferred_failure_keeps_its_retry_window_in_the_next_batch(self) -> None:+ consumer = WatchRankConsumer(MagicMock())+ message = _result_message()++ with (+ patch(f"{_CONSUMER}.rank_observation", AsyncMock(side_effect=RuntimeError("postgres down"))),+ patch(f"{_CONSUMER}.close_old_connections"),+ ):+ for _ in range(2):+ assert async_to_sync(consumer.handle)(message, 0.0, time.monotonic()) == "deferred"diff --git a/products/replay_vision/backend/watch_feed.py b/products/replay_vision/backend/watch_feed.pyindex 4b5614574c78..192f539177e1 100644--- a/products/replay_vision/backend/watch_feed.py+++ b/products/replay_vision/backend/watch_feed.py@@ -5,6 +5,7 @@ own window), so "outlier" and "rare" mean unusual for that scanner lately, not a"""import re+import mathfrom collections import dequefrom datetime import datetimefrom statistics import fmean, stdev@@ -88,6 +89,8 @@ _FRICTION_RE = re.compile(class WatchFeedEntry:observation_id: UUIDreason: dict[str, Any]+ # Set only by rankers that space a recording's cards apart; None opts the entry out.+ session_id: str | None = None@frozen@@ -391,6 +394,77 @@ def _trim_filler(ranked: list[WatchFeedEntry]) -> list[WatchFeedEntry]:return kept+# One incident can put many near-identical watchable sessions on one scanner, and a pure score+# sort would fill the top of the feed with them. Hold each scanner to this share of the evidence+# tier as it is placed (same intent as WATCH_FEED_MAX_SIGNAL_SHARE in the weighted ranker), with a+# floor so a short feed is not over-constrained; the overflow trails the tier instead of leaving it.+MAX_SCANNER_SHARE = 0.4+_SCANNER_SPREAD_FLOOR = 3+++def spread_scanners(ordered: list[tuple[Any, WatchFeedEntry]]) -> list[WatchFeedEntry]:+ """Hold each scanner to MAX_SCANNER_SHARE of the evidence tier as it is placed.++ Checked per place rather than over the whole list because the view slices the head: a feed of 5+ obeys the same share as a feed of 50. Overflow trails the tier in probability order — reordering+ for breadth is the job here, shortening the feed is not.+ """+ placed: list[WatchFeedEntry] = []+ deferred: list[WatchFeedEntry] = []+ counts: dict[Any, int] = {}+ for scanner_id, entry in ordered:+ allowed = max(_SCANNER_SPREAD_FLOOR, math.ceil(MAX_SCANNER_SHARE * (len(placed) + 1)))+ if counts.get(scanner_id, 0) >= allowed:+ deferred.append(entry)+ else:+ counts[scanner_id] = counts.get(scanner_id, 0) + 1+ placed.append(entry)+ return placed + deferred+++# Several scanners can pick one recording. Its cards stay this many places apart, so the feed never+# shows the same session twice in a row.+SESSION_SPACING = 5+++def space_sessions(entries: list[WatchFeedEntry]) -> list[WatchFeedEntry]:+ """Keep each session's cards at least SESSION_SPACING places apart, reordering but never dropping one.++ A held card returns at the first place its session allows, so score order holds otherwise. A feed+ with too few other sessions to space every repeat still ends with every card, in score order.+ """+ placed: list[WatchFeedEntry] = []+ waiting: list[WatchFeedEntry] = []+ for entry in entries:+ waiting.append(entry)+ while waiting:+ recent = {placed_entry.session_id for placed_entry in placed[-(SESSION_SPACING - 1) :]} - {None}+ ready = next((held for held in waiting if held.session_id not in recent), None)+ if ready is None:+ break+ waiting.remove(ready)+ placed.append(ready)+ return placed + waiting+++def finish_feed(+ picks: list[tuple[float, Any, Any, WatchFeedEntry]], filler_rows: list[dict[str, Any]]+) -> list[WatchFeedEntry]:+ """Order `(score, created_at, scanner_id, entry)` picks with the scanner spread and session spacing,+ then pad with `filler_rows` newest and unviewed first, trimmed to the feed's floor."""+ ordered = sorted(picks, key=lambda item: (item[0], item[1]), reverse=True)+ filler: list[tuple[bool, Any, WatchFeedEntry]] = []+ for row in filler_rows:+ viewed = bool(row.get("feed_viewed"))+ entry = WatchFeedEntry(observation_id=row["id"], reason={"kind": "recent" if viewed else "unviewed_recent"})+ filler.append((not viewed, row["created_at"], entry))+ filler.sort(key=lambda item: (item[0], item[1]), reverse=True)+ return _trim_filler(+ space_sessions(spread_scanners([(scanner_id, entry) for _, _, scanner_id, entry in ordered]))+ + [entry for *_, entry in filler]+ )++def rank_watch_feed_candidates(rows: list[dict[str, Any]]) -> list[WatchFeedEntry]:"""Rank candidate rows (`id`, `scanner_id`, `created_at`, `scanner_result`, `feed_viewed`) mostwatchable first by a blended score, then newest.diff --git a/products/replay_vision/backend/watch_rank.py b/products/replay_vision/backend/watch_rank.pynew file mode 100644index 000000000000..2dc4d7d0f3f8--- /dev/null+++ b/products/replay_vision/backend/watch_rank.py@@ -0,0 +1,370 @@+"""What to watch ranking from observation embeddings.++A consumer of the embedding worker's results ranks each observation once, from the vector the scan+already embeds for search, so the feed needs no model call and no sweep. Each vector is compared with fixed anchor sentences: the+closest issue anchor names the category, and its lead over the closest routine anchor is the score.+The feed turns scores into a percentile within each scanner when it loads, so a scanner that sees+errors all the time does not fill the feed.+"""++import asyncio+from dataclasses import asdict+from datetime import datetime, timedelta+from typing import Any, Literal++from django.core.cache import cache+from django.utils import timezone++import numpy as np++from posthog.api.embedding_worker import async_generate_embedding+from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.client.connection import ClickHouseUser+from posthog.clickhouse.query_tagging import Product, tags_context+from posthog.clickhouse.workload import Workload+from posthog.dataclasses import frozen+from posthog.models.team import Team++from products.replay_vision.backend.embeddings import (+ OBSERVATION_EMBEDDING_MODEL,+ OBSERVATION_EMBEDDINGS_TABLE,+ OBSERVATION_SCOPE_PREWHERE,+ observation_scope_params,+)+from products.replay_vision.backend.models.replay_observation import ReplayObservation+from products.replay_vision.backend.watch_feed import WatchFeedEntry, finish_feed++WatchCategory = Literal["failure", "struggle", "revenue", "routine"]++# Bump when an anchor changes: scores from different anchor sets do not compare, so the feed+# ignores ranks written under another version until they age out of the window.+ANCHOR_VERSION = 1++ANCHORS: dict[WatchCategory, tuple[str, ...]] = {+ "failure": (+ "The user saw an error message and the action they tried failed.",+ "A request failed or the page crashed, showing an error to the user.",+ "The user clicked a button and nothing happened; the action silently did not work.",+ "The page got stuck loading and the user waited on a spinner that never finished.",+ "The user was blocked by a dead end, a 404 page, or an access denied message.",+ ),+ "struggle": (+ "The user repeated the same action several times because it did not give the result they expected.",+ "The user rage clicked and gave up on the task without finishing.",+ "The user hit a validation error or confusing state in a form and abandoned it.",+ "The user was stuck: they tried several different ways to do one thing and none worked.",+ "The user worked around a broken or missing feature to get their task done.",+ ),+ "revenue": (+ "The user went to billing to cancel their subscription or downgrade their plan.",+ "The user tried to upgrade or pay but the upgrade or payment failed.",+ "The user hit a usage limit that stopped a feature from working.",+ "The user contacted support frustrated about a problem.",+ ),+ "routine": (+ "The user browsed normally and completed what they came to do without any issues.",+ "A routine session with nothing notable: normal navigation and expected behavior.",+ "The user viewed a few pages and left; nothing went wrong.",+ "Nothing stands out in this session.",+ "Routine session with no errors or friction; the user did what they intended.",+ "The user browsed the pricing page and documentation normally.",+ "The user successfully completed the task without problems.",+ "The user explored different sections of the website to learn about the product.",+ "The user read through a product or documentation page.",+ "The user looked through a list and opened a few items to review them.",+ "The user started something and left before finishing, with no sign of a problem.",+ "The user never reached the part of the product this check is about.",+ ),+}+ISSUE_CATEGORIES: tuple[WatchCategory, ...] = ("failure", "struggle", "revenue")++# Struggle picks rated worth watching less often than failure and revenue picks in a blind rating.+CATEGORY_WEIGHT: dict[str, float] = {"failure": 1.0, "revenue": 1.0, "struggle": 0.85}++# Cosine distance below which two observations from one scanner describe the same thing.+DUPLICATE_MAX_DISTANCE = 0.1+DUPLICATE_LOOKBACK = timedelta(days=7)+_DUPLICATE_QUERY_TIMEOUT_S = 10++# The feed surfaces a scanner's top fifth by score, once the scanner has enough ranked rows to+# make a percentile mean something.+WATCH_PICK_MIN_PERCENTILE = 0.8+WATCH_PICK_MIN_SCANNER_ROWS = 10+WATCH_PICK_SEEN_PENALTY = 0.3++_ANCHOR_CACHE_TTL_S = 7 * 24 * 3600+++@frozen+class WatchRank:+ category: WatchCategory+ contrast: float+ duplicate_of: str | None+ anchor_version: int = ANCHOR_VERSION++ def to_json(self) -> dict[str, Any]:+ return asdict(self)+++_ANCHOR_CACHE_KEY = f"replay-vision:watch-rank-anchors:v{ANCHOR_VERSION}:{OBSERVATION_EMBEDDING_MODEL.value}"+++def normalize(matrix: np.ndarray) -> np.ndarray:+ return matrix / np.linalg.norm(matrix, axis=-1, keepdims=True)+++# The anchors never change within a version, so each process keeps them after the first read.+_anchors_in_process: dict[WatchCategory, np.ndarray] | None = None+++async def anchor_vectors(team_id: int) -> dict[WatchCategory, np.ndarray]:+ """Unit vectors for every anchor, embedded once and cached across teams.++ Embedded under the calling team because the worker gates every call on a team's AI opt-in, and+ a team that ranks observations has already passed that gate.+ """+ global _anchors_in_process+ if _anchors_in_process is None:+ _anchors_in_process = await _load_anchor_vectors(team_id)+ return _anchors_in_process+++async def _load_anchor_vectors(team_id: int) -> dict[WatchCategory, np.ndarray]:+ vectors: dict[WatchCategory, list[list[float]]] | None = await cache.aget(_ANCHOR_CACHE_KEY)+ if vectors is None:+ team = await Team.objects.aget(id=team_id)+ embedded = await asyncio.gather(+ *(+ async_generate_embedding(team, text, model=OBSERVATION_EMBEDDING_MODEL.value)+ for texts in ANCHORS.values()+ for text in texts+ )+ )+ rows = iter(response.embedding for response in embedded)+ vectors = {category: [next(rows) for _ in texts] for category, texts in ANCHORS.items()}+ await cache.aset(_ANCHOR_CACHE_KEY, vectors, _ANCHOR_CACHE_TTL_S)+ anchors = {category: normalize(np.array(rows, dtype=float)) for category, rows in vectors.items()}+ return anchors+++def classify(vector: list[float] | np.ndarray, anchors: dict[WatchCategory, np.ndarray]) -> tuple[WatchCategory, float]:+ """The closest issue category and its lead over the closest routine anchor, given unit anchors.++ A negative lead means the observation reads more like routine than like any issue, and the+ category is then routine.+ """+ unit = normalize(np.asarray(vector, dtype=float))+ closest = {category: float(np.max(anchors[category] @ unit)) for category in anchors}+ category = max(ISSUE_CATEGORIES, key=lambda c: closest[c])+ contrast = round(closest[category] - closest["routine"], 4)+ return (category if contrast > 0 else "routine"), contrast+++# Caps the vectors one duplicate lookup reads to the scanner's newest rows, so a busy scanner stays cheap.+DUPLICATE_CANDIDATE_ROWS = 5_000+++def nearest_earlier_observation(+ *, team_id: int, scanner_id: str, observation_id: str, vector: list[float], now: datetime+) -> tuple[str, float] | None:+ """The scanner's closest other observation in the lookback window, with its cosine distance."""+ with tags_context(product=Product.REPLAY_VISION, query_type="replay_vision_watch_rank_duplicate"):+ # nosemgrep: clickhouse-fstring-param-audit - table name and scope are module constants+ rows = sync_execute(+ f"""+ SELECT document_id, min(cosineDistance(embedding, %(embedding)s)) AS distance+ FROM {OBSERVATION_EMBEDDINGS_TABLE}+ PREWHERE {OBSERVATION_SCOPE_PREWHERE}+ AND timestamp >= ifNull(+ (+ SELECT minOrNull(timestamp)+ FROM (+ SELECT timestamp+ FROM {OBSERVATION_EMBEDDINGS_TABLE}+ PREWHERE {OBSERVATION_SCOPE_PREWHERE} AND timestamp >= %(since)s+ ORDER BY timestamp DESC+ LIMIT %(candidate_cap)s+ )+ ),+ %(since)s+ )+ WHERE document_id != %(observation_id)s+ GROUP BY document_id+ ORDER BY distance ASC+ LIMIT 1+ """,+ {+ **observation_scope_params(team_id, [scanner_id]),+ "embedding": vector,+ "since": now - DUPLICATE_LOOKBACK,+ "observation_id": observation_id,+ "candidate_cap": DUPLICATE_CANDIDATE_ROWS,+ },+ team_id=team_id,+ readonly=True,+ ch_user=ClickHouseUser.REPLAY_VISION,+ settings={"max_execution_time": _DUPLICATE_QUERY_TIMEOUT_S},+ workload=Workload.OFFLINE,+ )+ if not rows:+ return None+ document_id, distance = rows[0]+ return str(document_id), float(distance)+++def duplicate_root(observation_id: str, neighbour_id: str, neighbour_rank: dict[str, Any] | None) -> str | None:+ """The first observation of a duplicate group, so the group shares one id however long it grows.++ An unranked neighbour can be ranking at the same moment and pick this observation in turn, so the+ smaller id roots the pair and the two never point at each other.+ """+ if neighbour_rank is None:+ return neighbour_id if neighbour_id < observation_id else None+ root = neighbour_rank.get("duplicate_of")+ return root if isinstance(root, str) and root else neighbour_id+++def current_rank(row: dict[str, Any]) -> dict[str, Any] | None:+ """The row's rank when it comes from the current anchors, else None."""+ rank = row.get("watch_rank")+ if not isinstance(rank, dict) or rank.get("anchor_version") != ANCHOR_VERSION:+ return None+ contrast = rank.get("contrast")+ if not isinstance(contrast, int | float) or isinstance(contrast, bool):+ return None+ return rank+++@frozen+class RankRequest:+ team_id: int+ scanner_id: str+ observation_id: str+ vector: list[float]+++async def rank_observation(request: RankRequest) -> Literal["ranked", "already_ranked", "missing"]:+ """Rank one observation from its embedding and save the rank on the row, unless it has a current one.++ Keeping an existing rank makes a redelivered result harmless, and stops a second pass from moving a+ duplicate link in a way that could make two observations point at each other.+ """+ row = await (+ ReplayObservation.objects.filter(team_id=request.team_id, id=request.observation_id)+ .values("watch_rank")+ .afirst()+ )+ # A deleted observation, or a deleted team, leaves nothing to rank.+ if row is None:+ return "missing"+ if current_rank({"watch_rank": row["watch_rank"]}) is not None:+ return "already_ranked"+ category, contrast = classify(request.vector, await anchor_vectors(request.team_id))+ duplicate_of: str | None = None+ # The feed only groups issue rows, so a routine row skips the ClickHouse lookup.+ if category != "routine":+ nearest = await asyncio.to_thread(+ nearest_earlier_observation,+ team_id=request.team_id,+ scanner_id=request.scanner_id,+ observation_id=request.observation_id,+ vector=request.vector,+ now=timezone.now(),+ )+ if nearest is not None and nearest[1] <= DUPLICATE_MAX_DISTANCE:+ neighbour_rank = await (+ ReplayObservation.objects.filter(team_id=request.team_id, id=nearest[0])+ .values_list("watch_rank", flat=True)+ .afirst()+ )+ duplicate_of = duplicate_root(request.observation_id, nearest[0], neighbour_rank)+ rank = WatchRank(category=category, contrast=contrast, duplicate_of=duplicate_of)+ await ReplayObservation.objects.filter(team_id=request.team_id, id=request.observation_id).aupdate(+ watch_rank=rank.to_json()+ )+ return "ranked"+++def _scanner_percentiles(rows: list[dict[str, Any]]) -> dict[str, float]:+ """Each ranked row's score as a percentile among its scanner's ranked rows in the window."""+ by_scanner: dict[Any, list[tuple[float, str]]] = {}+ for row in rows:+ if rank := current_rank(row):+ by_scanner.setdefault(row["scanner_id"], []).append((float(rank["contrast"]), str(row["id"])))+ percentiles: dict[str, float] = {}+ for scored in by_scanner.values():+ if len(scored) < WATCH_PICK_MIN_SCANNER_ROWS:+ continue+ scored.sort()+ for position, (_, observation_id) in enumerate(scored):+ percentiles[observation_id] = (position + 0.5) / len(scored)+ return percentiles+++def _group_root(observation_id: str, parents: dict[str, str]) -> str:+ """Follow `duplicate_of` links to the group's first observation.++ A root ranked after its copies can gain a root of its own, so a link can point at a member rather+ than at the root. The walk stops at a repeat, so a cycle cannot loop.+ """+ seen = {observation_id}+ while (parent := parents.get(observation_id)) is not None and parent not in seen:+ seen.add(parent)+ observation_id = parent+ return observation_id+++def rank_watch_feed_by_embedding(rows: list[dict[str, Any]]) -> list[WatchFeedEntry]:+ """Rank candidate rows (`id`, `scanner_id`, `session_id`, `created_at`, `watch_rank`, `feed_viewed`)+ on the embedding rank the watch rank consumer wrote.++ A pick is a row in its scanner's top fifth by score whose category is an issue. Near-duplicates+ collapse to their best-scoring row, which carries the size of the group. Picks sort by weighted+ percentile, viewed rows docked, one scanner held to a share of the picks, and one session's picks+ spaced apart. Every other row is filler, trimmed to the same floor as the other rankers.+ """+ percentiles = _scanner_percentiles(rows)+ parents = {+ str(row["id"]): str(rank["duplicate_of"])+ for row in rows+ if (rank := current_rank(row)) is not None and rank.get("duplicate_of")+ }+ best_in_group: dict[str, tuple[float, dict[str, Any]]] = {}+ group_members: dict[str, list[str]] = {}+ for row in rows:+ rank = current_rank(row)+ observation_id = str(row["id"])+ percentile = percentiles.get(observation_id)+ if rank is None or percentile is None or rank.get("category") not in ISSUE_CATEGORIES:+ continue+ # Group before the cutoff, so duplicates below it still count toward the group and never return as filler.+ group = _group_root(observation_id, parents)+ group_members.setdefault(group, []).append(observation_id)+ # Only a member above the cutoff can stand for the group, so the seen penalty never disqualifies it.+ if percentile < WATCH_PICK_MIN_PERCENTILE:+ continue+ score = percentile * CATEGORY_WEIGHT[rank["category"]]+ if row.get("feed_viewed"):+ score -= WATCH_PICK_SEEN_PENALTY+ if group not in best_in_group or score > best_in_group[group][0]:+ best_in_group[group] = (score, row)+ grouped_ids = {member for group in best_in_group for member in group_members[group]}++ picks: list[tuple[float, Any, Any, WatchFeedEntry]] = []+ for group, (score, row) in best_in_group.items():+ rank = row["watch_rank"]+ reason = {+ "kind": "embedding_pick",+ "watch_category": rank["category"],+ "similar_count": len(group_members[group]) - 1,+ }+ picks.append(+ (+ score,+ row["created_at"],+ row.get("scanner_id"),+ WatchFeedEntry(observation_id=row["id"], reason=reason, session_id=row["session_id"]),+ )+ )+ # A pick stands for its whole duplicate group, so no member returns as filler.+ return finish_feed(picks, [row for row in rows if str(row["id"]) not in grouped_ids])diff --git a/products/replay_vision/backend/watch_rank_backfill.py b/products/replay_vision/backend/watch_rank_backfill.pynew file mode 100644index 000000000000..de0e70a07fb1--- /dev/null+++ b/products/replay_vision/backend/watch_rank_backfill.py@@ -0,0 +1,201 @@+"""Rank a window of past observations from the embeddings search already stored.++The watch rank consumer ranks each new observation as its embedding lands. This fills in the observations from before+that shipped, so a team can move to the embedding ranker without waiting a week for ranks to build+up. It reads stored vectors from ClickHouse rather than embedding again, and finds near-duplicates in+memory per scanner, oldest first, the same way the consumer would have found them.+"""++from collections import defaultdict+from datetime import datetime+from itertools import batched+from typing import Any++from django.db.models import Count++import numpy as np+import structlog+from asgiref.sync import async_to_sync++from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.client.connection import ClickHouseUser+from posthog.clickhouse.query_tagging import Product, tags_context+from posthog.clickhouse.workload import Workload+from posthog.dataclasses import frozen++from products.replay_vision.backend.embeddings import (+ OBSERVATION_EMBEDDINGS_TABLE,+ OBSERVATION_SCOPE_PREWHERE,+ observation_scope_params,+)+from products.replay_vision.backend.models.replay_observation import ObservationStatus, ReplayObservation+from products.replay_vision.backend.watch_rank import (+ DUPLICATE_CANDIDATE_ROWS,+ DUPLICATE_MAX_DISTANCE,+ WatchCategory,+ WatchRank,+ anchor_vectors,+ classify,+ current_rank,+ duplicate_root,+ normalize,+)++logger = structlog.get_logger(__name__)++_VECTOR_CHUNK = 500+_QUERY_TIMEOUT_S = 60+++@frozen+class WatchRankBackfillResult:+ observations: int+ ranked: int+ without_embedding: int+++def teams_with_observations(since: datetime) -> list[int]:+ """Teams with succeeded observations since `since`, busiest first."""+ return list(+ ReplayObservation.objects.filter(status=ObservationStatus.SUCCEEDED, created_at__gte=since)+ .values("team_id")+ .annotate(n=Count("id"))+ .order_by("-n")+ .values_list("team_id", flat=True)+ )+++def _stored_vectors(+ team_id: int, scanner_id: str, observation_ids: list[str], since: datetime+) -> dict[str, np.ndarray]:+ """Each observation's stored unit vector, for one batch of a scanner's observations."""+ with tags_context(product=Product.REPLAY_VISION, query_type="replay_vision_watch_rank_backfill"):+ # nosemgrep: clickhouse-fstring-param-audit - table name and scope are module constants+ rows = sync_execute(+ f"""+ SELECT document_id, any(embedding)+ FROM {OBSERVATION_EMBEDDINGS_TABLE}+ PREWHERE {OBSERVATION_SCOPE_PREWHERE}+ AND timestamp >= %(since)s+ WHERE document_id IN %(ids)s+ GROUP BY document_id+ """,+ {**observation_scope_params(team_id, [scanner_id]), "since": since, "ids": observation_ids},+ team_id=team_id,+ readonly=True,+ ch_user=ClickHouseUser.REPLAY_VISION,+ settings={"max_execution_time": _QUERY_TIMEOUT_S},+ workload=Workload.OFFLINE,+ )+ return {str(document_id): normalize(np.asarray(embedding, dtype=np.float32)) for document_id, embedding in rows}+++@frozen+class _Neighbour:+ observation_id: str+ rank: dict[str, Any] | None+ distance: float+++class _RecentVectors:+ """A scanner's newest vectors and ranks, capped like the live duplicate lookup so memory and work stay bounded."""++ def __init__(self) -> None:+ self._capacity = DUPLICATE_CANDIDATE_ROWS+ self._matrix: np.ndarray | None = None+ self._ids: list[str] = []+ self._ranks: list[dict[str, Any] | None] = []+ self._next = 0++ def nearest(self, vector: np.ndarray) -> _Neighbour | None:+ if self._matrix is None or not self._ids:+ return None+ distances = 1.0 - self._matrix[: len(self._ids)] @ vector+ index = int(np.argmin(distances))+ return _Neighbour(observation_id=self._ids[index], rank=self._ranks[index], distance=float(distances[index]))++ def add(self, observation_id: str, vector: np.ndarray, rank: dict[str, Any] | None) -> None:+ if self._matrix is None:+ self._matrix = np.empty((self._capacity, vector.shape[0]), dtype=np.float32)+ self._matrix[self._next] = vector+ if len(self._ids) < self._capacity:+ self._ids.append(observation_id)+ self._ranks.append(rank)+ else:+ self._ids[self._next] = observation_id+ self._ranks[self._next] = rank+ self._next = (self._next + 1) % self._capacity+++def _rank_scanner(+ rows: list[dict[str, Any]],+ vectors: dict[str, np.ndarray],+ anchors: dict[WatchCategory, np.ndarray],+ recent: _RecentVectors,+) -> dict[str, dict[str, Any]]:+ """New ranks for the rows that lack a current one, walking oldest first.++ Pass the same `recent` across a scanner's batches, so a copy finds a match from an earlier batch.+ """+ written: dict[str, dict[str, Any]] = {}+ for row in rows:+ observation_id = str(row["id"])+ vector = vectors.get(observation_id)+ if vector is None:+ continue+ rank = current_rank(row)+ if rank is None:+ category, contrast = classify(vector, anchors)+ duplicate_of: str | None = None+ nearest = recent.nearest(vector) if category != "routine" else None+ if nearest is not None and nearest.distance <= DUPLICATE_MAX_DISTANCE:+ duplicate_of = duplicate_root(observation_id, nearest.observation_id, nearest.rank)+ rank = WatchRank(category=category, contrast=contrast, duplicate_of=duplicate_of).to_json()+ written[observation_id] = rank+ recent.add(observation_id, vector, rank)+ return written+++def backfill_team_watch_ranks(team_id: int, since: datetime, *, dry_run: bool = False) -> WatchRankBackfillResult:+ """Rank the team's succeeded observations since `since` that lack a rank from the current anchors.++ Works one scanner and one batch at a time, and saves each batch before the next, so a large team+ never holds its whole window in memory and an interrupted run keeps what it finished.+ """+ rows: list[dict[str, Any]] = [+ dict(row)+ for row in ReplayObservation.objects.filter(+ team_id=team_id, status=ObservationStatus.SUCCEEDED, created_at__gte=since+ )+ .order_by("created_at", "id")+ .values("id", "scanner_id", "watch_rank")+ ]+ by_scanner: dict[str, list[dict[str, Any]]] = defaultdict(list)+ for row in rows:+ by_scanner[str(row["scanner_id"])].append(row)+ anchors = async_to_sync(anchor_vectors)(team_id)+ ranked = 0+ embedded = 0+ for scanner_id, scanner_rows in by_scanner.items():+ recent = _RecentVectors()+ for batch in batched(scanner_rows, _VECTOR_CHUNK, strict=False):+ vectors = _stored_vectors(team_id, scanner_id, [str(row["id"]) for row in batch], since)+ embedded += len(vectors)+ written = _rank_scanner(list(batch), vectors, anchors, recent)+ ranked += len(written)+ if not dry_run and written:+ # The ids come from the team-scoped query above, so the update needs no team filter of its own.+ ReplayObservation.objects.bulk_update(+ [ReplayObservation(id=observation_id, watch_rank=rank) for observation_id, rank in written.items()],+ ["watch_rank"],+ )+ without_embedding = len(rows) - embedded+ logger.info(+ "replay_vision.watch_rank_backfill.team_done",+ team_id=team_id,+ observations=len(rows),+ ranked=ranked,+ without_embedding=without_embedding,+ dry_run=dry_run,+ )+ return WatchRankBackfillResult(observations=len(rows), ranked=ranked, without_embedding=without_embedding)diff --git a/products/replay_vision/backend/watch_rank_consumer.py b/products/replay_vision/backend/watch_rank_consumer.pynew file mode 100644index 000000000000..8ce94d2a6d1c--- /dev/null+++ b/products/replay_vision/backend/watch_rank_consumer.py@@ -0,0 +1,235 @@+"""Ranks observations for What to watch from the embedding worker's results topic.++The scan emits each observation's text to the embedding worker over Kafka. The worker publishes every+embedded document to the results topic, and this consumer ranks the Replay Vision ones from their+vectors. A worker outage only delays ranks, and a database outage shorter than the retry window too.+"""++import time+import signal+import asyncio+from collections.abc import Sequence+from typing import Any, Literal, Protocol++from django.conf import settings+from django.db import close_old_connections++import orjson+import structlog+from asgiref.sync import sync_to_async+from confluent_kafka import Consumer, KafkaException, Message, TopicPartition+from prometheus_client import Counter++from posthog.kafka_client import helper+from posthog.kafka_client.client import _build_sasl_config, _KafkaSecurityProtocol+from posthog.kafka_client.routing import get_profile_settings+from posthog.kafka_client.topics import KAFKA_DOCUMENT_EMBEDDING_RESULTS_TOPIC+from posthog.temporal.common.liveness_tracker import LivenessTracker++from products.replay_vision.backend.embeddings import (+ EMBEDDING_DOCUMENT_TYPE,+ EMBEDDING_PRODUCT,+ OBSERVATION_EMBEDDING_MODEL,+)+from products.replay_vision.backend.watch_rank import RankRequest, rank_observation++logger = structlog.get_logger(__name__)++CONSUMER_GROUP = "replay-vision-watch-rank"++_BATCH_SIZE = 50+_POLL_TIMEOUT_S = 1.0+# A batch stops after this long and goes back to Kafka, so the loop polls well inside the poll interval.+_BATCH_BUDGET_S = 240.0+_MAX_POLL_INTERVAL_S = 360.0+# ClickHouse loads the embeddings table from its own topic some seconds after the worker writes both.+_CLICKHOUSE_VISIBILITY_DELAY_S = 30.0+# How long one observation keeps failing, across batches, before the consumer moves past it.+_RETRY_WINDOW_S = 600.0+_RETRY_BACKOFF_S = 2.0+_RETRY_BACKOFF_MAX_S = 60.0++Outcome = Literal["skipped", "ranked", "already_ranked", "missing", "failed", "deferred", "interrupted"]++MESSAGES_TOTAL = Counter(+ "replay_vision_watch_rank_consumer_messages_total",+ "Embedding results the What to watch rank consumer read, by outcome.",+ labelnames=["outcome"],+)++# The worker writes compact JSON, and a quote inside a string value is escaped, so this marks our results.+_DOCUMENT_TYPE_MARKER = f'"document_type":"{EMBEDDING_DOCUMENT_TYPE}"'.encode()+++def parse_rank_request(value: bytes | None) -> RankRequest | None:+ """The rank request in one results message, or None when it is not a Replay Vision observation."""+ # Most messages belong to other products, and decoding their content and vectors is the expensive part.+ if value is None or _DOCUMENT_TYPE_MARKER not in value:+ return None+ try:+ payload = orjson.loads(value)+ except orjson.JSONDecodeError:+ return None+ if not isinstance(payload, dict):+ return None+ if payload.get("product") != EMBEDDING_PRODUCT or payload.get("document_type") != EMBEDDING_DOCUMENT_TYPE:+ return None+ metadata = payload.get("metadata")+ scanner_id = metadata.get("scanner_id") if isinstance(metadata, dict) else None+ vector = next(+ (+ result.get("embedding")+ for result in payload.get("results") or []+ if isinstance(result, dict)+ and result.get("model") == OBSERVATION_EMBEDDING_MODEL.value+ and result.get("outcome") == "success"+ ),+ None,+ )+ team_id = payload.get("team_id")+ observation_id = payload.get("document_id")+ if not isinstance(team_id, int) or not isinstance(scanner_id, str) or not isinstance(observation_id, str):+ return None+ # A zero vector has no direction, so the cosine scores would come out as NaN.+ if not isinstance(vector, list) or not any(vector):+ return None+ return RankRequest(team_id=team_id, scanner_id=scanner_id, observation_id=observation_id, vector=vector)+++class BatchConsumer(Protocol):+ def store_offsets(self, *, message: Message) -> None: ...++ def seek(self, partition: TopicPartition, /) -> None: ...+++class WatchRankConsumer:+ def __init__(self, liveness: LivenessTracker) -> None:+ self._liveness = liveness+ self._stop = asyncio.Event()+ # When each observation that is still failing first failed, so its retry window spans batches.+ self._first_failed_at: dict[str, float] = {}++ async def _wait(self, seconds: float) -> bool:+ """Sleep, unless shutdown comes first. True when the full time passed."""+ try:+ await asyncio.wait_for(self._stop.wait(), timeout=max(seconds, 0))+ except TimeoutError:+ return True+ return False++ async def handle(self, value: bytes | None, produced_at: float, batch_deadline: float) -> Outcome:+ """Rank the observation in one message, retrying failures until the batch or retry window ends."""+ request = parse_rank_request(value)+ if request is None:+ return "skipped"+ if not await self._wait(produced_at + _CLICKHOUSE_VISIBILITY_DELAY_S - time.time()):+ return "interrupted"+ backoff = _RETRY_BACKOFF_S+ while True:+ # Async ORM calls share one thread, so a connection that broke must be replaced before a retry.+ await sync_to_async(close_old_connections)()+ try:+ outcome = await rank_observation(request)+ self._first_failed_at.pop(request.observation_id, None)+ return outcome+ except Exception:+ logger.exception(+ "replay_vision.watch_rank_consumer.rank_failed",+ team_id=request.team_id,+ observation_id=request.observation_id,+ )+ self._liveness.record_heartbeat()+ now = time.monotonic()+ give_up_at = self._first_failed_at.setdefault(request.observation_id, now) + _RETRY_WINDOW_S+ if now + backoff > give_up_at:+ # The backfill command can write a rank this gave up on.+ self._first_failed_at.pop(request.observation_id, None)+ return "failed"+ if now + backoff > batch_deadline:+ return "deferred"+ if not await self._wait(backoff):+ return "interrupted"+ backoff = min(backoff * 2, _RETRY_BACKOFF_MAX_S)++ async def process_batch(self, consumer: BatchConsumer, messages: Sequence[Message]) -> None:+ batch_deadline = time.monotonic() + _BATCH_BUDGET_S+ for index, message in enumerate(messages):+ if self._stop.is_set():+ return+ if message.error():+ logger.warning("replay_vision.watch_rank_consumer.kafka_error", error=str(message.error()))+ continue+ _, produced_at_ms = message.timestamp()+ outcome = await self.handle(message.value(), produced_at_ms / 1000, batch_deadline)+ self._liveness.record_heartbeat()+ if outcome == "interrupted":+ return+ if outcome == "deferred":+ self._rewind(consumer, messages[index:])+ return+ MESSAGES_TOTAL.labels(outcome=outcome).inc()+ try:+ consumer.store_offsets(message=message)+ except KafkaException:+ logger.warning("replay_vision.watch_rank_consumer.offset_store_failed", partition=message.partition())+ self._rewind(consumer, messages[index + 1 :])+ return++ @staticmethod+ def _rewind(consumer: BatchConsumer, unhandled: Sequence[Message]) -> None:+ """Seek each partition back to its first unhandled message, so the next poll reads it again."""+ first: dict[tuple[str, int], int] = {}+ for message in unhandled:+ topic, partition = message.topic(), message.partition()+ if topic is not None and partition is not None:+ first.setdefault((topic, partition), message.offset() or 0)+ for (topic, partition), offset in first.items():+ try:+ consumer.seek(TopicPartition(topic, partition, offset))+ except KafkaException:+ # A rebalance took the partition, and its new owner starts from the last committed offset.+ continue++ @staticmethod+ def _consumer_config() -> dict[str, Any]:+ profile = get_profile_settings(topic=KAFKA_DOCUMENT_EMBEDDING_RESULTS_TOPIC)+ security_protocol = profile.security_protocol+ # Self-hosted base64 cert mode supplies its own SSL material, so SASL credentials must not attach.+ if settings.KAFKA_BASE64_KEYS:+ security_protocol = _KafkaSecurityProtocol.SSL+ config: dict[str, Any] = {+ "bootstrap.servers": ",".join(profile.hosts),+ "group.id": CONSUMER_GROUP,+ # Ranking skips rows that already have a current rank, so reading old results again is harmless.+ "auto.offset.reset": "earliest",+ # Offsets are stored only after a message is handled, and committed in the background.+ "enable.auto.commit": True,+ "enable.auto.offset.store": False,+ "max.poll.interval.ms": int(_MAX_POLL_INTERVAL_S * 1000),+ # The worker writes results inside Kafka transactions, and an aborted one must never rank.+ "isolation.level": "read_committed",+ "security.protocol": security_protocol or _KafkaSecurityProtocol.PLAINTEXT,+ }+ config.update(+ _build_sasl_config(security_protocol, profile.sasl_mechanism, profile.sasl_user, profile.sasl_password)+ )+ if settings.KAFKA_BASE64_KEYS:+ config.update(helper.ssl_cert_config())+ return config++ async def run(self) -> None: # pragma: no cover - exercised in dogfood+ loop = asyncio.get_running_loop()+ for sig in (signal.SIGINT, signal.SIGTERM):+ loop.add_signal_handler(sig, self._stop.set)++ consumer = Consumer(self._consumer_config())+ consumer.subscribe([KAFKA_DOCUMENT_EMBEDDING_RESULTS_TOPIC])+ logger.info("replay_vision.watch_rank_consumer.started")+ try:+ while not self._stop.is_set():+ self._liveness.record_heartbeat()+ messages = await asyncio.to_thread(consumer.consume, _BATCH_SIZE, _POLL_TIMEOUT_S)+ await self.process_batch(consumer, messages)+ finally:+ consumer.close()+ logger.info("replay_vision.watch_rank_consumer.stopped")diff --git a/products/replay_vision/frontend/generated/api.schemas.ts b/products/replay_vision/frontend/generated/api.schemas.tsindex 547447029c7d..ae8506b988b9 100644--- a/products/replay_vision/frontend/generated/api.schemas.ts+++ b/products/replay_vision/frontend/generated/api.schemas.ts@@ -2984,6 +2984,7 @@ export interface SuggestTagsResponseApi {* * `novel_summary` - Novel Summary* * `friction` - Friction* * `jev_watchable` - Jev Watchable+ * * `embedding_pick` - Embedding Pick* * `unviewed_recent` - Unviewed Recent* * `recent` - Recent*/@@ -2999,6 +3000,7 @@ export const WatchFeedReasonEnumApi = {NovelSummary: 'novel_summary',Friction: 'friction',JevWatchable: 'jev_watchable',+ EmbeddingPick: 'embedding_pick',UnviewedRecent: 'unviewed_recent',Recent: 'recent',} as const@@ -3042,11 +3044,24 @@ export const JevWatchReasonEnumApi = {Success: 'success',} as const+/**+ * * `failure` - Failure+ * * `struggle` - Struggle+ * * `revenue` - Revenue+ */+export type WatchFeedCategoryEnumApi = (typeof WatchFeedCategoryEnumApi)[keyof typeof WatchFeedCategoryEnumApi]++export const WatchFeedCategoryEnumApi = {+ Failure: 'failure',+ Struggle: 'struggle',+ Revenue: 'revenue',+} as const+/*** Machine-readable reason an observation made the feed; the frontend renders the copy.*/export interface WatchFeedReasonApi {- /** Highest-priority rule the observation satisfied: `signal_emitted` (it pushed a signal), `unusual_verdict` (a monitor answer that is the minority for that scanner this window), `verdict_yes` (a monitor hit, when the window is too thin to know which answer is unusual), `outlier_score` (far from the scanner's window average), `rare_tag` (a tag uncommon for the scanner this window), `novel_summary` (a summary that reads unlike the scanner's other sessions this window), `notable` (the scan itself judged the session worth watching), `friction` (the scan describes errors, retries, or dead ends), `jev_watchable` (the decision model judged the session worth watching; teams on the Jev ranker experiment only), `unviewed_recent` (new to you), `recent` (nothing special, newest available).+ /** Highest-priority rule the observation satisfied: `signal_emitted` (it pushed a signal), `unusual_verdict` (a monitor answer that is the minority for that scanner this window), `verdict_yes` (a monitor hit, when the window is too thin to know which answer is unusual), `outlier_score` (far from the scanner's window average), `rare_tag` (a tag uncommon for the scanner this window), `novel_summary` (a summary that reads unlike the scanner's other sessions this window), `notable` (the scan itself judged the session worth watching), `friction` (the scan describes errors, retries, or dead ends), `jev_watchable` (the decision model judged the session worth watching; teams on the Jev ranker experiment only), `embedding_pick` (the session reads like a failure, struggle, or revenue moment and ranks in its scanner's top fifth; teams on the embedding ranker only), `unviewed_recent` (new to you), `recent` (nothing special, newest available).** * `signal_emitted` - Signal Emitted* * `unusual_verdict` - Unusual Verdict@@ -3057,6 +3072,7 @@ export interface WatchFeedReasonApi {* * `novel_summary` - Novel Summary* * `friction` - Friction* * `jev_watchable` - Jev Watchable+ * * `embedding_pick` - Embedding Pick* * `unviewed_recent` - Unviewed Recent* * `recent` - Recent */kind: WatchFeedReasonEnumApi@@ -3128,6 +3144,17 @@ export interface WatchFeedReasonApi {* @nullable*/tag_share?: number | null+ /** What the session reads like, for `embedding_pick`.+ *+ * * `failure` - Failure+ * * `struggle` - Struggle+ * * `revenue` - Revenue */+ watch_category?: WatchFeedCategoryEnumApi+ /**+ * Other sessions from the same scanner in the window that describe nearly the same thing, for `embedding_pick`. The feed shows one session per group.+ * @nullable+ */+ similar_count?: number | null}/**@@ -3141,14 +3168,16 @@ export interface WatchFeedItemApi {}/**- * * `weighted-score` - weighted-score- * * `jev` - jev+ * * `weighted-score` - Weighted Score+ * * `jev` - Jev+ * * `embedding` - Embedding*/-export type RankerEnumApi = (typeof RankerEnumApi)[keyof typeof RankerEnumApi]+export type WatchFeedRankerEnumApi = (typeof WatchFeedRankerEnumApi)[keyof typeof WatchFeedRankerEnumApi]-export const RankerEnumApi = {+export const WatchFeedRankerEnumApi = {WeightedScore: 'weighted-score',Jev: 'jev',+ Embedding: 'embedding',} as const/**@@ -3157,13 +3186,14 @@ export const RankerEnumApi = {export interface WatchFeedResponseApi {/** Succeeded observations in the window worth watching, most interesting first, each carrying the reason it ranked. Every observation that carries a finding is returned; observations that carry none (`unviewed_recent`, `recent`) are returned only to pad a near-empty feed to three items, so a quiet window answers with a handful of rows rather than a full page of newest clips. */results: WatchFeedItemApi[]- /** Which ranker ordered this feed: `jev` ranks on the decision model's cached judgments, `weighted-score` on the deterministic blend. The arm is decided server-side per team, so clients read it from here rather than evaluating the flag themselves.+ /** Which ranker ordered this feed: `jev` ranks on the decision model's cached judgments, `embedding` on the rank the watch rank consumer wrote from the observation's embedding, `weighted-score` on the deterministic blend. The arm is decided server-side per team, so clients read it from here rather than evaluating the flag themselves.*- * * `weighted-score` - weighted-score- * * `jev` - jev */- ranker: RankerEnumApi+ * * `weighted-score` - Weighted Score+ * * `jev` - Jev+ * * `embedding` - Embedding */+ ranker: WatchFeedRankerEnumApi/**- * The team's variant of the `vision-watch-feed-ranker` experiment flag (`control`, `jev-shadow`, `jev`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from control. Clients report it on the feed-viewed event as `$feature/vision-watch-feed-ranker`, which is the exposure the experiment counts.+ * The team's variant of the `vision-watch-feed-ranker` experiment flag (`control`, `jev-shadow`, `jev`, `embedding`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from control. Clients report it on the feed-viewed event as `$feature/vision-watch-feed-ranker`, which is the exposure the experiment counts.* @nullable*/ranker_variant: string | nulldiff --git a/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.test.ts b/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.test.tsindex da626799a3c3..ea0f5ab50ddf 100644--- a/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.test.ts+++ b/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.test.ts@@ -125,6 +125,16 @@ describe('WatchFeedCard helpers', () => {reason: { kind: 'rare_tag' } as WatchFeedReasonApi,expected: 'Tagged something uncommon for this scanner lately.',},+ {+ name: 'names the category of an embedding pick with no similar sessions',+ reason: { kind: 'embedding_pick', watch_category: 'failure', similar_count: 0 } as WatchFeedReasonApi,+ expected: 'Something failed in this session.',+ },+ {+ name: 'counts the similar sessions behind an embedding pick',+ reason: { kind: 'embedding_pick', watch_category: 'struggle', similar_count: 1 } as WatchFeedReasonApi,+ expected: 'The user struggled in this session. 1 other session from this scanner looks similar.',+ },{name: 'renders a sentence for a reason kind this frontend does not know',reason: { kind: 'some_future_kind' } as unknown as WatchFeedReasonApi,diff --git a/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.tsx b/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.tsxindex 5330af3699c3..f04afddb0897 100644--- a/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.tsx+++ b/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.tsx@@ -17,6 +17,7 @@ import { UnviewedObservationTag } from '../../components/UnviewedObservationTag'import type {JevWatchReasonEnumApi,ReplayObservationApi,+ WatchFeedCategoryEnumApi,WatchFeedItemApi,WatchFeedReasonApi,} from '../../generated/api.schemas'@@ -49,6 +50,12 @@ const MAX_SHOWN_HEADLINES = 3const joinWithAnd = (parts: string[]): string =>parts.length > 1 ? `${parts.slice(0, -1).join(', ')}, and ${parts[parts.length - 1]}` : parts[0]+const WATCH_CATEGORY_COPY: Record<WatchFeedCategoryEnumApi, string> = {+ failure: 'Something failed in this session.',+ struggle: 'The user struggled in this session.',+ revenue: 'This session touches billing or a plan limit.',+}+/** What the reason kind says about the session. Never the scan's notability sentence: the row's title* already leads with it. */export function watchReasonCopy(reason: WatchFeedReasonApi): string {@@ -133,6 +140,16 @@ export function watchReasonCopy(reason: WatchFeedReasonApi): string {return 'The session shows signs of friction, like errors, retries, or dead ends.'case 'jev_watchable':return 'The decision model judged this session worth watching.'+ case 'embedding_pick': {+ const category = reason.watch_category+ ? WATCH_CATEGORY_COPY[reason.watch_category]+ : 'This session stands out for this scanner.'+ const similar = reason.similar_count ?? 0+ if (similar === 0) {+ return category+ }+ return `${category} ${similar} other session${similar === 1 ? ' from this scanner looks' : 's from this scanner look'} similar.`+ }case 'unviewed_recent':return 'New since you last looked.'case 'recent':diff --git a/products/replay_vision/frontend/replay_scanners/watchFeedLogic.ts b/products/replay_vision/frontend/replay_scanners/watchFeedLogic.tsindex 986eee0ce243..9b23e14849f0 100644--- a/products/replay_vision/frontend/replay_scanners/watchFeedLogic.ts+++ b/products/replay_vision/frontend/replay_scanners/watchFeedLogic.ts@@ -15,7 +15,7 @@ import { visionScannersWatchFeedRetrieve } from '../generated/api'import type { ReplayScannerApi, VisionQuotaApi, WatchFeedResponseApi } from '../generated/api.schemas'import type { VisionScannersWatchFeedRetrieveParams, WatchFeedItemApi } from '../generated/api.schemas'import type { ScannerTypeEnumApi } from '../generated/api.schemas'-import type { RankerEnumApi } from '../generated/api.schemas'+import type { WatchFeedRankerEnumApi } from '../generated/api.schemas'import type { FeedDismissalReasonEnumApi } from '../generated/api.schemas'import { visionQuotaLogic } from '../logics/visionQuotaLogic'import { visionScannersListLogic } from '../logics/visionScannersListLogic'@@ -139,7 +139,7 @@ export interface watchFeedLogicActions {ranker: WatchFeedRanker,variant: string | null) => {- ranker: RankerEnumApi+ ranker: WatchFeedRankerEnumApivariant: string | null}setScannerIdsFilter: (scannerIds: string[]) => {diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 0ce3db17b019..395b50059e0b 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -100220,18 +100220,6 @@ export namespace Schemas {code_usage_billing_active: boolean;}- /**- * * `weighted-score` - weighted-score- * * `jev` - jev- */- export type RankerEnum = typeof RankerEnum[keyof typeof RankerEnum];--- export const RankerEnum = {- WeightedScore: 'weighted-score',- Jev: 'jev',- } as const;-export interface RankingOutcomeInfo {/** Name of the outcome in the `ranking` field's predictions.*@@ -120369,6 +120357,20 @@ export namespace Schemas {run_id: string;}+ /**+ * * `failure` - Failure+ * * `struggle` - Struggle+ * * `revenue` - Revenue+ */+ export type WatchFeedCategoryEnum = typeof WatchFeedCategoryEnum[keyof typeof WatchFeedCategoryEnum];+++ export const WatchFeedCategoryEnum = {+ Failure: 'failure',+ Struggle: 'struggle',+ Revenue: 'revenue',+ } as const;+/*** * `signal_emitted` - Signal Emitted* * `unusual_verdict` - Unusual Verdict@@ -120379,6 +120381,7 @@ export namespace Schemas {* * `novel_summary` - Novel Summary* * `friction` - Friction* * `jev_watchable` - Jev Watchable+ * * `embedding_pick` - Embedding Pick* * `unviewed_recent` - Unviewed Recent* * `recent` - Recent*/@@ -120395,6 +120398,7 @@ export namespace Schemas {NovelSummary: 'novel_summary',Friction: 'friction',JevWatchable: 'jev_watchable',+ EmbeddingPick: 'embedding_pick',UnviewedRecent: 'unviewed_recent',Recent: 'recent',} as const;@@ -120413,7 +120417,7 @@ export namespace Schemas {* Machine-readable reason an observation made the feed; the frontend renders the copy.*/export interface WatchFeedReason {- /** Highest-priority rule the observation satisfied: `signal_emitted` (it pushed a signal), `unusual_verdict` (a monitor answer that is the minority for that scanner this window), `verdict_yes` (a monitor hit, when the window is too thin to know which answer is unusual), `outlier_score` (far from the scanner's window average), `rare_tag` (a tag uncommon for the scanner this window), `novel_summary` (a summary that reads unlike the scanner's other sessions this window), `notable` (the scan itself judged the session worth watching), `friction` (the scan describes errors, retries, or dead ends), `jev_watchable` (the decision model judged the session worth watching; teams on the Jev ranker experiment only), `unviewed_recent` (new to you), `recent` (nothing special, newest available).+ /** Highest-priority rule the observation satisfied: `signal_emitted` (it pushed a signal), `unusual_verdict` (a monitor answer that is the minority for that scanner this window), `verdict_yes` (a monitor hit, when the window is too thin to know which answer is unusual), `outlier_score` (far from the scanner's window average), `rare_tag` (a tag uncommon for the scanner this window), `novel_summary` (a summary that reads unlike the scanner's other sessions this window), `notable` (the scan itself judged the session worth watching), `friction` (the scan describes errors, retries, or dead ends), `jev_watchable` (the decision model judged the session worth watching; teams on the Jev ranker experiment only), `embedding_pick` (the session reads like a failure, struggle, or revenue moment and ranks in its scanner's top fifth; teams on the embedding ranker only), `unviewed_recent` (new to you), `recent` (nothing special, newest available).** * `signal_emitted` - Signal Emitted* * `unusual_verdict` - Unusual Verdict@@ -120424,6 +120428,7 @@ export namespace Schemas {* * `novel_summary` - Novel Summary* * `friction` - Friction* * `jev_watchable` - Jev Watchable+ * * `embedding_pick` - Embedding Pick* * `unviewed_recent` - Unviewed Recent* * `recent` - Recent */kind: WatchFeedReasonEnum;@@ -120495,6 +120500,17 @@ export namespace Schemas {* @nullable*/tag_share?: number | null;+ /** What the session reads like, for `embedding_pick`.+ *+ * * `failure` - Failure+ * * `struggle` - Struggle+ * * `revenue` - Revenue */+ watch_category?: WatchFeedCategoryEnum;+ /**+ * Other sessions from the same scanner in the window that describe nearly the same thing, for `embedding_pick`. The feed shows one session per group.+ * @nullable+ */+ similar_count?: number | null;}/**@@ -120507,19 +120523,34 @@ export namespace Schemas {reason: WatchFeedReason;}+ /**+ * * `weighted-score` - Weighted Score+ * * `jev` - Jev+ * * `embedding` - Embedding+ */+ export type WatchFeedRankerEnum = typeof WatchFeedRankerEnum[keyof typeof WatchFeedRankerEnum];+++ export const WatchFeedRankerEnum = {+ WeightedScore: 'weighted-score',+ Jev: 'jev',+ Embedding: 'embedding',+ } as const;+/*** Response of GET /vision/scanners/watch_feed/.*/export interface WatchFeedResponse {/** Succeeded observations in the window worth watching, most interesting first, each carrying the reason it ranked. Every observation that carries a finding is returned; observations that carry none (`unviewed_recent`, `recent`) are returned only to pad a near-empty feed to three items, so a quiet window answers with a handful of rows rather than a full page of newest clips. */results: WatchFeedItem[];- /** Which ranker ordered this feed: `jev` ranks on the decision model's cached judgments, `weighted-score` on the deterministic blend. The arm is decided server-side per team, so clients read it from here rather than evaluating the flag themselves.+ /** Which ranker ordered this feed: `jev` ranks on the decision model's cached judgments, `embedding` on the rank the watch rank consumer wrote from the observation's embedding, `weighted-score` on the deterministic blend. The arm is decided server-side per team, so clients read it from here rather than evaluating the flag themselves.*- * * `weighted-score` - weighted-score- * * `jev` - jev */- ranker: RankerEnum;+ * * `weighted-score` - Weighted Score+ * * `jev` - Jev+ * * `embedding` - Embedding */+ ranker: WatchFeedRankerEnum;/**- * The team's variant of the `vision-watch-feed-ranker` experiment flag (`control`, `jev-shadow`, `jev`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from control. Clients report it on the feed-viewed event as `$feature/vision-watch-feed-ranker`, which is the exposure the experiment counts.+ * The team's variant of the `vision-watch-feed-ranker` experiment flag (`control`, `jev-shadow`, `jev`, `embedding`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from control. Clients report it on the feed-viewed event as `$feature/vision-watch-feed-ranker`, which is the exposure the experiment counts.* @nullable*/ranker_variant: string | null;