Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-vision): rank what to watch from observation embeddings (#122748)

Tue Haulund77c5f52cfec9
diff --git a/bin/mprocs.yaml b/bin/mprocs.yaml
index 6cbcb453f342..403d094d1030 100755
--- a/bin/mprocs.yaml
+++ b/bin/mprocs.yaml
@@ -313,6 +313,16 @@ procs:
layer: Product services
tech: Rust
+ replay-vision-watch-rank:
+ shell: |-
+ bin/wait-for-docker && \
+ python manage.py run_replay_vision_watch_rank_consumer --health-port 8092 --metrics-port 8093
+ capability: embedding_service
+ ready_pattern: 'replay_vision.watch_rank_consumer.started'
+ groups:
+ layer: Product services
+ tech: Python
+
feature-flags:
shell: |-
bin/wait-for-docker && \
diff --git a/products/replay_vision/backend/api/scanners.py b/products/replay_vision/backend/api/scanners.py
index b4455a9f2cf7..713fc56577dd 100644
--- a/products/replay_vision/backend/api/scanners.py
+++ b/products/replay_vision/backend/api/scanners.py
@@ -188,6 +188,7 @@ from products.replay_vision.backend.temporal.constants import VISION_SIGNALS_SOU
from products.replay_vision.backend.temporal.metrics import record_estimate_outcome, record_scanner_limit_reached
from products.replay_vision.backend.temporal.read_meter_types import current_sweep_throttle_factor
from products.replay_vision.backend.watch_feed import rank_watch_feed_candidates
+from products.replay_vision.backend.watch_rank import rank_watch_feed_by_embedding
from products.signals.backend.facade.api import get_outcomes_for_signal_source_slice
# Date is set by the schedule at trigger time, not by the user — strip on save.
@@ -1644,6 +1645,18 @@ WATCH_FEED_PER_SCANNER_CAP = 100
WATCH_FEED_BYPASS_CHUNK = 200
+class WatchFeedRanker(models.TextChoices):
+ WEIGHTED_SCORE = "weighted-score"
+ JEV = "jev"
+ EMBEDDING = "embedding"
+
+
+class WatchFeedCategory(models.TextChoices):
+ FAILURE = "failure"
+ STRUGGLE = "struggle"
+ REVENUE = "revenue"
+
+
class WatchFeedReason(models.TextChoices):
SIGNAL_EMITTED = "signal_emitted"
UNUSUAL_VERDICT = "unusual_verdict"
@@ -1654,6 +1667,7 @@ class WatchFeedReason(models.TextChoices):
NOVEL_SUMMARY = "novel_summary"
FRICTION = "friction"
JEV_WATCHABLE = "jev_watchable"
+ EMBEDDING_PICK = "embedding_pick"
UNVIEWED_RECENT = "unviewed_recent"
RECENT = "recent"
@@ -1762,6 +1776,8 @@ class WatchFeedReasonSerializer(serializers.Serializer):
"session worth watching), `friction` (the scan describes errors, retries, or dead ends), "
"`jev_watchable` (the decision model judged the session worth watching; teams on the "
"Jev ranker experiment only), "
+ "`embedding_pick` (the session reads like a failure, struggle, or revenue moment and ranks in "
+ "its scanner's top fifth; teams on the embedding ranker only), "
"`unviewed_recent` (new to you), `recent` (nothing special, newest available)."
),
)
@@ -1834,6 +1850,19 @@ class WatchFeedReasonSerializer(serializers.Serializer):
allow_null=True,
help_text="Share (0-1) of the scanner's window observations carrying `tag`, for `rare_tag`.",
)
+ watch_category = serializers.ChoiceField(
+ choices=WatchFeedCategory.choices,
+ required=False,
+ help_text="What the session reads like, for `embedding_pick`.",
+ )
+ similar_count = serializers.IntegerField(
+ required=False,
+ allow_null=True,
+ help_text=(
+ "Other sessions from the same scanner in the window that describe nearly the same thing, for "
+ "`embedding_pick`. The feed shows one session per group."
+ ),
+ )
class WatchFeedItemSerializer(serializers.Serializer):
@@ -1856,9 +1885,10 @@ class WatchFeedResponseSerializer(serializers.Serializer):
),
)
ranker = serializers.ChoiceField(
- choices=["weighted-score", "jev"],
+ choices=WatchFeedRanker.choices,
help_text=(
"Which ranker ordered this feed: `jev` ranks on the decision model's cached judgments, "
+ "`embedding` on the rank the watch rank consumer wrote from the observation's embedding, "
"`weighted-score` on the deterministic blend. The arm is decided server-side per team, so "
"clients read it from here rather than evaluating the flag themselves."
),
@@ -1867,7 +1897,7 @@ class WatchFeedResponseSerializer(serializers.Serializer):
allow_null=True,
help_text=(
"The team's variant of the `vision-watch-feed-ranker` experiment flag (`control`, `jev-shadow`, "
- "`jev`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from "
+ "`jev`, `embedding`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from "
"control. Clients report it on the feed-viewed event as `$feature/vision-watch-feed-ranker`, "
"which is the exposure the experiment counts."
),
@@ -2611,7 +2641,7 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi
),
)
.filter(scanner_recency_rank__lte=WATCH_FEED_PER_SCANNER_CAP)
- .values("id", "scanner_id", "created_at", "scanner_result", "feed_viewed")
+ .values("id", "scanner_id", "session_id", "created_at", "scanner_result", "watch_rank", "feed_viewed")
.order_by("-created_at", "-id")[:WATCH_FEED_CANDIDATE_CAP]
)
# The flag selects one of two independent rankers; nothing is blended between them. Shadow
@@ -2620,8 +2650,11 @@ class ReplayScannerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, vi
# ranker that ordered it, so the shadow arm reads as weighted-score to the client, and the
# flag variant separately, which the client reports as the experiment's exposure.
variant = watch_feed_ranker_variant(self.team_id, self.team.uuid)
- ranker = "jev" if ranker_mode(variant) == "jev" else "weighted-score"
- if ranker == "jev":
+ mode = ranker_mode(variant)
+ ranker = mode if mode in ("jev", "embedding") else "weighted-score"
+ if ranker == "embedding":
+ ranked = rank_watch_feed_by_embedding(candidate_rows)[: params["limit"]]
+ elif ranker == "jev":
ranks = load_watch_ranks(self.team_id, allowed_ids)
probabilities = ranks.probabilities
jev_rows = list(candidate_rows)
diff --git a/products/replay_vision/backend/embeddings.py b/products/replay_vision/backend/embeddings.py
index ecdf6b741621..905e3679370b 100644
--- a/products/replay_vision/backend/embeddings.py
+++ b/products/replay_vision/backend/embeddings.py
@@ -4,8 +4,30 @@ A search only finds rows written with the same (model, product, document_type) t
import from here rather than defining their own copies.
"""
+from typing import Any
+
from posthog.schema import EmbeddingModelName
OBSERVATION_EMBEDDING_MODEL = EmbeddingModelName.TEXT_EMBEDDING_3_LARGE_3072
EMBEDDING_PRODUCT = "replay-vision"
EMBEDDING_DOCUMENT_TYPE = "replay-observation"
+
+# The physical table, read directly because the HogQL table pushes only the team filter down to storage.
+OBSERVATION_EMBEDDINGS_TABLE = (
+ f"distributed_posthog_document_embeddings_{OBSERVATION_EMBEDDING_MODEL.value.replace('-', '_')}"
+)
+
+# One team's observation embeddings from the given scanners; bind its values with `observation_scope_params`.
+OBSERVATION_SCOPE_PREWHERE = """team_id = %(team_id)s
+ AND product = %(product)s
+ AND document_type = %(document_type)s
+ AND JSONExtractString(metadata, 'scanner_id') IN %(scanner_ids)s"""
+
+
+def observation_scope_params(team_id: int, scanner_ids: list[str]) -> dict[str, Any]:
+ return {
+ "team_id": team_id,
+ "product": EMBEDDING_PRODUCT,
+ "document_type": EMBEDDING_DOCUMENT_TYPE,
+ "scanner_ids": scanner_ids,
+ }
diff --git a/products/replay_vision/backend/jev_watch_feed.py b/products/replay_vision/backend/jev_watch_feed.py
index f6567a7b3e61..206dd34f5947 100644
--- a/products/replay_vision/backend/jev_watch_feed.py
+++ b/products/replay_vision/backend/jev_watch_feed.py
@@ -51,12 +51,12 @@ from products.ml_inference.backend.facade.contracts import (
NoulAnswer,
)
from products.ml_inference.backend.facade.enums import DecisionQuestionType
-from products.replay_vision.backend.watch_feed import NOTABLE_MIN_SCORE, WatchFeedEntry, _trim_filler
+from products.replay_vision.backend.watch_feed import NOTABLE_MIN_SCORE, WatchFeedEntry, finish_feed
logger = structlog.get_logger(__name__)
WATCH_FEED_RANKER_FLAG = "vision-watch-feed-ranker"
-RankerMode = Literal["weighted-score", "jev-shadow", "jev"]
+RankerMode = Literal["weighted-score", "jev-shadow", "jev", "embedding"]
# The JevK5 build PostHog hosts on the AI gateway, which is the only backend the ml_inference
# facade reaches. The vendor (typesafe.ai) is not approved for customer data, and observation prose
# is customer data, so never point this at a vendor model id.
@@ -88,12 +88,6 @@ JEV_SEEN_PENALTY = 0.3
# bar empties the tier. Recalibrate from the judged event's watchability quantiles and top_scored
# sample, and bump the cache version below with any change here.
JEV_WATCHABLE_MIN = 0.3
-# One incident can put many near-identical watchable sessions on one scanner, and a pure probability
-# sort would fill the top of the feed with them. Hold each scanner to this share of the evidence
-# tier as it is placed (same intent as WATCH_FEED_MAX_SIGNAL_SHARE in the weighted ranker), with a
-# floor so a short feed is not over-constrained; the overflow trails the tier instead of leaving it.
-JEV_MAX_SCANNER_SHARE = 0.4
-_SCANNER_SPREAD_FLOOR = 3
# Observations per Jev request: the request carries the chunk as shared state and one question per
# observation, so a judgment is relative to its chunk, not to the whole window at once. Half the
# facade's per-request question cap: a request's prompt grows with its state, and a full 32-entry
@@ -268,9 +262,8 @@ def watch_feed_ranker_variant(team_id: int, team_uuid: UUID | str) -> str | None
Keyed on the project's uuid: both regions evaluate this one flag, and team ids repeat between
them. A region without the decision service has no sweep to fill the cache, so its teams take
- no part rather than land on an empty `jev` feed. Any flag failure also reads as None."""
- if not decision_api.decisions_available_here():
- return None
+ no part in the Jev arms rather than land on an empty `jev` feed. The embedding arm needs no
+ decision service. Any flag failure also reads as None."""
value = get_feature_flag_or_none(
WATCH_FEED_RANKER_FLAG,
# Only a person-property condition reads the distinct id; a project-aggregated flag buckets
@@ -280,11 +273,17 @@ def watch_feed_ranker_variant(team_id: int, team_uuid: UUID | str) -> str | None
group_properties={"project": {"id": team_id, "uuid": str(team_uuid)}},
send_feature_flag_events=False,
)
- return value if isinstance(value, str) else None
+ if not isinstance(value, str):
+ return None
+ if value in ("jev", "jev-shadow") and not decision_api.decisions_available_here():
+ return None
+ return value
def ranker_mode(variant: str | None) -> RankerMode:
- """The ranker a flag variant selects. Any variant but the two Jev arms is the default arm."""
+ """The ranker a flag variant selects. Any variant but the Jev and embedding arms is the default arm."""
+ if variant == "embedding":
+ return "embedding"
if variant == "jev-shadow":
return "jev-shadow"
if variant == "jev":
@@ -818,26 +817,6 @@ def _scan_notability_reason(row: dict[str, Any]) -> str | None:
return reason if isinstance(reason, str) and reason.strip() else None
-def _spread_scanners(ordered: list[tuple[Any, WatchFeedEntry]]) -> list[WatchFeedEntry]:
- """Hold each scanner to JEV_MAX_SCANNER_SHARE of the evidence tier as it is placed.
-
- Checked per place rather than over the whole list because the view slices the head: a feed of 5
- obeys the same share as a feed of 50. Overflow trails the tier in probability order — reordering
- for breadth is the job here, shortening the feed is not.
- """
- placed: list[WatchFeedEntry] = []
- deferred: list[WatchFeedEntry] = []
- counts: dict[Any, int] = {}
- for scanner_id, entry in ordered:
- allowed = max(_SCANNER_SPREAD_FLOOR, math.ceil(JEV_MAX_SCANNER_SHARE * (len(placed) + 1)))
- if counts.get(scanner_id, 0) >= allowed:
- deferred.append(entry)
- else:
- counts[scanner_id] = counts.get(scanner_id, 0) + 1
- placed.append(entry)
- return placed + deferred
-
-
def rank_watch_feed_by_jev(
rows: list[dict[str, Any]], probabilities: dict[str, float], reasons: dict[str, str] | None = None
) -> list[WatchFeedEntry]:
@@ -854,7 +833,7 @@ def rank_watch_feed_by_jev(
of them.
"""
watchable: list[tuple[float, Any, Any, WatchFeedEntry]] = []
- filler: list[tuple[bool, Any, WatchFeedEntry]] = []
+ filler_rows: list[dict[str, Any]] = []
for row in rows:
probability = probabilities.get(str(row["id"]))
viewed = bool(row.get("feed_viewed"))
@@ -871,11 +850,5 @@ def rank_watch_feed_by_jev(
(probability - (JEV_SEEN_PENALTY if viewed else 0.0), row["created_at"], row.get("scanner_id"), entry)
)
else:
- entry = WatchFeedEntry(observation_id=row["id"], reason={"kind": "recent" if viewed else "unviewed_recent"})
- filler.append((not viewed, row["created_at"], entry))
- watchable.sort(key=lambda item: (item[0], item[1]), reverse=True)
- filler.sort(key=lambda item: (item[0], item[1]), reverse=True)
- return _trim_filler(
- _spread_scanners([(scanner_id, entry) for _, _, scanner_id, entry in watchable])
- + [entry for *_, entry in filler]
- )
+ filler_rows.append(row)
+ return finish_feed(watchable, filler_rows)
diff --git a/products/replay_vision/backend/management/commands/backfill_replay_vision_watch_ranks.py b/products/replay_vision/backend/management/commands/backfill_replay_vision_watch_ranks.py
new file mode 100644
index 000000000000..378d91a69469
--- /dev/null
+++ b/products/replay_vision/backend/management/commands/backfill_replay_vision_watch_ranks.py
@@ -0,0 +1,33 @@
+from datetime import timedelta
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandParser
+from django.utils import timezone
+
+from products.replay_vision.backend.watch_rank import DUPLICATE_LOOKBACK
+from products.replay_vision.backend.watch_rank_backfill import backfill_team_watch_ranks, teams_with_observations
+
+
+class Command(BaseCommand):
+ help = "Rank recent Replay Vision observations for What to watch from the embeddings search already stored"
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument("--team-id", type=int, action="append", help="Only this team; repeat for more")
+ parser.add_argument("--days", type=int, default=DUPLICATE_LOOKBACK.days, help="How far back to rank, in days")
+ parser.add_argument("--dry-run", action="store_true", help="Count what would be ranked, write nothing")
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ since = timezone.now() - timedelta(days=options["days"])
+ team_ids = options["team_id"] or teams_with_observations(since)
+ verb = "Would rank" if options["dry_run"] else "Ranked"
+ for team_id in team_ids:
+ try:
+ result = backfill_team_watch_ranks(team_id, since, dry_run=bool(options["dry_run"]))
+ except Exception as e:
+ # One team without AI data processing, or a slow query, must not stop the rest.
+ self.stderr.write(f"Team {team_id}: skipped ({e.__class__.__name__}: {e})")
+ continue
+ self.stdout.write(
+ f"Team {team_id}: {verb} {result.ranked} of {result.observations} observations, "
+ f"{result.without_embedding} had no stored embedding."
+ )
diff --git a/products/replay_vision/backend/management/commands/run_replay_vision_watch_rank_consumer.py b/products/replay_vision/backend/management/commands/run_replay_vision_watch_rank_consumer.py
new file mode 100644
index 000000000000..dfb72cd6813e
--- /dev/null
+++ b/products/replay_vision/backend/management/commands/run_replay_vision_watch_rank_consumer.py
@@ -0,0 +1,38 @@
+import asyncio
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandParser
+
+from prometheus_client import start_http_server
+
+from posthog.temporal.common.health_server import HealthCheckServer
+from posthog.temporal.common.liveness_tracker import LivenessTracker
+
+from products.replay_vision.backend.watch_rank_consumer import WatchRankConsumer
+
+
+class Command(BaseCommand):
+ help = "Rank Replay Vision observations for What to watch from the embedding worker's results topic"
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument("--health-port", type=int, default=8090, help="Port for /healthz and /readyz")
+ parser.add_argument(
+ "--health-max-idle-seconds",
+ type=float,
+ default=120.0,
+ help="Seconds without a consume loop heartbeat before the health checks fail",
+ )
+ parser.add_argument("--metrics-port", type=int, default=8001, help="Port for Prometheus metrics")
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ start_http_server(options["metrics_port"])
+ asyncio.run(self._run(options["health_port"], options["health_max_idle_seconds"]))
+
+ async def _run(self, health_port: int, max_idle_seconds: float) -> None:
+ liveness = LivenessTracker()
+ health = HealthCheckServer(health_port, liveness, max_idle_seconds=max_idle_seconds)
+ await health.start()
+ try:
+ await WatchRankConsumer(liveness).run()
+ finally:
+ await health.stop()
diff --git a/products/replay_vision/backend/migrations/0126_replayobservation_watch_rank.py b/products/replay_vision/backend/migrations/0126_replayobservation_watch_rank.py
new file mode 100644
index 000000000000..82cd0240f0bc
--- /dev/null
+++ b/products/replay_vision/backend/migrations/0126_replayobservation_watch_rank.py
@@ -0,0 +1,21 @@
+# Generated by Django 5.2.18 on 2026-10-23 14:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("replay_vision", "0125_replayobservationfeeddismissal"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="replayobservation",
+ name="watch_rank",
+ field=models.JSONField(
+ blank=True,
+ help_text="What to watch ranking derived from the observation's embedding; see `watch_rank.WatchRank`.",
+ null=True,
+ ),
+ ),
+ ]
diff --git a/products/replay_vision/backend/migrations/max_migration.txt b/products/replay_vision/backend/migrations/max_migration.txt
index 094e746900bc..52d3ffafe31f 100644
--- a/products/replay_vision/backend/migrations/max_migration.txt
+++ b/products/replay_vision/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0125_replayobservationfeeddismissal
+0126_replayobservation_watch_rank
diff --git a/products/replay_vision/backend/models/replay_observation.py b/products/replay_vision/backend/models/replay_observation.py
index ee66a413b9af..5601a4757a11 100644
--- a/products/replay_vision/backend/models/replay_observation.py
+++ b/products/replay_vision/backend/models/replay_observation.py
@@ -98,6 +98,11 @@ class ReplayObservation(UUIDModel):
default=dict,
help_text="Result data persisted on success (model output, signals count); see `temporal.types.ScannerResult`.",
)
+ watch_rank = models.JSONField(
+ null=True,
+ blank=True,
+ help_text="What to watch ranking derived from the observation's embedding; see `watch_rank.WatchRank`.",
+ )
created_task_id = models.UUIDField(
null=True,
blank=True,
diff --git a/products/replay_vision/backend/search.py b/products/replay_vision/backend/search.py
index 9459f758027d..99f2c51ddc1a 100644
--- a/products/replay_vision/backend/search.py
+++ b/products/replay_vision/backend/search.py
@@ -31,9 +31,10 @@ from posthog.utils import relative_date_parse_with_delta_mapping
from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.replay_vision.backend.embeddings import (
- EMBEDDING_DOCUMENT_TYPE,
- EMBEDDING_PRODUCT,
OBSERVATION_EMBEDDING_MODEL,
+ OBSERVATION_EMBEDDINGS_TABLE,
+ OBSERVATION_SCOPE_PREWHERE,
+ observation_scope_params,
)
from products.replay_vision.backend.models.replay_observation import (
ObservationStatus,
@@ -78,11 +79,6 @@ _QUERY_VECTOR_EXECUTOR = ThreadPoolExecutor(max_workers=8, thread_name_prefix="r
# Warming gets its own pool so a burst of page views never queues a live search behind it.
_WARM_VECTOR_EXECUTOR = ThreadPoolExecutor(max_workers=2, thread_name_prefix="replay-vision-query-vector-warm")
-_EMBEDDINGS_TABLE = f"distributed_posthog_document_embeddings_{OBSERVATION_EMBEDDING_MODEL.value.replace('-', '_')}"
-_SCOPE_PREWHERE = """team_id = %(team_id)s
- AND product = %(product)s
- AND document_type = %(document_type)s
- AND JSONExtractString(metadata, 'scanner_id') IN %(scanner_ids)s"""
_CANDIDATE_QUERY_TYPE = "replay_vision_search_candidates"
_RANK_QUERY_TYPE = "replay_vision_search_rank"
@@ -214,7 +210,7 @@ def candidate_cutoff(team: Team, scanner_ids: list[str], filters: ObservationSea
SELECT minOrNull(timestamp)
FROM (
SELECT timestamp
- FROM {_EMBEDDINGS_TABLE}
+ FROM {OBSERVATION_EMBEDDINGS_TABLE}
PREWHERE {scope}
ORDER BY timestamp DESC
LIMIT %(candidate_cap)s
@@ -230,17 +226,13 @@ def candidate_cutoff(team: Team, scanner_ids: list[str], filters: ObservationSea
def _scope_params(team: Team, scanner_ids: list[str]) -> dict[str, Any]:
- return {
- "team_id": team.id,
- "product": EMBEDDING_PRODUCT,
- "document_type": EMBEDDING_DOCUMENT_TYPE,
- "scanner_ids": scanner_ids,
- "candidate_cap": _MAX_CANDIDATE_ROWS,
- }
+ return {**observation_scope_params(team.id, scanner_ids), "candidate_cap": _MAX_CANDIDATE_ROWS}
def _scope_clause(filters: ObservationSearchFilters, params: dict[str, Any]) -> str:
- return _SCOPE_PREWHERE + "".join(f"\n AND {clause}" for clause in filters.where_clauses(params))
+ return OBSERVATION_SCOPE_PREWHERE + "".join(
+ f"\n AND {clause}" for clause in filters.where_clauses(params)
+ )
def rank_observations(
@@ -287,7 +279,7 @@ def rank_observations(
document_id,
cosineDistance(embedding, %(embedding)s) AS row_distance,
substring(content, 1, %(snippet_chars)s) AS snippet
- FROM {_EMBEDDINGS_TABLE}
+ FROM {OBSERVATION_EMBEDDINGS_TABLE}
PREWHERE {scope}
AND timestamp >= %(cutoff)s
WHERE row_distance <= %(max_distance)s
diff --git a/products/replay_vision/backend/temporal/activities/embed_observation.py b/products/replay_vision/backend/temporal/activities/embed_observation.py
index 5ecad96cffb4..ec2fdcbaa2ce 100644
--- a/products/replay_vision/backend/temporal/activities/embed_observation.py
+++ b/products/replay_vision/backend/temporal/activities/embed_observation.py
@@ -6,6 +6,8 @@ via `metadata.scanner_id` for later natural-language search. monitor/classifier/
document under `rendering="summary"`. Empty content is skipped. Failures surface to the workflow, which
logs and swallows them — the observation is already marked succeeded.
+The worker's results topic feeds What to watch ranking, see `watch_rank_consumer`.
+
The structured result (monitor `verdict`, scorer `score`, classifier `tags`) is stamped into `metadata` so
search can filter by exact outcome inside the same ClickHouse ranking query, with no second-pass lookup.
"""
diff --git a/products/replay_vision/backend/temporal/jev_watch_rank/activities.py b/products/replay_vision/backend/temporal/jev_watch_rank/activities.py
index c220d7f6f4f0..87ed89d4715a 100644
--- a/products/replay_vision/backend/temporal/jev_watch_rank/activities.py
+++ b/products/replay_vision/backend/temporal/jev_watch_rank/activities.py
@@ -392,7 +392,7 @@ async def _judge_watch_ranks(inputs: JevWatchRankSweepInputs) -> JevWatchRankSwe
continue
mode = await asyncio.to_thread(watch_feed_ranker, team_id, team_uuid)
teams_evaluated += 1
- if mode == "weighted-score":
+ if mode not in ("jev", "jev-shadow"):
continue
# The scan prose the sweep sends to the model derives from recordings, so a revoked consent
# stops the judgments even while the flag stays on.
diff --git a/products/replay_vision/backend/tests/test_api.py b/products/replay_vision/backend/tests/test_api.py
index 1945dd1c4959..5ffa3ef63a49 100644
--- a/products/replay_vision/backend/tests/test_api.py
+++ b/products/replay_vision/backend/tests/test_api.py
@@ -76,6 +76,7 @@ from products.replay_vision.backend.tests.helpers import (
seed_scanner_spend,
snapshot_for as _snapshot_for,
)
+from products.replay_vision.backend.watch_rank import ANCHOR_VERSION
from products.signals.backend.facade.api import (
SignalSourceSliceOutcomes,
SignalSourceSlicePullRequest,
@@ -5108,7 +5109,7 @@ class TestWatchFeedAPI(_VisionAPITestCase):
resp = self.client.get(self.feed_url)
self.assertEqual(resp.json()["results"][0]["reason"], {"kind": "unviewed_recent"})
- def test_the_flag_switches_the_whole_ranker_between_weighted_score_and_jev(self) -> None:
+ def test_the_flag_switches_the_whole_ranker_between_arms(self) -> None:
# The two rankers are independent: a cached Jev probability must not move the weighted-score
# feed (weighted-score and jev-shadow arms), and the jev arm must rank on the cached
# probabilities alone, with unjudged and judged-low rows in the recency filler tier.
@@ -5152,6 +5153,29 @@ class TestWatchFeedAPI(_VisionAPITestCase):
self.assertEqual(items[1]["reason"], {"kind": "unviewed_recent"})
self.assertEqual(items[2]["reason"], {"kind": "unviewed_recent"})
+ # The embedding arm ranks on the rank the scan wrote, once the scanner has enough ranked rows.
+ for index in range(10):
+ filler = self._succeeded_observation(scanner, f"routine-{index}", 50 + index, self._monitor_result("no"))
+ ReplayObservation.objects.filter(id=filler.id).update(
+ watch_rank={
+ "category": "routine",
+ "contrast": -0.1,
+ "duplicate_of": None,
+ "anchor_version": ANCHOR_VERSION,
+ }
+ )
+ ReplayObservation.objects.filter(id=jev_low.id).update(
+ watch_rank={"category": "failure", "contrast": 0.2, "duplicate_of": None, "anchor_version": ANCHOR_VERSION}
+ )
+ with patch(ranker, return_value="embedding"):
+ resp = self.client.get(self.feed_url)
+ items = resp.json()["results"]
+ self.assertEqual(resp.json()["ranker"], "embedding")
+ self.assertEqual(items[0]["observation"]["session_id"], "jev-low")
+ self.assertEqual(
+ items[0]["reason"], {"kind": "embedding_pick", "watch_category": "failure", "similar_count": 0}
+ )
+
def test_the_jev_arm_surfaces_a_watchable_row_the_recency_slice_cut_off(self) -> None:
# The candidate query keeps only each scanner's newest rows, so on a high-volume scanner a
# watchable row from days ago never reaches the weighted feed. The jev arm must fetch it
diff --git a/products/replay_vision/backend/tests/test_jev_watch_feed.py b/products/replay_vision/backend/tests/test_jev_watch_feed.py
index a0d34d987936..da33e45ccf20 100644
--- a/products/replay_vision/backend/tests/test_jev_watch_feed.py
+++ b/products/replay_vision/backend/tests/test_jev_watch_feed.py
@@ -104,22 +104,25 @@ class TestWatchFeedRankerFlag(SimpleTestCase):
("unknown_variant", "something-else", "weighted-score"),
("shadow", "jev-shadow", "jev-shadow"),
("graduated", "jev", "jev"),
+ ("embedding", "embedding", "embedding"),
]
)
def test_only_known_variants_leave_the_default_arm(self, _name: str, flag_value: object, expected: str) -> None:
with patch(_FLAG, return_value=flag_value), patch(_REGION, return_value=True):
assert watch_feed_ranker(1, "project-uuid") == expected
- def test_the_flag_keys_on_the_project_uuid_and_skips_regions_without_jev(self) -> None:
+ def test_the_flag_keys_on_the_project_uuid_and_keeps_jev_arms_out_of_regions_without_jev(self) -> None:
# Both regions evaluate this one flag and team ids repeat between them, so only the uuid is unique.
with patch(_FLAG, return_value="jev") as flag, patch(_REGION, return_value=True):
assert watch_feed_ranker_variant(2, "project-uuid") == "jev"
assert flag.call_args.kwargs["groups"] == {"project": "project-uuid"}
# A region with no sweep to fill the cache must not land a team on an empty jev feed.
- with patch(_FLAG, return_value="jev") as flag, patch(_REGION, return_value=False):
+ with patch(_FLAG, return_value="jev"), patch(_REGION, return_value=False):
assert watch_feed_ranker_variant(2, "project-uuid") is None
assert watch_feed_ranker(2, "project-uuid") == "weighted-score"
- flag.assert_not_called()
+ # The embedding arm needs no decision service.
+ with patch(_FLAG, return_value="embedding"), patch(_REGION, return_value=False):
+ assert watch_feed_ranker(2, "project-uuid") == "embedding"
class TestJudgeScannerWindow(SimpleTestCase):
@@ -571,12 +574,13 @@ class TestJevWatchRankSweep(BaseTest):
flag = f"{activities}.watch_feed_ranker"
region = f"{activities}.decision_api.decisions_available_here"
- with patch(flag, return_value="weighted-score"), patch(region, return_value=True), patch(_API) as api:
- result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())
- # A team on the default arm costs no Jev calls and gets no cache entry.
- api.decide_when_available.assert_not_called()
- assert result.teams_enrolled == 0
- assert load_watch_ranks(self.team.id, [scanner.id]).probabilities == {}
+ # A team on the default or embedding arm costs no Jev calls and gets no cache entry.
+ for arm in ("weighted-score", "embedding"):
+ with patch(flag, return_value=arm), patch(region, return_value=True), patch(_API) as api:
+ result = async_to_sync(_judge_watch_ranks)(JevWatchRankSweepInputs())
+ api.decide_when_available.assert_not_called()
+ assert result.teams_enrolled == 0
+ assert load_watch_ranks(self.team.id, [scanner.id]).probabilities == {}
with (
patch(flag, side_effect=self._flag_arm("jev-shadow")),
diff --git a/products/replay_vision/backend/tests/test_watch_rank.py b/products/replay_vision/backend/tests/test_watch_rank.py
new file mode 100644
index 000000000000..d1381e8c23ba
--- /dev/null
+++ b/products/replay_vision/backend/tests/test_watch_rank.py
@@ -0,0 +1,192 @@
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from django.test import SimpleTestCase
+
+import numpy as np
+from parameterized import parameterized
+
+from products.replay_vision.backend.watch_feed import SESSION_SPACING
+from products.replay_vision.backend.watch_rank import (
+ ANCHOR_VERSION,
+ WatchCategory,
+ classify,
+ duplicate_root,
+ rank_watch_feed_by_embedding,
+)
+from products.replay_vision.backend.watch_rank_backfill import _rank_scanner, _RecentVectors
+
+_NOW = datetime(2026, 10, 23, 12, 0, tzinfo=UTC)
+
+
+def _row(
+ observation_id: str,
+ contrast: float | None,
+ *,
+ category: str = "failure",
+ scanner: str = "scanner-a",
+ duplicate_of: str | None = None,
+ anchor_version: int = ANCHOR_VERSION,
+ minutes_ago: int = 0,
+ viewed: bool = False,
+ session: str | None = None,
+) -> dict[str, Any]:
+ rank = (
+ None
+ if contrast is None
+ else {
+ "category": category,
+ "contrast": contrast,
+ "duplicate_of": duplicate_of,
+ "anchor_version": anchor_version,
+ }
+ )
+ return {
+ "id": observation_id,
+ "scanner_id": scanner,
+ "session_id": session or f"session-{observation_id}",
+ "created_at": _NOW - timedelta(minutes=minutes_ago),
+ "watch_rank": rank,
+ "feed_viewed": viewed,
+ }
+
+
+class TestClassify(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("closest_to_an_issue", [1.0, 0.1, 0.0], "failure", True),
+ ("closest_to_routine", [0.1, 0.0, 1.0], "routine", False),
+ ]
+ )
+ def test_routine_wins_over_the_closest_issue_when_it_is_closer(
+ self, _name: str, vector: list[float], category: str, positive: bool
+ ) -> None:
+ anchors: dict[WatchCategory, np.ndarray] = {
+ "failure": np.array([[1.0, 0.0, 0.0]]),
+ "struggle": np.array([[0.0, 1.0, 0.0]]),
+ "revenue": np.array([[0.0, -1.0, 0.0]]),
+ "routine": np.array([[0.0, 0.0, 1.0]]),
+ }
+ got_category, contrast = classify(vector, anchors)
+ assert got_category == category
+ assert (contrast > 0) is positive
+
+
+class TestDuplicateRoot(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("unranked_smaller_neighbour_is_the_root", "obs-b", "obs-a", None, "obs-a"),
+ ("unranked_larger_neighbour_defers_to_this_one", "obs-a", "obs-b", None, None),
+ ("ranked_neighbour_passes_on_its_root", "obs-a", "obs-b", {"duplicate_of": "obs-0"}, "obs-0"),
+ ("ranked_root_neighbour_is_the_root", "obs-a", "obs-b", {"duplicate_of": None}, "obs-b"),
+ ]
+ )
+ def test_two_concurrent_copies_agree_on_one_root(
+ self,
+ _name: str,
+ observation_id: str,
+ neighbour_id: str,
+ neighbour_rank: dict[str, Any] | None,
+ expected: str | None,
+ ) -> None:
+ assert duplicate_root(observation_id, neighbour_id, neighbour_rank) == expected
+
+
+class TestRankWatchFeedByEmbedding(SimpleTestCase):
+ def test_picks_the_scanner_top_issues_once_per_duplicate_group(self) -> None:
+ rows = [
+ # scanner-a has enough ranked rows for a percentile; its top two are one duplicate group.
+ _row("dup-root", 0.30),
+ _row("dup-copy", 0.32, duplicate_of="dup-root", minutes_ago=50),
+ _row("dup-low", -0.01, duplicate_of="dup-root"),
+ # A copy that points at another copy, as when its root was ranked after it, still joins the group.
+ _row("dup-chain", 0.02, duplicate_of="dup-copy", minutes_ago=3),
+ _row("routine-top", 0.40, category="routine", minutes_ago=50),
+ *[_row(f"a-{index}", 0.01 * index, minutes_ago=index + 1) for index in range(10)],
+ # A rank from older anchors does not compare with current scores.
+ _row("stale", 0.90, anchor_version=ANCHOR_VERSION - 1),
+ # scanner-b is too thin for a percentile, so its top row is only filler.
+ _row("b-top", 0.50, scanner="scanner-b", minutes_ago=1),
+ ]
+
+ ranked = rank_watch_feed_by_embedding(rows)
+
+ assert ranked[0].observation_id == "dup-copy"
+ assert ranked[0].reason == {"kind": "embedding_pick", "watch_category": "failure", "similar_count": 3}
+ picks = [entry.observation_id for entry in ranked if entry.reason["kind"] == "embedding_pick"]
+ assert picks == ["dup-copy"]
+ # Collapsed copies, even those below the cutoff, must not come back as filler.
+ assert {"dup-root", "dup-low", "dup-chain"}.isdisjoint(entry.observation_id for entry in ranked)
+
+ def test_a_viewed_pick_still_stands_for_its_group_over_a_copy_below_the_cutoff(self) -> None:
+ rows = [
+ _row("seen-top", 0.9, viewed=True),
+ _row("other-top", 0.8, minutes_ago=1),
+ _row("mid-copy", 0.7, duplicate_of="seen-top", minutes_ago=2),
+ *[_row(f"a-{index}", 0.01 * index, minutes_ago=index + 3) for index in range(7)],
+ ]
+
+ ranked = rank_watch_feed_by_embedding(rows)
+
+ picks = {
+ str(entry.observation_id): entry.reason for entry in ranked if entry.reason["kind"] == "embedding_pick"
+ }
+ assert picks["seen-top"]["similar_count"] == 1
+ assert "mid-copy" not in {entry.observation_id for entry in ranked}
+
+ def test_two_scanners_picking_one_session_land_apart_without_dropping_a_pick(self) -> None:
+ rows = [
+ _row(f"{scanner}-{index}", index / 20, scanner=scanner, minutes_ago=index)
+ for scanner in ("scanner-a", "scanner-b", "scanner-c")
+ for index in range(9)
+ ] + [
+ _row("a-top", 0.9, scanner="scanner-a", session="shared"),
+ _row("b-top", 0.9, scanner="scanner-b", session="shared", minutes_ago=1),
+ _row("c-top", 0.9, scanner="scanner-c", minutes_ago=2),
+ ]
+
+ ranked = [str(entry.observation_id) for entry in rank_watch_feed_by_embedding(rows)]
+
+ assert ranked[0] == "a-top"
+ assert ranked.index("b-top") - ranked.index("a-top") >= SESSION_SPACING
+ assert {"a-top", "b-top", "c-top", "scanner-a-8", "scanner-b-8", "scanner-c-8"} <= set(ranked)
+
+ def test_rows_without_a_rank_fall_back_to_the_filler_floor(self) -> None:
+ rows = [_row(f"row-{index}", None, minutes_ago=index, viewed=index == 0) for index in range(20)]
+
+ ranked = rank_watch_feed_by_embedding(rows)
+
+ assert [entry.observation_id for entry in ranked] == ["row-1", "row-2", "row-3"]
+
+
+class TestBackfillRankScanner(SimpleTestCase):
+ def test_walks_oldest_first_so_a_copy_joins_the_group_of_an_already_ranked_row(self) -> None:
+ anchors: dict[WatchCategory, np.ndarray] = {
+ "failure": np.array([[1.0, 0.0, 0.0]]),
+ "struggle": np.array([[0.0, 1.0, 0.0]]),
+ "revenue": np.array([[0.0, -1.0, 0.0]]),
+ "routine": np.array([[0.0, 0.0, 1.0]]),
+ }
+ error = np.array([1.0, 0.0, 0.0])
+ current = {"category": "failure", "contrast": 1.0, "duplicate_of": "root", "anchor_version": ANCHOR_VERSION}
+ rows: list[dict[str, Any]] = [
+ {"id": "ranked", "watch_rank": current},
+ {"id": "no-vector", "watch_rank": None},
+ {"id": "copy", "watch_rank": None},
+ {"id": "stale", "watch_rank": {**current, "anchor_version": ANCHOR_VERSION - 1}},
+ {"id": "calm", "watch_rank": None},
+ ]
+ vectors = {"ranked": error, "copy": error, "stale": error, "calm": np.array([0.0, 0.0, 1.0])}
+
+ # Two batches that share the recent vectors, as the backfill runs them.
+ recent = _RecentVectors()
+ written = {
+ **_rank_scanner(rows[:2], vectors, anchors, recent),
+ **_rank_scanner(rows[2:], vectors, anchors, recent),
+ }
+
+ assert set(written) == {"copy", "stale", "calm"}
+ assert written["copy"]["duplicate_of"] == "root"
+ assert written["stale"]["duplicate_of"] == "root"
+ assert written["calm"]["category"] == "routine"
+ assert written["calm"]["duplicate_of"] is None
diff --git a/products/replay_vision/backend/tests/test_watch_rank_consumer.py b/products/replay_vision/backend/tests/test_watch_rank_consumer.py
new file mode 100644
index 000000000000..a45aa5ca6804
--- /dev/null
+++ b/products/replay_vision/backend/tests/test_watch_rank_consumer.py
@@ -0,0 +1,196 @@
+import json
+import math
+import time
+from typing import Any
+
+from posthog.test.base import BaseTest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from django.test import SimpleTestCase
+
+import numpy as np
+from asgiref.sync import async_to_sync
+from confluent_kafka import KafkaException
+from parameterized import parameterized
+
+from products.replay_vision.backend.embeddings import (
+ EMBEDDING_DOCUMENT_TYPE,
+ EMBEDDING_PRODUCT,
+ OBSERVATION_EMBEDDING_MODEL,
+)
+from products.replay_vision.backend.models.replay_observation import ObservationTrigger, ReplayObservation
+from products.replay_vision.backend.models.replay_scanner import ReplayScanner, ScannerModel, ScannerType
+from products.replay_vision.backend.tests.helpers import snapshot_for
+from products.replay_vision.backend.watch_rank import ANCHOR_VERSION, RankRequest
+from products.replay_vision.backend.watch_rank_consumer import WatchRankConsumer, parse_rank_request
+
+_WATCH_RANK = "products.replay_vision.backend.watch_rank"
+_CURRENT_RANK = {"category": "struggle", "contrast": 0.2, "duplicate_of": None, "anchor_version": ANCHOR_VERSION}
+_CONSUMER = "products.replay_vision.backend.watch_rank_consumer"
+
+
+def _result_message(
+ *,
+ team_id: int = 7,
+ observation_id: str = "obs-1",
+ scanner_id: str | None = "scanner-1",
+ product: str = EMBEDDING_PRODUCT,
+ outcome: str = "success",
+ vector: list[float] | None = None,
+) -> bytes:
+ result: dict[str, Any] = {"model": OBSERVATION_EMBEDDING_MODEL.value, "outcome": outcome}
+ if outcome == "success":
+ result["embedding"] = [1.0, 0.1, 0.0] if vector is None else vector
+ else:
+ result["error"] = "provider refused"
+ return json.dumps(
+ {
+ "team_id": team_id,
+ "product": product,
+ "document_type": EMBEDDING_DOCUMENT_TYPE,
+ "rendering": "reasoning",
+ "document_id": observation_id,
+ "timestamp": "2026-10-24T12:00:00Z",
+ "content": "the payment form errored",
+ "models": [OBSERVATION_EMBEDDING_MODEL.value],
+ "metadata": {} if scanner_id is None else {"scanner_id": scanner_id},
+ "results": [result],
+ },
+ separators=(",", ":"),
+ ).encode()
+
+
+class TestParseRankRequest(SimpleTestCase):
+ def test_reads_an_observation_result(self) -> None:
+ assert parse_rank_request(_result_message()) == RankRequest(
+ team_id=7, scanner_id="scanner-1", observation_id="obs-1", vector=[1.0, 0.1, 0.0]
+ )
+
+ @parameterized.expand(
+ [
+ ("another_product", _result_message(product="error-tracking")),
+ ("failed_embedding", _result_message(outcome="failure")),
+ ("no_scanner", _result_message(scanner_id=None)),
+ ("zero_vector", _result_message(vector=[0.0, 0.0, 0.0])),
+ ("not_json", b'{"document_type":"replay-observation"'),
+ ]
+ )
+ def test_skips_what_it_cannot_rank(self, _name: str, value: bytes) -> None:
+ assert parse_rank_request(value) is None
+
+
+class TestWatchRankConsumer(BaseTest):
+ _ANCHORS = {
+ "failure": np.array([[1.0, 0.0, 0.0]]),
+ "struggle": np.array([[0.0, 1.0, 0.0]]),
+ "revenue": np.array([[0.0, -1.0, 0.0]]),
+ "routine": np.array([[0.0, 0.0, 1.0]]),
+ }
+
+ def _observation(self, scanner: ReplayScanner, session_id: str, **overrides: Any) -> ReplayObservation:
+ return ReplayObservation.objects.create(
+ scanner=scanner,
+ team=self.team,
+ session_id=session_id,
+ triggered_by=ObservationTrigger.ON_DEMAND,
+ scanner_snapshot=snapshot_for(scanner),
+ **overrides,
+ )
+
+ @parameterized.expand(
+ [
+ (
+ "duplicate_points_at_the_group_root",
+ None,
+ None,
+ "ranked",
+ {"category": "failure", "duplicate_of": "root-id"},
+ ),
+ ("ranking_failure_leaves_no_rank", None, RuntimeError("clickhouse down"), "failed", None),
+ ("redelivery_keeps_the_current_rank", _CURRENT_RANK, None, "already_ranked", _CURRENT_RANK),
+ ("deleted_observation_is_skipped", None, None, "missing", None),
+ ]
+ )
+ def test_ranks_the_observation_in_a_result(
+ self,
+ _name: str,
+ existing_rank: dict[str, Any] | None,
+ nearest_error: Exception | None,
+ outcome: str,
+ expected_rank: dict[str, Any] | None,
+ ) -> None:
+ scanner = ReplayScanner.objects.create(
+ team=self.team,
+ name="s",
+ scanner_type=ScannerType.MONITOR,
+ scanner_config={"prompt": "p"},
+ model=ScannerModel.GEMINI_3_8_FLASH,
+ )
+ neighbour = self._observation(scanner, "sess-earlier", watch_rank={"duplicate_of": "root-id"})
+ observation = self._observation(scanner, "sess-new", watch_rank=existing_rank)
+ nearest = MagicMock(side_effect=nearest_error, return_value=(str(neighbour.id), 0.05))
+ message = _result_message(team_id=self.team.id, observation_id=str(observation.id), scanner_id=str(scanner.id))
+ if outcome == "missing":
+ observation.delete()
+
+ with (
+ patch(f"{_WATCH_RANK}.anchor_vectors", AsyncMock(return_value=self._ANCHORS)),
+ patch(f"{_WATCH_RANK}.nearest_earlier_observation", nearest),
+ patch(f"{_CONSUMER}._RETRY_WINDOW_S", 0),
+ # It would close the connection that holds the test transaction.
+ patch(f"{_CONSUMER}.close_old_connections"),
+ ):
+ assert async_to_sync(WatchRankConsumer(MagicMock()).handle)(message, 0.0, math.inf) == outcome
+
+ if outcome == "missing":
+ return
+ observation.refresh_from_db()
+ if expected_rank is None:
+ assert observation.watch_rank is None
+ else:
+ assert observation.watch_rank is not None
+ assert expected_rank.items() <= observation.watch_rank.items()
+
+ @parameterized.expand(
+ [
+ # A revoked partition stops the batch, and the rest of it is read again rather than skipped.
+ ("revoked_partition", KafkaException("partition revoked"), "ranked", [0], 1),
+ # A message still failing at the end of the batch budget goes back to Kafka unstored.
+ ("deferred_retry", None, "deferred", [], 0),
+ ]
+ )
+ def test_unfinished_messages_are_read_again(
+ self, _name: str, store_error: Exception | None, outcome: str, stored: list[int], rewind_from: int
+ ) -> None:
+ messages = [
+ MagicMock(
+ **{
+ "error.return_value": None,
+ "timestamp.return_value": (1, 0),
+ "value.return_value": b"{}",
+ "topic.return_value": "results",
+ "partition.return_value": 0,
+ "offset.return_value": offset,
+ }
+ )
+ for offset in range(3)
+ ]
+ consumer = MagicMock(**{"store_offsets.side_effect": store_error})
+
+ with patch.object(WatchRankConsumer, "handle", AsyncMock(return_value=outcome)):
+ async_to_sync(WatchRankConsumer(MagicMock()).process_batch)(consumer, messages)
+
+ assert [c.kwargs["message"] for c in consumer.store_offsets.call_args_list] == [messages[i] for i in stored]
+ (seek,) = consumer.seek.call_args_list
+ assert seek.args[0].offset == rewind_from
+
+ def test_a_deferred_failure_keeps_its_retry_window_in_the_next_batch(self) -> None:
+ consumer = WatchRankConsumer(MagicMock())
+ message = _result_message()
+
+ with (
+ patch(f"{_CONSUMER}.rank_observation", AsyncMock(side_effect=RuntimeError("postgres down"))),
+ patch(f"{_CONSUMER}.close_old_connections"),
+ ):
+ for _ in range(2):
+ assert async_to_sync(consumer.handle)(message, 0.0, time.monotonic()) == "deferred"
diff --git a/products/replay_vision/backend/watch_feed.py b/products/replay_vision/backend/watch_feed.py
index 4b5614574c78..192f539177e1 100644
--- a/products/replay_vision/backend/watch_feed.py
+++ b/products/replay_vision/backend/watch_feed.py
@@ -5,6 +5,7 @@ own window), so "outlier" and "rare" mean unusual for that scanner lately, not a
"""
import re
+import math
from collections import deque
from datetime import datetime
from statistics import fmean, stdev
@@ -88,6 +89,8 @@ _FRICTION_RE = re.compile(
class WatchFeedEntry:
observation_id: UUID
reason: dict[str, Any]
+ # Set only by rankers that space a recording's cards apart; None opts the entry out.
+ session_id: str | None = None
@frozen
@@ -391,6 +394,77 @@ def _trim_filler(ranked: list[WatchFeedEntry]) -> list[WatchFeedEntry]:
return kept
+# One incident can put many near-identical watchable sessions on one scanner, and a pure score
+# sort would fill the top of the feed with them. Hold each scanner to this share of the evidence
+# tier as it is placed (same intent as WATCH_FEED_MAX_SIGNAL_SHARE in the weighted ranker), with a
+# floor so a short feed is not over-constrained; the overflow trails the tier instead of leaving it.
+MAX_SCANNER_SHARE = 0.4
+_SCANNER_SPREAD_FLOOR = 3
+
+
+def spread_scanners(ordered: list[tuple[Any, WatchFeedEntry]]) -> list[WatchFeedEntry]:
+ """Hold each scanner to MAX_SCANNER_SHARE of the evidence tier as it is placed.
+
+ Checked per place rather than over the whole list because the view slices the head: a feed of 5
+ obeys the same share as a feed of 50. Overflow trails the tier in probability order — reordering
+ for breadth is the job here, shortening the feed is not.
+ """
+ placed: list[WatchFeedEntry] = []
+ deferred: list[WatchFeedEntry] = []
+ counts: dict[Any, int] = {}
+ for scanner_id, entry in ordered:
+ allowed = max(_SCANNER_SPREAD_FLOOR, math.ceil(MAX_SCANNER_SHARE * (len(placed) + 1)))
+ if counts.get(scanner_id, 0) >= allowed:
+ deferred.append(entry)
+ else:
+ counts[scanner_id] = counts.get(scanner_id, 0) + 1
+ placed.append(entry)
+ return placed + deferred
+
+
+# Several scanners can pick one recording. Its cards stay this many places apart, so the feed never
+# shows the same session twice in a row.
+SESSION_SPACING = 5
+
+
+def space_sessions(entries: list[WatchFeedEntry]) -> list[WatchFeedEntry]:
+ """Keep each session's cards at least SESSION_SPACING places apart, reordering but never dropping one.
+
+ A held card returns at the first place its session allows, so score order holds otherwise. A feed
+ with too few other sessions to space every repeat still ends with every card, in score order.
+ """
+ placed: list[WatchFeedEntry] = []
+ waiting: list[WatchFeedEntry] = []
+ for entry in entries:
+ waiting.append(entry)
+ while waiting:
+ recent = {placed_entry.session_id for placed_entry in placed[-(SESSION_SPACING - 1) :]} - {None}
+ ready = next((held for held in waiting if held.session_id not in recent), None)
+ if ready is None:
+ break
+ waiting.remove(ready)
+ placed.append(ready)
+ return placed + waiting
+
+
+def finish_feed(
+ picks: list[tuple[float, Any, Any, WatchFeedEntry]], filler_rows: list[dict[str, Any]]
+) -> list[WatchFeedEntry]:
+ """Order `(score, created_at, scanner_id, entry)` picks with the scanner spread and session spacing,
+ then pad with `filler_rows` newest and unviewed first, trimmed to the feed's floor."""
+ ordered = sorted(picks, key=lambda item: (item[0], item[1]), reverse=True)
+ filler: list[tuple[bool, Any, WatchFeedEntry]] = []
+ for row in filler_rows:
+ viewed = bool(row.get("feed_viewed"))
+ entry = WatchFeedEntry(observation_id=row["id"], reason={"kind": "recent" if viewed else "unviewed_recent"})
+ filler.append((not viewed, row["created_at"], entry))
+ filler.sort(key=lambda item: (item[0], item[1]), reverse=True)
+ return _trim_filler(
+ space_sessions(spread_scanners([(scanner_id, entry) for _, _, scanner_id, entry in ordered]))
+ + [entry for *_, entry in filler]
+ )
+
+
def rank_watch_feed_candidates(rows: list[dict[str, Any]]) -> list[WatchFeedEntry]:
"""Rank candidate rows (`id`, `scanner_id`, `created_at`, `scanner_result`, `feed_viewed`) most
watchable first by a blended score, then newest.
diff --git a/products/replay_vision/backend/watch_rank.py b/products/replay_vision/backend/watch_rank.py
new file mode 100644
index 000000000000..2dc4d7d0f3f8
--- /dev/null
+++ b/products/replay_vision/backend/watch_rank.py
@@ -0,0 +1,370 @@
+"""What to watch ranking from observation embeddings.
+
+A consumer of the embedding worker's results ranks each observation once, from the vector the scan
+already embeds for search, so the feed needs no model call and no sweep. Each vector is compared with fixed anchor sentences: the
+closest issue anchor names the category, and its lead over the closest routine anchor is the score.
+The feed turns scores into a percentile within each scanner when it loads, so a scanner that sees
+errors all the time does not fill the feed.
+"""
+
+import asyncio
+from dataclasses import asdict
+from datetime import datetime, timedelta
+from typing import Any, Literal
+
+from django.core.cache import cache
+from django.utils import timezone
+
+import numpy as np
+
+from posthog.api.embedding_worker import async_generate_embedding
+from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.client.connection import ClickHouseUser
+from posthog.clickhouse.query_tagging import Product, tags_context
+from posthog.clickhouse.workload import Workload
+from posthog.dataclasses import frozen
+from posthog.models.team import Team
+
+from products.replay_vision.backend.embeddings import (
+ OBSERVATION_EMBEDDING_MODEL,
+ OBSERVATION_EMBEDDINGS_TABLE,
+ OBSERVATION_SCOPE_PREWHERE,
+ observation_scope_params,
+)
+from products.replay_vision.backend.models.replay_observation import ReplayObservation
+from products.replay_vision.backend.watch_feed import WatchFeedEntry, finish_feed
+
+WatchCategory = Literal["failure", "struggle", "revenue", "routine"]
+
+# Bump when an anchor changes: scores from different anchor sets do not compare, so the feed
+# ignores ranks written under another version until they age out of the window.
+ANCHOR_VERSION = 1
+
+ANCHORS: dict[WatchCategory, tuple[str, ...]] = {
+ "failure": (
+ "The user saw an error message and the action they tried failed.",
+ "A request failed or the page crashed, showing an error to the user.",
+ "The user clicked a button and nothing happened; the action silently did not work.",
+ "The page got stuck loading and the user waited on a spinner that never finished.",
+ "The user was blocked by a dead end, a 404 page, or an access denied message.",
+ ),
+ "struggle": (
+ "The user repeated the same action several times because it did not give the result they expected.",
+ "The user rage clicked and gave up on the task without finishing.",
+ "The user hit a validation error or confusing state in a form and abandoned it.",
+ "The user was stuck: they tried several different ways to do one thing and none worked.",
+ "The user worked around a broken or missing feature to get their task done.",
+ ),
+ "revenue": (
+ "The user went to billing to cancel their subscription or downgrade their plan.",
+ "The user tried to upgrade or pay but the upgrade or payment failed.",
+ "The user hit a usage limit that stopped a feature from working.",
+ "The user contacted support frustrated about a problem.",
+ ),
+ "routine": (
+ "The user browsed normally and completed what they came to do without any issues.",
+ "A routine session with nothing notable: normal navigation and expected behavior.",
+ "The user viewed a few pages and left; nothing went wrong.",
+ "Nothing stands out in this session.",
+ "Routine session with no errors or friction; the user did what they intended.",
+ "The user browsed the pricing page and documentation normally.",
+ "The user successfully completed the task without problems.",
+ "The user explored different sections of the website to learn about the product.",
+ "The user read through a product or documentation page.",
+ "The user looked through a list and opened a few items to review them.",
+ "The user started something and left before finishing, with no sign of a problem.",
+ "The user never reached the part of the product this check is about.",
+ ),
+}
+ISSUE_CATEGORIES: tuple[WatchCategory, ...] = ("failure", "struggle", "revenue")
+
+# Struggle picks rated worth watching less often than failure and revenue picks in a blind rating.
+CATEGORY_WEIGHT: dict[str, float] = {"failure": 1.0, "revenue": 1.0, "struggle": 0.85}
+
+# Cosine distance below which two observations from one scanner describe the same thing.
+DUPLICATE_MAX_DISTANCE = 0.1
+DUPLICATE_LOOKBACK = timedelta(days=7)
+_DUPLICATE_QUERY_TIMEOUT_S = 10
+
+# The feed surfaces a scanner's top fifth by score, once the scanner has enough ranked rows to
+# make a percentile mean something.
+WATCH_PICK_MIN_PERCENTILE = 0.8
+WATCH_PICK_MIN_SCANNER_ROWS = 10
+WATCH_PICK_SEEN_PENALTY = 0.3
+
+_ANCHOR_CACHE_TTL_S = 7 * 24 * 3600
+
+
+@frozen
+class WatchRank:
+ category: WatchCategory
+ contrast: float
+ duplicate_of: str | None
+ anchor_version: int = ANCHOR_VERSION
+
+ def to_json(self) -> dict[str, Any]:
+ return asdict(self)
+
+
+_ANCHOR_CACHE_KEY = f"replay-vision:watch-rank-anchors:v{ANCHOR_VERSION}:{OBSERVATION_EMBEDDING_MODEL.value}"
+
+
+def normalize(matrix: np.ndarray) -> np.ndarray:
+ return matrix / np.linalg.norm(matrix, axis=-1, keepdims=True)
+
+
+# The anchors never change within a version, so each process keeps them after the first read.
+_anchors_in_process: dict[WatchCategory, np.ndarray] | None = None
+
+
+async def anchor_vectors(team_id: int) -> dict[WatchCategory, np.ndarray]:
+ """Unit vectors for every anchor, embedded once and cached across teams.
+
+ Embedded under the calling team because the worker gates every call on a team's AI opt-in, and
+ a team that ranks observations has already passed that gate.
+ """
+ global _anchors_in_process
+ if _anchors_in_process is None:
+ _anchors_in_process = await _load_anchor_vectors(team_id)
+ return _anchors_in_process
+
+
+async def _load_anchor_vectors(team_id: int) -> dict[WatchCategory, np.ndarray]:
+ vectors: dict[WatchCategory, list[list[float]]] | None = await cache.aget(_ANCHOR_CACHE_KEY)
+ if vectors is None:
+ team = await Team.objects.aget(id=team_id)
+ embedded = await asyncio.gather(
+ *(
+ async_generate_embedding(team, text, model=OBSERVATION_EMBEDDING_MODEL.value)
+ for texts in ANCHORS.values()
+ for text in texts
+ )
+ )
+ rows = iter(response.embedding for response in embedded)
+ vectors = {category: [next(rows) for _ in texts] for category, texts in ANCHORS.items()}
+ await cache.aset(_ANCHOR_CACHE_KEY, vectors, _ANCHOR_CACHE_TTL_S)
+ anchors = {category: normalize(np.array(rows, dtype=float)) for category, rows in vectors.items()}
+ return anchors
+
+
+def classify(vector: list[float] | np.ndarray, anchors: dict[WatchCategory, np.ndarray]) -> tuple[WatchCategory, float]:
+ """The closest issue category and its lead over the closest routine anchor, given unit anchors.
+
+ A negative lead means the observation reads more like routine than like any issue, and the
+ category is then routine.
+ """
+ unit = normalize(np.asarray(vector, dtype=float))
+ closest = {category: float(np.max(anchors[category] @ unit)) for category in anchors}
+ category = max(ISSUE_CATEGORIES, key=lambda c: closest[c])
+ contrast = round(closest[category] - closest["routine"], 4)
+ return (category if contrast > 0 else "routine"), contrast
+
+
+# Caps the vectors one duplicate lookup reads to the scanner's newest rows, so a busy scanner stays cheap.
+DUPLICATE_CANDIDATE_ROWS = 5_000
+
+
+def nearest_earlier_observation(
+ *, team_id: int, scanner_id: str, observation_id: str, vector: list[float], now: datetime
+) -> tuple[str, float] | None:
+ """The scanner's closest other observation in the lookback window, with its cosine distance."""
+ with tags_context(product=Product.REPLAY_VISION, query_type="replay_vision_watch_rank_duplicate"):
+ # nosemgrep: clickhouse-fstring-param-audit - table name and scope are module constants
+ rows = sync_execute(
+ f"""
+ SELECT document_id, min(cosineDistance(embedding, %(embedding)s)) AS distance
+ FROM {OBSERVATION_EMBEDDINGS_TABLE}
+ PREWHERE {OBSERVATION_SCOPE_PREWHERE}
+ AND timestamp >= ifNull(
+ (
+ SELECT minOrNull(timestamp)
+ FROM (
+ SELECT timestamp
+ FROM {OBSERVATION_EMBEDDINGS_TABLE}
+ PREWHERE {OBSERVATION_SCOPE_PREWHERE} AND timestamp >= %(since)s
+ ORDER BY timestamp DESC
+ LIMIT %(candidate_cap)s
+ )
+ ),
+ %(since)s
+ )
+ WHERE document_id != %(observation_id)s
+ GROUP BY document_id
+ ORDER BY distance ASC
+ LIMIT 1
+ """,
+ {
+ **observation_scope_params(team_id, [scanner_id]),
+ "embedding": vector,
+ "since": now - DUPLICATE_LOOKBACK,
+ "observation_id": observation_id,
+ "candidate_cap": DUPLICATE_CANDIDATE_ROWS,
+ },
+ team_id=team_id,
+ readonly=True,
+ ch_user=ClickHouseUser.REPLAY_VISION,
+ settings={"max_execution_time": _DUPLICATE_QUERY_TIMEOUT_S},
+ workload=Workload.OFFLINE,
+ )
+ if not rows:
+ return None
+ document_id, distance = rows[0]
+ return str(document_id), float(distance)
+
+
+def duplicate_root(observation_id: str, neighbour_id: str, neighbour_rank: dict[str, Any] | None) -> str | None:
+ """The first observation of a duplicate group, so the group shares one id however long it grows.
+
+ An unranked neighbour can be ranking at the same moment and pick this observation in turn, so the
+ smaller id roots the pair and the two never point at each other.
+ """
+ if neighbour_rank is None:
+ return neighbour_id if neighbour_id < observation_id else None
+ root = neighbour_rank.get("duplicate_of")
+ return root if isinstance(root, str) and root else neighbour_id
+
+
+def current_rank(row: dict[str, Any]) -> dict[str, Any] | None:
+ """The row's rank when it comes from the current anchors, else None."""
+ rank = row.get("watch_rank")
+ if not isinstance(rank, dict) or rank.get("anchor_version") != ANCHOR_VERSION:
+ return None
+ contrast = rank.get("contrast")
+ if not isinstance(contrast, int | float) or isinstance(contrast, bool):
+ return None
+ return rank
+
+
+@frozen
+class RankRequest:
+ team_id: int
+ scanner_id: str
+ observation_id: str
+ vector: list[float]
+
+
+async def rank_observation(request: RankRequest) -> Literal["ranked", "already_ranked", "missing"]:
+ """Rank one observation from its embedding and save the rank on the row, unless it has a current one.
+
+ Keeping an existing rank makes a redelivered result harmless, and stops a second pass from moving a
+ duplicate link in a way that could make two observations point at each other.
+ """
+ row = await (
+ ReplayObservation.objects.filter(team_id=request.team_id, id=request.observation_id)
+ .values("watch_rank")
+ .afirst()
+ )
+ # A deleted observation, or a deleted team, leaves nothing to rank.
+ if row is None:
+ return "missing"
+ if current_rank({"watch_rank": row["watch_rank"]}) is not None:
+ return "already_ranked"
+ category, contrast = classify(request.vector, await anchor_vectors(request.team_id))
+ duplicate_of: str | None = None
+ # The feed only groups issue rows, so a routine row skips the ClickHouse lookup.
+ if category != "routine":
+ nearest = await asyncio.to_thread(
+ nearest_earlier_observation,
+ team_id=request.team_id,
+ scanner_id=request.scanner_id,
+ observation_id=request.observation_id,
+ vector=request.vector,
+ now=timezone.now(),
+ )
+ if nearest is not None and nearest[1] <= DUPLICATE_MAX_DISTANCE:
+ neighbour_rank = await (
+ ReplayObservation.objects.filter(team_id=request.team_id, id=nearest[0])
+ .values_list("watch_rank", flat=True)
+ .afirst()
+ )
+ duplicate_of = duplicate_root(request.observation_id, nearest[0], neighbour_rank)
+ rank = WatchRank(category=category, contrast=contrast, duplicate_of=duplicate_of)
+ await ReplayObservation.objects.filter(team_id=request.team_id, id=request.observation_id).aupdate(
+ watch_rank=rank.to_json()
+ )
+ return "ranked"
+
+
+def _scanner_percentiles(rows: list[dict[str, Any]]) -> dict[str, float]:
+ """Each ranked row's score as a percentile among its scanner's ranked rows in the window."""
+ by_scanner: dict[Any, list[tuple[float, str]]] = {}
+ for row in rows:
+ if rank := current_rank(row):
+ by_scanner.setdefault(row["scanner_id"], []).append((float(rank["contrast"]), str(row["id"])))
+ percentiles: dict[str, float] = {}
+ for scored in by_scanner.values():
+ if len(scored) < WATCH_PICK_MIN_SCANNER_ROWS:
+ continue
+ scored.sort()
+ for position, (_, observation_id) in enumerate(scored):
+ percentiles[observation_id] = (position + 0.5) / len(scored)
+ return percentiles
+
+
+def _group_root(observation_id: str, parents: dict[str, str]) -> str:
+ """Follow `duplicate_of` links to the group's first observation.
+
+ A root ranked after its copies can gain a root of its own, so a link can point at a member rather
+ than at the root. The walk stops at a repeat, so a cycle cannot loop.
+ """
+ seen = {observation_id}
+ while (parent := parents.get(observation_id)) is not None and parent not in seen:
+ seen.add(parent)
+ observation_id = parent
+ return observation_id
+
+
+def rank_watch_feed_by_embedding(rows: list[dict[str, Any]]) -> list[WatchFeedEntry]:
+ """Rank candidate rows (`id`, `scanner_id`, `session_id`, `created_at`, `watch_rank`, `feed_viewed`)
+ on the embedding rank the watch rank consumer wrote.
+
+ A pick is a row in its scanner's top fifth by score whose category is an issue. Near-duplicates
+ collapse to their best-scoring row, which carries the size of the group. Picks sort by weighted
+ percentile, viewed rows docked, one scanner held to a share of the picks, and one session's picks
+ spaced apart. Every other row is filler, trimmed to the same floor as the other rankers.
+ """
+ percentiles = _scanner_percentiles(rows)
+ parents = {
+ str(row["id"]): str(rank["duplicate_of"])
+ for row in rows
+ if (rank := current_rank(row)) is not None and rank.get("duplicate_of")
+ }
+ best_in_group: dict[str, tuple[float, dict[str, Any]]] = {}
+ group_members: dict[str, list[str]] = {}
+ for row in rows:
+ rank = current_rank(row)
+ observation_id = str(row["id"])
+ percentile = percentiles.get(observation_id)
+ if rank is None or percentile is None or rank.get("category") not in ISSUE_CATEGORIES:
+ continue
+ # Group before the cutoff, so duplicates below it still count toward the group and never return as filler.
+ group = _group_root(observation_id, parents)
+ group_members.setdefault(group, []).append(observation_id)
+ # Only a member above the cutoff can stand for the group, so the seen penalty never disqualifies it.
+ if percentile < WATCH_PICK_MIN_PERCENTILE:
+ continue
+ score = percentile * CATEGORY_WEIGHT[rank["category"]]
+ if row.get("feed_viewed"):
+ score -= WATCH_PICK_SEEN_PENALTY
+ if group not in best_in_group or score > best_in_group[group][0]:
+ best_in_group[group] = (score, row)
+ grouped_ids = {member for group in best_in_group for member in group_members[group]}
+
+ picks: list[tuple[float, Any, Any, WatchFeedEntry]] = []
+ for group, (score, row) in best_in_group.items():
+ rank = row["watch_rank"]
+ reason = {
+ "kind": "embedding_pick",
+ "watch_category": rank["category"],
+ "similar_count": len(group_members[group]) - 1,
+ }
+ picks.append(
+ (
+ score,
+ row["created_at"],
+ row.get("scanner_id"),
+ WatchFeedEntry(observation_id=row["id"], reason=reason, session_id=row["session_id"]),
+ )
+ )
+ # A pick stands for its whole duplicate group, so no member returns as filler.
+ return finish_feed(picks, [row for row in rows if str(row["id"]) not in grouped_ids])
diff --git a/products/replay_vision/backend/watch_rank_backfill.py b/products/replay_vision/backend/watch_rank_backfill.py
new file mode 100644
index 000000000000..de0e70a07fb1
--- /dev/null
+++ b/products/replay_vision/backend/watch_rank_backfill.py
@@ -0,0 +1,201 @@
+"""Rank a window of past observations from the embeddings search already stored.
+
+The watch rank consumer ranks each new observation as its embedding lands. This fills in the observations from before
+that shipped, so a team can move to the embedding ranker without waiting a week for ranks to build
+up. It reads stored vectors from ClickHouse rather than embedding again, and finds near-duplicates in
+memory per scanner, oldest first, the same way the consumer would have found them.
+"""
+
+from collections import defaultdict
+from datetime import datetime
+from itertools import batched
+from typing import Any
+
+from django.db.models import Count
+
+import numpy as np
+import structlog
+from asgiref.sync import async_to_sync
+
+from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.client.connection import ClickHouseUser
+from posthog.clickhouse.query_tagging import Product, tags_context
+from posthog.clickhouse.workload import Workload
+from posthog.dataclasses import frozen
+
+from products.replay_vision.backend.embeddings import (
+ OBSERVATION_EMBEDDINGS_TABLE,
+ OBSERVATION_SCOPE_PREWHERE,
+ observation_scope_params,
+)
+from products.replay_vision.backend.models.replay_observation import ObservationStatus, ReplayObservation
+from products.replay_vision.backend.watch_rank import (
+ DUPLICATE_CANDIDATE_ROWS,
+ DUPLICATE_MAX_DISTANCE,
+ WatchCategory,
+ WatchRank,
+ anchor_vectors,
+ classify,
+ current_rank,
+ duplicate_root,
+ normalize,
+)
+
+logger = structlog.get_logger(__name__)
+
+_VECTOR_CHUNK = 500
+_QUERY_TIMEOUT_S = 60
+
+
+@frozen
+class WatchRankBackfillResult:
+ observations: int
+ ranked: int
+ without_embedding: int
+
+
+def teams_with_observations(since: datetime) -> list[int]:
+ """Teams with succeeded observations since `since`, busiest first."""
+ return list(
+ ReplayObservation.objects.filter(status=ObservationStatus.SUCCEEDED, created_at__gte=since)
+ .values("team_id")
+ .annotate(n=Count("id"))
+ .order_by("-n")
+ .values_list("team_id", flat=True)
+ )
+
+
+def _stored_vectors(
+ team_id: int, scanner_id: str, observation_ids: list[str], since: datetime
+) -> dict[str, np.ndarray]:
+ """Each observation's stored unit vector, for one batch of a scanner's observations."""
+ with tags_context(product=Product.REPLAY_VISION, query_type="replay_vision_watch_rank_backfill"):
+ # nosemgrep: clickhouse-fstring-param-audit - table name and scope are module constants
+ rows = sync_execute(
+ f"""
+ SELECT document_id, any(embedding)
+ FROM {OBSERVATION_EMBEDDINGS_TABLE}
+ PREWHERE {OBSERVATION_SCOPE_PREWHERE}
+ AND timestamp >= %(since)s
+ WHERE document_id IN %(ids)s
+ GROUP BY document_id
+ """,
+ {**observation_scope_params(team_id, [scanner_id]), "since": since, "ids": observation_ids},
+ team_id=team_id,
+ readonly=True,
+ ch_user=ClickHouseUser.REPLAY_VISION,
+ settings={"max_execution_time": _QUERY_TIMEOUT_S},
+ workload=Workload.OFFLINE,
+ )
+ return {str(document_id): normalize(np.asarray(embedding, dtype=np.float32)) for document_id, embedding in rows}
+
+
+@frozen
+class _Neighbour:
+ observation_id: str
+ rank: dict[str, Any] | None
+ distance: float
+
+
+class _RecentVectors:
+ """A scanner's newest vectors and ranks, capped like the live duplicate lookup so memory and work stay bounded."""
+
+ def __init__(self) -> None:
+ self._capacity = DUPLICATE_CANDIDATE_ROWS
+ self._matrix: np.ndarray | None = None
+ self._ids: list[str] = []
+ self._ranks: list[dict[str, Any] | None] = []
+ self._next = 0
+
+ def nearest(self, vector: np.ndarray) -> _Neighbour | None:
+ if self._matrix is None or not self._ids:
+ return None
+ distances = 1.0 - self._matrix[: len(self._ids)] @ vector
+ index = int(np.argmin(distances))
+ return _Neighbour(observation_id=self._ids[index], rank=self._ranks[index], distance=float(distances[index]))
+
+ def add(self, observation_id: str, vector: np.ndarray, rank: dict[str, Any] | None) -> None:
+ if self._matrix is None:
+ self._matrix = np.empty((self._capacity, vector.shape[0]), dtype=np.float32)
+ self._matrix[self._next] = vector
+ if len(self._ids) < self._capacity:
+ self._ids.append(observation_id)
+ self._ranks.append(rank)
+ else:
+ self._ids[self._next] = observation_id
+ self._ranks[self._next] = rank
+ self._next = (self._next + 1) % self._capacity
+
+
+def _rank_scanner(
+ rows: list[dict[str, Any]],
+ vectors: dict[str, np.ndarray],
+ anchors: dict[WatchCategory, np.ndarray],
+ recent: _RecentVectors,
+) -> dict[str, dict[str, Any]]:
+ """New ranks for the rows that lack a current one, walking oldest first.
+
+ Pass the same `recent` across a scanner's batches, so a copy finds a match from an earlier batch.
+ """
+ written: dict[str, dict[str, Any]] = {}
+ for row in rows:
+ observation_id = str(row["id"])
+ vector = vectors.get(observation_id)
+ if vector is None:
+ continue
+ rank = current_rank(row)
+ if rank is None:
+ category, contrast = classify(vector, anchors)
+ duplicate_of: str | None = None
+ nearest = recent.nearest(vector) if category != "routine" else None
+ if nearest is not None and nearest.distance <= DUPLICATE_MAX_DISTANCE:
+ duplicate_of = duplicate_root(observation_id, nearest.observation_id, nearest.rank)
+ rank = WatchRank(category=category, contrast=contrast, duplicate_of=duplicate_of).to_json()
+ written[observation_id] = rank
+ recent.add(observation_id, vector, rank)
+ return written
+
+
+def backfill_team_watch_ranks(team_id: int, since: datetime, *, dry_run: bool = False) -> WatchRankBackfillResult:
+ """Rank the team's succeeded observations since `since` that lack a rank from the current anchors.
+
+ Works one scanner and one batch at a time, and saves each batch before the next, so a large team
+ never holds its whole window in memory and an interrupted run keeps what it finished.
+ """
+ rows: list[dict[str, Any]] = [
+ dict(row)
+ for row in ReplayObservation.objects.filter(
+ team_id=team_id, status=ObservationStatus.SUCCEEDED, created_at__gte=since
+ )
+ .order_by("created_at", "id")
+ .values("id", "scanner_id", "watch_rank")
+ ]
+ by_scanner: dict[str, list[dict[str, Any]]] = defaultdict(list)
+ for row in rows:
+ by_scanner[str(row["scanner_id"])].append(row)
+ anchors = async_to_sync(anchor_vectors)(team_id)
+ ranked = 0
+ embedded = 0
+ for scanner_id, scanner_rows in by_scanner.items():
+ recent = _RecentVectors()
+ for batch in batched(scanner_rows, _VECTOR_CHUNK, strict=False):
+ vectors = _stored_vectors(team_id, scanner_id, [str(row["id"]) for row in batch], since)
+ embedded += len(vectors)
+ written = _rank_scanner(list(batch), vectors, anchors, recent)
+ ranked += len(written)
+ if not dry_run and written:
+ # The ids come from the team-scoped query above, so the update needs no team filter of its own.
+ ReplayObservation.objects.bulk_update(
+ [ReplayObservation(id=observation_id, watch_rank=rank) for observation_id, rank in written.items()],
+ ["watch_rank"],
+ )
+ without_embedding = len(rows) - embedded
+ logger.info(
+ "replay_vision.watch_rank_backfill.team_done",
+ team_id=team_id,
+ observations=len(rows),
+ ranked=ranked,
+ without_embedding=without_embedding,
+ dry_run=dry_run,
+ )
+ return WatchRankBackfillResult(observations=len(rows), ranked=ranked, without_embedding=without_embedding)
diff --git a/products/replay_vision/backend/watch_rank_consumer.py b/products/replay_vision/backend/watch_rank_consumer.py
new file mode 100644
index 000000000000..8ce94d2a6d1c
--- /dev/null
+++ b/products/replay_vision/backend/watch_rank_consumer.py
@@ -0,0 +1,235 @@
+"""Ranks observations for What to watch from the embedding worker's results topic.
+
+The scan emits each observation's text to the embedding worker over Kafka. The worker publishes every
+embedded document to the results topic, and this consumer ranks the Replay Vision ones from their
+vectors. A worker outage only delays ranks, and a database outage shorter than the retry window too.
+"""
+
+import time
+import signal
+import asyncio
+from collections.abc import Sequence
+from typing import Any, Literal, Protocol
+
+from django.conf import settings
+from django.db import close_old_connections
+
+import orjson
+import structlog
+from asgiref.sync import sync_to_async
+from confluent_kafka import Consumer, KafkaException, Message, TopicPartition
+from prometheus_client import Counter
+
+from posthog.kafka_client import helper
+from posthog.kafka_client.client import _build_sasl_config, _KafkaSecurityProtocol
+from posthog.kafka_client.routing import get_profile_settings
+from posthog.kafka_client.topics import KAFKA_DOCUMENT_EMBEDDING_RESULTS_TOPIC
+from posthog.temporal.common.liveness_tracker import LivenessTracker
+
+from products.replay_vision.backend.embeddings import (
+ EMBEDDING_DOCUMENT_TYPE,
+ EMBEDDING_PRODUCT,
+ OBSERVATION_EMBEDDING_MODEL,
+)
+from products.replay_vision.backend.watch_rank import RankRequest, rank_observation
+
+logger = structlog.get_logger(__name__)
+
+CONSUMER_GROUP = "replay-vision-watch-rank"
+
+_BATCH_SIZE = 50
+_POLL_TIMEOUT_S = 1.0
+# A batch stops after this long and goes back to Kafka, so the loop polls well inside the poll interval.
+_BATCH_BUDGET_S = 240.0
+_MAX_POLL_INTERVAL_S = 360.0
+# ClickHouse loads the embeddings table from its own topic some seconds after the worker writes both.
+_CLICKHOUSE_VISIBILITY_DELAY_S = 30.0
+# How long one observation keeps failing, across batches, before the consumer moves past it.
+_RETRY_WINDOW_S = 600.0
+_RETRY_BACKOFF_S = 2.0
+_RETRY_BACKOFF_MAX_S = 60.0
+
+Outcome = Literal["skipped", "ranked", "already_ranked", "missing", "failed", "deferred", "interrupted"]
+
+MESSAGES_TOTAL = Counter(
+ "replay_vision_watch_rank_consumer_messages_total",
+ "Embedding results the What to watch rank consumer read, by outcome.",
+ labelnames=["outcome"],
+)
+
+# The worker writes compact JSON, and a quote inside a string value is escaped, so this marks our results.
+_DOCUMENT_TYPE_MARKER = f'"document_type":"{EMBEDDING_DOCUMENT_TYPE}"'.encode()
+
+
+def parse_rank_request(value: bytes | None) -> RankRequest | None:
+ """The rank request in one results message, or None when it is not a Replay Vision observation."""
+ # Most messages belong to other products, and decoding their content and vectors is the expensive part.
+ if value is None or _DOCUMENT_TYPE_MARKER not in value:
+ return None
+ try:
+ payload = orjson.loads(value)
+ except orjson.JSONDecodeError:
+ return None
+ if not isinstance(payload, dict):
+ return None
+ if payload.get("product") != EMBEDDING_PRODUCT or payload.get("document_type") != EMBEDDING_DOCUMENT_TYPE:
+ return None
+ metadata = payload.get("metadata")
+ scanner_id = metadata.get("scanner_id") if isinstance(metadata, dict) else None
+ vector = next(
+ (
+ result.get("embedding")
+ for result in payload.get("results") or []
+ if isinstance(result, dict)
+ and result.get("model") == OBSERVATION_EMBEDDING_MODEL.value
+ and result.get("outcome") == "success"
+ ),
+ None,
+ )
+ team_id = payload.get("team_id")
+ observation_id = payload.get("document_id")
+ if not isinstance(team_id, int) or not isinstance(scanner_id, str) or not isinstance(observation_id, str):
+ return None
+ # A zero vector has no direction, so the cosine scores would come out as NaN.
+ if not isinstance(vector, list) or not any(vector):
+ return None
+ return RankRequest(team_id=team_id, scanner_id=scanner_id, observation_id=observation_id, vector=vector)
+
+
+class BatchConsumer(Protocol):
+ def store_offsets(self, *, message: Message) -> None: ...
+
+ def seek(self, partition: TopicPartition, /) -> None: ...
+
+
+class WatchRankConsumer:
+ def __init__(self, liveness: LivenessTracker) -> None:
+ self._liveness = liveness
+ self._stop = asyncio.Event()
+ # When each observation that is still failing first failed, so its retry window spans batches.
+ self._first_failed_at: dict[str, float] = {}
+
+ async def _wait(self, seconds: float) -> bool:
+ """Sleep, unless shutdown comes first. True when the full time passed."""
+ try:
+ await asyncio.wait_for(self._stop.wait(), timeout=max(seconds, 0))
+ except TimeoutError:
+ return True
+ return False
+
+ async def handle(self, value: bytes | None, produced_at: float, batch_deadline: float) -> Outcome:
+ """Rank the observation in one message, retrying failures until the batch or retry window ends."""
+ request = parse_rank_request(value)
+ if request is None:
+ return "skipped"
+ if not await self._wait(produced_at + _CLICKHOUSE_VISIBILITY_DELAY_S - time.time()):
+ return "interrupted"
+ backoff = _RETRY_BACKOFF_S
+ while True:
+ # Async ORM calls share one thread, so a connection that broke must be replaced before a retry.
+ await sync_to_async(close_old_connections)()
+ try:
+ outcome = await rank_observation(request)
+ self._first_failed_at.pop(request.observation_id, None)
+ return outcome
+ except Exception:
+ logger.exception(
+ "replay_vision.watch_rank_consumer.rank_failed",
+ team_id=request.team_id,
+ observation_id=request.observation_id,
+ )
+ self._liveness.record_heartbeat()
+ now = time.monotonic()
+ give_up_at = self._first_failed_at.setdefault(request.observation_id, now) + _RETRY_WINDOW_S
+ if now + backoff > give_up_at:
+ # The backfill command can write a rank this gave up on.
+ self._first_failed_at.pop(request.observation_id, None)
+ return "failed"
+ if now + backoff > batch_deadline:
+ return "deferred"
+ if not await self._wait(backoff):
+ return "interrupted"
+ backoff = min(backoff * 2, _RETRY_BACKOFF_MAX_S)
+
+ async def process_batch(self, consumer: BatchConsumer, messages: Sequence[Message]) -> None:
+ batch_deadline = time.monotonic() + _BATCH_BUDGET_S
+ for index, message in enumerate(messages):
+ if self._stop.is_set():
+ return
+ if message.error():
+ logger.warning("replay_vision.watch_rank_consumer.kafka_error", error=str(message.error()))
+ continue
+ _, produced_at_ms = message.timestamp()
+ outcome = await self.handle(message.value(), produced_at_ms / 1000, batch_deadline)
+ self._liveness.record_heartbeat()
+ if outcome == "interrupted":
+ return
+ if outcome == "deferred":
+ self._rewind(consumer, messages[index:])
+ return
+ MESSAGES_TOTAL.labels(outcome=outcome).inc()
+ try:
+ consumer.store_offsets(message=message)
+ except KafkaException:
+ logger.warning("replay_vision.watch_rank_consumer.offset_store_failed", partition=message.partition())
+ self._rewind(consumer, messages[index + 1 :])
+ return
+
+ @staticmethod
+ def _rewind(consumer: BatchConsumer, unhandled: Sequence[Message]) -> None:
+ """Seek each partition back to its first unhandled message, so the next poll reads it again."""
+ first: dict[tuple[str, int], int] = {}
+ for message in unhandled:
+ topic, partition = message.topic(), message.partition()
+ if topic is not None and partition is not None:
+ first.setdefault((topic, partition), message.offset() or 0)
+ for (topic, partition), offset in first.items():
+ try:
+ consumer.seek(TopicPartition(topic, partition, offset))
+ except KafkaException:
+ # A rebalance took the partition, and its new owner starts from the last committed offset.
+ continue
+
+ @staticmethod
+ def _consumer_config() -> dict[str, Any]:
+ profile = get_profile_settings(topic=KAFKA_DOCUMENT_EMBEDDING_RESULTS_TOPIC)
+ security_protocol = profile.security_protocol
+ # Self-hosted base64 cert mode supplies its own SSL material, so SASL credentials must not attach.
+ if settings.KAFKA_BASE64_KEYS:
+ security_protocol = _KafkaSecurityProtocol.SSL
+ config: dict[str, Any] = {
+ "bootstrap.servers": ",".join(profile.hosts),
+ "group.id": CONSUMER_GROUP,
+ # Ranking skips rows that already have a current rank, so reading old results again is harmless.
+ "auto.offset.reset": "earliest",
+ # Offsets are stored only after a message is handled, and committed in the background.
+ "enable.auto.commit": True,
+ "enable.auto.offset.store": False,
+ "max.poll.interval.ms": int(_MAX_POLL_INTERVAL_S * 1000),
+ # The worker writes results inside Kafka transactions, and an aborted one must never rank.
+ "isolation.level": "read_committed",
+ "security.protocol": security_protocol or _KafkaSecurityProtocol.PLAINTEXT,
+ }
+ config.update(
+ _build_sasl_config(security_protocol, profile.sasl_mechanism, profile.sasl_user, profile.sasl_password)
+ )
+ if settings.KAFKA_BASE64_KEYS:
+ config.update(helper.ssl_cert_config())
+ return config
+
+ async def run(self) -> None: # pragma: no cover - exercised in dogfood
+ loop = asyncio.get_running_loop()
+ for sig in (signal.SIGINT, signal.SIGTERM):
+ loop.add_signal_handler(sig, self._stop.set)
+
+ consumer = Consumer(self._consumer_config())
+ consumer.subscribe([KAFKA_DOCUMENT_EMBEDDING_RESULTS_TOPIC])
+ logger.info("replay_vision.watch_rank_consumer.started")
+ try:
+ while not self._stop.is_set():
+ self._liveness.record_heartbeat()
+ messages = await asyncio.to_thread(consumer.consume, _BATCH_SIZE, _POLL_TIMEOUT_S)
+ await self.process_batch(consumer, messages)
+ finally:
+ consumer.close()
+ logger.info("replay_vision.watch_rank_consumer.stopped")
diff --git a/products/replay_vision/frontend/generated/api.schemas.ts b/products/replay_vision/frontend/generated/api.schemas.ts
index 547447029c7d..ae8506b988b9 100644
--- a/products/replay_vision/frontend/generated/api.schemas.ts
+++ b/products/replay_vision/frontend/generated/api.schemas.ts
@@ -2984,6 +2984,7 @@ export interface SuggestTagsResponseApi {
* * `novel_summary` - Novel Summary
* * `friction` - Friction
* * `jev_watchable` - Jev Watchable
+ * * `embedding_pick` - Embedding Pick
* * `unviewed_recent` - Unviewed Recent
* * `recent` - Recent
*/
@@ -2999,6 +3000,7 @@ export const WatchFeedReasonEnumApi = {
NovelSummary: 'novel_summary',
Friction: 'friction',
JevWatchable: 'jev_watchable',
+ EmbeddingPick: 'embedding_pick',
UnviewedRecent: 'unviewed_recent',
Recent: 'recent',
} as const
@@ -3042,11 +3044,24 @@ export const JevWatchReasonEnumApi = {
Success: 'success',
} as const
+/**
+ * * `failure` - Failure
+ * * `struggle` - Struggle
+ * * `revenue` - Revenue
+ */
+export type WatchFeedCategoryEnumApi = (typeof WatchFeedCategoryEnumApi)[keyof typeof WatchFeedCategoryEnumApi]
+
+export const WatchFeedCategoryEnumApi = {
+ Failure: 'failure',
+ Struggle: 'struggle',
+ Revenue: 'revenue',
+} as const
+
/**
* Machine-readable reason an observation made the feed; the frontend renders the copy.
*/
export interface WatchFeedReasonApi {
- /** Highest-priority rule the observation satisfied: `signal_emitted` (it pushed a signal), `unusual_verdict` (a monitor answer that is the minority for that scanner this window), `verdict_yes` (a monitor hit, when the window is too thin to know which answer is unusual), `outlier_score` (far from the scanner's window average), `rare_tag` (a tag uncommon for the scanner this window), `novel_summary` (a summary that reads unlike the scanner's other sessions this window), `notable` (the scan itself judged the session worth watching), `friction` (the scan describes errors, retries, or dead ends), `jev_watchable` (the decision model judged the session worth watching; teams on the Jev ranker experiment only), `unviewed_recent` (new to you), `recent` (nothing special, newest available).
+ /** Highest-priority rule the observation satisfied: `signal_emitted` (it pushed a signal), `unusual_verdict` (a monitor answer that is the minority for that scanner this window), `verdict_yes` (a monitor hit, when the window is too thin to know which answer is unusual), `outlier_score` (far from the scanner's window average), `rare_tag` (a tag uncommon for the scanner this window), `novel_summary` (a summary that reads unlike the scanner's other sessions this window), `notable` (the scan itself judged the session worth watching), `friction` (the scan describes errors, retries, or dead ends), `jev_watchable` (the decision model judged the session worth watching; teams on the Jev ranker experiment only), `embedding_pick` (the session reads like a failure, struggle, or revenue moment and ranks in its scanner's top fifth; teams on the embedding ranker only), `unviewed_recent` (new to you), `recent` (nothing special, newest available).
*
* * `signal_emitted` - Signal Emitted
* * `unusual_verdict` - Unusual Verdict
@@ -3057,6 +3072,7 @@ export interface WatchFeedReasonApi {
* * `novel_summary` - Novel Summary
* * `friction` - Friction
* * `jev_watchable` - Jev Watchable
+ * * `embedding_pick` - Embedding Pick
* * `unviewed_recent` - Unviewed Recent
* * `recent` - Recent */
kind: WatchFeedReasonEnumApi
@@ -3128,6 +3144,17 @@ export interface WatchFeedReasonApi {
* @nullable
*/
tag_share?: number | null
+ /** What the session reads like, for `embedding_pick`.
+ *
+ * * `failure` - Failure
+ * * `struggle` - Struggle
+ * * `revenue` - Revenue */
+ watch_category?: WatchFeedCategoryEnumApi
+ /**
+ * Other sessions from the same scanner in the window that describe nearly the same thing, for `embedding_pick`. The feed shows one session per group.
+ * @nullable
+ */
+ similar_count?: number | null
}
/**
@@ -3141,14 +3168,16 @@ export interface WatchFeedItemApi {
}
/**
- * * `weighted-score` - weighted-score
- * * `jev` - jev
+ * * `weighted-score` - Weighted Score
+ * * `jev` - Jev
+ * * `embedding` - Embedding
*/
-export type RankerEnumApi = (typeof RankerEnumApi)[keyof typeof RankerEnumApi]
+export type WatchFeedRankerEnumApi = (typeof WatchFeedRankerEnumApi)[keyof typeof WatchFeedRankerEnumApi]
-export const RankerEnumApi = {
+export const WatchFeedRankerEnumApi = {
WeightedScore: 'weighted-score',
Jev: 'jev',
+ Embedding: 'embedding',
} as const
/**
@@ -3157,13 +3186,14 @@ export const RankerEnumApi = {
export interface WatchFeedResponseApi {
/** Succeeded observations in the window worth watching, most interesting first, each carrying the reason it ranked. Every observation that carries a finding is returned; observations that carry none (`unviewed_recent`, `recent`) are returned only to pad a near-empty feed to three items, so a quiet window answers with a handful of rows rather than a full page of newest clips. */
results: WatchFeedItemApi[]
- /** Which ranker ordered this feed: `jev` ranks on the decision model's cached judgments, `weighted-score` on the deterministic blend. The arm is decided server-side per team, so clients read it from here rather than evaluating the flag themselves.
+ /** Which ranker ordered this feed: `jev` ranks on the decision model's cached judgments, `embedding` on the rank the watch rank consumer wrote from the observation's embedding, `weighted-score` on the deterministic blend. The arm is decided server-side per team, so clients read it from here rather than evaluating the flag themselves.
*
- * * `weighted-score` - weighted-score
- * * `jev` - jev */
- ranker: RankerEnumApi
+ * * `weighted-score` - Weighted Score
+ * * `jev` - Jev
+ * * `embedding` - Embedding */
+ ranker: WatchFeedRankerEnumApi
/**
- * The team's variant of the `vision-watch-feed-ranker` experiment flag (`control`, `jev-shadow`, `jev`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from control. Clients report it on the feed-viewed event as `$feature/vision-watch-feed-ranker`, which is the exposure the experiment counts.
+ * The team's variant of the `vision-watch-feed-ranker` experiment flag (`control`, `jev-shadow`, `jev`, `embedding`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from control. Clients report it on the feed-viewed event as `$feature/vision-watch-feed-ranker`, which is the exposure the experiment counts.
* @nullable
*/
ranker_variant: string | null
diff --git a/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.test.ts b/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.test.ts
index da626799a3c3..ea0f5ab50ddf 100644
--- a/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.test.ts
+++ b/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.test.ts
@@ -125,6 +125,16 @@ describe('WatchFeedCard helpers', () => {
reason: { kind: 'rare_tag' } as WatchFeedReasonApi,
expected: 'Tagged something uncommon for this scanner lately.',
},
+ {
+ name: 'names the category of an embedding pick with no similar sessions',
+ reason: { kind: 'embedding_pick', watch_category: 'failure', similar_count: 0 } as WatchFeedReasonApi,
+ expected: 'Something failed in this session.',
+ },
+ {
+ name: 'counts the similar sessions behind an embedding pick',
+ reason: { kind: 'embedding_pick', watch_category: 'struggle', similar_count: 1 } as WatchFeedReasonApi,
+ expected: 'The user struggled in this session. 1 other session from this scanner looks similar.',
+ },
{
name: 'renders a sentence for a reason kind this frontend does not know',
reason: { kind: 'some_future_kind' } as unknown as WatchFeedReasonApi,
diff --git a/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.tsx b/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.tsx
index 5330af3699c3..f04afddb0897 100644
--- a/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.tsx
+++ b/products/replay_vision/frontend/replay_scanners/components/WatchFeedCard.tsx
@@ -17,6 +17,7 @@ import { UnviewedObservationTag } from '../../components/UnviewedObservationTag'
import type {
JevWatchReasonEnumApi,
ReplayObservationApi,
+ WatchFeedCategoryEnumApi,
WatchFeedItemApi,
WatchFeedReasonApi,
} from '../../generated/api.schemas'
@@ -49,6 +50,12 @@ const MAX_SHOWN_HEADLINES = 3
const joinWithAnd = (parts: string[]): string =>
parts.length > 1 ? `${parts.slice(0, -1).join(', ')}, and ${parts[parts.length - 1]}` : parts[0]
+const WATCH_CATEGORY_COPY: Record<WatchFeedCategoryEnumApi, string> = {
+ failure: 'Something failed in this session.',
+ struggle: 'The user struggled in this session.',
+ revenue: 'This session touches billing or a plan limit.',
+}
+
/** What the reason kind says about the session. Never the scan's notability sentence: the row's title
* already leads with it. */
export function watchReasonCopy(reason: WatchFeedReasonApi): string {
@@ -133,6 +140,16 @@ export function watchReasonCopy(reason: WatchFeedReasonApi): string {
return 'The session shows signs of friction, like errors, retries, or dead ends.'
case 'jev_watchable':
return 'The decision model judged this session worth watching.'
+ case 'embedding_pick': {
+ const category = reason.watch_category
+ ? WATCH_CATEGORY_COPY[reason.watch_category]
+ : 'This session stands out for this scanner.'
+ const similar = reason.similar_count ?? 0
+ if (similar === 0) {
+ return category
+ }
+ return `${category} ${similar} other session${similar === 1 ? ' from this scanner looks' : 's from this scanner look'} similar.`
+ }
case 'unviewed_recent':
return 'New since you last looked.'
case 'recent':
diff --git a/products/replay_vision/frontend/replay_scanners/watchFeedLogic.ts b/products/replay_vision/frontend/replay_scanners/watchFeedLogic.ts
index 986eee0ce243..9b23e14849f0 100644
--- a/products/replay_vision/frontend/replay_scanners/watchFeedLogic.ts
+++ b/products/replay_vision/frontend/replay_scanners/watchFeedLogic.ts
@@ -15,7 +15,7 @@ import { visionScannersWatchFeedRetrieve } from '../generated/api'
import type { ReplayScannerApi, VisionQuotaApi, WatchFeedResponseApi } from '../generated/api.schemas'
import type { VisionScannersWatchFeedRetrieveParams, WatchFeedItemApi } from '../generated/api.schemas'
import type { ScannerTypeEnumApi } from '../generated/api.schemas'
-import type { RankerEnumApi } from '../generated/api.schemas'
+import type { WatchFeedRankerEnumApi } from '../generated/api.schemas'
import type { FeedDismissalReasonEnumApi } from '../generated/api.schemas'
import { visionQuotaLogic } from '../logics/visionQuotaLogic'
import { visionScannersListLogic } from '../logics/visionScannersListLogic'
@@ -139,7 +139,7 @@ export interface watchFeedLogicActions {
ranker: WatchFeedRanker,
variant: string | null
) => {
- ranker: RankerEnumApi
+ ranker: WatchFeedRankerEnumApi
variant: string | null
}
setScannerIdsFilter: (scannerIds: string[]) => {
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 0ce3db17b019..395b50059e0b 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -100220,18 +100220,6 @@ export namespace Schemas {
code_usage_billing_active: boolean;
}
- /**
- * * `weighted-score` - weighted-score
- * * `jev` - jev
- */
- export type RankerEnum = typeof RankerEnum[keyof typeof RankerEnum];
-
-
- export const RankerEnum = {
- WeightedScore: 'weighted-score',
- Jev: 'jev',
- } as const;
-
export interface RankingOutcomeInfo {
/** Name of the outcome in the `ranking` field's predictions.
*
@@ -120369,6 +120357,20 @@ export namespace Schemas {
run_id: string;
}
+ /**
+ * * `failure` - Failure
+ * * `struggle` - Struggle
+ * * `revenue` - Revenue
+ */
+ export type WatchFeedCategoryEnum = typeof WatchFeedCategoryEnum[keyof typeof WatchFeedCategoryEnum];
+
+
+ export const WatchFeedCategoryEnum = {
+ Failure: 'failure',
+ Struggle: 'struggle',
+ Revenue: 'revenue',
+ } as const;
+
/**
* * `signal_emitted` - Signal Emitted
* * `unusual_verdict` - Unusual Verdict
@@ -120379,6 +120381,7 @@ export namespace Schemas {
* * `novel_summary` - Novel Summary
* * `friction` - Friction
* * `jev_watchable` - Jev Watchable
+ * * `embedding_pick` - Embedding Pick
* * `unviewed_recent` - Unviewed Recent
* * `recent` - Recent
*/
@@ -120395,6 +120398,7 @@ export namespace Schemas {
NovelSummary: 'novel_summary',
Friction: 'friction',
JevWatchable: 'jev_watchable',
+ EmbeddingPick: 'embedding_pick',
UnviewedRecent: 'unviewed_recent',
Recent: 'recent',
} as const;
@@ -120413,7 +120417,7 @@ export namespace Schemas {
* Machine-readable reason an observation made the feed; the frontend renders the copy.
*/
export interface WatchFeedReason {
- /** Highest-priority rule the observation satisfied: `signal_emitted` (it pushed a signal), `unusual_verdict` (a monitor answer that is the minority for that scanner this window), `verdict_yes` (a monitor hit, when the window is too thin to know which answer is unusual), `outlier_score` (far from the scanner's window average), `rare_tag` (a tag uncommon for the scanner this window), `novel_summary` (a summary that reads unlike the scanner's other sessions this window), `notable` (the scan itself judged the session worth watching), `friction` (the scan describes errors, retries, or dead ends), `jev_watchable` (the decision model judged the session worth watching; teams on the Jev ranker experiment only), `unviewed_recent` (new to you), `recent` (nothing special, newest available).
+ /** Highest-priority rule the observation satisfied: `signal_emitted` (it pushed a signal), `unusual_verdict` (a monitor answer that is the minority for that scanner this window), `verdict_yes` (a monitor hit, when the window is too thin to know which answer is unusual), `outlier_score` (far from the scanner's window average), `rare_tag` (a tag uncommon for the scanner this window), `novel_summary` (a summary that reads unlike the scanner's other sessions this window), `notable` (the scan itself judged the session worth watching), `friction` (the scan describes errors, retries, or dead ends), `jev_watchable` (the decision model judged the session worth watching; teams on the Jev ranker experiment only), `embedding_pick` (the session reads like a failure, struggle, or revenue moment and ranks in its scanner's top fifth; teams on the embedding ranker only), `unviewed_recent` (new to you), `recent` (nothing special, newest available).
*
* * `signal_emitted` - Signal Emitted
* * `unusual_verdict` - Unusual Verdict
@@ -120424,6 +120428,7 @@ export namespace Schemas {
* * `novel_summary` - Novel Summary
* * `friction` - Friction
* * `jev_watchable` - Jev Watchable
+ * * `embedding_pick` - Embedding Pick
* * `unviewed_recent` - Unviewed Recent
* * `recent` - Recent */
kind: WatchFeedReasonEnum;
@@ -120495,6 +120500,17 @@ export namespace Schemas {
* @nullable
*/
tag_share?: number | null;
+ /** What the session reads like, for `embedding_pick`.
+ *
+ * * `failure` - Failure
+ * * `struggle` - Struggle
+ * * `revenue` - Revenue */
+ watch_category?: WatchFeedCategoryEnum;
+ /**
+ * Other sessions from the same scanner in the window that describe nearly the same thing, for `embedding_pick`. The feed shows one session per group.
+ * @nullable
+ */
+ similar_count?: number | null;
}
/**
@@ -120507,19 +120523,34 @@ export namespace Schemas {
reason: WatchFeedReason;
}
+ /**
+ * * `weighted-score` - Weighted Score
+ * * `jev` - Jev
+ * * `embedding` - Embedding
+ */
+ export type WatchFeedRankerEnum = typeof WatchFeedRankerEnum[keyof typeof WatchFeedRankerEnum];
+
+
+ export const WatchFeedRankerEnum = {
+ WeightedScore: 'weighted-score',
+ Jev: 'jev',
+ Embedding: 'embedding',
+ } as const;
+
/**
* Response of GET /vision/scanners/watch_feed/.
*/
export interface WatchFeedResponse {
/** Succeeded observations in the window worth watching, most interesting first, each carrying the reason it ranked. Every observation that carries a finding is returned; observations that carry none (`unviewed_recent`, `recent`) are returned only to pad a near-empty feed to three items, so a quiet window answers with a handful of rows rather than a full page of newest clips. */
results: WatchFeedItem[];
- /** Which ranker ordered this feed: `jev` ranks on the decision model's cached judgments, `weighted-score` on the deterministic blend. The arm is decided server-side per team, so clients read it from here rather than evaluating the flag themselves.
+ /** Which ranker ordered this feed: `jev` ranks on the decision model's cached judgments, `embedding` on the rank the watch rank consumer wrote from the observation's embedding, `weighted-score` on the deterministic blend. The arm is decided server-side per team, so clients read it from here rather than evaluating the flag themselves.
*
- * * `weighted-score` - weighted-score
- * * `jev` - jev */
- ranker: RankerEnum;
+ * * `weighted-score` - Weighted Score
+ * * `jev` - Jev
+ * * `embedding` - Embedding */
+ ranker: WatchFeedRankerEnum;
/**
- * The team's variant of the `vision-watch-feed-ranker` experiment flag (`control`, `jev-shadow`, `jev`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from control. Clients report it on the feed-viewed event as `$feature/vision-watch-feed-ranker`, which is the exposure the experiment counts.
+ * The team's variant of the `vision-watch-feed-ranker` experiment flag (`control`, `jev-shadow`, `jev`, `embedding`), or null when the team takes no part. Unlike `ranker`, it tells the shadow arm from control. Clients report it on the feed-viewed event as `$feature/vision-watch-feed-ranker`, which is the exposure the experiment counts.
* @nullable
*/
ranker_variant: string | null;