:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-vision): store narrow observation ranks and build highlight feed windows (#125980)
Tue Haulund461bff09b845
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yamlindex f4904ec389c9..35fe5738df0c 100644--- a/.semgrep/rules/security/idor-team-scoped-models.yaml+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml@@ -350,12 +350,15 @@ rules:|QueueSchedulerState|QuickFilter|QuickFilterContext+ |ReplayHighlightFeedEntry+ |ReplayHighlightFeedState|ReplayObservation|ReplayObservationAnchorSimilarity|ReplayObservationFeedDismissal|ReplayObservationFeedImpression|ReplayObservationLabel|ReplayObservationMedia+ |ReplayObservationRank|ReplayObservationRequest|ReplayObservationUsage|ReplayObservationView@@ -839,12 +842,15 @@ rules:|QueueSchedulerState|QuickFilter|QuickFilterContext+ |ReplayHighlightFeedEntry+ |ReplayHighlightFeedState|ReplayObservation|ReplayObservationAnchorSimilarity|ReplayObservationFeedDismissal|ReplayObservationFeedImpression|ReplayObservationLabel|ReplayObservationMedia+ |ReplayObservationRank|ReplayObservationRequest|ReplayObservationUsage|ReplayObservationViewdiff --git a/products/replay_vision/backend/highlight_builds.py b/products/replay_vision/backend/highlight_builds.pynew file mode 100644index 000000000000..be2a085c03df--- /dev/null+++ b/products/replay_vision/backend/highlight_builds.py@@ -0,0 +1,200 @@+"""Ranks a team's highlight feed windows from its rank rows, and stores each preset window as a build that page loads+read instead of ranking the window themselves.++A row's percentile is its place among every ranked row its scanner has in the window, so one statement places all four+preset windows from a single read of the longest one. A build keeps only rows that clear the percentile cut, up to a+bounded number of places per scanner, which is all a page load can reach.+"""++from collections.abc import Sequence+from dataclasses import fields+from datetime import datetime, timedelta+from typing import Any++from django.db import connection, transaction+from django.utils import timezone++from posthog.dataclasses import frozen+from posthog.models.scoping.manager import resolve_effective_team_id++from products.replay_vision.backend.models.replay_highlight_feed import (+ HighlightWindow,+ ReplayHighlightFeedEntry,+ ReplayHighlightFeedState,+)+from products.replay_vision.backend.models.replay_observation_rank import RankCategory++# The feed surfaces a scanner's top fifth by score, once the scanner has enough ranked rows to make a percentile mean+# something.+MIN_PERCENTILE = 0.8+MIN_SCANNER_ROWS = 10+# Page loads read places round-robin across scanners, so no load reaches deeper than this into one scanner.+MAX_PLACES_PER_SCANNER = 500+ISSUE_CODES = (RankCategory.FAILURE, RankCategory.STRUGGLE, RankCategory.REVENUE)+# Builds a reader may still be scrolling stay this long after a newer build replaces them.+BUILD_RETENTION = timedelta(minutes=30)+_STATEMENT_TIMEOUT = "120s"+_LOCK_TIMEOUT = "10s"+# Namespaces the per-team advisory lock that keeps two builds of one team from running at once.+_BUILD_LOCK_KEY = 0x52564842+++@frozen+class RankedEntry:+ window_days: int+ place: int+ observation_id: Any+ scanner_id: Any+ created_at: datetime+ category: int+ experiment_id: int+ percentile: float+ group_root: Any+++@frozen+class WindowBounds:+ window_days: int+ start: datetime+ end: datetime+++def _ranked_sql(+ team_id: int,+ anchor_version: int,+ windows: list[WindowBounds],+ scanner_ids: Sequence[Any] | None = None,+ ranked_before: datetime | None = None,+) -> tuple[str, list[Any]]:+ """A SELECT of every kept entry of `windows`, ranked from the team's rank rows, in `RankedEntry` field order.+ `scanner_ids` and `ranked_before` narrow the rows before ranking."""+ window_values = ", ".join(["(%s::smallint, %s::timestamptz, %s::timestamptz)"] * len(windows))+ narrowing = ""+ narrowing_params: list[Any] = []+ if scanner_ids is not None:+ narrowing += " AND scanner_id = ANY(%s::uuid[])"+ narrowing_params.append([str(scanner_id) for scanner_id in scanner_ids])+ if ranked_before is not None:+ narrowing += " AND ranked_at <= %s"+ narrowing_params.append(ranked_before)+ sql = f"""+ WITH bounds(window_days, window_start, window_end) AS (VALUES {window_values}),+ ranked_rows AS MATERIALIZED (+ SELECT observation_id, scanner_id, created_at, contrast, category, experiment_id,+ COALESCE(feed_group_id, observation_id) AS group_root+ FROM replay_vision_replayobservationrank+ WHERE team_id = %s AND anchor_version = %s+ AND created_at >= (SELECT min(window_start) FROM bounds)+ AND created_at <= (SELECT max(window_end) FROM bounds){narrowing}+ ),+ windowed AS (+ SELECT bounds.window_days, ranked_rows.*+ FROM ranked_rows+ JOIN bounds ON ranked_rows.created_at >= bounds.window_start AND ranked_rows.created_at <= bounds.window_end+ ),+ placed AS (+ SELECT windowed.*, ((scanner_rows - place + 1) - 0.5) / scanner_rows::double precision AS percentile+ FROM (+ SELECT windowed.*,+ count(*) OVER scanner_window AS scanner_rows,+ row_number() OVER (scanner_window ORDER BY contrast DESC, observation_id DESC) AS place+ FROM windowed+ WINDOW scanner_window AS (PARTITION BY window_days, scanner_id)+ ) windowed+ )+ SELECT window_days, place, observation_id, scanner_id, created_at, category, experiment_id, percentile,+ group_root+ FROM placed+ WHERE scanner_rows >= %s AND place <= %s AND percentile >= %s+ """+ params: list[Any] = [value for bounds in windows for value in (bounds.window_days, bounds.start, bounds.end)]+ params += [+ resolve_effective_team_id(team_id),+ anchor_version,+ *narrowing_params,+ MIN_SCANNER_ROWS,+ MAX_PLACES_PER_SCANNER,+ MIN_PERCENTILE,+ ]+ return sql, params+++def rank_window(+ team_id: int,+ anchor_version: int,+ start: datetime,+ end: datetime,+ scanner_ids: Sequence[Any] | None = None,+ ranked_before: datetime | None = None,+) -> list[RankedEntry]:+ """Rank one arbitrary window the way a build ranks a preset one, without storing it. `scanner_ids` ranks only those+ scanners, which ranks each the same since a percentile is within its scanner."""+ if scanner_ids is not None and not scanner_ids:+ return []+ window = WindowBounds(window_days=0, start=start, end=end)+ sql, params = _ranked_sql(team_id, anchor_version, [window], scanner_ids, ranked_before)+ with connection.cursor() as cursor:+ # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (fixed SQL, every value is a parameter)+ cursor.execute(sql, params)+ names = [field.name for field in fields(RankedEntry)]+ return [RankedEntry(**dict(zip(names, row))) for row in cursor.fetchall()]+++def preset_windows(now: datetime) -> list[WindowBounds]:+ return [+ WindowBounds(window_days=days, start=now - timedelta(days=days), end=now) for days in HighlightWindow.values+ ]+++def build_highlight_feed(team_id: int, anchor_version: int, *, wait: bool = True, force: bool = False) -> int | None:+ """Rank the team's preset windows as of now, store them as a new build, and make it current. Returns the current+ generation, or None when `wait` is off and another build of the team holds the lock.++ A team built since its ranks last changed keeps its build unless `force` is set, so a build that waited on another+ one does not repeat it. The state row is held only for the final swap, so rank writes never wait on the ranking.+ """+ row_team_id = resolve_effective_team_id(team_id)+ with transaction.atomic():+ with connection.cursor() as cursor:+ # A stuck statement or lock fails the build instead of stalling every build behind it.+ cursor.execute(+ "SELECT set_config('statement_timeout', %s, true), set_config('lock_timeout', %s, true)",+ [_STATEMENT_TIMEOUT, _LOCK_TIMEOUT],+ )+ cursor.execute(+ "SELECT pg_advisory_xact_lock(%s, %s)" if wait else "SELECT pg_try_advisory_xact_lock(%s, %s)",+ [_BUILD_LOCK_KEY, row_team_id],+ )+ if cursor.fetchone()[0] is False:+ return None+ state, _ = ReplayHighlightFeedState.objects.for_team(team_id).get_or_create(team_id=row_team_id)+ if not force and state.built_at is not None and state.changes <= state.built_changes:+ return state.generation+ seen_changes = state.changes+ generation = state.generation + 1+ sql, params = _ranked_sql(team_id, anchor_version, preset_windows(timezone.now()))+ with connection.cursor() as cursor:+ cursor.execute(+ # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (fixed SQL, every value is a parameter)+ f"""+ INSERT INTO replay_vision_replayhighlightfeedentry (+ team_id, generation, window_days, place, observation_id, scanner_id, created_at, category,+ experiment_id, percentile, group_root+ )+ SELECT %s, %s, ranked.* FROM ({sql}) ranked+ """,+ [row_team_id, generation, *params],+ )+ built_at = timezone.now()+ # A build stays while a reader who started on it may still be scrolling, and the newest one always stays.+ kept = [+ build+ for build in state.builds+ if datetime.fromisoformat(build[1]) > built_at - BUILD_RETENTION or build[0] == state.generation+ ]+ kept.append([generation, built_at.isoformat()])+ ReplayHighlightFeedEntry.objects.for_team(team_id).exclude(generation__in=[build[0] for build in kept]).delete()+ ReplayHighlightFeedState.objects.for_team(team_id).filter(team_id=row_team_id).update(+ generation=generation, built_at=built_at, builds=kept, built_changes=seen_changes+ )+ return generationdiff --git a/products/replay_vision/backend/management/commands/backfill_replay_vision_observation_ranks.py b/products/replay_vision/backend/management/commands/backfill_replay_vision_observation_ranks.pynew file mode 100644index 000000000000..fbb34b7d02f4--- /dev/null+++ b/products/replay_vision/backend/management/commands/backfill_replay_vision_observation_ranks.py@@ -0,0 +1,36 @@+from datetime import timedelta+from typing import Any++from django.core.management.base import BaseCommand, CommandParser+from django.utils import timezone++from products.replay_vision.backend.highlight_builds import build_highlight_feed+from products.replay_vision.backend.models.replay_highlight_feed import HighlightWindow+from products.replay_vision.backend.observation_ranks import copy_team_ranks+from products.replay_vision.backend.watch_rank import ANCHOR_VERSION+from products.replay_vision.backend.watch_rank_backfill import teams_with_observations+++class Command(BaseCommand):+ help = "Copy Replay Vision observations' ranks into rank rows, then build each team's highlight feed from them"++ def add_arguments(self, parser: CommandParser) -> None:+ parser.add_argument("--team-id", type=int, action="append", help="Only this team; repeat for more")+ parser.add_argument(+ "--days", type=int, default=max(HighlightWindow.values), help="How far back to copy, in days"+ )+ parser.add_argument("--dry-run", action="store_true", help="Count the ranks it would copy, write nothing")++ def handle(self, *args: Any, **options: Any) -> None:+ since = timezone.now() - timedelta(days=options["days"])+ dry_run = bool(options["dry_run"])+ for team_id in options["team_id"] or teams_with_observations(since):+ try:+ copied = copy_team_ranks(team_id, ANCHOR_VERSION, since, dry_run=dry_run)+ if not dry_run:+ build_highlight_feed(team_id, ANCHOR_VERSION)+ except Exception as e:+ # One team's slow query must not stop the rest.+ self.stderr.write(f"Team {team_id}: skipped ({e.__class__.__name__}: {e})")+ continue+ self.stdout.write(f"Team {team_id}: {'would copy' if dry_run else 'copied'} {copied} ranks.")diff --git a/products/replay_vision/backend/migrations/0142_replay_highlight_rank_tables.py b/products/replay_vision/backend/migrations/0142_replay_highlight_rank_tables.pynew file mode 100644index 000000000000..05b257cf7fe3--- /dev/null+++ b/products/replay_vision/backend/migrations/0142_replay_highlight_rank_tables.py@@ -0,0 +1,167 @@+# Generated by Django 5.2.18 on 2026-10-29 23:46++import django.db.models.deletion+import django.db.models.functions.datetime+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("posthog", "1415_organizationaiterms"),+ ("replay_vision", "0141_replayobservation_watch_rank_help_text"),+ ]++ operations = [+ migrations.CreateModel(+ name="ReplayHighlightFeedState",+ fields=[+ (+ "team",+ models.OneToOneField(+ db_constraint=False,+ on_delete=django.db.models.deletion.CASCADE,+ primary_key=True,+ related_name="+",+ serialize=False,+ to="posthog.team",+ ),+ ),+ ("generation", models.PositiveBigIntegerField(default=0)),+ ("built_at", models.DateTimeField(blank=True, null=True)),+ ("builds", models.JSONField(default=list)),+ ("changes", models.PositiveBigIntegerField(default=0)),+ ("built_changes", models.PositiveBigIntegerField(default=0)),+ ("attempted_at", models.DateTimeField(blank=True, null=True)),+ ],+ ),+ migrations.CreateModel(+ name="ReplayHighlightFeedEntry",+ fields=[+ ("id", models.BigAutoField(primary_key=True, serialize=False)),+ (+ "window_days",+ models.PositiveSmallIntegerField(+ choices=[+ (1, "Last 24 hours"),+ (7, "Last 7 days"),+ (30, "Last 30 days"),+ (90, "Last 90 days"),+ ]+ ),+ ),+ ("generation", models.PositiveBigIntegerField()),+ ("place", models.PositiveIntegerField()),+ ("observation_id", models.UUIDField()),+ ("scanner_id", models.UUIDField()),+ ("created_at", models.DateTimeField()),+ ("category", models.PositiveSmallIntegerField()),+ ("experiment_id", models.IntegerField(default=0)),+ ("percentile", models.FloatField()),+ ("group_root", models.UUIDField()),+ (+ "team",+ models.ForeignKey(+ db_constraint=False,+ db_index=False,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="+",+ to="posthog.team",+ ),+ ),+ ],+ options={+ "indexes": [+ models.Index(+ fields=["team", "generation", "window_days", "place"],+ name="rv_highlight_entry_read_idx",+ ),+ models.Index(fields=["observation_id"], name="rv_highlight_entry_obs_idx"),+ ],+ },+ ),+ migrations.CreateModel(+ name="ReplayObservationRank",+ fields=[+ (+ "observation",+ models.OneToOneField(+ db_constraint=False,+ on_delete=django.db.models.deletion.CASCADE,+ primary_key=True,+ related_name="+",+ serialize=False,+ to="replay_vision.replayobservation",+ ),+ ),+ ("created_at", models.DateTimeField()),+ ("anchor_version", models.PositiveSmallIntegerField()),+ ("contrast", models.FloatField()),+ (+ "category",+ models.PositiveSmallIntegerField(+ choices=[+ (1, "Failure"),+ (2, "Struggle"),+ (3, "Revenue"),+ (4, "Routine"),+ ]+ ),+ ),+ ("feed_group_id", models.UUIDField(blank=True, null=True)),+ ("search_theme_id", models.UUIDField(blank=True, null=True)),+ ("person_key", models.BigIntegerField()),+ ("experiment_id", models.IntegerField(default=0)),+ (+ "ranked_at",+ models.DateTimeField(db_default=django.db.models.functions.datetime.Now()),+ ),+ (+ "scanner",+ models.ForeignKey(+ db_constraint=False,+ db_index=False,+ on_delete=django.db.models.deletion.DO_NOTHING,+ related_name="+",+ to="replay_vision.replayscanner",+ ),+ ),+ (+ "team",+ models.ForeignKey(+ db_constraint=False,+ db_index=False,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="+",+ to="posthog.team",+ ),+ ),+ ],+ options={+ "indexes": [+ models.Index(+ fields=["team", "anchor_version", "created_at"],+ include=(+ "observation",+ "scanner",+ "contrast",+ "category",+ "feed_group_id",+ "person_key",+ "experiment_id",+ ),+ name="rv_rank_team_window_idx",+ ),+ models.Index(+ condition=models.Q(("feed_group_id__isnull", False)),+ fields=["feed_group_id"],+ name="rv_rank_feed_group_idx",+ ),+ models.Index(+ condition=models.Q(("search_theme_id__isnull", False)),+ fields=["search_theme_id", "created_at"],+ name="rv_rank_search_theme_idx",+ ),+ ],+ },+ ),+ ]diff --git a/products/replay_vision/backend/migrations/0143_replay_highlight_autovacuum.py b/products/replay_vision/backend/migrations/0143_replay_highlight_autovacuum.pynew file mode 100644index 000000000000..9bc5616e2a10--- /dev/null+++ b/products/replay_vision/backend/migrations/0143_replay_highlight_autovacuum.py@@ -0,0 +1,22 @@+from django.db import migrations+++class Migration(migrations.Migration):+ dependencies = [+ ("replay_vision", "0142_replay_highlight_rank_tables"),+ ]++ operations = [+ # Vacuum after a small share of new or replaced rows, so the visibility map keeps index-only reads off the heap.+ migrations.RunSQL(+ sql="""+ ALTER TABLE replay_vision_replayobservationrank SET (+ autovacuum_vacuum_insert_scale_factor = 0.02, autovacuum_vacuum_scale_factor = 0.02+ );+ ALTER TABLE replay_vision_replayhighlightfeedentry SET (+ autovacuum_vacuum_insert_scale_factor = 0.02, autovacuum_vacuum_scale_factor = 0.02+ );+ """,+ reverse_sql=migrations.RunSQL.noop,+ ),+ ]diff --git a/products/replay_vision/backend/migrations/max_migration.txt b/products/replay_vision/backend/migrations/max_migration.txtindex 2e883a49afae..c62685845af1 100644--- a/products/replay_vision/backend/migrations/max_migration.txt+++ b/products/replay_vision/backend/migrations/max_migration.txt@@ -1 +1 @@-0141_replayobservation_watch_rank_help_text+0143_replay_highlight_autovacuumdiff --git a/products/replay_vision/backend/models/__init__.py b/products/replay_vision/backend/models/__init__.pyindex 337dea9ea9cb..73043029c224 100644--- a/products/replay_vision/backend/models/__init__.py+++ b/products/replay_vision/backend/models/__init__.py@@ -1,9 +1,14 @@+from products.replay_vision.backend.models.replay_highlight_feed import (+ ReplayHighlightFeedEntry,+ ReplayHighlightFeedState,+)from products.replay_vision.backend.models.replay_observation import ReplayObservationfrom products.replay_vision.backend.models.replay_observation_anchor_similarity import ReplayObservationAnchorSimilarityfrom products.replay_vision.backend.models.replay_observation_feed_dismissal import ReplayObservationFeedDismissalfrom products.replay_vision.backend.models.replay_observation_feed_impression import ReplayObservationFeedImpressionfrom products.replay_vision.backend.models.replay_observation_label import ReplayObservationLabelfrom products.replay_vision.backend.models.replay_observation_media import ReplayObservationMedia+from products.replay_vision.backend.models.replay_observation_rank import ReplayObservationRankfrom products.replay_vision.backend.models.replay_observation_request import ReplayObservationRequestfrom products.replay_vision.backend.models.replay_observation_usage import ReplayObservationUsagefrom products.replay_vision.backend.models.replay_observation_view import ReplayObservationView@@ -24,6 +29,9 @@ from products.replay_vision.backend.models.vision_alert import ()__all__ = [+ "ReplayHighlightFeedEntry",+ "ReplayHighlightFeedState",+ "ReplayObservationRank","ReplayObservation","ReplayObservationAnchorSimilarity","ReplayObservationFeedDismissal",diff --git a/products/replay_vision/backend/models/replay_highlight_feed.py b/products/replay_vision/backend/models/replay_highlight_feed.pynew file mode 100644index 000000000000..72bcafc9b240--- /dev/null+++ b/products/replay_vision/backend/models/replay_highlight_feed.py@@ -0,0 +1,60 @@+from django.db import models++from posthog.models.scoping.root_mixin import TeamScopedRootMixin+++class HighlightWindow(models.IntegerChoices):+ """The feed's preset windows, in days."""++ DAY = 1, "Last 24 hours"+ WEEK = 7, "Last 7 days"+ MONTH = 30, "Last 30 days"+ QUARTER = 90, "Last 90 days"+++class ReplayHighlightFeedState(TeamScopedRootMixin, models.Model):+ """Which stored build of a team's highlight feed is current, and whether ranks changed since it was built."""++ team = models.OneToOneField(+ "posthog.Team", on_delete=models.CASCADE, primary_key=True, related_name="+", db_constraint=False+ )+ generation = models.PositiveBigIntegerField(default=0)+ built_at = models.DateTimeField(null=True, blank=True)+ # Each kept build as `[generation, ISO time it became current]`, so a load stays on the build its first page read.+ builds = models.JSONField(default=list)+ # Counts asks for a rebuild; a build records the count it saw, so an ask that lands during a build is not lost.+ changes = models.PositiveBigIntegerField(default=0)+ built_changes = models.PositiveBigIntegerField(default=0)+ # The last time a sweep tried to build the team, so a team whose builds keep failing waits its turn.+ attempted_at = models.DateTimeField(null=True, blank=True)++ class Meta:+ app_label = "replay_vision"+++class ReplayHighlightFeedEntry(TeamScopedRootMixin, models.Model):+ """One ranked candidate of a team's highlight feed for one window, as of one build. A page load filters and+ orders these instead of ranking the window itself."""++ id = models.BigAutoField(primary_key=True)+ team = models.ForeignKey(+ "posthog.Team", on_delete=models.CASCADE, related_name="+", db_constraint=False, db_index=False+ )+ window_days = models.PositiveSmallIntegerField(choices=HighlightWindow.choices)+ generation = models.PositiveBigIntegerField()+ # The row's place in its scanner by score in this window, best first.+ place = models.PositiveIntegerField()+ observation_id = models.UUIDField()+ scanner_id = models.UUIDField()+ created_at = models.DateTimeField()+ category = models.PositiveSmallIntegerField()+ experiment_id = models.IntegerField(default=0)+ percentile = models.FloatField()+ group_root = models.UUIDField()++ class Meta:+ app_label = "replay_vision"+ indexes = [+ models.Index(fields=["team", "generation", "window_days", "place"], name="rv_highlight_entry_read_idx"),+ models.Index(fields=["observation_id"], name="rv_highlight_entry_obs_idx"),+ ]diff --git a/products/replay_vision/backend/models/replay_observation_rank.py b/products/replay_vision/backend/models/replay_observation_rank.pynew file mode 100644index 000000000000..044089ad8674--- /dev/null+++ b/products/replay_vision/backend/models/replay_observation_rank.py@@ -0,0 +1,78 @@+from django.db import models+from django.db.models.functions import Now++from posthog.models.scoping.root_mixin import TeamScopedRootMixin+++class RankCategory(models.IntegerChoices):+ FAILURE = 1, "Failure"+ STRUGGLE = 2, "Struggle"+ REVENUE = 3, "Revenue"+ ROUTINE = 4, "Routine"+++class ReplayObservationRank(TeamScopedRootMixin, models.Model):+ """An observation's highlight rank, kept apart from the wide observation row so the feed reads only what ranking+ needs. A row is written once, when the observation is ranked under the current anchors."""++ # No database constraint, so the recording delete, which removes observations in raw SQL, never fails on a row here.+ observation = models.OneToOneField(+ "replay_vision.ReplayObservation",+ on_delete=models.CASCADE,+ primary_key=True,+ related_name="+",+ db_constraint=False,+ )+ team = models.ForeignKey(+ "posthog.Team", on_delete=models.CASCADE, related_name="+", db_constraint=False, db_index=False+ )+ # The observation's own cascade removes the row, so deleting a scanner does not scan this table for it.+ scanner = models.ForeignKey(+ "replay_vision.ReplayScanner",+ on_delete=models.DO_NOTHING,+ related_name="+",+ db_constraint=False,+ db_index=False,+ )+ # The observation's own creation time, so a window filter never reads the observation row.+ created_at = models.DateTimeField()+ # Ranks from another anchor set do not compare, so readers ignore rows written under another version.+ anchor_version = models.PositiveSmallIntegerField()+ contrast = models.FloatField()+ category = models.PositiveSmallIntegerField(choices=RankCategory.choices)+ feed_group_id = models.UUIDField(null=True, blank=True)+ search_theme_id = models.UUIDField(null=True, blank=True)+ # A stable hash of the observation's person, or of its session when it has no person, so group reach counts+ # people without reading the observation.+ person_key = models.BigIntegerField()+ # The experiment the observation was scanned for, or 0, so readers can keep experiment-scoped rows to the viewers+ # allowed to read them.+ experiment_id = models.IntegerField(default=0)+ # When the rank was written, so a load can leave out rows ranked after its first page.+ ranked_at = models.DateTimeField(db_default=Now())++ class Meta:+ app_label = "replay_vision"+ indexes = [+ models.Index(+ fields=["team", "anchor_version", "created_at"],+ include=[+ "observation",+ "scanner",+ "contrast",+ "category",+ "feed_group_id",+ "person_key",+ "experiment_id",+ ],+ name="rv_rank_team_window_idx",+ ),+ models.Index(+ fields=["feed_group_id"], name="rv_rank_feed_group_idx", condition=models.Q(feed_group_id__isnull=False)+ ),+ models.Index(+ fields=["search_theme_id", "created_at"],+ name="rv_rank_search_theme_idx",+ condition=models.Q(search_theme_id__isnull=False),+ ),+ ]diff --git a/products/replay_vision/backend/observation_ranks.py b/products/replay_vision/backend/observation_ranks.pynew file mode 100644index 000000000000..030e963ebeb5--- /dev/null+++ b/products/replay_vision/backend/observation_ranks.py@@ -0,0 +1,207 @@+"""Writes each ranked observation's narrow rank row, which the highlight feed reads instead of the observation.++The person key and experiment scope come from the observation row inside the same statement, so every writer derives+them the same way.+"""++import uuid+from collections.abc import Mapping+from datetime import datetime, timedelta+from itertools import batched+from typing import Any++from django.db import connection, transaction+from django.utils import timezone++from posthog.dataclasses import frozen+from posthog.models.scoping.manager import resolve_effective_team_id++from products.replay_vision.backend.models.replay_observation_rank import RankCategory++_WRITE_CHUNK = 500+_COPY_SLICE = timedelta(days=1)++CATEGORY_CODES: dict[str, int] = {category.name.lower(): category.value for category in RankCategory}++# A snapshot that names an experiment as anything but a plain integer gets -1, which matches no experiment, so the row+# stays hidden from every viewer rather than readable by all.+_EXPERIMENT_SQL = """+ CASE+ WHEN o.scanner_snapshot #>> '{experiment_targeting,experiment_id}' ~ '^-?[0-9]{1,9}$'+ THEN (o.scanner_snapshot #>> '{experiment_targeting,experiment_id}')::int+ WHEN o.scanner_snapshot #>> '{scanner_config,experiment_id}' ~ '^-?[0-9]{1,9}$'+ THEN (o.scanner_snapshot #>> '{scanner_config,experiment_id}')::int+ WHEN o.scanner_snapshot #> '{experiment_targeting,experiment_id}' IS NOT NULL+ OR o.scanner_snapshot #> '{scanner_config,experiment_id}' IS NOT NULL+ THEN -1+ ELSE 0+ END+"""+_PERSON_SQL = "hashtextextended(COALESCE(NULLIF(o.distinct_id, ''), o.session_id), 0)"+_RANK_COLUMNS = (+ "observation_id, team_id, scanner_id, created_at, anchor_version, contrast, category, feed_group_id, "+ "search_theme_id, person_key, experiment_id"+)+++@frozen+class NewRank:+ contrast: float+ category: str+++def record_ranks(team_id: int, anchor_version: int, ranks: Mapping[str, NewRank]) -> int:+ """Write a rank row for each ranked observation, in the caller's transaction. An observation that already has one+ under `anchor_version` keeps it, so a redelivered message changes nothing, and one ranked under other anchors is+ ranked afresh, clusters included. Returns how many rows it wrote."""+ row_team_id = resolve_effective_team_id(team_id)+ inserted = 0+ for chunk in batched(ranks.items(), _WRITE_CHUNK, strict=False):+ params: list[Any] = []+ for observation_id, rank in chunk:+ params += [observation_id, rank.contrast, CATEGORY_CODES[rank.category]]+ values = ", ".join(["(%s::uuid, %s::double precision, %s::smallint)"] * len(chunk))+ with connection.cursor() as cursor:+ cursor.execute(+ # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (fixed SQL, every value is a parameter)+ f"""+ INSERT INTO replay_vision_replayobservationrank ({_RANK_COLUMNS})+ SELECT o.id, %s, o.scanner_id, o.created_at, %s, r.contrast, r.category, NULL, NULL,+ {_PERSON_SQL}, {_EXPERIMENT_SQL}+ FROM (VALUES {values}) AS r(observation_id, contrast, category)+ JOIN replay_vision_replayobservation o ON o.id = r.observation_id AND o.team_id = %s+ ON CONFLICT (observation_id) DO UPDATE SET+ anchor_version = EXCLUDED.anchor_version,+ contrast = EXCLUDED.contrast,+ category = EXCLUDED.category,+ feed_group_id = NULL,+ search_theme_id = NULL,+ ranked_at = now()+ WHERE replay_vision_replayobservationrank.anchor_version <> EXCLUDED.anchor_version+ """,+ [row_team_id, anchor_version, *params, team_id],+ )+ inserted += cursor.rowcount+ if inserted:+ mark_changed(team_id)+ return inserted+++def set_rank_clusters(team_id: int, clusters: Mapping[str, tuple[uuid.UUID, uuid.UUID | None]]) -> None:+ """Record each observation's feed group and search theme on its rank row."""+ row_team_id = resolve_effective_team_id(team_id)+ updated = 0+ for chunk in batched(clusters.items(), _WRITE_CHUNK, strict=False):+ params: list[Any] = []+ for observation_id, (group_id, theme_id) in chunk:+ params += [observation_id, group_id, theme_id]+ values = ", ".join(["(%s::uuid, %s::uuid, %s::uuid)"] * len(chunk))+ with connection.cursor() as cursor:+ cursor.execute(+ # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (fixed SQL, every value is a parameter)+ f"""+ UPDATE replay_vision_replayobservationrank rank+ SET feed_group_id = c.group_id, search_theme_id = COALESCE(c.theme_id, rank.search_theme_id)+ FROM (VALUES {values}) AS c(observation_id, group_id, theme_id)+ WHERE rank.observation_id = c.observation_id AND rank.team_id = %s+ """,+ [*params, row_team_id],+ )+ updated += cursor.rowcount+ if updated:+ mark_changed(team_id)+++def copy_team_ranks(team_id: int, anchor_version: int, since: datetime, *, dry_run: bool = False) -> int:+ """Copy the team's current ranks since `since` from the observations' rank JSON into rank rows, filling in a+ feed group or search theme an existing row lacks. Returns how many rows it wrote, or how many ranks a write would+ read."""+ source = """+ FROM replay_vision_replayobservation o+ WHERE o.team_id = %(observation_team_id)s+ AND o.created_at >= %(since)s+ AND o.created_at < %(until)s+ AND o.status = 'succeeded'+ AND o.watch_rank IS NOT NULL+ AND o.watch_rank -> 'anchor_version' = to_jsonb(%(anchor_version)s::int)+ AND jsonb_typeof(o.watch_rank -> 'contrast') = 'number'+ AND o.watch_rank ->> 'category' = ANY(%(categories)s)+ """+ params: dict[str, Any] = {+ "team_id": resolve_effective_team_id(team_id),+ "observation_team_id": team_id,+ "since": since,+ "until": timezone.now(),+ "anchor_version": anchor_version,+ "categories": list(CATEGORY_CODES),+ }+ if dry_run:+ with connection.cursor() as cursor:+ # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (fixed SQL, every value is a parameter)+ cursor.execute(f"SELECT count(*) {source}", params)+ return cursor.fetchone()[0]+ written = 0+ end = params["until"]+ # One day per statement, so the rank rows an upsert locks are released before the next day's rows.+ while since < end:+ params |= {"since": since, "until": min(since + _COPY_SLICE, end)}+ written += _copy_slice(team_id, source, params)+ since += _COPY_SLICE+ _delete_orphaned_ranks(team_id)+ return written+++def _delete_orphaned_ranks(team_id: int) -> None:+ # A recording deletion removes observations in raw SQL, so rank rows it missed are removed here.+ with connection.cursor() as cursor:+ cursor.execute(+ """+ DELETE FROM replay_vision_replayobservationrank rank+ WHERE rank.team_id = %s+ AND NOT EXISTS (SELECT 1 FROM replay_vision_replayobservation o WHERE o.id = rank.observation_id)+ """,+ [resolve_effective_team_id(team_id)],+ )+++def _copy_slice(team_id: int, source: str, params: dict[str, Any]) -> int:+ category = "CASE o.watch_rank ->> 'category' " + " ".join(+ f"WHEN '{name}' THEN {code}" for name, code in CATEGORY_CODES.items()+ )+ with transaction.atomic(), connection.cursor() as cursor:+ cursor.execute(+ # nosemgrep: python.django.security.audit.raw-query.avoid-raw-sql (fixed SQL, every value is a parameter)+ f"""+ INSERT INTO replay_vision_replayobservationrank ({_RANK_COLUMNS})+ SELECT o.id, %(team_id)s, o.scanner_id, o.created_at, %(anchor_version)s,+ (o.watch_rank ->> 'contrast')::double precision, {category} END,+ (o.watch_rank ->> 'group')::uuid, (o.watch_rank ->> 'theme')::uuid,+ {_PERSON_SQL}, {_EXPERIMENT_SQL}+ {source}+ ON CONFLICT (observation_id) DO UPDATE SET+ feed_group_id = COALESCE(replay_vision_replayobservationrank.feed_group_id, EXCLUDED.feed_group_id),+ search_theme_id = COALESCE(replay_vision_replayobservationrank.search_theme_id, EXCLUDED.search_theme_id)+ WHERE (replay_vision_replayobservationrank.feed_group_id IS NULL AND EXCLUDED.feed_group_id IS NOT NULL)+ OR (replay_vision_replayobservationrank.search_theme_id IS NULL AND EXCLUDED.search_theme_id IS NOT NULL)+ """,+ params,+ )+ written = cursor.rowcount+ # Each committed day asks for a rebuild, so a copy stopped partway still reaches the feed.+ if written:+ mark_changed(team_id)+ return written+++def mark_changed(team_id: int) -> None:+ """Ask for a rebuild of the team's highlight feed, in the caller's transaction. Every ask counts, so one that lands+ while a build ranks is still due after it; a build never holds the state row while it ranks."""+ with connection.cursor() as cursor:+ cursor.execute(+ """+ INSERT INTO replay_vision_replayhighlightfeedstate (team_id, generation, builds, changes, built_changes)+ VALUES (%s, 0, '[]'::jsonb, 1, 0)+ ON CONFLICT (team_id) DO UPDATE SET changes = replay_vision_replayhighlightfeedstate.changes + 1+ """,+ [resolve_effective_team_id(team_id)],+ )diff --git a/products/replay_vision/backend/tests/test_highlight_builds.py b/products/replay_vision/backend/tests/test_highlight_builds.pynew file mode 100644index 000000000000..5606b74b4dc7--- /dev/null+++ b/products/replay_vision/backend/tests/test_highlight_builds.py@@ -0,0 +1,232 @@+import uuid+from datetime import UTC, datetime, timedelta+from typing import Any++import time_machine+from posthog.test.base import BaseTest++from parameterized import parameterized++from posthog.models import Team++from products.replay_vision.backend.highlight_builds import BUILD_RETENTION, build_highlight_feed, rank_window+from products.replay_vision.backend.models.replay_highlight_feed import (+ ReplayHighlightFeedEntry,+ ReplayHighlightFeedState,+)+from products.replay_vision.backend.models.replay_observation import (+ ObservationStatus,+ ObservationTrigger,+ ReplayObservation,+)+from products.replay_vision.backend.models.replay_observation_rank import RankCategory, ReplayObservationRank+from products.replay_vision.backend.models.replay_scanner import ReplayScanner, ScannerModel, ScannerType+from products.replay_vision.backend.observation_ranks import NewRank, copy_team_ranks, record_ranks+from products.replay_vision.backend.tests.helpers import snapshot_for+from products.replay_vision.backend.watch_rank import ANCHOR_VERSION++_NOW = datetime(2026, 10, 20, 12, tzinfo=UTC)+++@time_machine.travel(_NOW, tick=False)+class TestHighlightBuilds(BaseTest):+ def setUp(self) -> None:+ super().setUp()+ self.scanner = self._scanner("checkout")++ def _scanner(self, name: str) -> ReplayScanner:+ return ReplayScanner.objects.create(+ team=self.team,+ name=name,+ scanner_type=ScannerType.MONITOR,+ scanner_config={"prompt": "p"},+ model=ScannerModel.GEMINI_3_8_FLASH,+ )++ def _observation(+ self, created_at: datetime, *, scanner: ReplayScanner | None = None, person: str = "", **fields: Any+ ) -> ReplayObservation:+ scanner = scanner or self.scanner+ fields.setdefault("scanner_snapshot", snapshot_for(scanner))+ fields.setdefault("completed_at", created_at)+ fields.setdefault("team", self.team)+ observation = ReplayObservation.objects.create(+ scanner=scanner,+ session_id=f"sess-{uuid.uuid4()}",+ distinct_id=person,+ triggered_by=ObservationTrigger.ON_DEMAND,+ status=ObservationStatus.SUCCEEDED,+ **fields,+ )+ ReplayObservation.objects.filter(id=observation.id).update(created_at=created_at)+ return observation++ def _ranked(self, count: int, *, age: timedelta, top: float = 1.0) -> list[str]:+ ids = []+ for index in range(count):+ observation = self._observation(_NOW - age - timedelta(minutes=index))+ ids.append(str(observation.id))+ record_ranks(+ self.team.id, ANCHOR_VERSION, {ids[-1]: NewRank(contrast=top - index / 100, category="failure")}+ )+ return ids++ def test_a_build_keeps_each_windows_top_fifth_with_percentiles(self) -> None:+ recent = self._ranked(10, age=timedelta(hours=1))+ older = self._ranked(10, age=timedelta(days=3), top=0.5)++ build_highlight_feed(self.team.id, ANCHOR_VERSION)++ entries = {+ (entry.window_days, str(entry.observation_id)): (entry.place, round(entry.percentile, 3))+ for entry in ReplayHighlightFeedEntry.objects.for_team(self.team.id)+ }+ # The day window has ten rows, so its best two clear the cut; the week window places all twenty.+ assert {key: value for key, value in entries.items() if key[0] == 1} == {+ (1, recent[0]): (1, 0.95),+ (1, recent[1]): (2, 0.85),+ }+ assert {key: value for key, value in entries.items() if key[0] == 7} == {+ (7, recent[0]): (1, 0.975),+ (7, recent[1]): (2, 0.925),+ (7, recent[2]): (3, 0.875),+ (7, recent[3]): (4, 0.825),+ }+ assert older[0] not in {observation_id for _, observation_id in entries}++ @parameterized.expand(+ [+ ("too_few_rows_for_a_percentile", 9, 0),+ ("enough_rows", 10, 2),+ ]+ )+ def test_a_scanner_needs_enough_rows_to_be_placed(self, _name: str, rows: int, kept: int) -> None:+ self._ranked(rows, age=timedelta(hours=1))++ build_highlight_feed(self.team.id, ANCHOR_VERSION)++ assert ReplayHighlightFeedEntry.objects.for_team(self.team.id).filter(window_days=1).count() == kept++ @parameterized.expand(+ [+ ("targeted", {"experiment_targeting": {"experiment_id": 9}}, 9),+ ("config", {"scanner_config": {"experiment_id": 4}}, 4),+ # A value no viewer's experiment can match keeps the row hidden from everyone.+ ("unparseable", {"experiment_targeting": {"experiment_id": 9.5}}, -1),+ ("untargeted", {}, 0),+ ]+ )+ def test_a_rank_row_records_the_experiment_its_scan_was_scoped_to(+ self, _name: str, snapshot: dict[str, Any], experiment_id: int+ ) -> None:+ observation = self._observation(_NOW, scanner_snapshot=snapshot)++ record_ranks(self.team.id, ANCHOR_VERSION, {str(observation.id): NewRank(contrast=0.1, category="failure")})++ assert ReplayObservationRank.objects.for_team(self.team.id).get().experiment_id == experiment_id++ def test_an_arbitrary_window_ranks_like_a_build_and_narrows_to_its_scanners(self) -> None:+ self._ranked(10, age=timedelta(hours=1))+ other = self._scanner("other")+ for index in range(10):+ observation = self._observation(_NOW - timedelta(hours=2, minutes=index), scanner=other)+ record_ranks(self.team.id, ANCHOR_VERSION, {str(observation.id): NewRank(contrast=0.5, category="failure")})++ build_highlight_feed(self.team.id, ANCHOR_VERSION)+ ranked = rank_window(self.team.id, ANCHOR_VERSION, _NOW - timedelta(days=1), _NOW, [self.scanner.id])++ stored = ReplayHighlightFeedEntry.objects.for_team(self.team.id).filter(+ window_days=1, scanner_id=self.scanner.id+ )+ assert sorted((entry.observation_id, entry.place) for entry in ranked) == sorted(+ (entry.observation_id, entry.place) for entry in stored+ )+ assert rank_window(self.team.id, ANCHOR_VERSION, _NOW - timedelta(days=1), _NOW, []) == []++ def test_a_build_keeps_recent_builds_and_drops_ones_past_retention(self) -> None:+ self._ranked(10, age=timedelta(hours=1))++ build_highlight_feed(self.team.id, ANCHOR_VERSION, force=True)+ with time_machine.travel(_NOW + BUILD_RETENTION + timedelta(minutes=1), tick=False):+ build_highlight_feed(self.team.id, ANCHOR_VERSION, force=True)+ build_highlight_feed(self.team.id, ANCHOR_VERSION, force=True)++ state = ReplayHighlightFeedState.objects.for_team(self.team.id).get()+ assert set(+ ReplayHighlightFeedEntry.objects.for_team(self.team.id).values_list("generation", flat=True).distinct()+ ) == {2, 3}+ assert [build[0] for build in state.builds] == [2, 3]++ def test_a_team_built_since_its_last_change_keeps_its_build(self) -> None:+ self._ranked(10, age=timedelta(hours=1))+ first = build_highlight_feed(self.team.id, ANCHOR_VERSION)++ unchanged = build_highlight_feed(self.team.id, ANCHOR_VERSION)+ self._ranked(1, age=timedelta(minutes=5))+ changed = build_highlight_feed(self.team.id, ANCHOR_VERSION)++ assert (first, unchanged, changed) == (1, 1, 2)++ @parameterized.expand(+ [+ ("redelivery_keeps_the_first_rank", ANCHOR_VERSION, 0, 0.4, RankCategory.FAILURE),+ ("new_anchors_rank_afresh", ANCHOR_VERSION + 1, 1, 0.9, RankCategory.STRUGGLE),+ ]+ )+ def test_a_second_rank_replaces_only_a_rank_from_other_anchors(+ self, _name: str, second_version: int, written: int, contrast: float, category: int+ ) -> None:+ observation = self._observation(_NOW)+ record_ranks(self.team.id, ANCHOR_VERSION, {str(observation.id): NewRank(contrast=0.4, category="failure")})+ ReplayObservationRank.objects.for_team(self.team.id).update(feed_group_id=uuid.uuid4())++ again = record_ranks(+ self.team.id, second_version, {str(observation.id): NewRank(contrast=0.9, category="struggle")}+ )++ rank = ReplayObservationRank.objects.for_team(self.team.id).get(observation_id=observation.id)+ assert (again, rank.contrast, rank.category, rank.feed_group_id is None) == (+ written,+ contrast,+ category,+ written == 1,+ )++ def test_a_child_environments_ranks_reach_its_projects_builds(self) -> None:+ child = Team.objects.create(organization=self.organization, project=self.team.project, parent_team=self.team)+ scanner = ReplayScanner.objects.create(+ team=child,+ name="child",+ scanner_type=ScannerType.MONITOR,+ scanner_config={"prompt": "p"},+ model=ScannerModel.GEMINI_3_8_FLASH,+ )+ for index in range(10):+ observation = self._observation(_NOW - timedelta(hours=1, minutes=index), scanner=scanner, team=child)+ record_ranks(+ child.id, ANCHOR_VERSION, {str(observation.id): NewRank(contrast=1 - index / 100, category="failure")}+ )++ build_highlight_feed(child.id, ANCHOR_VERSION)++ assert ReplayHighlightFeedEntry.objects.for_team(self.team.id).filter(window_days=1).count() == 2++ def test_copy_fills_a_missing_group_from_the_rank_json_once(self) -> None:+ group = uuid.uuid4()+ created_at = _NOW - timedelta(minutes=1)+ observation = self._observation(+ created_at,+ watch_rank={"category": "struggle", "contrast": 0.7, "anchor_version": ANCHOR_VERSION, "group": str(group)},+ )+ self._observation(created_at, watch_rank={"category": "failure", "contrast": 0.7, "anchor_version": 99})++ copied = copy_team_ranks(self.team.id, ANCHOR_VERSION, _NOW - timedelta(days=1))+ again = copy_team_ranks(self.team.id, ANCHOR_VERSION, _NOW - timedelta(days=1))++ rank = ReplayObservationRank.objects.for_team(self.team.id).get()+ assert (copied, again) == (1, 0)+ assert (rank.observation_id, rank.category, rank.feed_group_id) == (+ observation.id,+ RankCategory.STRUGGLE,+ group,+ )diff --git a/products/replay_vision/backend/tests/test_watch_rank_consumer.py b/products/replay_vision/backend/tests/test_watch_rank_consumer.pyindex 1edcc2f07e08..24619bbe1b7a 100644--- a/products/replay_vision/backend/tests/test_watch_rank_consumer.py+++ b/products/replay_vision/backend/tests/test_watch_rank_consumer.py@@ -24,6 +24,7 @@ from products.replay_vision.backend.embeddings import ()from products.replay_vision.backend.models.replay_observation import ObservationTrigger, ReplayObservationfrom products.replay_vision.backend.models.replay_observation_anchor_similarity import ReplayObservationAnchorSimilarity+from products.replay_vision.backend.models.replay_observation_rank import ReplayObservationRankfrom products.replay_vision.backend.models.replay_scanner import ReplayScanner, ScannerModel, ScannerTypefrom products.replay_vision.backend.tests.helpers import axis_anchors, snapshot_forfrom products.replay_vision.backend.watch_rank import ANCHOR_VERSION, RankRequest@@ -136,6 +137,9 @@ class TestWatchRankConsumer(BaseTest):if outcome == "missing":return+ assert ReplayObservationRank.objects.for_team(self.team.id).filter(observation_id=observation.id).exists() == (+ outcome == "ranked"+ )observation.refresh_from_db()if expected_rank is None:assert observation.watch_rank is Nonediff --git a/products/replay_vision/backend/watch_rank.py b/products/replay_vision/backend/watch_rank.pyindex cb2b5df02879..900299f30284 100644--- a/products/replay_vision/backend/watch_rank.py+++ b/products/replay_vision/backend/watch_rank.py@@ -17,7 +17,7 @@ from typing import Any, Literal, castfrom django.core.cache import cachefrom django.core.exceptions import EmptyResultSet-from django.db import connections+from django.db import connections, transactionfrom django.db.models import CharField, Count, Exists, F, QuerySet, Value, Windowfrom django.db.models.expressions import RawSQLfrom django.db.models.fields.json import KeyTextTransform, KeyTransform@@ -43,6 +43,7 @@ from products.replay_vision.backend.models.replay_observation import (watch_rank_contrast,)from products.replay_vision.backend.models.replay_observation_anchor_similarity import ReplayObservationAnchorSimilarity+from products.replay_vision.backend.observation_ranks import NewRank, record_ranksfrom products.replay_vision.backend.scanner_access import accessible_observationsWatchCategory = Literal["failure", "struggle", "revenue", "routine"]@@ -251,14 +252,19 @@ async def rank_observation(request: RankRequest) -> Literal["ranked", "already_rif current_rank({"watch_rank": row["watch_rank"]}) is not None:return "already_ranked"reading = classify(request.vector, await anchor_vectors(request.team_id))- # The similarities land first, so a rank on the row means its similarities are there too.- await sync_to_async(save_anchor_similarities)(request.team_id, {request.observation_id: reading.similarities})- await ReplayObservation.objects.filter(team_id=request.team_id, id=request.observation_id).aupdate(- watch_rank=reading.rank.to_json()- )+ await sync_to_async(_save_rank)(request.team_id, request.observation_id, reading)return "ranked"+def _save_rank(team_id: int, observation_id: str, reading: AnchorReading) -> None:+ # One transaction, so a rank on the row always comes with its similarities.+ rank = reading.rank+ with transaction.atomic():+ save_anchor_similarities(team_id, {observation_id: reading.similarities})+ ReplayObservation.objects.filter(team_id=team_id, id=observation_id).update(watch_rank=rank.to_json())+ record_ranks(team_id, ANCHOR_VERSION, {observation_id: NewRank(contrast=rank.contrast, category=rank.category)})++def people_boost(people: int) -> float:"""How much a pick's score grows with the number of people its feed group reached."""return min(HIGHLIGHT_PICK_PEOPLE_BOOST_CAP, 1 + HIGHLIGHT_PICK_PEOPLE_BOOST * math.log(max(people, 1)))diff --git a/products/replay_vision/backend/watch_rank_backfill.py b/products/replay_vision/backend/watch_rank_backfill.pyindex fbb683da4d56..d18050a77ecd 100644--- a/products/replay_vision/backend/watch_rank_backfill.py+++ b/products/replay_vision/backend/watch_rank_backfill.py@@ -36,6 +36,7 @@ from products.replay_vision.backend.embeddings import (team_embeddings_params,)from products.replay_vision.backend.models.replay_observation import ObservationStatus, ReplayObservation+from products.replay_vision.backend.observation_ranks import NewRank, record_ranksfrom products.replay_vision.backend.watch_rank import (ANCHOR_VERSION,AnchorReading,@@ -185,14 +186,26 @@ def backfill_team_watch_ranks(team_id: int, since: datetime, *, dry_run: bool =.exclude(watch_rank__anchor_version=ANCHOR_VERSION).values_list("id", flat=True))+ unranked = {+ observation_id: reading.rank+ for observation_id, reading in written.items()+ if UUID(observation_id) in still_unranked+ }ReplayObservation.objects.bulk_update([- ReplayObservation(id=observation_id, watch_rank=reading.rank.to_json())- for observation_id, reading in written.items()- if UUID(observation_id) in still_unranked+ ReplayObservation(id=observation_id, watch_rank=rank.to_json())+ for observation_id, rank in unranked.items()],["watch_rank"],)+ record_ranks(+ team_id,+ ANCHOR_VERSION,+ {+ observation_id: NewRank(contrast=rank.contrast, category=rank.category)+ for observation_id, rank in unranked.items()+ },+ )without_embedding = len(rows) - embeddedlogger.info("replay_vision.watch_rank_backfill.team_done",diff --git a/products/replay_vision/backend/watch_semantics.py b/products/replay_vision/backend/watch_semantics.pyindex b664d51e9621..7420eb7c2254 100644--- a/products/replay_vision/backend/watch_semantics.py+++ b/products/replay_vision/backend/watch_semantics.py@@ -30,6 +30,7 @@ from products.replay_vision.backend.models.replay_scanner_semantics import (ReplayScannerSearchTheme,ReplayScannerSemanticCenter,)+from products.replay_vision.backend.observation_ranks import set_rank_clustersfrom products.replay_vision.backend.scanner_access import scanners_for_reading_observationsfrom products.replay_vision.backend.semantic_space import (SEMANTIC_DIMENSIONS,@@ -169,6 +170,13 @@ def assign_semantics_batch(for observation_id, assignment in assigned.items()},)+ set_rank_clusters(+ team_id,+ {+ observation_id: (assignment.group_id, assignment.theme.theme_id if assignment.theme else None)+ for observation_id, assignment in assigned.items()+ },+ )except Exception:feed_group_cache.forget(scanner_id)raise