Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-vision): cluster observations into feed groups and search themes (#123809)

Tue Haulund5ae738264b7f
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yaml
index d9d72e06a5df..c1f7fac28f58 100644
--- a/.semgrep/rules/security/idor-team-scoped-models.yaml
+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml
@@ -359,6 +359,9 @@ rules:
|ReplayObservationView
|ReplayScanner
|ReplayScannerBackfill
+ |ReplayScannerFeedGroup
+ |ReplayScannerSearchTheme
+ |ReplayScannerSemanticCenter
|ReplayVisionLearnedRuleset
|Repo
|RepoRoutingRule
@@ -839,6 +842,9 @@ rules:
|ReplayObservationView
|ReplayScanner
|ReplayScannerBackfill
+ |ReplayScannerFeedGroup
+ |ReplayScannerSearchTheme
+ |ReplayScannerSemanticCenter
|ReplayVisionLearnedRuleset
|Repo
|RepoRoutingRule
diff --git a/products/replay_vision/backend/api/observations.py b/products/replay_vision/backend/api/observations.py
index f25424563ea5..b08de8c6746f 100644
--- a/products/replay_vision/backend/api/observations.py
+++ b/products/replay_vision/backend/api/observations.py
@@ -8,9 +8,9 @@ from zoneinfo import ZoneInfo
from django.conf import settings
from django.db import transaction
-from django.db.models import Case, CharField, IntegerField, Q, QuerySet, Subquery, Value, When
+from django.db.models import Case, IntegerField, Q, QuerySet, Subquery, Value, When
from django.db.models.fields.json import KeyTextTransform, KeyTransform
-from django.db.models.functions import Cast, Coalesce
+from django.db.models.functions import Cast
from django.http.response import HttpResponseBase
from django.utils.timezone import now
@@ -60,6 +60,7 @@ from products.replay_vision.backend.models.replay_observation import (
annotate_output_number,
hydrate_for_serialization,
jsonb_typeof,
+ watch_group_root,
)
from products.replay_vision.backend.models.replay_observation_feed_dismissal import (
FeedDismissalReason,
@@ -905,9 +906,7 @@ class ReplayObservationFilter(django_filters.FilterSet):
) -> QuerySet[ReplayObservation]:
if self._team is None:
return queryset.none()
- root = Coalesce(
- KeyTextTransform("duplicate_of", "watch_rank"), Cast("id", CharField()), output_field=CharField()
- )
+ root = watch_group_root()
current = Q(
status=ObservationStatus.SUCCEEDED, watch_rank__isnull=False, watch_rank__anchor_version=ANCHOR_VERSION
)
diff --git a/products/replay_vision/backend/gemini_client.py b/products/replay_vision/backend/gemini_client.py
index 5be9ad4eb3bd..4a163cb9c54c 100644
--- a/products/replay_vision/backend/gemini_client.py
+++ b/products/replay_vision/backend/gemini_client.py
@@ -21,12 +21,15 @@ from posthog.llm.gateway_client import (
from posthog.models import Team
from posthog.ph_client import feature_enabled_or_false
-from products.replay_vision.backend.temporal.gemini import gemini_api_key
-
AI_PRODUCT = "replay_vision"
GATEWAY_FLAG = "replay-vision-ai-gateway"
+def gemini_api_key() -> str:
+ """Replay Vision's dedicated key (own GCP project), falling back to the shared key where unset."""
+ return settings.REPLAY_VISION_GEMINI_API_KEY or settings.GEMINI_API_KEY
+
+
async def replay_gemini_client(team_id: int, *, properties: Mapping[str, Any] | None = None) -> "ReplayGemini":
base_properties = dict(properties or {})
if await _gateway_enabled(team_id) and (
diff --git a/products/replay_vision/backend/learned_rules.py b/products/replay_vision/backend/learned_rules.py
index f49a7de19af1..8c5b99be5399 100644
--- a/products/replay_vision/backend/learned_rules.py
+++ b/products/replay_vision/backend/learned_rules.py
@@ -29,6 +29,7 @@ from posthog.models.team import Team
from posthog.models.team.extensions import get_or_create_team_extension
from products.replay_vision.backend.distinct_ids import replay_vision_distinct_id
+from products.replay_vision.backend.gemini_client import gemini_api_key
from products.replay_vision.backend.models.replay_observation import ObservationStatus, ReplayObservation
from products.replay_vision.backend.models.replay_observation_label import ReplayObservationLabel
from products.replay_vision.backend.models.replay_scanner import ReplayScanner, ScannerOrigin
@@ -39,7 +40,6 @@ from products.replay_vision.backend.models.replay_vision_learned_ruleset import
from products.replay_vision.backend.models.team_replay_vision_config import TeamReplayVisionConfig
from products.replay_vision.backend.observation_formatting import describe_output, explanation_text, read_output
from products.replay_vision.backend.scanner_access import snapshot_experiment_scope_q
-from products.replay_vision.backend.temporal.gemini import gemini_api_key
from products.replay_vision.backend.temporal.team_context import sanitize_prompt_text
from ee.hogai.utils.untrusted import neutralize_markup
diff --git a/products/replay_vision/backend/management/commands/backfill_replay_vision_semantics.py b/products/replay_vision/backend/management/commands/backfill_replay_vision_semantics.py
new file mode 100644
index 000000000000..dbf344ba435f
--- /dev/null
+++ b/products/replay_vision/backend/management/commands/backfill_replay_vision_semantics.py
@@ -0,0 +1,33 @@
+from datetime import timedelta
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandParser
+from django.utils import timezone
+
+from products.replay_vision.backend.watch_rank_backfill import teams_with_observations
+from products.replay_vision.backend.watch_semantics import SEMANTIC_RETENTION
+from products.replay_vision.backend.watch_semantics_backfill import backfill_team_semantics
+
+
+class Command(BaseCommand):
+ help = "Group ranked Replay Vision observations into feed groups and named search themes"
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument("--team-id", type=int, action="append", help="Only this team; repeat for more")
+ parser.add_argument("--days", type=int, default=SEMANTIC_RETENTION.days, help="How far back to read, in days")
+ parser.add_argument("--dry-run", action="store_true", help="Count what would be grouped, write nothing")
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ since = timezone.now() - timedelta(days=options["days"])
+ team_ids = options["team_id"] or teams_with_observations(since)
+ for team_id in team_ids:
+ try:
+ result = backfill_team_semantics(team_id, since, dry_run=bool(options["dry_run"]))
+ except Exception as e:
+ # One team without AI data processing, or a slow query, must not stop the rest.
+ self.stderr.write(f"Team {team_id}: skipped ({e.__class__.__name__}: {e})")
+ continue
+ self.stdout.write(
+ f"Team {team_id}: {result.observations} issue observations, "
+ f"{result.assigned} grouped, {result.named} themes named."
+ )
diff --git a/products/replay_vision/backend/management/commands/backfill_replay_vision_watch_ranks.py b/products/replay_vision/backend/management/commands/backfill_replay_vision_watch_ranks.py
index 378d91a69469..4ce8e84feb6f 100644
--- a/products/replay_vision/backend/management/commands/backfill_replay_vision_watch_ranks.py
+++ b/products/replay_vision/backend/management/commands/backfill_replay_vision_watch_ranks.py
@@ -4,8 +4,11 @@ from typing import Any
from django.core.management.base import BaseCommand, CommandParser
from django.utils import timezone
-from products.replay_vision.backend.watch_rank import DUPLICATE_LOOKBACK
-from products.replay_vision.backend.watch_rank_backfill import backfill_team_watch_ranks, teams_with_observations
+from products.replay_vision.backend.watch_rank_backfill import (
+ WATCH_RANK_BACKFILL_DAYS,
+ backfill_team_watch_ranks,
+ teams_with_observations,
+)
class Command(BaseCommand):
@@ -13,7 +16,7 @@ class Command(BaseCommand):
def add_arguments(self, parser: CommandParser) -> None:
parser.add_argument("--team-id", type=int, action="append", help="Only this team; repeat for more")
- parser.add_argument("--days", type=int, default=DUPLICATE_LOOKBACK.days, help="How far back to rank, in days")
+ parser.add_argument("--days", type=int, default=WATCH_RANK_BACKFILL_DAYS, help="How far back to rank, in days")
parser.add_argument("--dry-run", action="store_true", help="Count what would be ranked, write nothing")
def handle(self, *args: Any, **options: Any) -> None:
diff --git a/products/replay_vision/backend/migrations/0133_scanner_semantics.py b/products/replay_vision/backend/migrations/0133_scanner_semantics.py
new file mode 100644
index 000000000000..474e5f8c27ba
--- /dev/null
+++ b/products/replay_vision/backend/migrations/0133_scanner_semantics.py
@@ -0,0 +1,82 @@
+# Generated by Django 5.2.18 on 2026-10-27 10:50
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+import posthog.uuidt
+
+
+def _scanner() -> models.ForeignKey:
+ return models.ForeignKey(
+ db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name="+", to="replay_vision.replayscanner"
+ )
+
+
+def _team() -> models.ForeignKey:
+ return models.ForeignKey(
+ db_constraint=False, on_delete=django.db.models.deletion.CASCADE, related_name="+", to="posthog.team"
+ )
+
+
+def _uuid_id() -> models.UUIDField:
+ return models.UUIDField(default=posthog.uuidt.uuid7, editable=False, primary_key=True, serialize=False)
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("posthog", "1413_event_filter_pipeline_override"),
+ ("replay_vision", "0132_ranked_pick_indexes"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="ReplayScannerFeedGroup",
+ fields=[
+ ("id", _uuid_id()),
+ ("scanner", _scanner()),
+ ("team", _team()),
+ ("vector_sum", models.BinaryField()),
+ ("member_count", models.PositiveIntegerField(default=0)),
+ ("last_member_at", models.DateTimeField()),
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ],
+ options={
+ "indexes": [models.Index(fields=["scanner", "-last_member_at"], name="rv_feed_group_recent_idx")],
+ },
+ ),
+ migrations.CreateModel(
+ name="ReplayScannerSearchTheme",
+ fields=[
+ ("id", _uuid_id()),
+ ("scanner", _scanner()),
+ ("team", _team()),
+ ("vector_sum", models.BinaryField()),
+ ("member_count", models.PositiveIntegerField(default=0)),
+ ("name", models.CharField(blank=True, default="", max_length=128)),
+ ("rename_at_member_count", models.PositiveIntegerField(default=5)),
+ ("last_member_at", models.DateTimeField()),
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ],
+ options={
+ "indexes": [models.Index(fields=["scanner", "-last_member_at"], name="rv_theme_scanner_recent_idx")],
+ },
+ ),
+ migrations.CreateModel(
+ name="ReplayScannerSemanticCenter",
+ fields=[
+ (
+ "scanner",
+ models.OneToOneField(
+ on_delete=django.db.models.deletion.CASCADE,
+ primary_key=True,
+ related_name="+",
+ serialize=False,
+ to="replay_vision.replayscanner",
+ ),
+ ),
+ ("team", _team()),
+ ("vector_sum", models.BinaryField()),
+ ("member_count", models.PositiveIntegerField(default=0)),
+ ],
+ ),
+ ]
diff --git a/products/replay_vision/backend/migrations/0134_feed_group_index.py b/products/replay_vision/backend/migrations/0134_feed_group_index.py
new file mode 100644
index 000000000000..1e4b850b0d9d
--- /dev/null
+++ b/products/replay_vision/backend/migrations/0134_feed_group_index.py
@@ -0,0 +1,32 @@
+import django.db.models.fields.json
+import django.db.models.functions.comparison
+from django.db import migrations, models
+
+from posthog.migration_helpers import SafeAddIndexConcurrently, SafeRemoveIndexConcurrently
+
+
+class Migration(migrations.Migration):
+ atomic = False
+
+ dependencies = [
+ ("replay_vision", "0133_scanner_semantics"),
+ ]
+
+ operations = [
+ SafeAddIndexConcurrently(
+ model_name="replayobservation",
+ index=models.Index(
+ django.db.models.functions.comparison.Coalesce(
+ django.db.models.fields.json.KeyTextTransform("group", "watch_rank"),
+ django.db.models.functions.comparison.Cast("id", models.CharField()),
+ output_field=models.CharField(),
+ ),
+ condition=models.Q(("status", "succeeded"), ("watch_rank__isnull", False)),
+ name="rlo_ranked_feed_group_idx",
+ ),
+ ),
+ SafeRemoveIndexConcurrently(
+ model_name="replayobservation",
+ name="rlo_ranked_group_idx",
+ ),
+ ]
diff --git a/products/replay_vision/backend/migrations/max_migration.txt b/products/replay_vision/backend/migrations/max_migration.txt
index 9fd42bb17ea8..a2c14b64d517 100644
--- a/products/replay_vision/backend/migrations/max_migration.txt
+++ b/products/replay_vision/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0132_ranked_pick_indexes
+0134_feed_group_index
diff --git a/products/replay_vision/backend/models/__init__.py b/products/replay_vision/backend/models/__init__.py
index 942516461587..3d21a6d46221 100644
--- a/products/replay_vision/backend/models/__init__.py
+++ b/products/replay_vision/backend/models/__init__.py
@@ -8,6 +8,11 @@ from products.replay_vision.backend.models.replay_observation_usage import Repla
from products.replay_vision.backend.models.replay_observation_view import ReplayObservationView
from products.replay_vision.backend.models.replay_scanner import ReplayScanner
from products.replay_vision.backend.models.replay_scanner_backfill import ReplayScannerBackfill
+from products.replay_vision.backend.models.replay_scanner_semantics import (
+ ReplayScannerFeedGroup,
+ ReplayScannerSearchTheme,
+ ReplayScannerSemanticCenter,
+)
from products.replay_vision.backend.models.replay_vision_learned_ruleset import ReplayVisionLearnedRuleset
from products.replay_vision.backend.models.replay_vision_org_admission_budget import ReplayVisionOrgAdmissionBudget
from products.replay_vision.backend.models.team_replay_vision_config import TeamReplayVisionConfig
@@ -28,6 +33,9 @@ __all__ = [
"ReplayObservationView",
"ReplayScanner",
"ReplayScannerBackfill",
+ "ReplayScannerFeedGroup",
+ "ReplayScannerSearchTheme",
+ "ReplayScannerSemanticCenter",
"ReplayVisionLearnedRuleset",
"ReplayVisionOrgAdmissionBudget",
"TeamReplayVisionConfig",
diff --git a/products/replay_vision/backend/models/replay_observation.py b/products/replay_vision/backend/models/replay_observation.py
index 96aba7598d7d..1cdbeabf55c8 100644
--- a/products/replay_vision/backend/models/replay_observation.py
+++ b/products/replay_vision/backend/models/replay_observation.py
@@ -48,8 +48,8 @@ def watch_rank_contrast() -> Cast:
def watch_group_root() -> Coalesce:
- """A ranked row's duplicate group: the observation it duplicates, or itself. `rlo_ranked_group_idx` indexes it."""
- return Coalesce(KeyTextTransform("duplicate_of", "watch_rank"), Cast("id", CharField()), output_field=CharField())
+ """A ranked row's feed group, or the row itself when it has none. `rlo_ranked_feed_group_idx` indexes it."""
+ return Coalesce(KeyTextTransform("group", "watch_rank"), Cast("id", CharField()), output_field=CharField())
class ReplayObservation(UUIDModel):
@@ -200,10 +200,10 @@ class ReplayObservation(UUIDModel):
name="rlo_ranked_contrast_idx",
condition=models.Q(status="succeeded", watch_rank__isnull=False, watch_rank__anchor_version=1),
),
- # Finds the members of the feed picks' duplicate groups, to count the sessions and people each one reached.
+ # Finds the members of the feed picks' groups, to count the sessions and people each one reached.
models.Index(
watch_group_root(),
- name="rlo_ranked_group_idx",
+ name="rlo_ranked_feed_group_idx",
condition=models.Q(status="succeeded", watch_rank__isnull=False),
),
# Serves the per-backfill observation filter and progress counts; partial since live rows never match.
diff --git a/products/replay_vision/backend/models/replay_scanner_semantics.py b/products/replay_vision/backend/models/replay_scanner_semantics.py
new file mode 100644
index 000000000000..0255954d8b84
--- /dev/null
+++ b/products/replay_vision/backend/models/replay_scanner_semantics.py
@@ -0,0 +1,61 @@
+from django.db import models
+
+from posthog.models.scoping.root_mixin import TeamScopedRootMixin
+from posthog.models.utils import UUIDModel
+
+# A theme waits for this many members before its first name, so a one-off session never costs a model call.
+THEME_FIRST_RENAME_AT = 5
+
+
+class ReplayScannerSemanticCenter(TeamScopedRootMixin, models.Model):
+ """A scanner's running sum of issue observation vectors. Themes and feed groups compare vectors with this mean
+ taken out, because every observation of one scanner shares its question's wording."""
+
+ scanner = models.OneToOneField(
+ "replay_vision.ReplayScanner", on_delete=models.CASCADE, primary_key=True, related_name="+"
+ )
+ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+", db_constraint=False)
+ vector_sum = models.BinaryField()
+ member_count = models.PositiveIntegerField(default=0)
+
+ class Meta:
+ app_label = "replay_vision"
+
+
+class ReplayScannerSearchTheme(TeamScopedRootMixin, UUIDModel):
+ """A broad cluster of one scanner's issue observations, named as a search the Search tab suggests."""
+
+ # The scanner and recency index serves every read, so the foreign key needs no index of its own.
+ scanner = models.ForeignKey(
+ "replay_vision.ReplayScanner", on_delete=models.CASCADE, related_name="+", db_index=False
+ )
+ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+", db_constraint=False)
+ vector_sum = models.BinaryField()
+ member_count = models.PositiveIntegerField(default=0)
+ name = models.CharField(max_length=128, blank=True, default="")
+ # The member count that triggers the next naming, so a theme is renamed as it grows rather than on every member.
+ rename_at_member_count = models.PositiveIntegerField(default=THEME_FIRST_RENAME_AT)
+ last_member_at = models.DateTimeField()
+ created_at = models.DateTimeField(auto_now_add=True)
+
+ class Meta:
+ app_label = "replay_vision"
+ indexes = [models.Index(fields=["scanner", "-last_member_at"], name="rv_theme_scanner_recent_idx")]
+
+
+class ReplayScannerFeedGroup(TeamScopedRootMixin, UUIDModel):
+ """A tight cluster of one scanner's issue observations that tell the same story, shown as one feed card."""
+
+ # The scanner and recency index serves every read, so the foreign key needs no index of its own.
+ scanner = models.ForeignKey(
+ "replay_vision.ReplayScanner", on_delete=models.CASCADE, related_name="+", db_index=False
+ )
+ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+", db_constraint=False)
+ vector_sum = models.BinaryField()
+ member_count = models.PositiveIntegerField(default=0)
+ last_member_at = models.DateTimeField()
+ created_at = models.DateTimeField(auto_now_add=True)
+
+ class Meta:
+ app_label = "replay_vision"
+ indexes = [models.Index(fields=["scanner", "-last_member_at"], name="rv_feed_group_recent_idx")]
diff --git a/products/replay_vision/backend/semantic_space.py b/products/replay_vision/backend/semantic_space.py
new file mode 100644
index 000000000000..36b046ed4b2b
--- /dev/null
+++ b/products/replay_vision/backend/semantic_space.py
@@ -0,0 +1,66 @@
+"""Shared vector maths for search themes and feed groups.
+
+Both compare a scanner's issue observations on a short prefix of their embedding, with the scanner's running mean
+taken out, because every observation of one scanner shares the wording of its question and would otherwise look
+alike. A young scanner's mean is mostly its own few rows, so until the mean has enough members both compare plain
+vectors at a plain similarity.
+"""
+
+import numpy as np
+
+from posthog.dataclasses import frozen
+
+# text-embedding-3 vectors keep their meaning when cut short and renormalised, and a short one keeps the clusters
+# cheap to hold and compare.
+SEMANTIC_DIMENSIONS = 512
+CENTRE_MIN_MEMBERS = 50
+
+
+@frozen
+class SemanticSpace:
+ """One observation's vector and the scanner mean it is compared against."""
+
+ vector: np.ndarray
+ mean: np.ndarray
+ centring: bool
+
+
+def semantic_vector(vector: list[float] | np.ndarray) -> np.ndarray:
+ head = np.zeros(SEMANTIC_DIMENSIONS, dtype=np.float32)
+ values = np.asarray(vector[:SEMANTIC_DIMENSIONS], dtype=np.float32)
+ head[: len(values)] = values
+ return unit(head)
+
+
+def unit(vector: np.ndarray) -> np.ndarray:
+ norm = np.linalg.norm(vector, axis=-1, keepdims=True)
+ return np.divide(vector, norm, out=np.zeros_like(vector), where=norm > 0)
+
+
+def pack(vector: np.ndarray) -> bytes:
+ return vector.astype(np.float32).tobytes()
+
+
+def unpack(data: bytes | memoryview) -> np.ndarray:
+ return np.frombuffer(bytes(data), dtype=np.float32)
+
+
+def scanner_space(center_sum: np.ndarray, center_count: int, vector: np.ndarray) -> SemanticSpace:
+ """The space for one new vector, given the scanner's sum and count before it joins."""
+ mean = (center_sum + vector) / (center_count + 1)
+ # A vector that matches the scanner mean has no direction once centred, so it compares plain like a young scanner.
+ centring = center_count >= CENTRE_MIN_MEMBERS and bool(np.linalg.norm(vector - mean) > 1e-6)
+ return SemanticSpace(vector=vector, mean=mean if centring else np.zeros_like(vector), centring=centring)
+
+
+def nearest_cluster(
+ sums: np.ndarray, counts: np.ndarray, space: SemanticSpace, centred_threshold: float, raw_threshold: float
+) -> int | None:
+ """The index of the cluster whose mean is closest to the vector, or None when none is close enough."""
+ if not len(counts):
+ return None
+ centroids = unit(sums / counts[:, None] - space.mean)
+ similarities = centroids @ unit(space.vector - space.mean)
+ best = int(np.argmax(similarities))
+ threshold = centred_threshold if space.centring else raw_threshold
+ return best if similarities[best] >= threshold else None
diff --git a/products/replay_vision/backend/temporal/activities/cleanup_gemini_file.py b/products/replay_vision/backend/temporal/activities/cleanup_gemini_file.py
index 522ae0287544..57e9490c5e95 100644
--- a/products/replay_vision/backend/temporal/activities/cleanup_gemini_file.py
+++ b/products/replay_vision/backend/temporal/activities/cleanup_gemini_file.py
@@ -3,8 +3,8 @@
from google.genai import Client as RawGenAIClient
from temporalio import activity
+from products.replay_vision.backend.gemini_client import gemini_api_key
from products.replay_vision.backend.temporal.decorators import track_activity
-from products.replay_vision.backend.temporal.gemini import gemini_api_key
from products.replay_vision.backend.temporal.gemini_cleanup_sweep.tracking import delete_and_untrack
from products.replay_vision.backend.temporal.types import CleanupGeminiFileInputs
diff --git a/products/replay_vision/backend/temporal/gemini.py b/products/replay_vision/backend/temporal/gemini.py
index 55728778003f..c13b9d92e62e 100644
--- a/products/replay_vision/backend/temporal/gemini.py
+++ b/products/replay_vision/backend/temporal/gemini.py
@@ -1,6 +1,4 @@
-"""Gemini API key resolution and error classification for Replay Vision."""
-
-from django.conf import settings
+"""Gemini error classification for Replay Vision."""
import httpx
import aiohttp
@@ -9,12 +7,6 @@ from google.genai.errors import APIError
from products.replay_vision.backend.error_kinds import FailureKind
-
-def gemini_api_key() -> str:
- """Replay Vision's dedicated key (own GCP project), falling back to the shared key where unset."""
- return settings.REPLAY_VISION_GEMINI_API_KEY or settings.GEMINI_API_KEY
-
-
# 408 request timeout and 409 conflict join the usual rate-limit/5xx set: all clear on their own.
_TRANSIENT_STATUS_CODES = frozenset({408, 409, 429, 500, 502, 503, 504})
diff --git a/products/replay_vision/backend/temporal/gemini_cleanup_sweep/activities.py b/products/replay_vision/backend/temporal/gemini_cleanup_sweep/activities.py
index a8964d59ff37..0cc85b0dd7da 100644
--- a/products/replay_vision/backend/temporal/gemini_cleanup_sweep/activities.py
+++ b/products/replay_vision/backend/temporal/gemini_cleanup_sweep/activities.py
@@ -11,8 +11,8 @@ from temporalio.service import RPCError, RPCStatusCode
from posthog.temporal.common.client import async_connect
from posthog.temporal.common.heartbeat import Heartbeater
+from products.replay_vision.backend.gemini_client import gemini_api_key
from products.replay_vision.backend.temporal.decorators import track_activity
-from products.replay_vision.backend.temporal.gemini import gemini_api_key
from products.replay_vision.backend.temporal.gemini_cleanup_sweep.constants import (
DELETE_CONCURRENCY,
DESCRIBE_CONCURRENCY,
diff --git a/products/replay_vision/backend/tests/test_api.py b/products/replay_vision/backend/tests/test_api.py
index 566e38a19d9d..4ce4fba16bd9 100644
--- a/products/replay_vision/backend/tests/test_api.py
+++ b/products/replay_vision/backend/tests/test_api.py
@@ -2257,13 +2257,11 @@ class TestReplayObservationViewSet(_VisionAPITestCase):
}
root_id = str(rows["root"].id)
- def rank(
- duplicate_of: str | None, category: str = "failure", anchor_version: int = ANCHOR_VERSION
- ) -> dict[str, Any]:
+ def rank(group: str | None, category: str = "failure", anchor_version: int = ANCHOR_VERSION) -> dict[str, Any]:
return {
"category": category,
"contrast": 0.3,
- "duplicate_of": duplicate_of,
+ "group": group,
"anchor_version": anchor_version,
}
@@ -5214,12 +5212,12 @@ class TestWatchFeedAPI(_VisionAPITestCase):
watch_rank={
"category": "routine",
"contrast": -0.1,
- "duplicate_of": None,
+ "group": None,
"anchor_version": ANCHOR_VERSION,
}
)
ReplayObservation.objects.filter(id=jev_low.id).update(
- watch_rank={"category": "failure", "contrast": 0.2, "duplicate_of": None, "anchor_version": ANCHOR_VERSION}
+ watch_rank={"category": "failure", "contrast": 0.2, "group": None, "anchor_version": ANCHOR_VERSION}
)
with patch(ranker, return_value="embedding"):
resp = self.client.get(self.feed_url)
@@ -5267,7 +5265,7 @@ class TestWatchFeedAPI(_VisionAPITestCase):
watch_rank={
"category": "failure",
"contrast": 1 - index / 20,
- "duplicate_of": None,
+ "group": None,
"anchor_version": ANCHOR_VERSION,
},
)
@@ -5283,13 +5281,13 @@ class TestWatchFeedAPI(_VisionAPITestCase):
def test_embedding_picks_reach_past_the_recency_slice_and_count_hidden_duplicates(self) -> None:
scanner = self._create_scanner(name="m")
- def ranked(session_id: str, minutes_ago: int, category: str, contrast: float, duplicate_of: str | None = None):
+ def ranked(session_id: str, minutes_ago: int, category: str, contrast: float, group: str | None = None):
obs = self._succeeded_observation(scanner, session_id, minutes_ago, self._monitor_result("no"))
ReplayObservation.objects.filter(id=obs.id).update(
watch_rank={
"category": category,
"contrast": contrast,
- "duplicate_of": duplicate_of,
+ "group": group,
"anchor_version": ANCHOR_VERSION,
}
)
@@ -5297,7 +5295,7 @@ class TestWatchFeedAPI(_VisionAPITestCase):
oldest = ranked("oldest-failure", 500, "failure", 0.9)
# A copy of the pick that scores below the cut still makes the group bigger.
- ranked("copy", 400, "failure", -0.5, duplicate_of=str(oldest.id))
+ ranked("copy", 400, "failure", -0.5, group=str(oldest.id))
for index in range(10):
ranked(f"routine-{index}", index, "routine", -0.1)
@@ -5315,13 +5313,13 @@ class TestWatchFeedAPI(_VisionAPITestCase):
def test_a_thin_embedding_feed_pads_by_group_and_never_pads_with_a_copy_of_a_pick(self) -> None:
scanner = self._create_scanner(name="m")
- def ranked(session_id: str, minutes_ago: int, category: str, contrast: float, duplicate_of: str | None = None):
+ def ranked(session_id: str, minutes_ago: int, category: str, contrast: float, group: str | None = None):
obs = self._succeeded_observation(scanner, session_id, minutes_ago, self._monitor_result("no"))
ReplayObservation.objects.filter(id=obs.id).update(
watch_rank={
"category": category,
"contrast": contrast,
- "duplicate_of": duplicate_of,
+ "group": group,
"anchor_version": ANCHOR_VERSION,
}
)
@@ -5329,8 +5327,8 @@ class TestWatchFeedAPI(_VisionAPITestCase):
root = ranked("pick", 500, "failure", 0.9)
# Two copies of the one finding: one above the cut, one newest of all and below it.
- ranked("copy-above", 400, "failure", 0.8, duplicate_of=str(root.id))
- ranked("copy-newest", 0, "failure", -0.9, duplicate_of=str(root.id))
+ ranked("copy-above", 400, "failure", 0.8, group=str(root.id))
+ ranked("copy-newest", 0, "failure", -0.9, group=str(root.id))
for index in range(10):
ranked(f"routine-{index}", 10 + index, "routine", -0.1)
@@ -5352,7 +5350,7 @@ class TestWatchFeedAPI(_VisionAPITestCase):
watch_rank={
"category": "failure",
"contrast": top - index / 100,
- "duplicate_of": None,
+ "group": None,
"anchor_version": ANCHOR_VERSION,
}
)
@@ -5382,7 +5380,7 @@ class TestWatchFeedAPI(_VisionAPITestCase):
watch_rank={
"category": "failure",
"contrast": 1 - index / 100,
- "duplicate_of": root_id if 0 < index < 5 else None,
+ "group": root_id if 0 < index < 5 else None,
"anchor_version": ANCHOR_VERSION,
}
)
@@ -5408,7 +5406,7 @@ class TestWatchFeedAPI(_VisionAPITestCase):
watch_rank={
"category": category,
"contrast": 0.5 - index * 0.01,
- "duplicate_of": None,
+ "group": None,
"anchor_version": ANCHOR_VERSION,
}
)
@@ -5960,7 +5958,7 @@ class TestWatchFeedAPI(_VisionAPITestCase):
watch_rank={
"category": "failure",
"contrast": index / 10,
- "duplicate_of": None,
+ "group": None,
"anchor_version": ANCHOR_VERSION,
}
)
diff --git a/products/replay_vision/backend/tests/test_watch_rank.py b/products/replay_vision/backend/tests/test_watch_rank.py
index 41399781b463..d496475a8345 100644
--- a/products/replay_vision/backend/tests/test_watch_rank.py
+++ b/products/replay_vision/backend/tests/test_watch_rank.py
@@ -18,10 +18,9 @@ from products.replay_vision.backend.watch_rank import (
ANCHOR_VERSION,
WatchCategory,
classify,
- duplicate_root,
rank_watch_feed_by_embedding,
)
-from products.replay_vision.backend.watch_rank_backfill import _rank_scanner, _RecentVectors, _stored_vectors
+from products.replay_vision.backend.watch_rank_backfill import _rank_scanner, _stored_vectors
_NOW = datetime(2026, 10, 23, 12, 0, tzinfo=UTC)
@@ -32,7 +31,7 @@ def _row(
*,
category: str = "failure",
scanner: str = "scanner-a",
- duplicate_of: str | None = None,
+ group: str | None = None,
anchor_version: int = ANCHOR_VERSION,
minutes_ago: int = 0,
viewed: bool = False,
@@ -44,9 +43,9 @@ def _row(
else {
"category": category,
"contrast": contrast,
- "duplicate_of": duplicate_of,
"anchor_version": anchor_version,
}
+ | ({"group": group} if group else {})
)
return {
"id": observation_id,
@@ -88,26 +87,6 @@ class TestPickIndexes(SimpleTestCase):
)
-class TestDuplicateRoot(SimpleTestCase):
- @parameterized.expand(
- [
- ("unranked_smaller_neighbour_is_the_root", "obs-b", "obs-a", None, "obs-a"),
- ("unranked_larger_neighbour_defers_to_this_one", "obs-a", "obs-b", None, None),
- ("ranked_neighbour_passes_on_its_root", "obs-a", "obs-b", {"duplicate_of": "obs-0"}, "obs-0"),
- ("ranked_root_neighbour_is_the_root", "obs-a", "obs-b", {"duplicate_of": None}, "obs-b"),
- ]
- )
- def test_two_concurrent_copies_agree_on_one_root(
- self,
- _name: str,
- observation_id: str,
- neighbour_id: str,
- neighbour_rank: dict[str, Any] | None,
- expected: str | None,
- ) -> None:
- assert duplicate_root(observation_id, neighbour_id, neighbour_rank) == expected
-
-
class TestRankWatchFeedByEmbedding(SimpleTestCase):
def test_a_finding_that_reached_more_people_outranks_a_stronger_one_off(self) -> None:
rows = [_row(f"a-{index}", 0.01 * index, minutes_ago=index + 1) for index in range(10)]
@@ -120,14 +99,13 @@ class TestRankWatchFeedByEmbedding(SimpleTestCase):
assert [entry.observation_id for entry in ranked][:2] == ["widespread", "one-off"]
assert ranked[0].reason["affected_people"] == 6
- def test_picks_the_scanner_top_issues_once_per_duplicate_group(self) -> None:
+ def test_picks_the_scanner_top_issues_once_per_feed_group(self) -> None:
rows = [
- # scanner-a has enough ranked rows for a percentile; its top two are one duplicate group.
- _row("dup-root", 0.30),
- _row("dup-copy", 0.32, duplicate_of="dup-root", minutes_ago=50),
- _row("dup-low", -0.01, duplicate_of="dup-root"),
- # A copy that points at another copy, as when its root was ranked after it, still joins the group.
- _row("dup-chain", 0.02, duplicate_of="dup-copy", minutes_ago=3),
+ # scanner-a has enough ranked rows for a percentile; its top two are one feed group.
+ _row("dup-root", 0.30, group="g1"),
+ _row("dup-copy", 0.32, group="g1", minutes_ago=50),
+ _row("dup-low", -0.01, group="g1"),
+ _row("dup-chain", 0.02, group="g1", minutes_ago=3),
_row("routine-top", 0.40, category="routine", minutes_ago=50),
*[_row(f"a-{index}", 0.01 * index, minutes_ago=index + 1) for index in range(10)],
# A rank from older anchors does not compare with current scores.
@@ -152,9 +130,9 @@ class TestRankWatchFeedByEmbedding(SimpleTestCase):
def test_a_viewed_pick_still_stands_for_its_group_over_a_copy_below_the_cutoff(self) -> None:
rows = [
- _row("seen-top", 0.9, viewed=True),
+ _row("seen-top", 0.9, viewed=True, group="g1"),
_row("other-top", 0.8, minutes_ago=1),
- _row("mid-copy", 0.7, duplicate_of="seen-top", minutes_ago=2),
+ _row("mid-copy", 0.7, group="g1", minutes_ago=2),
*[_row(f"a-{index}", 0.01 * index, minutes_ago=index + 3) for index in range(7)],
]
@@ -192,7 +170,7 @@ class TestRankWatchFeedByEmbedding(SimpleTestCase):
class TestBackfillRankScanner(SimpleTestCase):
- def test_walks_oldest_first_so_a_copy_joins_the_group_of_an_already_ranked_row(self) -> None:
+ def test_ranks_only_rows_without_a_current_rank(self) -> None:
anchors: dict[WatchCategory, np.ndarray] = {
"failure": np.array([[1.0, 0.0, 0.0]]),
"struggle": np.array([[0.0, 1.0, 0.0]]),
@@ -200,7 +178,7 @@ class TestBackfillRankScanner(SimpleTestCase):
"routine": np.array([[0.0, 0.0, 1.0]]),
}
error = np.array([1.0, 0.0, 0.0])
- current = {"category": "failure", "contrast": 1.0, "duplicate_of": "root", "anchor_version": ANCHOR_VERSION}
+ current = {"category": "failure", "contrast": 1.0, "anchor_version": ANCHOR_VERSION}
rows: list[dict[str, Any]] = [
{"id": "ranked", "watch_rank": current},
{"id": "no-vector", "watch_rank": None},
@@ -210,18 +188,11 @@ class TestBackfillRankScanner(SimpleTestCase):
]
vectors = {"ranked": error, "copy": error, "stale": error, "calm": np.array([0.0, 0.0, 1.0])}
- # Two batches that share the recent vectors, as the backfill runs them.
- recent = _RecentVectors()
- written = {
- **_rank_scanner(rows[:2], vectors, anchors, recent),
- **_rank_scanner(rows[2:], vectors, anchors, recent),
- }
+ written = _rank_scanner(rows, vectors, anchors)
assert set(written) == {"copy", "stale", "calm"}
- assert written["copy"]["duplicate_of"] == "root"
- assert written["stale"]["duplicate_of"] == "root"
+ assert written["copy"]["category"] == "failure"
assert written["calm"]["category"] == "routine"
- assert written["calm"]["duplicate_of"] is None
@parameterized.expand(
[
diff --git a/products/replay_vision/backend/tests/test_watch_rank_consumer.py b/products/replay_vision/backend/tests/test_watch_rank_consumer.py
index a45aa5ca6804..094e00b36f25 100644
--- a/products/replay_vision/backend/tests/test_watch_rank_consumer.py
+++ b/products/replay_vision/backend/tests/test_watch_rank_consumer.py
@@ -25,7 +25,7 @@ from products.replay_vision.backend.watch_rank import ANCHOR_VERSION, RankReques
from products.replay_vision.backend.watch_rank_consumer import WatchRankConsumer, parse_rank_request
_WATCH_RANK = "products.replay_vision.backend.watch_rank"
-_CURRENT_RANK = {"category": "struggle", "contrast": 0.2, "duplicate_of": None, "anchor_version": ANCHOR_VERSION}
+_CURRENT_RANK = {"category": "struggle", "contrast": 0.2, "anchor_version": ANCHOR_VERSION, "group": "g1"}
_CONSUMER = "products.replay_vision.backend.watch_rank_consumer"
@@ -99,14 +99,8 @@ class TestWatchRankConsumer(BaseTest):
@parameterized.expand(
[
- (
- "duplicate_points_at_the_group_root",
- None,
- None,
- "ranked",
- {"category": "failure", "duplicate_of": "root-id"},
- ),
- ("ranking_failure_leaves_no_rank", None, RuntimeError("clickhouse down"), "failed", None),
+ ("issue_row_gets_a_rank_and_a_feed_group", None, None, "ranked", {"category": "failure"}),
+ ("ranking_failure_leaves_no_rank", None, RuntimeError("cache down"), "failed", None),
("redelivery_keeps_the_current_rank", _CURRENT_RANK, None, "already_ranked", _CURRENT_RANK),
("deleted_observation_is_skipped", None, None, "missing", None),
]
@@ -115,7 +109,7 @@ class TestWatchRankConsumer(BaseTest):
self,
_name: str,
existing_rank: dict[str, Any] | None,
- nearest_error: Exception | None,
+ anchor_error: Exception | None,
outcome: str,
expected_rank: dict[str, Any] | None,
) -> None:
@@ -126,21 +120,18 @@ class TestWatchRankConsumer(BaseTest):
scanner_config={"prompt": "p"},
model=ScannerModel.GEMINI_3_8_FLASH,
)
- neighbour = self._observation(scanner, "sess-earlier", watch_rank={"duplicate_of": "root-id"})
observation = self._observation(scanner, "sess-new", watch_rank=existing_rank)
- nearest = MagicMock(side_effect=nearest_error, return_value=(str(neighbour.id), 0.05))
message = _result_message(team_id=self.team.id, observation_id=str(observation.id), scanner_id=str(scanner.id))
if outcome == "missing":
observation.delete()
with (
- patch(f"{_WATCH_RANK}.anchor_vectors", AsyncMock(return_value=self._ANCHORS)),
- patch(f"{_WATCH_RANK}.nearest_earlier_observation", nearest),
+ patch(f"{_WATCH_RANK}.anchor_vectors", AsyncMock(side_effect=anchor_error, return_value=self._ANCHORS)),
patch(f"{_CONSUMER}._RETRY_WINDOW_S", 0),
# It would close the connection that holds the test transaction.
patch(f"{_CONSUMER}.close_old_connections"),
):
- assert async_to_sync(WatchRankConsumer(MagicMock()).handle)(message, 0.0, math.inf) == outcome
+ assert async_to_sync(WatchRankConsumer(MagicMock()).handle)(message, math.inf) == outcome
if outcome == "missing":
return
@@ -150,6 +141,7 @@ class TestWatchRankConsumer(BaseTest):
else:
assert observation.watch_rank is not None
assert expected_rank.items() <= observation.watch_rank.items()
+ assert observation.watch_rank.get("group")
@parameterized.expand(
[
@@ -193,4 +185,4 @@ class TestWatchRankConsumer(BaseTest):
patch(f"{_CONSUMER}.close_old_connections"),
):
for _ in range(2):
- assert async_to_sync(consumer.handle)(message, 0.0, time.monotonic()) == "deferred"
+ assert async_to_sync(consumer.handle)(message, time.monotonic()) == "deferred"
diff --git a/products/replay_vision/backend/tests/test_watch_semantics.py b/products/replay_vision/backend/tests/test_watch_semantics.py
new file mode 100644
index 000000000000..d650a7f07d8f
--- /dev/null
+++ b/products/replay_vision/backend/tests/test_watch_semantics.py
@@ -0,0 +1,237 @@
+from datetime import timedelta
+from typing import Any
+
+from posthog.test.base import BaseTest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from django.test import SimpleTestCase
+from django.utils import timezone
+
+import numpy as np
+from asgiref.sync import async_to_sync
+from parameterized import parameterized
+
+from products.replay_vision.backend.models.replay_observation import ObservationTrigger, ReplayObservation
+from products.replay_vision.backend.models.replay_scanner import PromptValence, ReplayScanner, ScannerModel, ScannerType
+from products.replay_vision.backend.models.replay_scanner_semantics import (
+ ReplayScannerFeedGroup,
+ ReplayScannerSearchTheme,
+ ReplayScannerSemanticCenter,
+)
+from products.replay_vision.backend.semantic_space import SEMANTIC_DIMENSIONS, pack, semantic_vector
+from products.replay_vision.backend.tests.helpers import snapshot_for
+from products.replay_vision.backend.watch_feed_groups import feed_group_cache
+from products.replay_vision.backend.watch_rank import ANCHOR_VERSION
+from products.replay_vision.backend.watch_semantics import assign_semantics
+from products.replay_vision.backend.watch_themes import belongs_in_theme, finalize_label, name_theme
+
+_THEMES = "products.replay_vision.backend.watch_themes"
+
+
+def _rank(category: str = "failure") -> dict[str, Any]:
+ return {"category": category, "contrast": 0.2, "anchor_version": ANCHOR_VERSION}
+
+
+def _vector(*head: float) -> list[float]:
+ vector = np.zeros(3072)
+ vector[: len(head)] = head
+ return (vector / np.linalg.norm(vector)).tolist()
+
+
+class TestThemeRules(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("routine_never_themes", "routine", ScannerType.SUMMARIZER, "", {}, False),
+ ("summarizer_issue_themes", "struggle", ScannerType.SUMMARIZER, "", {}, True),
+ ("bad_news_monitor_keeps_yes", "failure", ScannerType.MONITOR, "bad", {"verdict": "yes"}, True),
+ ("bad_news_monitor_drops_no", "failure", ScannerType.MONITOR, "bad", {"verdict": "no"}, False),
+ ("good_news_monitor_keeps_no", "failure", ScannerType.MONITOR, "good", {"verdict": "no"}, True),
+ ("unjudged_monitor_keeps_every_answer", "failure", ScannerType.MONITOR, "", {"verdict": "no"}, True),
+ ]
+ )
+ def test_which_observations_join_a_theme(
+ self, _name: str, category: str, scanner_type: str, valence: str, output: dict[str, Any], expected: bool
+ ) -> None:
+ assert (
+ belongs_in_theme(category=category, scanner_type=scanner_type, valence=valence, output=output) is expected
+ )
+
+ @parameterized.expand(
+ [
+ ("normalised", " Card Declined At Checkout. ", "card declined at checkout"),
+ ("too_long", " ".join(["word"] * 11), ""),
+ ("identifier", "errors for [email protected]", ""),
+ ("empty", "", ""),
+ ]
+ )
+ def test_label_shape(self, _name: str, raw: str, expected: str) -> None:
+ assert finalize_label(raw) == expected
+
+
+class TestAssignSemantics(BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ feed_group_cache.clear()
+ self.scanner = self._scanner(ScannerType.SUMMARIZER)
+
+ def _scanner(self, scanner_type: str, valence: str = "", name: str = "checkout") -> ReplayScanner:
+ return ReplayScanner.objects.create(
+ team=self.team,
+ name=name,
+ scanner_type=scanner_type,
+ prompt_valence=valence,
+ scanner_config={"prompt": "What happened at checkout?"},
+ model=ScannerModel.GEMINI_3_8_FLASH,
+ )
+
+ def _observation(
+ self, category: str = "failure", output: dict[str, Any] | None = None, scanner: ReplayScanner | None = None
+ ) -> ReplayObservation:
+ scanner = scanner or self.scanner
+ return ReplayObservation.objects.create(
+ scanner=scanner,
+ team=self.team,
+ session_id=f"sess-{ReplayObservation.objects.count()}",
+ triggered_by=ObservationTrigger.ON_DEMAND,
+ scanner_snapshot=snapshot_for(scanner),
+ watch_rank=_rank(category),
+ scanner_result={"model_output": output or {"scanner_type": "summarizer", "summary": ""}},
+ )
+
+ def _assign(self, observation: ReplayObservation, vector: list[float]) -> Any:
+ return assign_semantics(
+ team_id=self.team.id,
+ scanner_id=str(observation.scanner_id),
+ observation_id=str(observation.id),
+ vector=vector,
+ now=timezone.now(),
+ )
+
+ def _counts(self, model: type[ReplayScannerFeedGroup] | type[ReplayScannerSearchTheme]) -> list[int]:
+ return sorted(model.objects.for_team(self.team.id).values_list("member_count", flat=True))
+
+ def test_close_observations_share_a_group_and_a_theme_and_a_redelivery_counts_once(self) -> None:
+ first, close, far = self._observation(), self._observation(), self._observation()
+ routine = self._observation(category="routine")
+
+ joined = [self._assign(first, _vector(1.0)), self._assign(close, _vector(1.0, 0.1))]
+ assert joined[0].group_id == joined[1].group_id
+ assert joined[0].theme.theme_id == joined[1].theme.theme_id
+ assert self._assign(first, _vector(1.0)) is None
+ assert self._assign(far, _vector(0.0, 1.0)).group_id != joined[0].group_id
+ assert self._assign(routine, _vector(1.0)) is None
+
+ assert self._counts(ReplayScannerFeedGroup) == [1, 2]
+ assert self._counts(ReplayScannerSearchTheme) == [1, 2]
+ first.refresh_from_db()
+ assert first.watch_rank is not None
+ assert first.watch_rank["group"] == str(joined[0].group_id)
+ assert first.watch_rank["theme"] == str(joined[0].theme.theme_id)
+
+ @parameterized.expand(
+ [
+ ("monitor_answer_the_scanner_is_not_looking_for", "unwanted_answer"),
+ # A theme name reaches every scanner reader, so a row only the experiment's viewers may read stays out.
+ ("observation_scanned_under_experiment_targeting", "experiment"),
+ # The current valence may describe a question of the opposite sense to the one this row answered.
+ ("monitor_row_scanned_under_an_earlier_prompt", "earlier_prompt"),
+ ]
+ )
+ def test_some_rows_get_a_feed_group_but_no_theme(self, _name: str, case: str) -> None:
+ if case == "experiment":
+ observation = self._observation()
+ snapshot = {**observation.scanner_snapshot, "experiment_targeting": {"experiment_id": 7}}
+ ReplayObservation.objects.filter(id=observation.id).update(scanner_snapshot=snapshot)
+ else:
+ monitor = self._scanner(ScannerType.MONITOR, PromptValence.BAD, name="checkout errors")
+ verdict = "no" if case == "unwanted_answer" else "yes"
+ observation = self._observation(output={"scanner_type": "monitor", "verdict": verdict}, scanner=monitor)
+ if case == "earlier_prompt":
+ monitor.scanner_config = {"prompt": "Did checkout succeed?"}
+ monitor.save(update_fields=["scanner_config"])
+
+ assignment = self._assign(observation, _vector(1.0))
+
+ assert assignment.theme is None
+ observation.refresh_from_db()
+ assert observation.watch_rank is not None
+ assert observation.watch_rank["group"] == str(assignment.group_id)
+ assert "theme" not in observation.watch_rank
+
+ @parameterized.expand(
+ [
+ # A young scanner's mean is its own few rows, so two near-identical rows share both clusters on raw vectors.
+ ("young_scanner_matches_raw", 0, 1),
+ # Once the scanner's shared wording is taken out, what tells the rows apart decides.
+ ("seasoned_scanner_matches_centred", 100, 2),
+ ]
+ )
+ def test_the_scanner_mean_is_taken_out_once_it_is_known(self, _name: str, history: int, clusters: int) -> None:
+ shared = np.zeros(SEMANTIC_DIMENSIONS, dtype=np.float32)
+ shared[0] = history
+ ReplayScannerSemanticCenter.objects.for_team(self.team.id).create(
+ scanner=self.scanner, team=self.team, vector_sum=shared.tobytes(), member_count=history
+ )
+ self._assign(self._observation(), _vector(1.0, 0.2))
+ self._assign(self._observation(), _vector(1.0, -0.2))
+ assert len(self._counts(ReplayScannerFeedGroup)) == clusters
+ assert len(self._counts(ReplayScannerSearchTheme)) == clusters
+
+ def test_a_group_written_by_another_process_is_joined_rather_than_duplicated(self) -> None:
+ self._assign(self._observation(), _vector(1.0))
+ ReplayScannerFeedGroup.objects.for_team(self.team.id).create(
+ team=self.team,
+ scanner=self.scanner,
+ vector_sum=pack(semantic_vector(_vector(0.0, 1.0))),
+ member_count=1,
+ last_member_at=timezone.now(),
+ )
+ center = ReplayScannerSemanticCenter.objects.for_team(self.team.id).get(scanner=self.scanner)
+ center.member_count += 1
+ center.save(update_fields=["member_count"])
+
+ self._assign(self._observation(), _vector(0.0, 1.0))
+
+ assert self._counts(ReplayScannerFeedGroup) == [1, 2]
+
+ def test_clusters_without_a_member_for_the_retention_window_are_pruned(self) -> None:
+ stale = timezone.now() - timedelta(days=91)
+ for model in (ReplayScannerFeedGroup, ReplayScannerSearchTheme):
+ model.objects.for_team(self.team.id).create(
+ team=self.team,
+ scanner=self.scanner,
+ vector_sum=pack(semantic_vector(_vector(0.0, 1.0))),
+ member_count=3,
+ last_member_at=stale,
+ )
+ with patch("products.replay_vision.backend.watch_semantics._PRUNE_EVERY", 1):
+ self._assign(self._observation(), _vector(1.0))
+
+ assert self._counts(ReplayScannerFeedGroup) == [1]
+ assert self._counts(ReplayScannerSearchTheme) == [1]
+
+ def test_a_theme_is_due_a_name_at_its_fifth_member(self) -> None:
+ due = [self._assign(self._observation(), _vector(1.0)).theme.name_due for _ in range(5)]
+ assert due == [False, False, False, False, True]
+
+ def test_naming_writes_the_label_once_per_doubling(self) -> None:
+ for _ in range(5):
+ assignment = self._assign(
+ self._observation(output={"scanner_type": "summarizer", "summary": "The card form rejected a card."}),
+ _vector(1.0),
+ )
+ theme_id = assignment.theme.theme_id
+ response = MagicMock(text='{"query": "Valid Card Rejected At Checkout."}')
+ client = MagicMock(aclose=AsyncMock())
+ client.models.generate_content = AsyncMock(return_value=response)
+ with (
+ patch(f"{_THEMES}.replay_gemini_client", AsyncMock(return_value=client)),
+ patch(f"{_THEMES}.is_ai_data_processing_approved", return_value=True),
+ ):
+ first = async_to_sync(name_theme)(self.team.id, theme_id, timezone.now())
+ again = async_to_sync(name_theme)(self.team.id, theme_id, timezone.now())
+
+ assert (first, again) == ("valid card rejected at checkout", None)
+ assert client.models.generate_content.await_count == 1
+ theme = ReplayScannerSearchTheme.objects.for_team(self.team.id).get(id=theme_id)
+ assert (theme.name, theme.rename_at_member_count) == ("valid card rejected at checkout", 10)
diff --git a/products/replay_vision/backend/watch_feed_groups.py b/products/replay_vision/backend/watch_feed_groups.py
new file mode 100644
index 000000000000..4fb86f7f467c
--- /dev/null
+++ b/products/replay_vision/backend/watch_feed_groups.py
@@ -0,0 +1,163 @@
+"""Feed groups: tight clusters of a scanner's issue observations that tell the same story, so the Highlights feed
+shows each story once with the number of people it reached.
+
+An observation joins the scanner's closest recent group, or starts one. A busy scanner has thousands of recent
+groups, so the consumer keeps each scanner's groups in memory instead of reading them from Postgres on every
+observation. Every assignment runs under the lock on the scanner's semantic centre and bumps its member count, so a
+cache whose count disagrees with the locked row missed a write from another process and reloads.
+"""
+
+import uuid
+from collections import OrderedDict
+from datetime import datetime, timedelta
+
+from django.db.models import F
+from django.db.models.functions import Greatest
+
+import numpy as np
+
+from products.replay_vision.backend.models.replay_scanner_semantics import ReplayScannerFeedGroup
+from products.replay_vision.backend.semantic_space import (
+ SEMANTIC_DIMENSIONS,
+ SemanticSpace,
+ nearest_cluster,
+ pack,
+ unpack,
+)
+
+FEED_GROUP_MIN_SIMILARITY = 0.65
+FEED_GROUP_MIN_RAW_SIMILARITY = 0.90
+# A group with no new member for this long stops taking members, so a story that returns later starts a new card.
+FEED_GROUP_ACTIVE_WINDOW = timedelta(days=30)
+MAX_CACHED_GROUPS_PER_SCANNER = 20_000
+# Spare capacity counts too, so the cap bounds the memory the arrays actually hold.
+MAX_CACHED_GROUPS = 100_000
+
+
+class _ScannerGroups:
+ """One scanner's active groups as arrays with spare capacity, and the centre member count they reflect."""
+
+ def __init__(self, groups: list[ReplayScannerFeedGroup], version: int) -> None:
+ self.size = len(groups)
+ capacity = max(16, self.size * 2)
+ self.ids: list[uuid.UUID] = [group.id for group in groups]
+ self._sums = np.zeros((capacity, SEMANTIC_DIMENSIONS), dtype=np.float32)
+ self._counts = np.zeros(capacity, dtype=np.float32)
+ self._last = np.zeros(capacity, dtype=np.float64)
+ for index, group in enumerate(groups):
+ self._sums[index] = unpack(group.vector_sum)
+ self._counts[index] = group.member_count
+ self._last[index] = group.last_member_at.timestamp()
+ self.version = version
+
+ def __len__(self) -> int:
+ return self.size
+
+ @property
+ def capacity(self) -> int:
+ return len(self._counts)
+
+ @property
+ def sums(self) -> np.ndarray:
+ return self._sums[: self.size]
+
+ @property
+ def counts(self) -> np.ndarray:
+ return self._counts[: self.size]
+
+ @property
+ def last(self) -> np.ndarray:
+ return self._last[: self.size]
+
+ def add(self, group_id: uuid.UUID, vector: np.ndarray, at: datetime) -> None:
+ if self.size == len(self._counts):
+ self._sums = np.concatenate([self._sums, np.zeros_like(self._sums)])
+ self._counts = np.concatenate([self._counts, np.zeros_like(self._counts)])
+ self._last = np.concatenate([self._last, np.zeros_like(self._last)])
+ self.ids.append(group_id)
+ self._sums[self.size] = vector
+ self._counts[self.size] = 1.0
+ self._last[self.size] = at.timestamp()
+ self.size += 1
+
+ def trim(self, now: datetime) -> None:
+ """Drop groups that stopped taking members, then the least recent ones above the per-scanner cap."""
+ keep = np.nonzero(self.last >= (now - FEED_GROUP_ACTIVE_WINDOW).timestamp())[0]
+ if len(keep) > MAX_CACHED_GROUPS_PER_SCANNER:
+ keep = keep[np.argsort(self.last[keep])[-MAX_CACHED_GROUPS_PER_SCANNER:]]
+ if len(keep) == self.size:
+ return
+ self.ids = [self.ids[i] for i in keep]
+ self._sums[: len(keep)] = self.sums[keep]
+ self._counts[: len(keep)] = self.counts[keep]
+ self._last[: len(keep)] = self.last[keep]
+ self.size = len(keep)
+
+
+class FeedGroupCache:
+ def __init__(self) -> None:
+ self._scanners: OrderedDict[str, _ScannerGroups] = OrderedDict()
+
+ def assign(self, *, team_id: int, scanner_id: str, space: SemanticSpace, version: int, now: datetime) -> uuid.UUID:
+ """Put one observation in its scanner's closest active group, or start a group. Call under the centre lock,
+ with the centre's member count before this observation joins."""
+ groups = self._load(team_id, scanner_id, version, now)
+ best = nearest_cluster(
+ groups.sums, groups.counts, space, FEED_GROUP_MIN_SIMILARITY, FEED_GROUP_MIN_RAW_SIMILARITY
+ )
+ if best is None:
+ group = ReplayScannerFeedGroup.objects.for_team(team_id).create(
+ team_id=team_id,
+ scanner_id=scanner_id,
+ vector_sum=pack(space.vector),
+ member_count=1,
+ last_member_at=now,
+ )
+ groups.add(group.id, space.vector, now)
+ group_id = group.id
+ else:
+ group_id = groups.ids[best]
+ groups.sums[best] += space.vector
+ groups.counts[best] += 1
+ groups.last[best] = max(groups.last[best], now.timestamp())
+ # The backfill replays old rows while the consumer adds new ones, so the latest member time never moves back.
+ ReplayScannerFeedGroup.objects.for_team(team_id).filter(id=group_id).update(
+ vector_sum=pack(groups.sums[best]),
+ member_count=F("member_count") + 1,
+ last_member_at=Greatest("last_member_at", now),
+ )
+ groups.version = version + 1
+ return group_id
+
+ def forget(self, scanner_id: str) -> None:
+ """Drop a scanner's groups, so an assignment whose transaction rolled back never leaves them ahead of Postgres."""
+ self._scanners.pop(scanner_id, None)
+
+ def _load(self, team_id: int, scanner_id: str, version: int, now: datetime) -> _ScannerGroups:
+ groups = self._scanners.get(scanner_id)
+ if groups is None or groups.version != version:
+ rows = list(
+ ReplayScannerFeedGroup.objects.for_team(team_id)
+ .filter(scanner_id=scanner_id, last_member_at__gte=now - FEED_GROUP_ACTIVE_WINDOW)
+ .order_by("-last_member_at")
+ .only("id", "vector_sum", "member_count", "last_member_at")[:MAX_CACHED_GROUPS_PER_SCANNER]
+ )
+ groups = _ScannerGroups(rows, version)
+ self._scanners[scanner_id] = groups
+ else:
+ groups.trim(now)
+ self._scanners.move_to_end(scanner_id)
+ self._evict()
+ return groups
+
+ def _evict(self) -> None:
+ total = sum(groups.capacity for groups in self._scanners.values())
+ while total > MAX_CACHED_GROUPS and len(self._scanners) > 1:
+ _, evicted = self._scanners.popitem(last=False)
+ total -= evicted.capacity
+
+ def clear(self) -> None:
+ self._scanners.clear()
+
+
+feed_group_cache = FeedGroupCache()
diff --git a/products/replay_vision/backend/watch_rank.py b/products/replay_vision/backend/watch_rank.py
index 7e91cb8913a8..cb7fedb60a2d 100644
--- a/products/replay_vision/backend/watch_rank.py
+++ b/products/replay_vision/backend/watch_rank.py
@@ -11,7 +11,6 @@ import math
import asyncio
from collections import Counter
from dataclasses import asdict
-from datetime import datetime, timedelta
from typing import Any, Literal
from django.core.cache import cache
@@ -21,26 +20,16 @@ from django.db.models import CharField, Count, Exists, F, QuerySet, Value, Windo
from django.db.models.expressions import RawSQL
from django.db.models.fields.json import KeyTextTransform, KeyTransform
from django.db.models.functions import Coalesce, NullIf, RowNumber
-from django.utils import timezone
import numpy as np
from posthog.api.embedding_worker import async_generate_embedding
-from posthog.clickhouse.client import sync_execute
-from posthog.clickhouse.client.connection import ClickHouseUser
-from posthog.clickhouse.query_tagging import Product, tags_context
-from posthog.clickhouse.workload import Workload
from posthog.dataclasses import frozen
from posthog.models.team import Team
from posthog.utils import dict_from_cursor_fetchall
from products.access_control.backend.facade.user_access_control import UserAccessControl
-from products.replay_vision.backend.embeddings import (
- OBSERVATION_EMBEDDING_MODEL,
- OBSERVATION_EMBEDDINGS_TABLE,
- OBSERVATION_SCOPE_PREWHERE,
- observation_scope_params,
-)
+from products.replay_vision.backend.embeddings import OBSERVATION_EMBEDDING_MODEL
from products.replay_vision.backend.models.replay_observation import (
ReplayObservation,
jsonb_typeof,
@@ -97,11 +86,6 @@ ISSUE_CATEGORIES: tuple[WatchCategory, ...] = ("failure", "struggle", "revenue")
# Struggle picks rated worth watching less often than failure and revenue picks in a blind rating.
CATEGORY_WEIGHT: dict[str, float] = {"failure": 1.0, "revenue": 1.0, "struggle": 0.85}
-# Cosine distance below which two observations from one scanner describe the same thing.
-DUPLICATE_MAX_DISTANCE = 0.1
-DUPLICATE_LOOKBACK = timedelta(days=7)
-_DUPLICATE_QUERY_TIMEOUT_S = 10
-
# The feed surfaces a scanner's top fifth by score, once the scanner has enough ranked rows to
# make a percentile mean something.
WATCH_PICK_MIN_PERCENTILE = 0.8
@@ -124,7 +108,6 @@ _ANCHOR_CACHE_TTL_S = 7 * 24 * 3600
class WatchRank:
category: WatchCategory
contrast: float
- duplicate_of: str | None
anchor_version: int = ANCHOR_VERSION
def to_json(self) -> dict[str, Any]:
@@ -185,70 +168,6 @@ def classify(vector: list[float] | np.ndarray, anchors: dict[WatchCategory, np.n
return (category if contrast > 0 else "routine"), contrast
-# Caps the vectors one duplicate lookup reads to the scanner's newest rows, so a busy scanner stays cheap.
-DUPLICATE_CANDIDATE_ROWS = 5_000
-
-
-def nearest_earlier_observation(
- *, team_id: int, scanner_id: str, observation_id: str, vector: list[float], now: datetime
-) -> tuple[str, float] | None:
- """The scanner's closest other observation in the lookback window, with its cosine distance."""
- with tags_context(product=Product.REPLAY_VISION, query_type="replay_vision_watch_rank_duplicate"):
- # nosemgrep: clickhouse-fstring-param-audit - table name and scope are module constants
- rows = sync_execute(
- f"""
- SELECT document_id, min(cosineDistance(embedding, %(embedding)s)) AS distance
- FROM {OBSERVATION_EMBEDDINGS_TABLE}
- PREWHERE {OBSERVATION_SCOPE_PREWHERE}
- AND timestamp >= ifNull(
- (
- SELECT minOrNull(timestamp)
- FROM (
- SELECT timestamp
- FROM {OBSERVATION_EMBEDDINGS_TABLE}
- PREWHERE {OBSERVATION_SCOPE_PREWHERE} AND timestamp >= %(since)s
- ORDER BY timestamp DESC
- LIMIT %(candidate_cap)s
- )
- ),
- %(since)s
- )
- WHERE document_id != %(observation_id)s
- GROUP BY document_id
- ORDER BY distance ASC
- LIMIT 1
- """,
- {
- **observation_scope_params(team_id, [scanner_id]),
- "embedding": vector,
- "since": now - DUPLICATE_LOOKBACK,
- "observation_id": observation_id,
- "candidate_cap": DUPLICATE_CANDIDATE_ROWS,
- },
- team_id=team_id,
- readonly=True,
- ch_user=ClickHouseUser.REPLAY_VISION,
- settings={"max_execution_time": _DUPLICATE_QUERY_TIMEOUT_S},
- workload=Workload.OFFLINE,
- )
- if not rows:
- return None
- document_id, distance = rows[0]
- return str(document_id), float(distance)
-
-
-def duplicate_root(observation_id: str, neighbour_id: str, neighbour_rank: dict[str, Any] | None) -> str | None:
- """The first observation of a duplicate group, so the group shares one id however long it grows.
-
- An unranked neighbour can be ranking at the same moment and pick this observation in turn, so the
- smaller id roots the pair and the two never point at each other.
- """
- if neighbour_rank is None:
- return neighbour_id if neighbour_id < observation_id else None
- root = neighbour_rank.get("duplicate_of")
- return root if isinstance(root, str) and root else neighbour_id
-
-
def current_rank(row: dict[str, Any]) -> dict[str, Any] | None:
"""The row's rank when it comes from the current anchors, else None."""
rank = row.get("watch_rank")
@@ -271,8 +190,8 @@ class RankRequest:
async def rank_observation(request: RankRequest) -> Literal["ranked", "already_ranked", "missing"]:
"""Rank one observation from its embedding and save the rank on the row, unless it has a current one.
- Keeping an existing rank makes a redelivered result harmless, and stops a second pass from moving a
- duplicate link in a way that could make two observations point at each other.
+ Keeping an existing rank makes a redelivered result harmless, and keeps the feed group and search theme
+ written next to it.
"""
row = await (
ReplayObservation.objects.filter(team_id=request.team_id, id=request.observation_id)
@@ -285,25 +204,7 @@ async def rank_observation(request: RankRequest) -> Literal["ranked", "already_r
if current_rank({"watch_rank": row["watch_rank"]}) is not None:
return "already_ranked"
category, contrast = classify(request.vector, await anchor_vectors(request.team_id))
- duplicate_of: str | None = None
- # The feed only groups issue rows, so a routine row skips the ClickHouse lookup.
- if category != "routine":
- nearest = await asyncio.to_thread(
- nearest_earlier_observation,
- team_id=request.team_id,
- scanner_id=request.scanner_id,
- observation_id=request.observation_id,
- vector=request.vector,
- now=timezone.now(),
- )
- if nearest is not None and nearest[1] <= DUPLICATE_MAX_DISTANCE:
- neighbour_rank = await (
- ReplayObservation.objects.filter(team_id=request.team_id, id=nearest[0])
- .values_list("watch_rank", flat=True)
- .afirst()
- )
- duplicate_of = duplicate_root(request.observation_id, nearest[0], neighbour_rank)
- rank = WatchRank(category=category, contrast=contrast, duplicate_of=duplicate_of)
+ rank = WatchRank(category=category, contrast=contrast)
await ReplayObservation.objects.filter(team_id=request.team_id, id=request.observation_id).aupdate(
watch_rank=rank.to_json()
)
@@ -326,26 +227,13 @@ def _scanner_percentiles(rows: list[dict[str, Any]]) -> dict[str, float]:
return percentiles
-def _group_root(observation_id: str, parents: dict[str, str]) -> str:
- """Follow `duplicate_of` links to the group's first observation.
-
- A root ranked after its copies can gain a root of its own, so a link can point at a member rather
- than at the root. The walk stops at a repeat, so a cycle cannot loop.
- """
- seen = {observation_id}
- while (parent := parents.get(observation_id)) is not None and parent not in seen:
- seen.add(parent)
- observation_id = parent
- return observation_id
-
-
def people_boost(people: int) -> float:
- """How much a pick's score grows with the number of people its duplicate group reached."""
+ """How much a pick's score grows with the number of people its feed group reached."""
return min(WATCH_PICK_PEOPLE_BOOST_CAP, 1 + WATCH_PICK_PEOPLE_BOOST * math.log(max(people, 1)))
-def _cap_duplicate_groups(rows: list[dict[str, Any]], cap: int) -> dict[Any, dict[str, Any]]:
- """The first `cap` rows, keeping a few per duplicate group so the ranker can still pick a readable, unseen one.
+def _cap_feed_groups(rows: list[dict[str, Any]], cap: int) -> dict[Any, dict[str, Any]]:
+ """The first `cap` rows, keeping a few per feed group so the ranker can still pick a readable, unseen one.
A group never spans scanners, so the pick order meets each group's rows best first.
"""
@@ -428,7 +316,7 @@ def select_embedding_picks(
team_id: int,
cap: int,
) -> list[dict[str, Any]]:
- """The window's pick rows for the embedding ranker, with each row's percentile and the reach of its duplicate group.
+ """The window's pick rows for the embedding ranker, with each row's percentile and the reach of its feed group.
The percentile is taken over every ranked row a scanner has in the window, not over a recency slice, so a
longer window reaches further back instead of re-ranking the same newest rows.
@@ -447,12 +335,12 @@ def select_embedding_picks(
# Bounds the fetch, since a client-side cursor buffers every row; only a window full of copies needs the rest.
fetch_limit = cap * (WATCH_PICK_GROUP_CANDIDATES + 1)
rows = _pick_window_rows(ranked, current_ranks, fetch_limit)
- window_values = _cap_duplicate_groups(rows, cap)
+ window_values = _cap_feed_groups(rows, cap)
if len(window_values) < cap and len(rows) == fetch_limit:
- window_values = _cap_duplicate_groups(_pick_window_rows(ranked, current_ranks, None), cap)
+ window_values = _cap_feed_groups(_pick_window_rows(ranked, current_ranks, None), cap)
if not window_values:
return []
- # Counted over readable rows only, so a group never reveals how many restricted sessions it holds. Duplicates
+ # Counted over readable rows only, so a group never reveals how many restricted sessions it holds. Members
# below the cutoff still count, and a person who hit the problem twice counts once.
person = Coalesce(NullIf("distinct_id", Value("")), "session_id", output_field=CharField())
roots = {row["watch_group_root"] for row in window_values.values()}
@@ -496,7 +384,7 @@ def select_embedding_filler(
team_id: int,
pick_roots: set[str],
) -> list[dict[str, Any]]:
- """The newest readable rows outside the picks' duplicate groups, unwatched first, enough to bring a thin feed up to its floor."""
+ """The newest readable rows outside the picks' feed groups, unwatched first, enough to bring a thin feed up to its floor."""
base = (
accessible_observations(access, team_id, walkable.filter(scanner_id__in=scanner_ids))
.annotate(watch_group_root=watch_group_root())
@@ -519,7 +407,7 @@ def rank_watch_feed_by_embedding(rows: list[dict[str, Any]]) -> list[WatchFeedEn
"""Rank candidate rows (`id`, `scanner_id`, `session_id`, `created_at`, `watch_rank`, `feed_viewed`,
and optionally `feed_impressions`) on the embedding rank the watch rank consumer wrote.
- A pick is a row in its scanner's top fifth by score whose category is an issue. Near-duplicates
+ A pick is a row in its scanner's top fifth by score whose category is an issue. Rows of one feed group
collapse to their best-scoring row, which carries the size of the group. Picks sort by weighted
percentile, decayed for each earlier impression, viewed rows docked, one scanner held to a share
of the picks, and one session's picks spaced apart. Every other row is filler, trimmed to the same
@@ -531,11 +419,6 @@ def rank_watch_feed_by_embedding(rows: list[dict[str, Any]]) -> list[WatchFeedEn
if any("watch_percentile" in row for row in rows)
else _scanner_percentiles(rows)
)
- parents = {
- str(row["id"]): str(rank["duplicate_of"])
- for row in rows
- if (rank := current_rank(row)) is not None and rank.get("duplicate_of")
- }
best_in_group: dict[str, tuple[float, dict[str, Any]]] = {}
group_members: dict[str, list[str]] = {}
for row in rows:
@@ -544,8 +427,8 @@ def rank_watch_feed_by_embedding(rows: list[dict[str, Any]]) -> list[WatchFeedEn
percentile = percentiles.get(observation_id)
if rank is None or percentile is None or rank.get("category") not in ISSUE_CATEGORIES:
continue
- # Group before the cutoff, so duplicates below it still count toward the group and never return as filler.
- group = _group_root(observation_id, parents)
+ # Group before the cutoff, so members below it still count toward the group and never return as filler.
+ group = str(rank.get("group") or observation_id)
group_members.setdefault(group, []).append(observation_id)
# Only a member above the cutoff can stand for the group, so the seen penalty never disqualifies it.
if percentile < WATCH_PICK_MIN_PERCENTILE:
@@ -579,5 +462,5 @@ def rank_watch_feed_by_embedding(rows: list[dict[str, Any]]) -> list[WatchFeedEn
WatchFeedEntry(observation_id=row["id"], reason=reason, session_id=row["session_id"]),
)
)
- # A pick stands for its whole duplicate group, so no member returns as filler.
+ # A pick stands for its whole feed group, so no member returns as filler.
return finish_feed(picks, [row for row in rows if str(row["id"]) not in grouped_ids])
diff --git a/products/replay_vision/backend/watch_rank_backfill.py b/products/replay_vision/backend/watch_rank_backfill.py
index 45dc261f1a55..b844e27f892b 100644
--- a/products/replay_vision/backend/watch_rank_backfill.py
+++ b/products/replay_vision/backend/watch_rank_backfill.py
@@ -2,8 +2,8 @@
The watch rank consumer ranks each new observation as its embedding lands. This fills in the observations from before
that shipped, so a team can move to the embedding ranker without waiting a week for ranks to build
-up. It reads stored vectors from ClickHouse rather than embedding again, and finds near-duplicates in
-memory per scanner, oldest first, the same way the consumer would have found them.
+up. It reads stored vectors from ClickHouse rather than embedding again. Feed groups and search themes
+come from the semantics backfill, which runs after this one.
"""
import time
@@ -12,7 +12,9 @@ from collections import defaultdict
from datetime import datetime
from itertools import batched
from typing import Any
+from uuid import UUID
+from django.db import transaction
from django.db.models import Count
import numpy as np
@@ -33,20 +35,19 @@ from products.replay_vision.backend.embeddings import (
)
from products.replay_vision.backend.models.replay_observation import ObservationStatus, ReplayObservation
from products.replay_vision.backend.watch_rank import (
- DUPLICATE_CANDIDATE_ROWS,
- DUPLICATE_MAX_DISTANCE,
+ ANCHOR_VERSION,
WatchCategory,
WatchRank,
anchor_vectors,
classify,
current_rank,
- duplicate_root,
normalize,
)
logger = structlog.get_logger(__name__)
_VECTOR_CHUNK = 500
+WATCH_RANK_BACKFILL_DAYS = 7
_QUERY_TIMEOUT_S = 60
_CAPACITY_RETRY_BASE_S = 15
_CAPACITY_RETRY_ATTEMPTS = 6
@@ -109,69 +110,18 @@ def _read_stored_vectors(
return {str(document_id): normalize(np.asarray(embedding, dtype=np.float32)) for document_id, embedding in rows}
-@frozen
-class _Neighbour:
- observation_id: str
- rank: dict[str, Any] | None
- distance: float
-
-
-class _RecentVectors:
- """A scanner's newest vectors and ranks, capped like the live duplicate lookup so memory and work stay bounded."""
-
- def __init__(self) -> None:
- self._capacity = DUPLICATE_CANDIDATE_ROWS
- self._matrix: np.ndarray | None = None
- self._ids: list[str] = []
- self._ranks: list[dict[str, Any] | None] = []
- self._next = 0
-
- def nearest(self, vector: np.ndarray) -> _Neighbour | None:
- if self._matrix is None or not self._ids:
- return None
- distances = 1.0 - self._matrix[: len(self._ids)] @ vector
- index = int(np.argmin(distances))
- return _Neighbour(observation_id=self._ids[index], rank=self._ranks[index], distance=float(distances[index]))
-
- def add(self, observation_id: str, vector: np.ndarray, rank: dict[str, Any] | None) -> None:
- if self._matrix is None:
- self._matrix = np.empty((self._capacity, vector.shape[0]), dtype=np.float32)
- self._matrix[self._next] = vector
- if len(self._ids) < self._capacity:
- self._ids.append(observation_id)
- self._ranks.append(rank)
- else:
- self._ids[self._next] = observation_id
- self._ranks[self._next] = rank
- self._next = (self._next + 1) % self._capacity
-
-
def _rank_scanner(
- rows: list[dict[str, Any]],
- vectors: dict[str, np.ndarray],
- anchors: dict[WatchCategory, np.ndarray],
- recent: _RecentVectors,
+ rows: list[dict[str, Any]], vectors: dict[str, np.ndarray], anchors: dict[WatchCategory, np.ndarray]
) -> dict[str, dict[str, Any]]:
- """New ranks for the rows that lack a current one, walking oldest first.
-
- Pass the same `recent` across a scanner's batches, so a copy finds a match from an earlier batch.
- """
+ """New ranks for the rows that lack a current one."""
written: dict[str, dict[str, Any]] = {}
for row in rows:
observation_id = str(row["id"])
vector = vectors.get(observation_id)
- if vector is None:
+ if vector is None or current_rank(row) is not None:
continue
- rank = current_rank(row)
- if rank is None:
- category, contrast = classify(vector, anchors)
- duplicate_of: str | None = None
- nearest = recent.nearest(vector) if category != "routine" else None
- if nearest is not None and nearest.distance <= DUPLICATE_MAX_DISTANCE:
- duplicate_of = duplicate_root(observation_id, nearest.observation_id, nearest.rank)
- rank = WatchRank(category=category, contrast=contrast, duplicate_of=duplicate_of).to_json()
- written[observation_id] = rank
- recent.add(observation_id, vector, rank)
+ category, contrast = classify(vector, anchors)
+ written[observation_id] = WatchRank(category=category, contrast=contrast).to_json()
return written
@@ -196,18 +146,29 @@ def backfill_team_watch_ranks(team_id: int, since: datetime, *, dry_run: bool =
ranked = 0
embedded = 0
for scanner_id, scanner_rows in by_scanner.items():
- recent = _RecentVectors()
for batch in batched(scanner_rows, _VECTOR_CHUNK, strict=False):
vectors = _stored_vectors(team_id, scanner_id, [str(row["id"]) for row in batch], since)
embedded += len(vectors)
- written = _rank_scanner(list(batch), vectors, anchors, recent)
+ written = _rank_scanner(list(batch), vectors, anchors)
ranked += len(written)
if not dry_run and written:
- # The ids come from the team-scoped query above, so the update needs no team filter of its own.
- ReplayObservation.objects.bulk_update(
- [ReplayObservation(id=observation_id, watch_rank=rank) for observation_id, rank in written.items()],
- ["watch_rank"],
- )
+ # Only rows still without a current rank, locked until the write, so a rank the consumer writes meanwhile
+ # waits for this one and a rank it already wrote, with its group, stays.
+ with transaction.atomic():
+ still_unranked = set(
+ ReplayObservation.objects.select_for_update()
+ .filter(team_id=team_id, id__in=list(written))
+ .exclude(watch_rank__anchor_version=ANCHOR_VERSION)
+ .values_list("id", flat=True)
+ )
+ ReplayObservation.objects.bulk_update(
+ [
+ ReplayObservation(id=observation_id, watch_rank=rank)
+ for observation_id, rank in written.items()
+ if UUID(observation_id) in still_unranked
+ ],
+ ["watch_rank"],
+ )
without_embedding = len(rows) - embedded
logger.info(
"replay_vision.watch_rank_backfill.team_done",
diff --git a/products/replay_vision/backend/watch_rank_consumer.py b/products/replay_vision/backend/watch_rank_consumer.py
index 8ce94d2a6d1c..e63104e2199c 100644
--- a/products/replay_vision/backend/watch_rank_consumer.py
+++ b/products/replay_vision/backend/watch_rank_consumer.py
@@ -6,13 +6,16 @@ vectors. A worker outage only delays ranks, and a database outage shorter than t
"""
import time
+import uuid
import signal
import asyncio
from collections.abc import Sequence
+from datetime import datetime
from typing import Any, Literal, Protocol
from django.conf import settings
from django.db import close_old_connections
+from django.utils import timezone
import orjson
import structlog
@@ -32,6 +35,8 @@ from products.replay_vision.backend.embeddings import (
OBSERVATION_EMBEDDING_MODEL,
)
from products.replay_vision.backend.watch_rank import RankRequest, rank_observation
+from products.replay_vision.backend.watch_semantics import assign_semantics
+from products.replay_vision.backend.watch_themes import name_theme
logger = structlog.get_logger(__name__)
@@ -42,10 +47,11 @@ _POLL_TIMEOUT_S = 1.0
# A batch stops after this long and goes back to Kafka, so the loop polls well inside the poll interval.
_BATCH_BUDGET_S = 240.0
_MAX_POLL_INTERVAL_S = 360.0
-# ClickHouse loads the embeddings table from its own topic some seconds after the worker writes both.
-_CLICKHOUSE_VISIBILITY_DELAY_S = 30.0
# How long one observation keeps failing, across batches, before the consumer moves past it.
_RETRY_WINDOW_S = 600.0
+_NAMING_DRAIN_S = 20.0
+# A due theme left unnamed while this many are in flight is named when its next member arrives.
+_MAX_NAMING_TASKS = 8
_RETRY_BACKOFF_S = 2.0
_RETRY_BACKOFF_MAX_S = 60.0
@@ -108,6 +114,8 @@ class WatchRankConsumer:
self._stop = asyncio.Event()
# When each observation that is still failing first failed, so its retry window spans batches.
self._first_failed_at: dict[str, float] = {}
+ # Naming calls run beside the loop, so a slow model call never holds up ranking. Kept to stop collection.
+ self._naming: set[asyncio.Task[Any]] = set()
async def _wait(self, seconds: float) -> bool:
"""Sleep, unless shutdown comes first. True when the full time passed."""
@@ -117,19 +125,19 @@ class WatchRankConsumer:
return True
return False
- async def handle(self, value: bytes | None, produced_at: float, batch_deadline: float) -> Outcome:
+ async def handle(self, value: bytes | None, batch_deadline: float) -> Outcome:
"""Rank the observation in one message, retrying failures until the batch or retry window ends."""
request = parse_rank_request(value)
if request is None:
return "skipped"
- if not await self._wait(produced_at + _CLICKHOUSE_VISIBILITY_DELAY_S - time.time()):
- return "interrupted"
backoff = _RETRY_BACKOFF_S
while True:
# Async ORM calls share one thread, so a connection that broke must be replaced before a retry.
await sync_to_async(close_old_connections)()
try:
outcome = await rank_observation(request)
+ if outcome != "missing":
+ await self._semantics(request)
self._first_failed_at.pop(request.observation_id, None)
return outcome
except Exception:
@@ -151,6 +159,32 @@ class WatchRankConsumer:
return "interrupted"
backoff = min(backoff * 2, _RETRY_BACKOFF_MAX_S)
+ async def _semantics(self, request: RankRequest) -> None:
+ """Put a ranked observation in a feed group and a theme, and name the theme in the background once it is due."""
+ now = timezone.now()
+ assignment = await sync_to_async(assign_semantics)(
+ team_id=request.team_id,
+ scanner_id=request.scanner_id,
+ observation_id=request.observation_id,
+ vector=request.vector,
+ now=now,
+ )
+ if assignment is None or assignment.theme is None or not assignment.theme.name_due:
+ return
+ if len(self._naming) >= _MAX_NAMING_TASKS:
+ return
+ task = asyncio.create_task(self._name(request.team_id, assignment.theme.theme_id, now))
+ self._naming.add(task)
+ task.add_done_callback(self._naming.discard)
+
+ @staticmethod
+ async def _name(team_id: int, theme_id: uuid.UUID, now: datetime) -> None:
+ try:
+ await name_theme(team_id, theme_id, now)
+ except Exception:
+ # A theme that missed its name is named again when it next doubles.
+ logger.exception("replay_vision.watch_rank_consumer.theme_naming_failed", team_id=team_id)
+
async def process_batch(self, consumer: BatchConsumer, messages: Sequence[Message]) -> None:
batch_deadline = time.monotonic() + _BATCH_BUDGET_S
for index, message in enumerate(messages):
@@ -159,8 +193,7 @@ class WatchRankConsumer:
if message.error():
logger.warning("replay_vision.watch_rank_consumer.kafka_error", error=str(message.error()))
continue
- _, produced_at_ms = message.timestamp()
- outcome = await self.handle(message.value(), produced_at_ms / 1000, batch_deadline)
+ outcome = await self.handle(message.value(), batch_deadline)
self._liveness.record_heartbeat()
if outcome == "interrupted":
return
@@ -232,4 +265,7 @@ class WatchRankConsumer:
await self.process_batch(consumer, messages)
finally:
consumer.close()
+ # A theme claimed for naming stays unnamed until it doubles, so in-flight names get a moment to finish.
+ if self._naming:
+ await asyncio.wait(self._naming, timeout=_NAMING_DRAIN_S)
logger.info("replay_vision.watch_rank_consumer.stopped")
diff --git a/products/replay_vision/backend/watch_semantics.py b/products/replay_vision/backend/watch_semantics.py
new file mode 100644
index 000000000000..0d7eb53819f6
--- /dev/null
+++ b/products/replay_vision/backend/watch_semantics.py
@@ -0,0 +1,151 @@
+"""Places each ranked issue observation in a feed group and, when it qualifies, a search theme.
+
+The embedding consumer and the semantics backfill both call `assign_semantics`. It locks the scanner's semantic
+centre for the whole assignment, so two observations of one scanner never race on its clusters, and writes both ids
+to the observation's rank in one update, so a redelivered message finds them already there.
+"""
+
+import uuid
+from datetime import datetime, timedelta
+from typing import Any
+
+from django.db import transaction
+from django.db.models import F, Func, JSONField, Value
+
+import numpy as np
+
+from posthog.dataclasses import frozen
+
+from products.replay_vision.backend.models.replay_observation import ReplayObservation
+from products.replay_vision.backend.models.replay_scanner import PromptValence, ReplayScanner, ScannerType
+from products.replay_vision.backend.models.replay_scanner_semantics import (
+ ReplayScannerFeedGroup,
+ ReplayScannerSearchTheme,
+ ReplayScannerSemanticCenter,
+)
+from products.replay_vision.backend.semantic_space import (
+ SEMANTIC_DIMENSIONS,
+ pack,
+ scanner_space,
+ semantic_vector,
+ unpack,
+)
+from products.replay_vision.backend.watch_feed_groups import feed_group_cache
+from products.replay_vision.backend.watch_rank import ISSUE_CATEGORIES, current_rank
+from products.replay_vision.backend.watch_themes import ThemeAssignment, assign_theme, belongs_in_theme
+
+# Clusters with no member for this long are deleted, since the feed's longest window no longer reaches them.
+SEMANTIC_RETENTION = timedelta(days=90)
+# Pruning runs on one in this many assignments of a scanner, which keeps it rare and still bounds the tables.
+_PRUNE_EVERY = 500
+
+
+@frozen
+class SemanticAssignment:
+ group_id: uuid.UUID
+ theme: ThemeAssignment | None
+
+
+def assign_semantics(
+ *, team_id: int, scanner_id: str, observation_id: str, vector: list[float], now: datetime
+) -> SemanticAssignment | None:
+ """None when the observation is not a ranked issue, already has its clusters, or is gone."""
+ scanner = (
+ ReplayScanner.objects.filter(team_id=team_id, id=scanner_id)
+ .values("scanner_type", "prompt_valence", "scanner_config")
+ .first()
+ )
+ if scanner is None:
+ return None
+ try:
+ with transaction.atomic():
+ center, _ = (
+ ReplayScannerSemanticCenter.objects.for_team(team_id)
+ .select_for_update()
+ .get_or_create(
+ scanner_id=scanner_id,
+ defaults={"team_id": team_id, "vector_sum": pack(np.zeros(SEMANTIC_DIMENSIONS, dtype=np.float32))},
+ )
+ )
+ row = (
+ ReplayObservation.objects.filter(team_id=team_id, id=observation_id)
+ .values("watch_rank", "scanner_result", "scanner_snapshot")
+ .first()
+ )
+ if row is None:
+ return None
+ rank = current_rank({"watch_rank": row["watch_rank"]})
+ if rank is None or rank.get("group") or rank.get("category") not in ISSUE_CATEGORIES:
+ return None
+
+ v = semantic_vector(vector)
+ center_sum = unpack(center.vector_sum)
+ space = scanner_space(center_sum, center.member_count, v)
+ group_id = feed_group_cache.assign(
+ team_id=team_id, scanner_id=scanner_id, space=space, version=center.member_count, now=now
+ )
+ theme = None
+ output = (row["scanner_result"] or {}).get("model_output") or {}
+ # A theme name surfaces as a suggestion to anyone who can read the scanner, so an observation scanned
+ # under experiment targeting, which only that experiment's viewers may read, never joins one.
+ if (
+ not _experiment_scoped(row["scanner_snapshot"])
+ and not _valence_unknown_for_row(
+ scanner["scanner_type"],
+ scanner["prompt_valence"],
+ scanner["scanner_config"],
+ row["scanner_snapshot"],
+ )
+ and belongs_in_theme(
+ category=rank["category"],
+ scanner_type=scanner["scanner_type"],
+ valence=scanner["prompt_valence"],
+ output=output,
+ )
+ ):
+ theme = assign_theme(team_id=team_id, scanner_id=scanner_id, space=space, now=now)
+
+ center.vector_sum = pack(center_sum + v)
+ center.member_count += 1
+ center.save(update_fields=["vector_sum", "member_count"])
+ if center.member_count % _PRUNE_EVERY == 0:
+ _prune(team_id, scanner_id, now)
+
+ keys = {"group": str(group_id)} | ({"theme": str(theme.theme_id)} if theme else {})
+ ReplayObservation.objects.filter(team_id=team_id, id=observation_id).update(
+ watch_rank=Func(
+ F("watch_rank"),
+ Value(keys, output_field=JSONField()),
+ template="%(expressions)s",
+ arg_joiner=" || ",
+ output_field=JSONField(),
+ )
+ )
+ except Exception:
+ feed_group_cache.forget(scanner_id)
+ raise
+ return SemanticAssignment(group_id=group_id, theme=theme)
+
+
+def _valence_unknown_for_row(
+ scanner_type: str, valence: str, scanner_config: dict[str, Any] | None, snapshot: dict[str, Any] | None
+) -> bool:
+ """A monitor's current valence describes its current prompt, so a row scanned under another prompt may answer a
+ question of the opposite sense and stays out of themes."""
+ if scanner_type != ScannerType.MONITOR or valence not in (PromptValence.GOOD, PromptValence.BAD):
+ return False
+ scanned_prompt = ((snapshot or {}).get("scanner_config") or {}).get("prompt")
+ return scanned_prompt != (scanner_config or {}).get("prompt")
+
+
+def _experiment_scoped(snapshot: dict[str, Any] | None) -> bool:
+ snapshot = snapshot or {}
+ targeting = snapshot.get("experiment_targeting") or {}
+ config = snapshot.get("scanner_config") or {}
+ return bool(targeting.get("experiment_id") or config.get("experiment_id"))
+
+
+def _prune(team_id: int, scanner_id: str, now: datetime) -> None:
+ cutoff = now - SEMANTIC_RETENTION
+ ReplayScannerFeedGroup.objects.for_team(team_id).filter(scanner_id=scanner_id, last_member_at__lt=cutoff).delete()
+ ReplayScannerSearchTheme.objects.for_team(team_id).filter(scanner_id=scanner_id, last_member_at__lt=cutoff).delete()
diff --git a/products/replay_vision/backend/watch_semantics_backfill.py b/products/replay_vision/backend/watch_semantics_backfill.py
new file mode 100644
index 000000000000..36bdfedacbce
--- /dev/null
+++ b/products/replay_vision/backend/watch_semantics_backfill.py
@@ -0,0 +1,86 @@
+"""Builds feed groups and search themes from observations ranked before the consumer assigned them, so the feed and
+the suggestions start from history instead of an empty slate."""
+
+from datetime import datetime
+
+from django.db.models import F
+from django.db.models.fields.json import KeyTextTransform
+from django.utils import timezone
+
+import structlog
+from asgiref.sync import async_to_sync
+
+from posthog.dataclasses import frozen
+
+from products.replay_vision.backend.models.replay_observation import ReplayObservation
+from products.replay_vision.backend.models.replay_scanner_semantics import ReplayScannerSearchTheme
+from products.replay_vision.backend.watch_rank import ANCHOR_VERSION, ISSUE_CATEGORIES
+from products.replay_vision.backend.watch_rank_backfill import _VECTOR_CHUNK, _stored_vectors
+from products.replay_vision.backend.watch_semantics import assign_semantics
+from products.replay_vision.backend.watch_themes import THEME_ACTIVE_WINDOW, name_theme
+
+logger = structlog.get_logger(__name__)
+
+
+@frozen
+class SemanticBackfillResult:
+ observations: int
+ assigned: int
+ named: int
+
+
+def backfill_team_semantics(team_id: int, since: datetime, *, dry_run: bool = False) -> SemanticBackfillResult:
+ """Replay a team's ranked issue observations oldest first, as the consumer would have seen them, then name
+ every active theme that reached its naming size."""
+ rows = (
+ ReplayObservation.objects.filter(
+ team_id=team_id,
+ status="succeeded",
+ created_at__gte=since,
+ watch_rank__anchor_version=ANCHOR_VERSION,
+ watch_rank__group__isnull=True,
+ )
+ .alias(category=KeyTextTransform("category", "watch_rank"))
+ .filter(category__in=ISSUE_CATEGORIES)
+ .order_by("scanner_id", "created_at")
+ .values_list("id", "scanner_id", "created_at")
+ )
+ by_scanner: dict[str, list[tuple[str, datetime]]] = {}
+ for observation_id, scanner_id, created_at in rows.iterator(chunk_size=2_000):
+ by_scanner.setdefault(str(scanner_id), []).append((str(observation_id), created_at))
+ observations = sum(len(scanner_rows) for scanner_rows in by_scanner.values())
+ if dry_run:
+ return SemanticBackfillResult(observations=observations, assigned=0, named=0)
+
+ assigned = 0
+ for scanner_id, scanner_rows in by_scanner.items():
+ for start in range(0, len(scanner_rows), _VECTOR_CHUNK):
+ chunk = scanner_rows[start : start + _VECTOR_CHUNK]
+ vectors = _stored_vectors(team_id, scanner_id, [observation_id for observation_id, _ in chunk], since)
+ for observation_id, created_at in chunk:
+ vector = vectors.get(observation_id)
+ if vector is None:
+ continue
+ assignment = assign_semantics(
+ team_id=team_id,
+ scanner_id=scanner_id,
+ observation_id=observation_id,
+ vector=vector.tolist(),
+ now=created_at,
+ )
+ assigned += assignment is not None
+
+ named = 0
+ now = timezone.now()
+ due = (
+ ReplayScannerSearchTheme.objects.for_team(team_id)
+ .filter(member_count__gte=F("rename_at_member_count"), last_member_at__gte=now - THEME_ACTIVE_WINDOW)
+ .order_by("-member_count")
+ .values_list("id", flat=True)
+ )
+ for theme_id in list(due):
+ try:
+ named += async_to_sync(name_theme)(team_id, theme_id, now) is not None
+ except Exception:
+ logger.exception("replay_vision.watch_semantics_backfill.naming_failed", team_id=team_id)
+ return SemanticBackfillResult(observations=observations, assigned=assigned, named=named)
diff --git a/products/replay_vision/backend/watch_themes.py b/products/replay_vision/backend/watch_themes.py
new file mode 100644
index 000000000000..de2bb398fb77
--- /dev/null
+++ b/products/replay_vision/backend/watch_themes.py
@@ -0,0 +1,227 @@
+"""Search themes: broad clusters of a scanner's issue observations, each named as a search the Search tab suggests.
+
+An issue observation joins the closest active theme of its scanner, or starts one when none is close. A theme is
+named by a small model when it reaches a few members and renamed each time it doubles, so model calls follow new
+findings rather than a schedule.
+"""
+
+import re
+import uuid
+from datetime import datetime, timedelta
+from typing import Any
+
+from django.core.cache import cache
+from django.db.models import F
+
+import numpy as np
+import structlog
+from asgiref.sync import sync_to_async
+from google.genai import types
+from pydantic import BaseModel, Field
+
+from posthog.dataclasses import frozen
+
+from products.replay_vision.backend.consent import is_ai_data_processing_approved
+from products.replay_vision.backend.distinct_ids import replay_vision_distinct_id
+from products.replay_vision.backend.gemini_client import AI_PRODUCT, replay_gemini_client
+from products.replay_vision.backend.models.replay_observation import ReplayObservation
+from products.replay_vision.backend.models.replay_scanner import PromptValence, ReplayScanner, ScannerType
+from products.replay_vision.backend.models.replay_scanner_semantics import ReplayScannerSearchTheme
+from products.replay_vision.backend.observation_formatting import explanation_text
+from products.replay_vision.backend.semantic_space import (
+ SEMANTIC_DIMENSIONS,
+ SemanticSpace,
+ nearest_cluster,
+ pack,
+ unpack,
+)
+from products.replay_vision.backend.watch_rank import ISSUE_CATEGORIES
+
+logger = structlog.get_logger(__name__)
+
+THEME_MIN_SIMILARITY = 0.40
+THEME_MIN_RAW_SIMILARITY = 0.85
+# A theme with no new member for this long no longer takes members or counts as a suggestion.
+THEME_ACTIVE_WINDOW = timedelta(days=30)
+# Bounds the themes one assignment reads while it holds the scanner's lock.
+MAX_ACTIVE_THEMES = 2_000
+THEME_NAME_SAMPLES = 8
+THEME_NAMING_DAILY_CAP = 100
+MAX_THEME_LABEL_WORDS = 10
+_THEME_MODEL = "gemini-3.5-flash-lite"
+_SAMPLE_CHARS = 400
+_IDENTIFIER = re.compile(r"https?://|www\.|@")
+
+_NAMING_PROMPT = """You name a theme for a semantic search over AI-written observations of session recordings. \
+The observations below come from one scanner, an AI check that looks for a specific thing in each session, and \
+an embedding model grouped them because they describe the same kind of situation.
+
+Write one search query that finds this theme: what these sessions have in common, phrased the way a person \
+would describe what they are looking for (e.g. "coupon rejected at checkout", "gave up during signup").
+
+Rules:
+- 3 to 8 words, lowercase, no trailing punctuation.
+- Name the shared situation, not one session's details. Prefer concrete product situations over generic \
+phrases like "frustrated users".
+- Never include names, emails, ids, URLs, or any other identifier from the observations.
+- Return an empty query when the observations share no clear situation.
+
+Scanner: {scanner_name}
+Scanner instructions: {scanner_prompt}
+
+Observations:
+{samples}"""
+
+
+class _ThemeName(BaseModel):
+ query: str = Field(description="One search query for the shared theme, or an empty string.")
+
+
+@frozen
+class ThemeAssignment:
+ theme_id: uuid.UUID
+ name_due: bool
+
+
+def belongs_in_theme(*, category: str, scanner_type: str, valence: str, output: dict[str, Any]) -> bool:
+ """An issue observation, and for a monitor with a known valence only the answer the team looks for, so a
+ monitor's many reasons why nothing went wrong never become a suggestion."""
+ if category not in ISSUE_CATEGORIES:
+ return False
+ if scanner_type == ScannerType.MONITOR:
+ verdict = output.get("verdict")
+ if valence == PromptValence.BAD:
+ return verdict == "yes"
+ if valence == PromptValence.GOOD:
+ return verdict == "no"
+ return True
+
+
+def assign_theme(*, team_id: int, scanner_id: str, space: SemanticSpace, now: datetime) -> ThemeAssignment:
+ """Put one observation in its scanner's closest active theme, or start a theme. Call under the centre lock."""
+ themes = list(
+ ReplayScannerSearchTheme.objects.for_team(team_id)
+ .filter(scanner_id=scanner_id, last_member_at__gte=now - THEME_ACTIVE_WINDOW)
+ .order_by("-last_member_at")
+ .only("id", "vector_sum", "member_count", "rename_at_member_count", "last_member_at")[:MAX_ACTIVE_THEMES]
+ )
+ best = nearest_cluster(
+ np.array([unpack(theme.vector_sum) for theme in themes], dtype=np.float32).reshape(-1, SEMANTIC_DIMENSIONS),
+ np.array([theme.member_count for theme in themes], dtype=np.float32),
+ space,
+ THEME_MIN_SIMILARITY,
+ THEME_MIN_RAW_SIMILARITY,
+ )
+ if best is None:
+ theme = ReplayScannerSearchTheme.objects.for_team(team_id).create(
+ team_id=team_id, scanner_id=scanner_id, vector_sum=pack(space.vector), member_count=1, last_member_at=now
+ )
+ else:
+ theme = themes[best]
+ theme.vector_sum = pack(unpack(theme.vector_sum) + space.vector)
+ theme.member_count += 1
+ # The backfill replays old rows while the consumer adds new ones, so the latest member time never moves back.
+ theme.last_member_at = max(theme.last_member_at, now)
+ theme.save(update_fields=["vector_sum", "member_count", "last_member_at"])
+ return ThemeAssignment(theme_id=theme.id, name_due=theme.member_count >= theme.rename_at_member_count)
+
+
+def _claim_naming(team_id: int, theme_id: uuid.UUID) -> bool:
+ """Move the theme's next naming to double its size, and say whether this caller won the right to name it now."""
+ return bool(
+ ReplayScannerSearchTheme.objects.for_team(team_id)
+ .filter(id=theme_id, member_count__gte=F("rename_at_member_count"))
+ .update(rename_at_member_count=F("member_count") * 2)
+ )
+
+
+def _naming_budget_key(team_id: int, now: datetime) -> str:
+ return f"replay_vision:theme_naming:{team_id}:{now:%Y-%m-%d}"
+
+
+def _reserve_naming_budget(team_id: int, now: datetime) -> bool:
+ """Take one naming from the project's daily budget in a single increment, so concurrent callers never overshoot."""
+ key = _naming_budget_key(team_id, now)
+ cache.add(key, 0, timeout=2 * 24 * 3600)
+ if cache.incr(key) <= THEME_NAMING_DAILY_CAP:
+ return True
+ _refund_naming_budget(team_id, now)
+ return False
+
+
+def _refund_naming_budget(team_id: int, now: datetime) -> None:
+ cache.decr(_naming_budget_key(team_id, now))
+
+
+def _naming_context(team_id: int, theme_id: uuid.UUID) -> tuple[ReplayScanner, list[str]] | None:
+ theme = ReplayScannerSearchTheme.objects.for_team(team_id).filter(id=theme_id).select_related("scanner").first()
+ if theme is None:
+ return None
+ rows = (
+ # The window keeps the walk to recent rows of the scanner, since the JSON key has no index of its own.
+ ReplayObservation.objects.filter(
+ team_id=team_id,
+ scanner_id=theme.scanner_id,
+ created_at__gte=theme.last_member_at - THEME_ACTIVE_WINDOW,
+ watch_rank__theme=str(theme_id),
+ )
+ .order_by("-created_at")
+ .values_list("scanner_result", flat=True)[:THEME_NAME_SAMPLES]
+ )
+ samples = [
+ text[:_SAMPLE_CHARS] for result in rows if (text := explanation_text((result or {}).get("model_output") or {}))
+ ]
+ return theme.scanner, samples
+
+
+def _save_name(team_id: int, theme_id: uuid.UUID, name: str) -> None:
+ ReplayScannerSearchTheme.objects.for_team(team_id).filter(id=theme_id).update(name=name)
+
+
+def finalize_label(raw: str) -> str:
+ """The model's query in the shape a suggestion takes, or empty when it breaks the prompt's rules."""
+ query = " ".join(raw.split()).strip().rstrip(".!?").lower()
+ if not query or len(query.split()) > MAX_THEME_LABEL_WORDS or _IDENTIFIER.search(query):
+ return ""
+ return query[:128]
+
+
+async def name_theme(team_id: int, theme_id: uuid.UUID, now: datetime) -> str | None:
+ """Name a theme that reached its naming size. None when another caller claimed it, or nothing could be named."""
+ # Consent and budget come before the claim, so a theme that cannot be named now is tried again on its next member.
+ if not await sync_to_async(is_ai_data_processing_approved)(team_id):
+ return None
+ if not await sync_to_async(_reserve_naming_budget)(team_id, now):
+ logger.info("replay_vision.watch_themes.naming_budget_spent", team_id=team_id)
+ return None
+ # A caller that loses the claim hands its reservation back, so two callers racing for one theme cost one call.
+ if not await sync_to_async(_claim_naming)(team_id, theme_id):
+ await sync_to_async(_refund_naming_budget)(team_id, now)
+ return None
+ context = await sync_to_async(_naming_context)(team_id, theme_id)
+ if context is None or not context[1]:
+ return None
+ scanner, samples = context
+ prompt = (scanner.scanner_config or {}).get("prompt") if isinstance(scanner.scanner_config, dict) else None
+ client = await replay_gemini_client(team_id, properties={"ai_product": AI_PRODUCT, "feature": "name_search_theme"})
+ try:
+ response = await client.models.generate_content(
+ model=_THEME_MODEL,
+ contents=_NAMING_PROMPT.format(
+ scanner_name=scanner.name,
+ scanner_prompt=(prompt or "")[:600],
+ samples="\n".join(f"- {sample}" for sample in samples),
+ ),
+ config=types.GenerateContentConfig(response_mime_type="application/json", response_schema=_ThemeName),
+ posthog_distinct_id=replay_vision_distinct_id(team_id),
+ posthog_trace_id=str(uuid.uuid4()),
+ posthog_properties={"$ai_span_name": "name_search_theme", "team_id": team_id},
+ posthog_groups={"project": str(team_id)},
+ )
+ finally:
+ await client.aclose()
+ label = finalize_label(_ThemeName.model_validate_json(response.text or "{}").query)
+ if not label:
+ return None
+ await sync_to_async(_save_name)(team_id, theme_id, label)
+ return label