Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-vision): store anchor similarities so the feed can be scored per viewer (#123935)

Tue Haulund40a43c48bf89
diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yaml
index c1f7fac28f58..592e882f1910 100644
--- a/.semgrep/rules/security/idor-team-scoped-models.yaml
+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml
@@ -350,6 +350,7 @@ rules:
|QuickFilter
|QuickFilterContext
|ReplayObservation
+ |ReplayObservationAnchorSimilarity
|ReplayObservationFeedDismissal
|ReplayObservationFeedImpression
|ReplayObservationLabel
@@ -833,6 +834,7 @@ rules:
|QuickFilter
|QuickFilterContext
|ReplayObservation
+ |ReplayObservationAnchorSimilarity
|ReplayObservationFeedDismissal
|ReplayObservationFeedImpression
|ReplayObservationLabel
diff --git a/products/replay_vision/backend/migrations/0135_observation_anchor_similarity.py b/products/replay_vision/backend/migrations/0135_observation_anchor_similarity.py
new file mode 100644
index 000000000000..a6dd834885fd
--- /dev/null
+++ b/products/replay_vision/backend/migrations/0135_observation_anchor_similarity.py
@@ -0,0 +1,41 @@
+# Generated by Django 5.2.18 on 2026-10-27 11:13
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("posthog", "1413_event_filter_pipeline_override"),
+ ("replay_vision", "0134_feed_group_index"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="ReplayObservationAnchorSimilarity",
+ fields=[
+ (
+ "observation",
+ models.OneToOneField(
+ db_constraint=False,
+ on_delete=django.db.models.deletion.CASCADE,
+ primary_key=True,
+ related_name="+",
+ serialize=False,
+ to="replay_vision.replayobservation",
+ ),
+ ),
+ ("anchor_version", models.PositiveSmallIntegerField()),
+ ("similarities", models.JSONField()),
+ (
+ "team",
+ models.ForeignKey(
+ db_constraint=False,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="+",
+ to="posthog.team",
+ ),
+ ),
+ ],
+ ),
+ ]
diff --git a/products/replay_vision/backend/migrations/max_migration.txt b/products/replay_vision/backend/migrations/max_migration.txt
index a2c14b64d517..ce37b4e04c40 100644
--- a/products/replay_vision/backend/migrations/max_migration.txt
+++ b/products/replay_vision/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0134_feed_group_index
+0135_observation_anchor_similarity
diff --git a/products/replay_vision/backend/models/__init__.py b/products/replay_vision/backend/models/__init__.py
index 3d21a6d46221..337dea9ea9cb 100644
--- a/products/replay_vision/backend/models/__init__.py
+++ b/products/replay_vision/backend/models/__init__.py
@@ -1,4 +1,5 @@
from products.replay_vision.backend.models.replay_observation import ReplayObservation
+from products.replay_vision.backend.models.replay_observation_anchor_similarity import ReplayObservationAnchorSimilarity
from products.replay_vision.backend.models.replay_observation_feed_dismissal import ReplayObservationFeedDismissal
from products.replay_vision.backend.models.replay_observation_feed_impression import ReplayObservationFeedImpression
from products.replay_vision.backend.models.replay_observation_label import ReplayObservationLabel
@@ -24,6 +25,7 @@ from products.replay_vision.backend.models.vision_alert import (
__all__ = [
"ReplayObservation",
+ "ReplayObservationAnchorSimilarity",
"ReplayObservationFeedDismissal",
"ReplayObservationFeedImpression",
"ReplayObservationLabel",
diff --git a/products/replay_vision/backend/models/replay_observation_anchor_similarity.py b/products/replay_vision/backend/models/replay_observation_anchor_similarity.py
new file mode 100644
index 000000000000..fc406ad74cf9
--- /dev/null
+++ b/products/replay_vision/backend/models/replay_observation_anchor_similarity.py
@@ -0,0 +1,24 @@
+from django.db import models
+
+from posthog.models.scoping.root_mixin import TeamScopedRootMixin
+
+
+class ReplayObservationAnchorSimilarity(TeamScopedRootMixin, models.Model):
+ """An observation's similarity to every watch anchor sentence, keyed by anchor, so a viewer's own choice of
+ anchors can be scored without reading the embedding back."""
+
+ # No database constraint, so the recording delete, which removes observations in raw SQL, never fails on a row here.
+ observation = models.OneToOneField(
+ "replay_vision.ReplayObservation",
+ on_delete=models.CASCADE,
+ primary_key=True,
+ related_name="+",
+ db_constraint=False,
+ )
+ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, related_name="+", db_constraint=False)
+ # Similarities from another anchor set do not compare, so readers ignore rows written under another version.
+ anchor_version = models.PositiveSmallIntegerField()
+ similarities = models.JSONField()
+
+ class Meta:
+ app_label = "replay_vision"
diff --git a/products/replay_vision/backend/tests/helpers.py b/products/replay_vision/backend/tests/helpers.py
index 41aed8f00a80..23dde1ede31a 100644
--- a/products/replay_vision/backend/tests/helpers.py
+++ b/products/replay_vision/backend/tests/helpers.py
@@ -2,6 +2,8 @@ from typing import Any
from django.utils import timezone
+import numpy as np
+
from posthog.models import Team, User
from posthog.models.utils import uuid7
@@ -15,6 +17,7 @@ from products.replay_vision.backend.models.replay_observation import (
from products.replay_vision.backend.models.replay_observation_usage import ReplayObservationUsage
from products.replay_vision.backend.models.replay_scanner import ReplayScanner
from products.replay_vision.backend.temporal.activities.create_observation import _build_scanner_snapshot
+from products.replay_vision.backend.watch_rank import ANCHORS, WatchCategory
def snapshot_for(scanner: ReplayScanner) -> dict[str, Any]:
@@ -78,3 +81,15 @@ def create_experiment(
start_date=timezone.now() if launched else None,
exposure_criteria={},
)
+
+
+def axis_anchors() -> dict[WatchCategory, np.ndarray]:
+ """Unit anchors in three dimensions: failure on x, struggle on y, revenue on -y, routine on z, one row per anchor
+ sentence so similarities come out keyed like real ones."""
+ axes = {
+ "failure": [1.0, 0.0, 0.0],
+ "struggle": [0.0, 1.0, 0.0],
+ "revenue": [0.0, -1.0, 0.0],
+ "routine": [0.0, 0.0, 1.0],
+ }
+ return {category: np.array([axes[category]] * len(ANCHORS[category])) for category in ANCHORS}
diff --git a/products/replay_vision/backend/tests/test_watch_rank.py b/products/replay_vision/backend/tests/test_watch_rank.py
index d496475a8345..b85803c322a1 100644
--- a/products/replay_vision/backend/tests/test_watch_rank.py
+++ b/products/replay_vision/backend/tests/test_watch_rank.py
@@ -13,13 +13,9 @@ from posthog.exceptions import ClickHouseAtCapacity
from products.replay_vision.backend import watch_rank_backfill
from products.replay_vision.backend.models.replay_observation import ReplayObservation
+from products.replay_vision.backend.tests.helpers import axis_anchors
from products.replay_vision.backend.watch_feed import SESSION_SPACING
-from products.replay_vision.backend.watch_rank import (
- ANCHOR_VERSION,
- WatchCategory,
- classify,
- rank_watch_feed_by_embedding,
-)
+from products.replay_vision.backend.watch_rank import ANCHOR_VERSION, ANCHORS, classify, rank_watch_feed_by_embedding
from products.replay_vision.backend.watch_rank_backfill import _rank_scanner, _stored_vectors
_NOW = datetime(2026, 10, 23, 12, 0, tzinfo=UTC)
@@ -67,15 +63,12 @@ class TestClassify(SimpleTestCase):
def test_routine_wins_over_the_closest_issue_when_it_is_closer(
self, _name: str, vector: list[float], category: str, positive: bool
) -> None:
- anchors: dict[WatchCategory, np.ndarray] = {
- "failure": np.array([[1.0, 0.0, 0.0]]),
- "struggle": np.array([[0.0, 1.0, 0.0]]),
- "revenue": np.array([[0.0, -1.0, 0.0]]),
- "routine": np.array([[0.0, 0.0, 1.0]]),
- }
- got_category, contrast = classify(vector, anchors)
- assert got_category == category
- assert (contrast > 0) is positive
+ anchors = axis_anchors()
+ reading = classify(vector, anchors)
+ assert reading.rank.category == category
+ assert (reading.rank.contrast > 0) is positive
+ assert set(reading.similarities) == {key for texts in ANCHORS.values() for key in texts}
+ assert reading.rank.scores["routine"] == max(reading.similarities[key] for key in ANCHORS["routine"])
class TestPickIndexes(SimpleTestCase):
@@ -171,12 +164,7 @@ class TestRankWatchFeedByEmbedding(SimpleTestCase):
class TestBackfillRankScanner(SimpleTestCase):
def test_ranks_only_rows_without_a_current_rank(self) -> None:
- anchors: dict[WatchCategory, np.ndarray] = {
- "failure": np.array([[1.0, 0.0, 0.0]]),
- "struggle": np.array([[0.0, 1.0, 0.0]]),
- "revenue": np.array([[0.0, -1.0, 0.0]]),
- "routine": np.array([[0.0, 0.0, 1.0]]),
- }
+ anchors = axis_anchors()
error = np.array([1.0, 0.0, 0.0])
current = {"category": "failure", "contrast": 1.0, "anchor_version": ANCHOR_VERSION}
rows: list[dict[str, Any]] = [
@@ -191,8 +179,8 @@ class TestBackfillRankScanner(SimpleTestCase):
written = _rank_scanner(rows, vectors, anchors)
assert set(written) == {"copy", "stale", "calm"}
- assert written["copy"]["category"] == "failure"
- assert written["calm"]["category"] == "routine"
+ assert written["copy"].rank.category == "failure"
+ assert written["calm"].rank.category == "routine"
@parameterized.expand(
[
diff --git a/products/replay_vision/backend/tests/test_watch_rank_consumer.py b/products/replay_vision/backend/tests/test_watch_rank_consumer.py
index 094e00b36f25..3bb5e9a5277f 100644
--- a/products/replay_vision/backend/tests/test_watch_rank_consumer.py
+++ b/products/replay_vision/backend/tests/test_watch_rank_consumer.py
@@ -8,7 +8,6 @@ from unittest.mock import AsyncMock, MagicMock, patch
from django.test import SimpleTestCase
-import numpy as np
from asgiref.sync import async_to_sync
from confluent_kafka import KafkaException
from parameterized import parameterized
@@ -19,8 +18,9 @@ from products.replay_vision.backend.embeddings import (
OBSERVATION_EMBEDDING_MODEL,
)
from products.replay_vision.backend.models.replay_observation import ObservationTrigger, ReplayObservation
+from products.replay_vision.backend.models.replay_observation_anchor_similarity import ReplayObservationAnchorSimilarity
from products.replay_vision.backend.models.replay_scanner import ReplayScanner, ScannerModel, ScannerType
-from products.replay_vision.backend.tests.helpers import snapshot_for
+from products.replay_vision.backend.tests.helpers import axis_anchors, snapshot_for
from products.replay_vision.backend.watch_rank import ANCHOR_VERSION, RankRequest
from products.replay_vision.backend.watch_rank_consumer import WatchRankConsumer, parse_rank_request
@@ -80,12 +80,7 @@ class TestParseRankRequest(SimpleTestCase):
class TestWatchRankConsumer(BaseTest):
- _ANCHORS = {
- "failure": np.array([[1.0, 0.0, 0.0]]),
- "struggle": np.array([[0.0, 1.0, 0.0]]),
- "revenue": np.array([[0.0, -1.0, 0.0]]),
- "routine": np.array([[0.0, 0.0, 1.0]]),
- }
+ _ANCHORS = axis_anchors()
def _observation(self, scanner: ReplayScanner, session_id: str, **overrides: Any) -> ReplayObservation:
return ReplayObservation.objects.create(
@@ -142,6 +137,10 @@ class TestWatchRankConsumer(BaseTest):
assert observation.watch_rank is not None
assert expected_rank.items() <= observation.watch_rank.items()
assert observation.watch_rank.get("group")
+ # A redelivery returns before ranking, so only a fresh rank writes the similarity row.
+ assert ReplayObservationAnchorSimilarity.objects.for_team(self.team.id).filter(
+ observation=observation
+ ).exists() == (outcome == "ranked")
@parameterized.expand(
[
diff --git a/products/replay_vision/backend/tests/test_watch_semantics.py b/products/replay_vision/backend/tests/test_watch_semantics.py
index d650a7f07d8f..2d91ca65f421 100644
--- a/products/replay_vision/backend/tests/test_watch_semantics.py
+++ b/products/replay_vision/backend/tests/test_watch_semantics.py
@@ -12,6 +12,7 @@ from asgiref.sync import async_to_sync
from parameterized import parameterized
from products.replay_vision.backend.models.replay_observation import ObservationTrigger, ReplayObservation
+from products.replay_vision.backend.models.replay_observation_anchor_similarity import ReplayObservationAnchorSimilarity
from products.replay_vision.backend.models.replay_scanner import PromptValence, ReplayScanner, ScannerModel, ScannerType
from products.replay_vision.backend.models.replay_scanner_semantics import (
ReplayScannerFeedGroup,
@@ -19,17 +20,19 @@ from products.replay_vision.backend.models.replay_scanner_semantics import (
ReplayScannerSemanticCenter,
)
from products.replay_vision.backend.semantic_space import SEMANTIC_DIMENSIONS, pack, semantic_vector
-from products.replay_vision.backend.tests.helpers import snapshot_for
+from products.replay_vision.backend.tests.helpers import axis_anchors, snapshot_for
from products.replay_vision.backend.watch_feed_groups import feed_group_cache
from products.replay_vision.backend.watch_rank import ANCHOR_VERSION
from products.replay_vision.backend.watch_semantics import assign_semantics
+from products.replay_vision.backend.watch_semantics_backfill import backfill_team_semantics
from products.replay_vision.backend.watch_themes import belongs_in_theme, finalize_label, name_theme
_THEMES = "products.replay_vision.backend.watch_themes"
+_BACKFILL = "products.replay_vision.backend.watch_semantics_backfill"
def _rank(category: str = "failure") -> dict[str, Any]:
- return {"category": category, "contrast": 0.2, "anchor_version": ANCHOR_VERSION}
+ return {"category": category, "contrast": 0.2, "anchor_version": ANCHOR_VERSION, "duplicate_of": "retired"}
def _vector(*head: float) -> list[float]:
@@ -119,14 +122,18 @@ class TestAssignSemantics(BaseTest):
assert joined[0].theme.theme_id == joined[1].theme.theme_id
assert self._assign(first, _vector(1.0)) is None
assert self._assign(far, _vector(0.0, 1.0)).group_id != joined[0].group_id
- assert self._assign(routine, _vector(1.0)) is None
+ # A routine row joins a feed group, so a viewer who counts it as an issue finds it grouped, but never a theme.
+ routine_assignment = self._assign(routine, _vector(1.0))
+ assert routine_assignment.group_id == joined[0].group_id
+ assert routine_assignment.theme is None
- assert self._counts(ReplayScannerFeedGroup) == [1, 2]
+ assert self._counts(ReplayScannerFeedGroup) == [1, 3]
assert self._counts(ReplayScannerSearchTheme) == [1, 2]
first.refresh_from_db()
assert first.watch_rank is not None
assert first.watch_rank["group"] == str(joined[0].group_id)
assert first.watch_rank["theme"] == str(joined[0].theme.theme_id)
+ assert "duplicate_of" not in first.watch_rank
@parameterized.expand(
[
@@ -194,6 +201,29 @@ class TestAssignSemantics(BaseTest):
assert self._counts(ReplayScannerFeedGroup) == [1, 2]
+ def test_the_backfill_fills_only_what_a_ranked_row_is_missing(self) -> None:
+ grouped = self._observation()
+ ReplayObservation.objects.filter(id=grouped.id).update(
+ watch_rank={**_rank(), "group": "kept"}, status="succeeded", completed_at=timezone.now()
+ )
+ vector = np.array([1.0, 0.0, 0.0], dtype=np.float32)
+ with (
+ patch(f"{_BACKFILL}._stored_vectors", return_value={str(grouped.id): vector}),
+ patch(f"{_BACKFILL}.anchor_vectors", AsyncMock(return_value=axis_anchors())),
+ ):
+ backfill_team_semantics(self.team.id, timezone.now() - timedelta(days=1))
+
+ grouped.refresh_from_db()
+ assert grouped.watch_rank is not None
+ assert grouped.watch_rank["group"] == "kept"
+ assert set(grouped.watch_rank["scores"]) == {"failure", "struggle", "revenue", "routine"}
+ assert (
+ ReplayObservationAnchorSimilarity.objects.for_team(self.team.id)
+ .filter(observation_id=grouped.id, anchor_version=ANCHOR_VERSION)
+ .exists()
+ )
+ assert self._counts(ReplayScannerFeedGroup) == []
+
def test_clusters_without_a_member_for_the_retention_window_are_pruned(self) -> None:
stale = timezone.now() - timedelta(days=91)
for model in (ReplayScannerFeedGroup, ReplayScannerSearchTheme):
diff --git a/products/replay_vision/backend/watch_rank.py b/products/replay_vision/backend/watch_rank.py
index cb7fedb60a2d..5b2f9d3be799 100644
--- a/products/replay_vision/backend/watch_rank.py
+++ b/products/replay_vision/backend/watch_rank.py
@@ -22,9 +22,11 @@ from django.db.models.fields.json import KeyTextTransform, KeyTransform
from django.db.models.functions import Coalesce, NullIf, RowNumber
import numpy as np
+from asgiref.sync import sync_to_async
from posthog.api.embedding_worker import async_generate_embedding
from posthog.dataclasses import frozen
+from posthog.models.scoping.manager import resolve_effective_team_id
from posthog.models.team import Team
from posthog.utils import dict_from_cursor_fetchall
@@ -36,6 +38,7 @@ from products.replay_vision.backend.models.replay_observation import (
watch_group_root,
watch_rank_contrast,
)
+from products.replay_vision.backend.models.replay_observation_anchor_similarity import ReplayObservationAnchorSimilarity
from products.replay_vision.backend.scanner_access import accessible_observations
from products.replay_vision.backend.watch_feed import WATCH_FEED_MIN_ITEMS, WatchFeedEntry, finish_feed
@@ -45,41 +48,42 @@ WatchCategory = Literal["failure", "struggle", "revenue", "routine"]
# ignores ranks written under another version until they age out of the window.
ANCHOR_VERSION = 1
-ANCHORS: dict[WatchCategory, tuple[str, ...]] = {
- "failure": (
- "The user saw an error message and the action they tried failed.",
- "A request failed or the page crashed, showing an error to the user.",
- "The user clicked a button and nothing happened; the action silently did not work.",
- "The page got stuck loading and the user waited on a spinner that never finished.",
- "The user was blocked by a dead end, a 404 page, or an access denied message.",
- ),
- "struggle": (
- "The user repeated the same action several times because it did not give the result they expected.",
- "The user rage clicked and gave up on the task without finishing.",
- "The user hit a validation error or confusing state in a form and abandoned it.",
- "The user was stuck: they tried several different ways to do one thing and none worked.",
- "The user worked around a broken or missing feature to get their task done.",
- ),
- "revenue": (
- "The user went to billing to cancel their subscription or downgrade their plan.",
- "The user tried to upgrade or pay but the upgrade or payment failed.",
- "The user hit a usage limit that stopped a feature from working.",
- "The user contacted support frustrated about a problem.",
- ),
- "routine": (
- "The user browsed normally and completed what they came to do without any issues.",
- "A routine session with nothing notable: normal navigation and expected behavior.",
- "The user viewed a few pages and left; nothing went wrong.",
- "Nothing stands out in this session.",
- "Routine session with no errors or friction; the user did what they intended.",
- "The user browsed the pricing page and documentation normally.",
- "The user successfully completed the task without problems.",
- "The user explored different sections of the website to learn about the product.",
- "The user read through a product or documentation page.",
- "The user looked through a list and opened a few items to review them.",
- "The user started something and left before finishing, with no sign of a problem.",
- "The user never reached the part of the product this check is about.",
- ),
+# Each sentence keeps its key for good: stored similarities and a viewer's anchor choices refer to anchors by key.
+ANCHORS: dict[WatchCategory, dict[str, str]] = {
+ "failure": {
+ "error_message": "The user saw an error message and the action they tried failed.",
+ "request_failed": "A request failed or the page crashed, showing an error to the user.",
+ "silent_click": "The user clicked a button and nothing happened; the action silently did not work.",
+ "endless_spinner": "The page got stuck loading and the user waited on a spinner that never finished.",
+ "dead_end": "The user was blocked by a dead end, a 404 page, or an access denied message.",
+ },
+ "struggle": {
+ "repeated_action": "The user repeated the same action several times because it did not give the result they expected.",
+ "rage_click": "The user rage clicked and gave up on the task without finishing.",
+ "form_abandoned": "The user hit a validation error or confusing state in a form and abandoned it.",
+ "stuck_trying": "The user was stuck: they tried several different ways to do one thing and none worked.",
+ "workaround": "The user worked around a broken or missing feature to get their task done.",
+ },
+ "revenue": {
+ "cancel_or_downgrade": "The user went to billing to cancel their subscription or downgrade their plan.",
+ "payment_failed": "The user tried to upgrade or pay but the upgrade or payment failed.",
+ "usage_limit": "The user hit a usage limit that stopped a feature from working.",
+ "frustrated_support": "The user contacted support frustrated about a problem.",
+ },
+ "routine": {
+ "completed_task": "The user browsed normally and completed what they came to do without any issues.",
+ "nothing_notable": "A routine session with nothing notable: normal navigation and expected behavior.",
+ "few_pages": "The user viewed a few pages and left; nothing went wrong.",
+ "nothing_stands_out": "Nothing stands out in this session.",
+ "no_friction": "Routine session with no errors or friction; the user did what they intended.",
+ "pricing_docs": "The user browsed the pricing page and documentation normally.",
+ "task_success": "The user successfully completed the task without problems.",
+ "exploring_site": "The user explored different sections of the website to learn about the product.",
+ "reading_docs": "The user read through a product or documentation page.",
+ "reviewing_list": "The user looked through a list and opened a few items to review them.",
+ "left_early": "The user started something and left before finishing, with no sign of a problem.",
+ "never_reached": "The user never reached the part of the product this check is about.",
+ },
}
ISSUE_CATEGORIES: tuple[WatchCategory, ...] = ("failure", "struggle", "revenue")
@@ -108,12 +112,20 @@ _ANCHOR_CACHE_TTL_S = 7 * 24 * 3600
class WatchRank:
category: WatchCategory
contrast: float
+ # The closest anchor's similarity per category, so a viewer's choice of categories can be scored from the row.
+ scores: dict[WatchCategory, float]
anchor_version: int = ANCHOR_VERSION
def to_json(self) -> dict[str, Any]:
return asdict(self)
+@frozen
+class AnchorReading:
+ rank: WatchRank
+ similarities: dict[str, float]
+
+
_ANCHOR_CACHE_KEY = f"replay-vision:watch-rank-anchors:v{ANCHOR_VERSION}:{OBSERVATION_EMBEDDING_MODEL.value}"
@@ -145,7 +157,7 @@ async def _load_anchor_vectors(team_id: int) -> dict[WatchCategory, np.ndarray]:
*(
async_generate_embedding(team, text, model=OBSERVATION_EMBEDDING_MODEL.value)
for texts in ANCHORS.values()
- for text in texts
+ for text in texts.values()
)
)
rows = iter(response.embedding for response in embedded)
@@ -155,17 +167,26 @@ async def _load_anchor_vectors(team_id: int) -> dict[WatchCategory, np.ndarray]:
return anchors
-def classify(vector: list[float] | np.ndarray, anchors: dict[WatchCategory, np.ndarray]) -> tuple[WatchCategory, float]:
+def classify(vector: list[float] | np.ndarray, anchors: dict[WatchCategory, np.ndarray]) -> AnchorReading:
"""The closest issue category and its lead over the closest routine anchor, given unit anchors.
A negative lead means the observation reads more like routine than like any issue, and the
category is then routine.
"""
unit = normalize(np.asarray(vector, dtype=float))
- closest = {category: float(np.max(anchors[category] @ unit)) for category in anchors}
+ by_category = {category: anchors[category] @ unit for category in ANCHORS}
+ closest = {category: float(np.max(sims)) for category, sims in by_category.items()}
category = max(ISSUE_CATEGORIES, key=lambda c: closest[c])
contrast = round(closest[category] - closest["routine"], 4)
- return (category if contrast > 0 else "routine"), contrast
+ rank = WatchRank(
+ category=category if contrast > 0 else "routine",
+ contrast=contrast,
+ scores={c: round(score, 4) for c, score in closest.items()},
+ )
+ similarities = {
+ key: round(float(sim), 4) for c, sims in by_category.items() for key, sim in zip(ANCHORS[c], sims, strict=True)
+ }
+ return AnchorReading(rank=rank, similarities=similarities)
def current_rank(row: dict[str, Any]) -> dict[str, Any] | None:
@@ -179,6 +200,25 @@ def current_rank(row: dict[str, Any]) -> dict[str, Any] | None:
return rank
+def save_anchor_similarities(team_id: int, similarities: dict[str, dict[str, float]]) -> None:
+ # A bulk insert skips the model's canonical-team rewrite, so the row carries the id reads are scoped to.
+ canonical_team_id = resolve_effective_team_id(team_id)
+ ReplayObservationAnchorSimilarity.objects.for_team(canonical_team_id, canonical=True).bulk_create(
+ [
+ ReplayObservationAnchorSimilarity(
+ observation_id=observation_id,
+ team_id=canonical_team_id,
+ anchor_version=ANCHOR_VERSION,
+ similarities=values,
+ )
+ for observation_id, values in similarities.items()
+ ],
+ update_conflicts=True,
+ unique_fields=["observation"],
+ update_fields=["anchor_version", "similarities"],
+ )
+
+
@frozen
class RankRequest:
team_id: int
@@ -203,10 +243,11 @@ async def rank_observation(request: RankRequest) -> Literal["ranked", "already_r
return "missing"
if current_rank({"watch_rank": row["watch_rank"]}) is not None:
return "already_ranked"
- category, contrast = classify(request.vector, await anchor_vectors(request.team_id))
- rank = WatchRank(category=category, contrast=contrast)
+ reading = classify(request.vector, await anchor_vectors(request.team_id))
+ # The similarities land first, so a rank on the row means its similarities are there too.
+ await sync_to_async(save_anchor_similarities)(request.team_id, {request.observation_id: reading.similarities})
await ReplayObservation.objects.filter(team_id=request.team_id, id=request.observation_id).aupdate(
- watch_rank=rank.to_json()
+ watch_rank=reading.rank.to_json()
)
return "ranked"
diff --git a/products/replay_vision/backend/watch_rank_backfill.py b/products/replay_vision/backend/watch_rank_backfill.py
index b844e27f892b..0169fcb5d35d 100644
--- a/products/replay_vision/backend/watch_rank_backfill.py
+++ b/products/replay_vision/backend/watch_rank_backfill.py
@@ -36,12 +36,13 @@ from products.replay_vision.backend.embeddings import (
from products.replay_vision.backend.models.replay_observation import ObservationStatus, ReplayObservation
from products.replay_vision.backend.watch_rank import (
ANCHOR_VERSION,
+ AnchorReading,
WatchCategory,
- WatchRank,
anchor_vectors,
classify,
current_rank,
normalize,
+ save_anchor_similarities,
)
logger = structlog.get_logger(__name__)
@@ -112,16 +113,15 @@ def _read_stored_vectors(
def _rank_scanner(
rows: list[dict[str, Any]], vectors: dict[str, np.ndarray], anchors: dict[WatchCategory, np.ndarray]
-) -> dict[str, dict[str, Any]]:
+) -> dict[str, AnchorReading]:
"""New ranks for the rows that lack a current one."""
- written: dict[str, dict[str, Any]] = {}
+ written: dict[str, AnchorReading] = {}
for row in rows:
observation_id = str(row["id"])
vector = vectors.get(observation_id)
if vector is None or current_rank(row) is not None:
continue
- category, contrast = classify(vector, anchors)
- written[observation_id] = WatchRank(category=category, contrast=contrast).to_json()
+ written[observation_id] = classify(vector, anchors)
return written
@@ -152,6 +152,9 @@ def backfill_team_watch_ranks(team_id: int, since: datetime, *, dry_run: bool =
written = _rank_scanner(list(batch), vectors, anchors)
ranked += len(written)
if not dry_run and written:
+ save_anchor_similarities(
+ team_id, {observation_id: reading.similarities for observation_id, reading in written.items()}
+ )
# Only rows still without a current rank, locked until the write, so a rank the consumer writes meanwhile
# waits for this one and a rank it already wrote, with its group, stays.
with transaction.atomic():
@@ -163,8 +166,8 @@ def backfill_team_watch_ranks(team_id: int, since: datetime, *, dry_run: bool =
)
ReplayObservation.objects.bulk_update(
[
- ReplayObservation(id=observation_id, watch_rank=rank)
- for observation_id, rank in written.items()
+ ReplayObservation(id=observation_id, watch_rank=reading.rank.to_json())
+ for observation_id, reading in written.items()
if UUID(observation_id) in still_unranked
],
["watch_rank"],
diff --git a/products/replay_vision/backend/watch_semantics.py b/products/replay_vision/backend/watch_semantics.py
index 0d7eb53819f6..2e4d35bc5f17 100644
--- a/products/replay_vision/backend/watch_semantics.py
+++ b/products/replay_vision/backend/watch_semantics.py
@@ -1,4 +1,7 @@
-"""Places each ranked issue observation in a feed group and, when it qualifies, a search theme.
+"""Places each ranked observation in a feed group and, when it qualifies, a search theme.
+
+Every ranked row gets a group, not only issues, so a viewer who counts different anchors as issues finds those rows
+already grouped.
The embedding consumer and the semantics backfill both call `assign_semantics`. It locks the scanner's semantic
centre for the whole assignment, so two observations of one scanner never race on its clusters, and writes both ids
@@ -31,7 +34,7 @@ from products.replay_vision.backend.semantic_space import (
unpack,
)
from products.replay_vision.backend.watch_feed_groups import feed_group_cache
-from products.replay_vision.backend.watch_rank import ISSUE_CATEGORIES, current_rank
+from products.replay_vision.backend.watch_rank import current_rank
from products.replay_vision.backend.watch_themes import ThemeAssignment, assign_theme, belongs_in_theme
# Clusters with no member for this long are deleted, since the feed's longest window no longer reaches them.
@@ -49,7 +52,7 @@ class SemanticAssignment:
def assign_semantics(
*, team_id: int, scanner_id: str, observation_id: str, vector: list[float], now: datetime
) -> SemanticAssignment | None:
- """None when the observation is not a ranked issue, already has its clusters, or is gone."""
+ """None when the observation has no current rank, already has its clusters, or is gone."""
scanner = (
ReplayScanner.objects.filter(team_id=team_id, id=scanner_id)
.values("scanner_type", "prompt_valence", "scanner_config")
@@ -75,7 +78,7 @@ def assign_semantics(
if row is None:
return None
rank = current_rank({"watch_rank": row["watch_rank"]})
- if rank is None or rank.get("group") or rank.get("category") not in ISSUE_CATEGORIES:
+ if rank is None or rank.get("group"):
return None
v = semantic_vector(vector)
@@ -111,15 +114,8 @@ def assign_semantics(
if center.member_count % _PRUNE_EVERY == 0:
_prune(team_id, scanner_id, now)
- keys = {"group": str(group_id)} | ({"theme": str(theme.theme_id)} if theme else {})
- ReplayObservation.objects.filter(team_id=team_id, id=observation_id).update(
- watch_rank=Func(
- F("watch_rank"),
- Value(keys, output_field=JSONField()),
- template="%(expressions)s",
- arg_joiner=" || ",
- output_field=JSONField(),
- )
+ merge_watch_rank(
+ team_id, observation_id, {"group": str(group_id)} | ({"theme": str(theme.theme_id)} if theme else {})
)
except Exception:
feed_group_cache.forget(scanner_id)
@@ -127,6 +123,23 @@ def assign_semantics(
return SemanticAssignment(group_id=group_id, theme=theme)
+def merge_watch_rank(team_id: int, observation_id: str, keys: dict[str, Any]) -> None:
+ """Add keys to an observation's rank without rewriting what another writer put there, and drop the retired
+ duplicate link."""
+ without_duplicate = Func(
+ F("watch_rank"), Value("duplicate_of"), template="(%(expressions)s)", arg_joiner=" - ", output_field=JSONField()
+ )
+ ReplayObservation.objects.filter(team_id=team_id, id=observation_id).update(
+ watch_rank=Func(
+ without_duplicate,
+ Value(keys, output_field=JSONField()),
+ template="%(expressions)s",
+ arg_joiner=" || ",
+ output_field=JSONField(),
+ )
+ )
+
+
def _valence_unknown_for_row(
scanner_type: str, valence: str, scanner_config: dict[str, Any] | None, snapshot: dict[str, Any] | None
) -> bool:
diff --git a/products/replay_vision/backend/watch_semantics_backfill.py b/products/replay_vision/backend/watch_semantics_backfill.py
index 36bdfedacbce..0eeab105695a 100644
--- a/products/replay_vision/backend/watch_semantics_backfill.py
+++ b/products/replay_vision/backend/watch_semantics_backfill.py
@@ -1,10 +1,9 @@
-"""Builds feed groups and search themes from observations ranked before the consumer assigned them, so the feed and
-the suggestions start from history instead of an empty slate."""
+"""Brings observations ranked before the consumer wrote everything it writes today up to date: category scores,
+anchor similarities, feed groups and search themes, so the feed and the suggestions start from history."""
from datetime import datetime
-from django.db.models import F
-from django.db.models.fields.json import KeyTextTransform
+from django.db.models import Exists, F, OuterRef, Q
from django.utils import timezone
import structlog
@@ -13,15 +12,25 @@ from asgiref.sync import async_to_sync
from posthog.dataclasses import frozen
from products.replay_vision.backend.models.replay_observation import ReplayObservation
+from products.replay_vision.backend.models.replay_observation_anchor_similarity import ReplayObservationAnchorSimilarity
from products.replay_vision.backend.models.replay_scanner_semantics import ReplayScannerSearchTheme
-from products.replay_vision.backend.watch_rank import ANCHOR_VERSION, ISSUE_CATEGORIES
+from products.replay_vision.backend.watch_rank import ANCHOR_VERSION, anchor_vectors, classify, save_anchor_similarities
from products.replay_vision.backend.watch_rank_backfill import _VECTOR_CHUNK, _stored_vectors
-from products.replay_vision.backend.watch_semantics import assign_semantics
+from products.replay_vision.backend.watch_semantics import assign_semantics, merge_watch_rank
from products.replay_vision.backend.watch_themes import THEME_ACTIVE_WINDOW, name_theme
logger = structlog.get_logger(__name__)
+@frozen
+class _Pending:
+ observation_id: str
+ created_at: datetime
+ needs_group: bool
+ needs_scores: bool
+ needs_similarities: bool
+
+
@frozen
class SemanticBackfillResult:
observations: int
@@ -30,45 +39,74 @@ class SemanticBackfillResult:
def backfill_team_semantics(team_id: int, since: datetime, *, dry_run: bool = False) -> SemanticBackfillResult:
- """Replay a team's ranked issue observations oldest first, as the consumer would have seen them, then name
- every active theme that reached its naming size."""
- rows = (
+ """Replay a team's ranked observations oldest first, as the consumer would have seen them, then name every
+ active theme that reached its naming size."""
+ has_similarities = Exists(
+ ReplayObservationAnchorSimilarity.objects.for_team(team_id).filter(
+ observation_id=OuterRef("id"), anchor_version=ANCHOR_VERSION
+ )
+ )
+ pending = (
ReplayObservation.objects.filter(
- team_id=team_id,
- status="succeeded",
- created_at__gte=since,
- watch_rank__anchor_version=ANCHOR_VERSION,
- watch_rank__group__isnull=True,
+ team_id=team_id, status="succeeded", created_at__gte=since, watch_rank__anchor_version=ANCHOR_VERSION
)
- .alias(category=KeyTextTransform("category", "watch_rank"))
- .filter(category__in=ISSUE_CATEGORIES)
- .order_by("scanner_id", "created_at")
- .values_list("id", "scanner_id", "created_at")
+ .annotate(has_similarities=has_similarities)
+ .filter(Q(watch_rank__group__isnull=True) | Q(watch_rank__scores__isnull=True) | Q(has_similarities=False))
)
- by_scanner: dict[str, list[tuple[str, datetime]]] = {}
- for observation_id, scanner_id, created_at in rows.iterator(chunk_size=2_000):
- by_scanner.setdefault(str(scanner_id), []).append((str(observation_id), created_at))
- observations = sum(len(scanner_rows) for scanner_rows in by_scanner.values())
+ scanner_ids = [str(scanner_id) for scanner_id in pending.values_list("scanner_id", flat=True).distinct()]
if dry_run:
- return SemanticBackfillResult(observations=observations, assigned=0, named=0)
+ return SemanticBackfillResult(observations=pending.count(), assigned=0, named=0)
+ anchors = async_to_sync(anchor_vectors)(team_id)
+ observations = 0
assigned = 0
- for scanner_id, scanner_rows in by_scanner.items():
+ # One scanner at a time, so memory stays at one scanner's pending rows however large the team's window is.
+ for scanner_id in scanner_ids:
+ scanner_rows = [
+ _Pending(
+ observation_id=str(observation_id),
+ created_at=created_at,
+ needs_group=not (rank or {}).get("group"),
+ needs_scores=(rank or {}).get("scores") is None,
+ needs_similarities=not has_row,
+ )
+ for observation_id, created_at, rank, has_row in pending.filter(scanner_id=scanner_id)
+ .order_by("created_at")
+ .values_list("id", "created_at", "watch_rank", "has_similarities")
+ .iterator(chunk_size=2_000)
+ ]
+ observations += len(scanner_rows)
for start in range(0, len(scanner_rows), _VECTOR_CHUNK):
chunk = scanner_rows[start : start + _VECTOR_CHUNK]
- vectors = _stored_vectors(team_id, scanner_id, [observation_id for observation_id, _ in chunk], since)
- for observation_id, created_at in chunk:
- vector = vectors.get(observation_id)
+ vectors = _stored_vectors(team_id, scanner_id, [row.observation_id for row in chunk], since)
+ readings = {
+ row.observation_id: classify(vectors[row.observation_id], anchors)
+ for row in chunk
+ if row.observation_id in vectors and (row.needs_scores or row.needs_similarities)
+ }
+ save_anchor_similarities(
+ team_id,
+ {
+ row.observation_id: readings[row.observation_id].similarities
+ for row in chunk
+ if row.needs_similarities and row.observation_id in readings
+ },
+ )
+ for row in chunk:
+ vector = vectors.get(row.observation_id)
if vector is None:
continue
- assignment = assign_semantics(
- team_id=team_id,
- scanner_id=scanner_id,
- observation_id=observation_id,
- vector=vector.tolist(),
- now=created_at,
- )
- assigned += assignment is not None
+ if row.needs_scores:
+ merge_watch_rank(team_id, row.observation_id, {"scores": readings[row.observation_id].rank.scores})
+ if row.needs_group:
+ assignment = assign_semantics(
+ team_id=team_id,
+ scanner_id=scanner_id,
+ observation_id=row.observation_id,
+ vector=vector.tolist(),
+ now=row.created_at,
+ )
+ assigned += assignment is not None
named = 0
now = timezone.now()