:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import uuid
from datetime import datetime, timedelta
from typing import Any
from django.utils.timezone import now
from drf_spectacular.utils import extend_schema, extend_schema_serializer, extend_schema_view
from rest_framework import serializers, viewsets
from rest_framework.exceptions import NotFound, ValidationError
from rest_framework.request import Request
from rest_framework.response import Response
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.rate_limit import ReplayVisionClustersBurstRateThrottle, ReplayVisionClustersSustainedRateThrottle
from products.replay_vision.backend.api.filters import split_csv, validate_date_bound
from products.replay_vision.backend.experiment_variants import UNATTRIBUTED_VARIANT
from products.replay_vision.backend.scanner_access import readable_scanner_ids_in_scope
from products.replay_vision.backend.search import parse_date_bound
from products.replay_vision.backend.semantic_clusters import (
EXAMPLES_PER_CLUSTER,
ClusterKind,
ClusterOrder,
ClusterPage,
ClusterScope,
ClusterStatus,
ClusterWindow,
cluster_scanner_id,
summarize_clusters,
)
def _check_date_bound(value: str) -> None:
validate_date_bound(value)
DEFAULT_CLUSTER_WINDOW = timedelta(days=30)
# Clusters with no member for this long are pruned, so a longer window would undercount.
MAX_CLUSTER_WINDOW = timedelta(days=90)
# Counted live across every scanner, so a window past this reads too many rows for one request.
MAX_CROSS_SCANNER_WINDOW = timedelta(days=30)
MAX_CLUSTER_LIMIT = 100
class ClusterQuerySerializer(serializers.Serializer):
scanner_id = serializers.UUIDField(
required=False,
help_text="Only this scanner's clusters. Defaults to every scanner you can read.",
)
date_from = serializers.CharField(
required=False,
validators=[_check_date_bound],
help_text=(
"Count members from this time: ISO 8601 or relative like `-7d`. Defaults to `-30d`. "
"The window spans at most 90 days with `scanner_id`, and at most 30 days without it."
),
)
date_to = serializers.CharField(
required=False,
validators=[_check_date_bound],
help_text="Count members up to this time: ISO 8601, relative, or `now`. Defaults to now.",
)
class GroupQuerySerializer(ClusterQuerySerializer):
include_routine = serializers.BooleanField(
default=False,
help_text=(
"Also count routine observations, so groups describe ordinary behavior and not only issues. "
"Use it for experiment scanners, where the question is how each variant's users behave."
),
)
variant = serializers.CharField(
required=False,
help_text=(
"Experiment scanners only: count only members attributed to these variant keys (comma-separated). "
f"`{UNATTRIBUTED_VARIANT}` matches members with no attributed variant."
),
)
by_variant = serializers.BooleanField(
default=False,
help_text="Split each group's window counts by experiment variant in `variants`. Use it on experiment scanners.",
)
class _PageQueryFields(serializers.Serializer):
order = serializers.ChoiceField(
choices=ClusterOrder.choices,
default=ClusterOrder.PEOPLE.value,
help_text=(
"`people` (distinct people affected), `observations` (member count), `growth` (members gained over "
"the same-length window before `date_from`), or `recent` (newest member first)."
),
)
limit = serializers.IntegerField(default=20, min_value=1, max_value=MAX_CLUSTER_LIMIT, help_text="Page size.")
offset = serializers.IntegerField(default=0, min_value=0, help_text="Rows to skip, for the next page.")
class ClusterListQuerySerializer(ClusterQuerySerializer, _PageQueryFields):
pass
class GroupListQuerySerializer(GroupQuerySerializer, _PageQueryFields):
pass
class VariantCountSerializer(serializers.Serializer):
key = serializers.CharField(help_text=f"The variant key, or `{UNATTRIBUTED_VARIANT}` for members with none.")
observation_count = serializers.IntegerField(help_text="The group's members in this variant in the window.")
people_count = serializers.IntegerField(help_text="Distinct people among them.")
class ClusterSerializer(serializers.Serializer):
id = serializers.UUIDField(
help_text="The cluster id. Pass it as `theme` or `group` to `vision-scanners-observations-list` to read members."
)
scanner_id = serializers.UUIDField(help_text="The scanner whose observations the cluster groups.")
scanner_name = serializers.CharField(help_text="The scanner's name.")
name = serializers.CharField(
allow_null=True,
help_text=(
"A short model-written search phrase for a theme. Null for a group, and for a theme too small to be "
"named; `example_summary` stands in."
),
)
status = serializers.ChoiceField(
choices=ClusterStatus.choices,
help_text=(
"Where the cluster stands, from its members: `emerging` (first member this week), `active`, `fading` "
"(last week under half of the week before) or `dormant` (no member for two weeks)."
),
)
observation_count = serializers.IntegerField(
help_text="Readable observations in the window: issue observations, plus routine ones with `include_routine`."
)
people_count = serializers.IntegerField(
help_text="Distinct people among them; a session without an identified person counts as one."
)
previous_observation_count = serializers.IntegerField(
help_text="Members in the same-length window just before `date_from`, to compare the trend against."
)
first_seen = serializers.DateTimeField(help_text="When the window's oldest member was observed.")
last_seen = serializers.DateTimeField(help_text="When the window's newest member was observed.")
example_observation_ids = serializers.ListField(
child=serializers.UUIDField(),
help_text=f"Up to {EXAMPLES_PER_CLUSTER} of the window's highest-ranked members, best first.",
)
example_summary = serializers.CharField(
help_text="One-line summary of the best example; the readable label for a group, which has no name."
)
variants = VariantCountSerializer(
many=True,
help_text=(
"The window's members split by experiment variant, largest first. Empty unless `by_variant` is set. "
"Compare each variant's share of the group with its share of the scanner's sessions, since variants "
"are sampled at different rates."
),
)
@extend_schema_serializer(many=False)
class ClusterListSerializer(serializers.Serializer):
results = ClusterSerializer(many=True, help_text="Clusters with at least one member in the window.")
has_more = serializers.BooleanField(help_text="Whether another page follows at `offset + limit`.")
class _ClusterViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):
"""Read-only summaries of one kind of cluster over the scanners the caller may read."""
kind: ClusterKind
query_serializer: type[ClusterQuerySerializer] = ClusterQuerySerializer
list_query_serializer: type[ClusterQuerySerializer] = ClusterListQuerySerializer
scope_object = "replay_scanner"
required_scopes = ["replay_scanner:read", "session_recording:read"]
scope_object_read_actions = ["list", "retrieve"]
throttle_classes = [ReplayVisionClustersBurstRateThrottle, ReplayVisionClustersSustainedRateThrottle]
def _window(self, params: dict[str, Any], scanner_id: uuid.UUID | None, current: datetime) -> ClusterWindow:
timezone_info = self.team.timezone_info
date_to = parse_date_bound(params.get("date_to") or "now", timezone_info, end_of_range=True, now=current)
date_from = (
parse_date_bound(params["date_from"], timezone_info, end_of_range=False, now=current)
if params.get("date_from")
else date_to - DEFAULT_CLUSTER_WINDOW
)
if date_from >= date_to:
raise ValidationError({"date_from": "Must be before `date_to`."})
# Three calendar months can run to 92 days, so the cap leaves room for `-3m`.
if date_to - date_from > MAX_CLUSTER_WINDOW + timedelta(days=3):
raise ValidationError({"date_from": f"The window can span at most {MAX_CLUSTER_WINDOW.days} days."})
if scanner_id is None and date_to - date_from > MAX_CROSS_SCANNER_WINDOW + timedelta(days=1):
raise ValidationError(
{
"date_from": f"Without `scanner_id`, the window can span at most {MAX_CROSS_SCANNER_WINDOW.days} days."
}
)
return ClusterWindow(date_from=date_from, date_to=date_to)
def _summarize(
self, params: dict[str, Any], scanner_id: uuid.UUID | None, cluster_id: uuid.UUID | None = None
) -> ClusterPage:
current = now()
return summarize_clusters(
kind=self.kind,
team_id=self.team_id,
access=self.user_access_control,
scanner_ids=readable_scanner_ids_in_scope(self.user_access_control, self.team_id, scanner_id),
window=self._window(params, scanner_id, current),
order=ClusterOrder(params.get("order", ClusterOrder.PEOPLE)),
limit=params.get("limit", 1),
offset=params.get("offset", 0),
cluster_id=cluster_id,
scope=ClusterScope(
include_routine=params.get("include_routine", False),
variants=tuple(split_csv(params.get("variant") or "")),
),
by_variant=params.get("by_variant", False),
now=current,
)
def list(self, request: Request, *args: Any, **kwargs: Any) -> Response:
params = self.list_query_serializer(data=request.query_params)
params.is_valid(raise_exception=True)
page = self._summarize(params.validated_data, params.validated_data.get("scanner_id"))
return Response(ClusterListSerializer({"results": page.results, "has_more": page.has_more}).data)
def retrieve(self, request: Request, *args: Any, pk: str, **kwargs: Any) -> Response:
params = self.query_serializer(data=request.query_params)
params.is_valid(raise_exception=True)
try:
cluster_id = uuid.UUID(pk)
except ValueError:
raise NotFound()
# A cluster lives on one scanner, so reading only that scanner keeps a 90-day window cheap.
scanner_id = cluster_scanner_id(self.kind, self.team_id, cluster_id)
asked = params.validated_data.get("scanner_id")
if scanner_id is None or (asked is not None and asked != scanner_id):
raise NotFound()
page = self._summarize(params.validated_data, scanner_id, cluster_id=cluster_id)
# Not-found also covers a cluster with no readable member in the window.
if not page.results:
raise NotFound()
return Response(ClusterSerializer(page.results[0]).data)
@extend_schema_view(
list=extend_schema(
description=(
"Recurring kinds of issue in replay observations, each a broad cluster of one scanner's issue "
"observations with a short model-written name. Use it to see what comes up often and what is growing."
),
parameters=[ClusterListQuerySerializer],
responses={200: ClusterListSerializer},
),
retrieve=extend_schema(
description="One search theme's counts and examples over the window.",
parameters=[ClusterQuerySerializer],
responses={200: ClusterSerializer},
),
)
class VisionThemeViewSet(_ClusterViewSet):
kind = ClusterKind.THEME
@extend_schema_view(
list=extend_schema(
description=(
"Groups of near-duplicate issue observations, each one scanner's sessions that tell the same story, "
"such as one bug many people hit. Narrower than themes. Use it to find the issues that reach the most "
"people."
),
parameters=[GroupListQuerySerializer],
responses={200: ClusterListSerializer},
),
retrieve=extend_schema(
description="One group's counts and examples over the window.",
parameters=[GroupQuerySerializer],
responses={200: ClusterSerializer},
),
)
class VisionGroupViewSet(_ClusterViewSet):
kind = ClusterKind.GROUP
query_serializer = GroupQuerySerializer
list_query_serializer = GroupListQuerySerializer