:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): rank the error tracking backfill by user impact (#127455)
Georgiy Tarasov3f9b46f9be53
diff --git a/products/error_tracking/backend/facade/api.py b/products/error_tracking/backend/facade/api.pyindex cccd2eabd619..5ea1785f6d30 100644--- a/products/error_tracking/backend/facade/api.py+++ b/products/error_tracking/backend/facade/api.py@@ -145,6 +145,18 @@ def list_issues_created_since(team_id: int, since: datetime, limit: int) -> listreturn [_to_issue_preview(issue) for issue in issues]+def list_active_issue_impact(team_id: int, days: int, limit: int) -> list[contracts.ErrorTrackingIssueImpact]:+ """Active issues seen in the last `days` days with their user, session and daily occurrence counts.++ Ordered by first seen, newest first. Runs one ClickHouse query.+ """+ from ..logic.issue_impact import ( # noqa: PLC0415 - keeps the HogQL query runners off the django.setup() path+ list_active_issue_impact as _list_active_issue_impact,+ )++ return _list_active_issue_impact(team_id=team_id, days=days, limit=limit)++def query_new_error_issues(period_start: datetime, period_end: datetime) -> QuerySet:# "New this week" = issue first created within the digest window. Only active issues# (not archived/resolved/suppressed) are worth surfacing as a new production error.diff --git a/products/error_tracking/backend/facade/contracts.py b/products/error_tracking/backend/facade/contracts.pyindex 34da46522463..f7ee592a1a5d 100644--- a/products/error_tracking/backend/facade/contracts.py+++ b/products/error_tracking/backend/facade/contracts.py@@ -128,6 +128,22 @@ class ErrorTrackingFingerprint:created_at: datetime+@dataclass(frozen=True)+class ErrorTrackingIssueImpact:+ id: UUID+ name: str | None+ description: str | None+ severity: str | None+ is_assigned: bool+ source: str | None+ first_seen: datetime+ last_seen: datetime+ occurrences: int+ users: int+ sessions: int+ daily_occurrences: tuple[int, ...]++@dataclass(frozen=True)class ErrorTrackingIssuePreview:id: UUIDdiff --git a/products/error_tracking/backend/hogql_queries/test/test_error_tracking_query_runner.py b/products/error_tracking/backend/hogql_queries/test/test_error_tracking_query_runner.pyindex ee465fa54b9e..960565ffddd8 100644--- a/products/error_tracking/backend/hogql_queries/test/test_error_tracking_query_runner.py+++ b/products/error_tracking/backend/hogql_queries/test/test_error_tracking_query_runner.py@@ -844,6 +844,27 @@ class TestErrorTrackingQueryRunner(ClickhouseTestMixin, NonAtomicBaseTestKeepIderesults = self._calculate(status="all")["results"]self.assertEqual([r["id"] for r in results], [self.issue_id_three, self.issue_id_two, self.issue_id_one])+ @time_machine.travel("2020-01-10T12:12:00", tick=False)+ def test_list_active_issue_impact_lists_newest_active_issues(self):+ from products.error_tracking.backend.facade import api++ self.team.test_account_filters = []+ self.team.save(update_fields=["test_account_filters"])+ resolved_issue = ErrorTrackingIssue.objects.get(id=self.issue_id_three)+ resolved_issue.status = ErrorTrackingIssue.Status.RESOLVED+ resolved_issue.save(update_fields=["status"])+ sync_issues_to_clickhouse(issue_ids=[self.issue_id_three], team_id=self.team.pk)++ impacts = api.list_active_issue_impact(team_id=self.team.pk, days=30, limit=10)++ self.assertEqual(+ [(str(impact.id), impact.users, impact.occurrences) for impact in impacts],+ [(self.issue_id_two, 1, 1), (self.issue_id_one, 2, 2)],+ )+ self.assertEqual(len(impacts[1].daily_occurrences), 30)+ self.assertEqual(sum(impacts[1].daily_occurrences), 2)+ self.assertEqual(impacts[1].last_seen, datetime(2020, 1, 10, 9, 11, tzinfo=UTC))+@time_machine.travel("2022-01-10T12:11:00", tick=False)def test_expired_postgres_state_uses_clickhouse_state(self):ErrorTrackingIssue.objects.filter(id=self.issue_id_one).update(diff --git a/products/error_tracking/backend/logic/issue_impact.py b/products/error_tracking/backend/logic/issue_impact.pynew file mode 100644index 000000000000..9fbeb48fddcb--- /dev/null+++ b/products/error_tracking/backend/logic/issue_impact.py@@ -0,0 +1,54 @@+from datetime import datetime+from typing import Any+from uuid import UUID++from posthog.schema import DateRange, ErrorTrackingQuery++from posthog.models.team.team import Team++from products.error_tracking.backend.facade import contracts+from products.error_tracking.backend.facade.queries import run_error_tracking_query+++def list_active_issue_impact(team_id: int, days: int, limit: int) -> list[contracts.ErrorTrackingIssueImpact]:+ """Active issues seen in the last `days` days, newest first seen first."""+ team = Team.objects.get(id=team_id)+ query = ErrorTrackingQuery(+ kind="ErrorTrackingQuery",+ dateRange=DateRange(date_from=f"-{days}d"),+ status="active",+ filterTestAccounts=True,+ volumeResolution=days,+ limit=limit,+ orderBy="first_seen",+ orderDirection="DESC",+ withAggregations=True,+ withFirstEvent=False,+ withLastEvent=False,+ tags={"productKey": "error_tracking"},+ )+ data = run_error_tracking_query(team, query)+ results = data.get("results") or []+ return [_to_issue_impact(row) for row in results if row.get("first_seen") and row.get("last_seen")]+++def _to_issue_impact(row: dict[str, Any]) -> contracts.ErrorTrackingIssueImpact:+ aggregations = row.get("aggregations") or {}+ return contracts.ErrorTrackingIssueImpact(+ id=UUID(row["id"]),+ name=row.get("name"),+ description=row.get("description"),+ severity=row.get("severity"),+ is_assigned=row.get("assignee") is not None,+ source=row.get("source"),+ first_seen=_parse_datetime(row["first_seen"]),+ last_seen=_parse_datetime(row["last_seen"]),+ occurrences=int(aggregations.get("occurrences") or 0),+ users=int(aggregations.get("users") or 0),+ sessions=int(aggregations.get("sessions") or 0),+ daily_occurrences=tuple(int(count or 0) for count in aggregations.get("volumeRange") or []),+ )+++def _parse_datetime(value: str | datetime) -> datetime:+ return value if isinstance(value, datetime) else datetime.fromisoformat(value)diff --git a/products/signals/backend/facade/api.py b/products/signals/backend/facade/api.pyindex 658f4a2e1e37..2647d8b593da 100644--- a/products/signals/backend/facade/api.py+++ b/products/signals/backend/facade/api.py@@ -866,10 +866,14 @@ async def emit_signal(sync_with_source: bool = False,source_match_signal_id: str | None = None,source_match_report_id: str | None = None,-) -> None:+) -> bool:"""Emit a signal for grouping and potential report generation, fire-and-forget.+ Returns True when the signal was dispatched to the pipeline, and False when it was skipped: the+ organization has not approved AI data processing, the source is off, the signal is already+ finalized, the team's steering filtered it, or its idempotency key was already used.+Active path:emit_signal() -> SignalEmitterWorkflow -> BufferSignalsWorkflow -> TeamSignalGroupingV2Workflow@@ -930,7 +934,7 @@ async def emit_signal(await database_sync_to_async(complete_signal_pipeline)(team_id=team.id, signal_id=signal_id, billing_exempt=system_source_exempt(source_product))- return+ return Falseis_enabled = await database_sync_to_async(SignalSourceConfig.is_source_enabled, thread_sensitive=False)(team.id, source_product, source_type@@ -940,7 +944,7 @@ async def emit_signal(await database_sync_to_async(complete_signal_pipeline)(team_id=team.id, signal_id=signal_id, billing_exempt=system_source_exempt(source_product))- return+ return Falsedescription_tokens = _token_count(description)if description_tokens > MAX_SIGNAL_DESCRIPTION_TOKENS:@@ -1000,7 +1004,7 @@ async def emit_signal(team_id=team.id, source_product=source_product, source_type=source_type, idempotency_key=idempotency_key)if await database_sync_to_async(finalized_signal_ids)(team_id=team.id, signal_ids=[signal_id]):- return+ return False# Below the started event on purpose: a filtered signal then has a top-of-funnel event to be# counted against, so a steering drop reads apart from a dispatch failure rather than as one.@@ -1023,7 +1027,7 @@ async def emit_signal(await database_sync_to_async(complete_signal_pipeline, thread_sensitive=False)(team_id=team.id, signal_id=signal_id, billing_exempt=system_source_exempt(source_product))- return+ return Falseclient = await async_connect()@@ -1078,11 +1082,12 @@ async def emit_signal(# A repeated idempotency key is a deliberate dedupe, so label it rather than leave it to# read as a dispatch failure in started - filtered - emitted.capture_emission_event("signal_data_source_filtered", {"filter_reason": "duplicate_emission"})- return+ return False# Fire the analytics event only after the signal is definitively queued so# Temporal/connection failures don't inflate the "signals emitted" metric.capture_emission_event("signal_emitted")+ return Truedef forward_report_discussion_note(diff --git a/products/signals/backend/temporal/backfill_error_tracking.py b/products/signals/backend/temporal/backfill_error_tracking.pyindex d5cd9aabd5e0..79907202297e 100644--- a/products/signals/backend/temporal/backfill_error_tracking.py+++ b/products/signals/backend/temporal/backfill_error_tracking.py@@ -1,5 +1,6 @@-from dataclasses import dataclass-from datetime import timedelta+import math+from dataclasses import dataclass, replace+from datetime import datetime, timedeltafrom uuid import UUIDimport posthoganalytics@@ -9,34 +10,106 @@ from temporalio.common import RetryPolicyfrom posthog.temporal.common.scoped import scoped_temporalfrom posthog.temporal.common.utils import close_db_connections+from products.error_tracking.backend.facade.contracts import ErrorTrackingIssueImpact+BACKFILL_WINDOW_DAYS = 30BACKFILL_ISSUE_LIMIT = 5---@dataclass+# The pool holds the newest issues and ranking happens in Python, because a pool cut by users would+# fill with long-running issues that are older than the window, and the backfill would emit nothing.+CANDIDATE_POOL_SIZE = 200+CANDIDATE_LIMIT = 15+MAX_CANDIDATES_PER_SOURCE = 2+MIN_USERS = 2+CRASH_LOOP_OCCURRENCES = 100+SEVERITY_MULTIPLIERS = {"critical": 1.5, "high": 1.25}+TREND_DAYS = 7+++@dataclass(frozen=True)class BackfillErrorTrackingInput:team_id: int+ # Only workflow code that stops at BACKFILL_ISSUE_LIMIT asks for spare candidates. Workflow code+ # without that stop emits every candidate it gets, so the default returns no spares.+ candidate_limit: int = BACKFILL_ISSUE_LIMIT-@dataclass+@dataclass(frozen=True)class ErrorTrackingIssueData:issue_id: strname: strdescription: strfingerprint: str+ # Defaults keep histories from runs that started before these fields existed replayable.+ users: int = 0+ occurrences: int = 0-@dataclass+@dataclass(frozen=True)class EmitBackfillSignalInput:team_id: intissue: ErrorTrackingIssueData+def impact_score(issue: ErrorTrackingIssueImpact, now: datetime) -> float:+ """How much fixing this issue matters to the team's users. Only the order of scores is meaningful."""+ # Users count three times as much as occurrences, because one user who hits an error in a+ # loop can produce thousands of occurrences.+ score = 3 * math.log1p(issue.users) + math.log1p(issue.occurrences)+ score *= _recency_multiplier(now - issue.last_seen)+ score *= _trend_multiplier(issue.daily_occurrences)+ score *= SEVERITY_MULTIPLIERS.get(issue.severity or "", 1.0)+ if issue.is_assigned:+ score *= 0.5+ if issue.users <= 1 and issue.occurrences >= CRASH_LOOP_OCCURRENCES:+ score *= 0.5+ return score+++def _recency_multiplier(since_last_seen: timedelta) -> float:+ if since_last_seen <= timedelta(days=3):+ return 1.0+ if since_last_seen <= timedelta(days=7):+ return 0.6+ return 0.3+++def _trend_multiplier(daily_occurrences: tuple[int, ...]) -> float:+ recent = sum(daily_occurrences[-TREND_DAYS:])+ prior = sum(daily_occurrences[-2 * TREND_DAYS : -TREND_DAYS])+ return 1.25 if recent > 2 * prior else 1.0+++def rank_backfill_candidates(+ issues: list[ErrorTrackingIssueImpact], now: datetime, limit: int = CANDIDATE_LIMIT+) -> list[ErrorTrackingIssueImpact]:+ """Issues first seen inside the window that are worth a report, highest impact first."""+ window_start = now - timedelta(days=BACKFILL_WINDOW_DAYS)+ eligible = [+ issue+ for issue in issues+ if issue.first_seen >= window_start+ and (issue.users >= MIN_USERS or (issue.severity or "") in SEVERITY_MULTIPLIERS)+ ]+ eligible.sort(key=lambda issue: (impact_score(issue, now), issue.users), reverse=True)++ ranked: list[ErrorTrackingIssueImpact] = []+ per_source: dict[str, int] = {}+ for issue in eligible:+ if issue.source:+ if per_source.get(issue.source, 0) >= MAX_CANDIDATES_PER_SOURCE:+ continue+ per_source[issue.source] = per_source.get(issue.source, 0) + 1+ ranked.append(issue)+ if len(ranked) >= limit:+ break+ return ranked++@activity.defn@scoped_temporal()@close_db_connectionsasync def fetch_error_tracking_issues_activity(input: BackfillErrorTrackingInput) -> list[ErrorTrackingIssueData]:- """Fetch the most recently created error tracking issues from the last 30 days."""+ """Fetch new error tracking issues from the last 30 days, ranked by impact on the team's users."""from django.utils import timezonefrom posthog.sync import database_sync_to_async@@ -44,33 +117,33 @@ async def fetch_error_tracking_issues_activity(input: BackfillErrorTrackingInputfrom products.error_tracking.backend.facade import api as error_tracking_apidef _fetch() -> list[ErrorTrackingIssueData]:- # Issue metadata and fingerprints both live in Postgres, so the backfill reads them- # directly instead of scanning events. The previous events scan also misattributed- # old issues with recent occurrences as newly created, and missed imported issues- # whose event timestamps predate the window.- previews = error_tracking_api.list_issues_created_since(- team_id=input.team_id,- since=timezone.now() - timedelta(days=BACKFILL_WINDOW_DAYS),- limit=BACKFILL_ISSUE_LIMIT,+ candidates = rank_backfill_candidates(+ error_tracking_api.list_active_issue_impact(+ team_id=input.team_id, days=BACKFILL_WINDOW_DAYS, limit=CANDIDATE_POOL_SIZE+ ),+ now=timezone.now(),+ limit=input.candidate_limit,)- if not previews:+ if not candidates:return []first_fingerprints: dict[UUID, str] = {fingerprint.issue_id: fingerprint.fingerprintfor fingerprint in error_tracking_api.list_first_fingerprints(- team_id=input.team_id, issue_ids=[preview.id for preview in previews]+ team_id=input.team_id, issue_ids=[candidate.id for candidate in candidates])}return [ErrorTrackingIssueData(- issue_id=str(preview.id),- name=preview.name or "Unknown",- description=preview.description or "",- fingerprint=first_fingerprints.get(preview.id, ""),+ issue_id=str(candidate.id),+ name=candidate.name or "Unknown",+ description=candidate.description or "",+ fingerprint=first_fingerprints.get(candidate.id, ""),+ users=candidate.users,+ occurrences=candidate.occurrences,)- for preview in previews+ for candidate in candidates]return await database_sync_to_async(_fetch)()@@ -79,8 +152,10 @@ async def fetch_error_tracking_issues_activity(input: BackfillErrorTrackingInput@activity.defn@scoped_temporal()@close_db_connections-async def emit_backfill_signal_activity(input: EmitBackfillSignalInput) -> None:- """Emit an issue_created signal for a single error tracking issue."""+# `None` stays in the hint because histories and workers from before the boolean result return `None`,+# and a `bool` hint makes Temporal fail to decode it on every replay.+async def emit_backfill_signal_activity(input: EmitBackfillSignalInput) -> bool | None:+ """Emit an issue_created signal for a single error tracking issue. Returns whether it was dispatched."""from posthog.models import Teamfrom products.signals.backend.facade.api import emit_signal@@ -91,8 +166,13 @@ async def emit_backfill_signal_activity(input: EmitBackfillSignalInput) -> None:f"New error tracking issue created - this particular exception was observed for the first time:\n"f"{input.issue.name}: {input.issue.description}\n")+ if input.issue.users or input.issue.occurrences:+ description += (+ f"Affected {input.issue.users} users across {input.issue.occurrences} occurrences "+ f"in the last {BACKFILL_WINDOW_DAYS} days.\n"+ )- await emit_signal(+ return await emit_signal(team=team,source_product="error_tracking",source_type="issue_created",@@ -117,17 +197,24 @@ class BackfillErrorTrackingWorkflow:await self._run_impl(input)async def _run_impl(self, input: BackfillErrorTrackingInput) -> None:+ stops_at_limit = workflow.patched("backfill-error-tracking-ranked-2026-11")issues = await workflow.execute_activity(fetch_error_tracking_issues_activity,- input,+ replace(input, candidate_limit=CANDIDATE_LIMIT) if stops_at_limit else input,start_to_close_timeout=timedelta(minutes=5),retry_policy=RetryPolicy(maximum_attempts=3),)+ emitted = 0for issue in issues:- await workflow.execute_activity(+ if stops_at_limit and emitted >= BACKFILL_ISSUE_LIMIT:+ break+ dispatched = await workflow.execute_activity(emit_backfill_signal_activity,EmitBackfillSignalInput(team_id=input.team_id, issue=issue),start_to_close_timeout=timedelta(minutes=2),retry_policy=RetryPolicy(maximum_attempts=3),)+ # `None` comes from an old worker, which emitted the signal, so it counts toward the limit.+ if dispatched is not False:+ emitted += 1diff --git a/products/signals/backend/test/test_backfill_error_tracking.py b/products/signals/backend/test/test_backfill_error_tracking.pynew file mode 100644index 000000000000..397690bf6894--- /dev/null+++ b/products/signals/backend/test/test_backfill_error_tracking.py@@ -0,0 +1,141 @@+import uuid+from datetime import UTC, datetime, timedelta++import pytest+from unittest.mock import AsyncMock, MagicMock, patch++from temporalio import activity+from temporalio.testing import ActivityEnvironment, WorkflowEnvironment+from temporalio.worker import UnsandboxedWorkflowRunner, Worker++from products.error_tracking.backend.facade.contracts import ErrorTrackingIssueImpact+from products.signals.backend.facade.api import validate_signal_input+from products.signals.backend.temporal.backfill_error_tracking import (+ BACKFILL_ISSUE_LIMIT,+ CANDIDATE_LIMIT,+ BackfillErrorTrackingInput,+ BackfillErrorTrackingWorkflow,+ EmitBackfillSignalInput,+ ErrorTrackingIssueData,+ emit_backfill_signal_activity,+ rank_backfill_candidates,+)++NOW = datetime(2026, 6, 30, tzinfo=UTC)+++def _issue(+ name: str,+ *,+ users: int = 10,+ occurrences: int = 50,+ first_seen_days_ago: int = 5,+ last_seen_days_ago: int = 0,+ severity: str | None = None,+ is_assigned: bool = False,+ source: str | None = None,+) -> ErrorTrackingIssueImpact:+ return ErrorTrackingIssueImpact(+ id=uuid.uuid4(),+ name=name,+ description=None,+ severity=severity,+ is_assigned=is_assigned,+ source=source,+ first_seen=NOW - timedelta(days=first_seen_days_ago),+ last_seen=NOW - timedelta(days=last_seen_days_ago),+ occurrences=occurrences,+ users=users,+ sessions=users,+ daily_occurrences=(),+ )+++def test_rank_orders_by_impact_and_drops_ineligible_issues():+ issues = [+ _issue("newest_but_tiny", users=2, occurrences=3, first_seen_days_ago=0),+ _issue("broad_and_ongoing", users=500, occurrences=2000, first_seen_days_ago=20),+ _issue("broad_but_stale", users=500, occurrences=2000, first_seen_days_ago=20, last_seen_days_ago=15),+ _issue("broad_but_assigned", users=500, occurrences=2000, is_assigned=True),+ _issue("single_user_crash_loop", users=1, occurrences=5000),+ _issue("single_user_critical", users=1, occurrences=4, severity="critical"),+ _issue("older_than_window", users=900, occurrences=9000, first_seen_days_ago=60),+ ]++ ranked = [issue.name for issue in rank_backfill_candidates(issues, NOW)]++ assert ranked == [+ "broad_and_ongoing",+ "broad_but_assigned",+ "broad_but_stale",+ "single_user_critical",+ "newest_but_tiny",+ ]+++def test_rank_caps_issues_sharing_a_source_file():+ issues = [_issue(f"same_file_{i}", users=100 - i, source="app/checkout.ts") for i in range(4)]+ issues.append(_issue("other_file", users=5, source="app/search.ts"))++ ranked = [issue.name for issue in rank_backfill_candidates(issues, NOW)]++ assert ranked == ["same_file_0", "same_file_1", "other_file"]++[email protected]("dispatched_result", [True, None])+async def test_workflow_replaces_steered_out_candidates_and_stops_at_limit(dispatched_result: bool | None):+ steered_out = {"issue_0", "issue_1"}+ emitted: list[str] = []++ @activity.defn(name="fetch_error_tracking_issues_activity")+ async def fake_fetch(input: BackfillErrorTrackingInput) -> list[ErrorTrackingIssueData]:+ return [+ ErrorTrackingIssueData(issue_id=f"issue_{i}", name="Error", description="", fingerprint="")+ for i in range(input.candidate_limit)+ ]++ @activity.defn(name="emit_backfill_signal_activity")+ async def fake_emit(input: EmitBackfillSignalInput) -> bool | None:+ emitted.append(input.issue.issue_id)+ return False if input.issue.issue_id in steered_out else dispatched_result++ async with await WorkflowEnvironment.start_time_skipping() as env:+ async with Worker(+ env.client,+ task_queue="backfill-error-tracking-test",+ workflows=[BackfillErrorTrackingWorkflow],+ activities=[fake_fetch, fake_emit],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ await env.client.execute_workflow(+ BackfillErrorTrackingWorkflow.run,+ BackfillErrorTrackingInput(team_id=1),+ id=f"backfill-error-tracking-{uuid.uuid4()}",+ task_queue="backfill-error-tracking-test",+ )++ assert CANDIDATE_LIMIT > BACKFILL_ISSUE_LIMIT + len(steered_out)+ assert emitted == [f"issue_{i}" for i in range(BACKFILL_ISSUE_LIMIT + len(steered_out))]+++async def test_emitted_signal_matches_the_error_tracking_contract():+ issue = ErrorTrackingIssueData(+ issue_id=str(uuid.uuid4()),+ name="TypeError",+ description="x is undefined",+ fingerprint="fp",+ users=3,+ occurrences=9,+ )+ emit = AsyncMock(return_value=True)++ with (+ patch("posthog.models.Team.objects.aget", AsyncMock(return_value=MagicMock())),+ patch("products.signals.backend.facade.api.emit_signal", emit),+ ):+ await ActivityEnvironment().run(emit_backfill_signal_activity, EmitBackfillSignalInput(team_id=1, issue=issue))++ kwargs = {key: value for key, value in emit.call_args.kwargs.items() if key != "team"}+ validate_signal_input(**kwargs, remediation=None)