:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(web-analytics): bound the traffic achievement's first pageview scan (#119731)
Jordan Mryyandf4c58eb23c7
diff --git a/products/web_analytics/backend/achievements/evaluators.py b/products/web_analytics/backend/achievements/evaluators.pyindex 38c9b65de054..ab6f26ed432b 100644--- a/products/web_analytics/backend/achievements/evaluators.py+++ b/products/web_analytics/backend/achievements/evaluators.py@@ -1,8 +1,9 @@import jsonimport hashlibfrom collections.abc import Callable-from dataclasses import dataclass+from dataclasses import dataclass, replacefrom datetime import UTC, date, datetime, time, timedelta+from time import monotonicfrom django.utils import timezone@@ -20,7 +21,7 @@ from posthog.models.team.team import Teamfrom posthog.models.user import Userfrom products.actions.backend.models.action import Action-from products.web_analytics.backend.achievements.definitions import STREAK_ARM_WEEKLY+from products.web_analytics.backend.achievements.definitions import STREAK_ARM_WEEKLY, TRACKS, TrackKeyfrom products.web_analytics.backend.achievements.query_concurrency import achievement_query_scopefrom products.web_analytics.backend.hogql_queries.web_lazy_precompute_common import test_account_filter_exprfrom products.web_analytics.backend.models import WebAnalyticsInteraction, WebAnalyticsVisit@@ -164,41 +165,147 @@ def _ingestion_window_expr(return ast.And(exprs=exprs)+ACHIEVEMENT_SCAN_LIMITS = (+ CHQueryErrorTooManyBytes,+ ClickHouseQueryTimeOut,+ ClickHouseEstimatedQueryExecutionTimeTooLong,+)+ACHIEVEMENT_SCAN_SETTINGS = HogQLGlobalSettings(timeout_overflow_mode="throw", read_overflow_mode="throw")+ACHIEVEMENT_MAX_EMPTY_CHUNKS = 20+TRAFFIC_TARGET_PAGEVIEWS = TRACKS[TrackKey.TRAFFIC].stages[-1].threshold+TRAFFIC_BOOTSTRAP_CHUNK_HOURS = 24+TRAFFIC_BOOTSTRAP_TARGET_ROWS = 50_000_000+TRAFFIC_BOOTSTRAP_MAX_CHUNK_HOURS = 24 * 365 * 60+TRAFFIC_BOOTSTRAP_TIME_BUDGET_SECONDS = 30+TRAFFIC_MIN_TIMESTAMP = datetime(1970, 1, 1, tzinfo=UTC)+++@frozen+class TrafficBootstrap:+ created_until: datetime+ counted: int = 0+ before: datetime | None = None+ chunk_hours: int = TRAFFIC_BOOTSTRAP_CHUNK_HOURS++ @property+ def done(self) -> bool:+ return self.counted >= TRAFFIC_TARGET_PAGEVIEWS or self.before == TRAFFIC_MIN_TIMESTAMP++ @classmethod+ def from_checkpoint(cls, checkpoint: dict[str, object]) -> "TrafficBootstrap | None":+ bootstrap = checkpoint.get("bootstrap")+ if not isinstance(bootstrap, dict):+ return None+ created_until = parse_zoned_datetime_string(bootstrap.get("created_until"))+ before = parse_zoned_datetime_string(bootstrap.get("before"))+ counted = bootstrap.get("counted")+ chunk_hours = bootstrap.get("chunk_hours")+ if (+ created_until is None+ or (bootstrap.get("before") is not None and before is None)+ or not isinstance(counted, int)+ or counted < 0+ or not isinstance(chunk_hours, int)+ or not 1 <= chunk_hours <= TRAFFIC_BOOTSTRAP_MAX_CHUNK_HOURS+ ):+ return None+ return cls(created_until=created_until, counted=counted, before=before, chunk_hours=chunk_hours)++ def to_checkpoint(self) -> dict[str, object]:+ if self.done:+ return {"counted_through": self.created_until.isoformat()}+ return {+ "bootstrap": {+ "created_until": self.created_until.isoformat(),+ "counted": self.counted,+ "before": self.before.isoformat() if self.before is not None else None,+ "chunk_hours": self.chunk_hours,+ }+ }+++def _count_pageviews(+ teams: list[Team],+ since: datetime | None,+ until: datetime,+ earliest_timestamp: datetime | None,+ latest_timestamp: datetime | None = None,+) -> int:+ total = 0+ for team in teams:+ query = parse_select(+ "SELECT count() FROM events WHERE and(event IN ('$pageview', '$screen'), {window}, {test})",+ placeholders={+ "window": _ingestion_window_expr(since, until, earliest_timestamp, latest_timestamp),+ "test": _test_account_filter_expr(team),+ },+ )+ response = execute_hogql_query(+ query=query,+ team=team,+ query_type="web_achievements_pageviews",+ # The team-scoped sweep has no acting user, and only an aggregate count leaves the query.+ bypass_warehouse_access_control=True,+ settings=ACHIEVEMENT_SCAN_SETTINGS,+ )+ if response.results:+ total += int(response.results[0][0] or 0)+ return total+++def _next_traffic_chunk_hours(chunk_hours: int, rows: int) -> int:+ if rows == 0:+ return min(chunk_hours * 2, TRAFFIC_BOOTSTRAP_MAX_CHUNK_HOURS)+ return max(1, min(chunk_hours * TRAFFIC_BOOTSTRAP_TARGET_ROWS // rows, TRAFFIC_BOOTSTRAP_MAX_CHUNK_HOURS))+++def _advance_traffic_bootstrap(teams: list[Team], bootstrap: TrafficBootstrap) -> TrafficBootstrap:+ deadline = monotonic() + TRAFFIC_BOOTSTRAP_TIME_BUDGET_SECONDS+ for _ in range(ACHIEVEMENT_MAX_EMPTY_CHUNKS):+ upper = bootstrap.before or bootstrap.created_until+ lower = max(TRAFFIC_MIN_TIMESTAMP, upper - timedelta(hours=bootstrap.chunk_hours))+ try:+ rows = _count_pageviews(+ teams,+ since=None,+ until=bootstrap.created_until,+ earliest_timestamp=None if lower == TRAFFIC_MIN_TIMESTAMP else lower,+ latest_timestamp=bootstrap.before,+ )+ except ACHIEVEMENT_SCAN_LIMITS:+ if bootstrap.chunk_hours <= 1:+ raise+ return replace(bootstrap, chunk_hours=max(1, bootstrap.chunk_hours // 2))+ bootstrap = replace(+ bootstrap,+ counted=bootstrap.counted + rows,+ before=lower,+ chunk_hours=_next_traffic_chunk_hours(bootstrap.chunk_hours, rows),+ )+ if bootstrap.done or rows or monotonic() >= deadline:+ break+ return bootstrap++def evaluate_cumulative_pageviews(ctx: EvalContext, prior: PriorProgress) -> TrackEvaluation:until = timezone.now() - INGESTION_LAG+ bootstrap = TrafficBootstrap.from_checkpoint(prior.checkpoint)since = parse_zoned_datetime_string(prior.checkpoint.get("counted_through")) or prior.last_computed_at- earliest_timestamp = since - LATE_ARRIVAL_LOOKBACK if since is not None else None- total = prior.value if since is not None else 0+ teams = _project_environment_teams(ctx.team)+ if bootstrap is None and since is not None:+ with achievement_query_scope(ctx.team.id):+ added = _count_pageviews(teams, since, until, since - LATE_ARRIVAL_LOOKBACK)+ return TrackEvaluation(value=prior.value + added, checkpoint={"counted_through": until.isoformat()})+with achievement_query_scope(ctx.team.id):- for team in _project_environment_teams(ctx.team):- query = parse_select(- "SELECT count() FROM events WHERE and(event IN ('$pageview', '$screen'), {window}, {test})",- placeholders={- "window": _ingestion_window_expr(since, until, earliest_timestamp),- "test": _test_account_filter_expr(team),- },- )- response = execute_hogql_query(- query=query,- team=team,- query_type="web_achievements_pageviews",- # The team-scoped sweep has no acting user, and only an aggregate count leaves the query.- bypass_warehouse_access_control=True,- )- if response.results:- total += int(response.results[0][0] or 0)- return TrackEvaluation(value=total, checkpoint={"counted_through": until.isoformat()})+ bootstrap = _advance_traffic_bootstrap(teams, bootstrap or TrafficBootstrap(created_until=until))+ return TrackEvaluation(value=bootstrap.counted, checkpoint=bootstrap.to_checkpoint(), complete=bootstrap.done)CONVERSIONS_LOOKBACK_DAYS = 90CONVERSIONS_BOOTSTRAP_CHUNK_DAYS = 7CONVERSIONS_MAX_TIMESTAMP = datetime(2299, 12, 31, 23, 59, 59, 999999, tzinfo=UTC)CONVERSIONS_FUTURE_CHUNK_MAX_HOURS = 24 * 365 * 50-CONVERSIONS_SCAN_LIMITS = (- CHQueryErrorTooManyBytes,- ClickHouseQueryTimeOut,- ClickHouseEstimatedQueryExecutionTimeTooLong,-)@frozen@@ -318,7 +425,7 @@ def _add_conversion_counts(query_type="web_achievements_conversions",# The team-scoped sweep has no acting user, and only per-day counts leave the query.bypass_warehouse_access_control=True,- settings=HogQLGlobalSettings(timeout_overflow_mode="throw", read_overflow_mode="throw"),+ settings=ACHIEVEMENT_SCAN_SETTINGS,)for row in response.results or []:day_counts = chunk_daily.setdefault(row[0].isoformat(), [0] * len(actions))@@ -343,7 +450,7 @@ def _advance_conversion_future(chunk_hours = bootstrap.chunk_hours# Empty timestamp ranges are cheap to skip with the event timestamp index. Grow those# ranges so a scan with no future-dated events does not take hundreds of daily sweeps.- for _ in range(20):+ for _ in range(ACHIEVEMENT_MAX_EMPTY_CHUNKS):chunk_end = min(next_start + timedelta(hours=chunk_hours), CONVERSIONS_MAX_TIMESTAMP)try:has_rows = _add_conversion_counts(@@ -356,7 +463,7 @@ def _advance_conversion_future(chunk_end,latest_inclusive=chunk_end == CONVERSIONS_MAX_TIMESTAMP,)- except CONVERSIONS_SCAN_LIMITS:+ except ACHIEVEMENT_SCAN_LIMITS:if chunk_hours <= 1:raisereturn ConversionBootstrap(@@ -402,7 +509,7 @@ def _advance_conversion_bootstrap(if bootstrap.phase == "tail":try:_add_conversion_counts(ctx, actions, daily, bootstrap.created_since, until, window_start)- except CONVERSIONS_SCAN_LIMITS:+ except ACHIEVEMENT_SCAN_LIMITS:return ConversionBootstrap(next_start=window_start,end=bootstrap.end,@@ -446,7 +553,7 @@ def _advance_conversion_bootstrap(next_start,chunk_end,)- except CONVERSIONS_SCAN_LIMITS:+ except ACHIEVEMENT_SCAN_LIMITS:if chunk_hours <= 1:raisereturn ConversionBootstrap(@@ -527,7 +634,7 @@ def evaluate_conversions(ctx: EvalContext, prior: PriorProgress) -> TrackEvaluatearliest_timestamp = max(window_start, since - LATE_ARRIVAL_LOOKBACK) if since is not None else window_starttry:_add_conversion_counts(ctx, actions, daily, since, until, earliest_timestamp)- except CONVERSIONS_SCAN_LIMITS:+ except ACHIEVEMENT_SCAN_LIMITS:if since is not None:raisedaily = {}diff --git a/products/web_analytics/backend/achievements/tasks.py b/products/web_analytics/backend/achievements/tasks.pyindex a8dfa4718c43..999bc2598d7a 100644--- a/products/web_analytics/backend/achievements/tasks.py+++ b/products/web_analytics/backend/achievements/tasks.py@@ -55,6 +55,8 @@ ACHIEVEMENTS_FLAG = "web-analytics-achievements"SWEEP_ACTIVE_WINDOW_DAYS = 7RECOMPUTE_INTERVAL = timedelta(hours=20)RECOMPUTE_EXPIRES_SECONDS = 5 * 60+RETRY_BACKOFF_BASE = timedelta(minutes=15)+RETRY_BACKOFF_MAX = timedelta(hours=24)TRANSIENT_ACHIEVEMENT_ERRORS = (ConcurrencyLimitExceeded, RedisError, *CH_TRANSIENT_ERRORS)TEAM_QUERY_TRACK_KEYS = [str(track.key) for track in TRACKS.values() if track.evaluator_key in INCREMENTAL_EVALUATORS]@@ -185,6 +187,8 @@ def persist_progress(if bump_last_computed_at:progress.last_computed_at = timezone.now()progress.state = state+ progress.failed_attempts = 0+ progress.retry_after = Noneprogress.save()@@ -216,14 +220,45 @@ def evaluate_track(return incremental_evaluator(ctx, prior)+def _retry_backoff(failed_attempts: int) -> timedelta:+ return min(RETRY_BACKOFF_BASE * 2 ** min(failed_attempts - 1, 10), RETRY_BACKOFF_MAX)+++def _due_q(now: datetime) -> Q:+ return (Q(last_computed_at__isnull=True) | Q(last_computed_at__lte=now - RECOMPUTE_INTERVAL)) & (+ Q(retry_after__isnull=True) | Q(retry_after__lte=now)+ )+++def _claim_attempt(progress: WebAnalyticsAchievementProgress) -> bool:+ now = timezone.now()+ failed_attempts = progress.failed_attempts + 1+ return bool(+ WebAnalyticsAchievementProgress.objects.for_team(progress.team_id, canonical=True)+ .filter(_due_q(now), pk=progress.pk, failed_attempts=progress.failed_attempts)+ .update(failed_attempts=failed_attempts, retry_after=now + _retry_backoff(failed_attempts))+ )+++def _release_attempt(progress: WebAnalyticsAchievementProgress) -> None:+ WebAnalyticsAchievementProgress.objects.for_team(progress.team_id, canonical=True).filter(pk=progress.pk).update(+ failed_attempts=progress.failed_attempts, retry_after=progress.retry_after+ )++def _recompute_track(ctx: EvalContext, track: TrackDefinition) -> None:progress = get_or_create_progress(ctx, track)if progress.current_stage >= len(track.stages):return- if track.evaluator_key in INCREMENTAL_EVALUATORS and not is_due(progress):+ incremental = track.evaluator_key in INCREMENTAL_EVALUATORS+ if incremental and (not is_due(progress) or not _claim_attempt(progress)):returntry:evaluation = evaluate_track(ctx, track, progress)+ except ConcurrencyLimitExceeded:+ if incremental:+ _release_attempt(progress)+ raiseexcept TRANSIENT_ACHIEVEMENT_ERRORS:raiseexcept Exception as e:@@ -330,7 +365,8 @@ def _send_unlock_notification(ctx: EvalContext, track: TrackDefinition, stage: idef due_team_ids(limit: int) -> list[int]:- active_since = timezone.now().date() - timedelta(days=SWEEP_ACTIVE_WINDOW_DAYS)+ now = timezone.now()+ active_since = now.date() - timedelta(days=SWEEP_ACTIVE_WINDOW_DAYS)active_team_ids = (WebAnalyticsVisit.objects.unscoped().filter(visit_date__gte=active_since).values("team_id").distinct())@@ -342,7 +378,7 @@ def due_team_ids(limit: int) -> list[int]:current_stage__lt=STAGE_COUNT,team_id__in=active_team_ids,)- .filter(Q(last_computed_at__isnull=True) | Q(last_computed_at__lte=timezone.now() - RECOMPUTE_INTERVAL))+ .filter(_due_q(now)).values("team_id").annotate(oldest_computed_at=Min("last_computed_at")).order_by(F("oldest_computed_at").asc(nulls_first=True), "team_id")diff --git a/products/web_analytics/backend/migrations/0017_webanalyticsachievementprogress_retry_backoff.py b/products/web_analytics/backend/migrations/0017_webanalyticsachievementprogress_retry_backoff.pynew file mode 100644index 000000000000..bbad0697ab61--- /dev/null+++ b/products/web_analytics/backend/migrations/0017_webanalyticsachievementprogress_retry_backoff.py@@ -0,0 +1,22 @@+# Generated by Django 5.2.18 on 2026-10-19 15:56++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("web_analytics", "0016_savedheatmap_last_viewed_idx"),+ ]++ operations = [+ migrations.AddField(+ model_name="webanalyticsachievementprogress",+ name="failed_attempts",+ field=models.PositiveSmallIntegerField(db_default=0, default=0),+ ),+ migrations.AddField(+ model_name="webanalyticsachievementprogress",+ name="retry_after",+ field=models.DateTimeField(blank=True, null=True),+ ),+ ]diff --git a/products/web_analytics/backend/migrations/max_migration.txt b/products/web_analytics/backend/migrations/max_migration.txtindex d8bfa7bbe71a..096e6c656cc8 100644--- a/products/web_analytics/backend/migrations/max_migration.txt+++ b/products/web_analytics/backend/migrations/max_migration.txt@@ -1 +1 @@-0016_savedheatmap_last_viewed_idx+0017_webanalyticsachievementprogress_retry_backoffdiff --git a/products/web_analytics/backend/models/web_analytics_achievement_progress.py b/products/web_analytics/backend/models/web_analytics_achievement_progress.pyindex ab25724e89d4..6e6fadec0c31 100644--- a/products/web_analytics/backend/models/web_analytics_achievement_progress.py+++ b/products/web_analytics/backend/models/web_analytics_achievement_progress.py@@ -18,6 +18,8 @@ class WebAnalyticsAchievementProgress(TeamScopedRootMixin, UUIDModel):progress_value = models.BigIntegerField(default=0)state = models.JSONField(default=dict)last_computed_at = models.DateTimeField(null=True, blank=True)+ failed_attempts = models.PositiveSmallIntegerField(default=0, db_default=0)+ retry_after = models.DateTimeField(null=True, blank=True)updated_at = models.DateTimeField(auto_now=True)class Meta:diff --git a/products/web_analytics/backend/test/test_achievements_evaluators.py b/products/web_analytics/backend/test/test_achievements_evaluators.pyindex 3215cf8e98ba..ef5fd5948de0 100644--- a/products/web_analytics/backend/test/test_achievements_evaluators.py+++ b/products/web_analytics/backend/test/test_achievements_evaluators.py@@ -19,6 +19,7 @@ from posthog.test.warehouse_access import WAREHOUSE_ACCESS_CONTROL_FLAG, filter_from products.actions.backend.models.action import Actionfrom products.web_analytics.backend.achievements.definitions import STREAK_ARM_DAILY, STREAK_ARM_WEEKLYfrom products.web_analytics.backend.achievements.evaluators import (+ INGESTION_LAG,EvalContext,PriorProgress,TrackEvaluation,@@ -147,6 +148,68 @@ class TestTeamEvaluators(ClickhouseTestMixin, APIBaseTest):assert evaluation.checkpoint is not Noneself.assertGreater(datetime.fromisoformat(str(evaluation.checkpoint["counted_through"])), watermark)+ @parameterized.expand(+ [+ ("walks_back_after_a_scan_limit", 100_000_000, True, 4, True),+ ("stops_at_the_top_stage", 2, False, 2, False),+ ]+ )+ def test_cumulative_pageviews_bootstraps_history_in_bounded_chunks(+ self, _name: str, target: int, fail_first_chunk: bool, expected: int, resumes: bool+ ) -> None:+ first_now = timezone.now()+ for timestamp in (+ first_now + timedelta(days=1),+ first_now - timedelta(hours=2),+ first_now - timedelta(days=5),+ first_now - timedelta(days=400),+ ):+ _create_event(+ team=self.team,+ event="$pageview",+ distinct_id="d1",+ timestamp=timestamp,+ created_at=first_now - timedelta(hours=2),+ )+ flush_persons_and_events()++ attempts = 0++ def fail_first_attempt(*args: object, **kwargs: object):+ nonlocal attempts+ attempts += 1+ if fail_first_chunk and attempts == 1:+ raise CHQueryErrorTooManyBytes("read limit", code=307)+ return execute_hogql_query(*args, **kwargs)++ evaluations: list[TrackEvaluation] = []+ prior = EMPTY_PRIOR+ with (+ patch("products.web_analytics.backend.achievements.evaluators.TRAFFIC_TARGET_PAGEVIEWS", target),+ patch(+ "products.web_analytics.backend.achievements.evaluators.execute_hogql_query",+ side_effect=fail_first_attempt,+ ),+ ):+ for sweep in range(20):+ with patch(+ "products.web_analytics.backend.achievements.evaluators.timezone.now",+ return_value=first_now + timedelta(minutes=5 * sweep),+ ):+ evaluation = evaluate_cumulative_pageviews(self._ctx(), prior)+ evaluations.append(evaluation)+ if evaluation.complete:+ break+ assert evaluation.checkpoint is not None+ prior = PriorProgress(value=0, last_computed_at=None, checkpoint=evaluation.checkpoint)++ final = evaluations[-1]+ self.assertTrue(final.complete)+ self.assertEqual(final.value, expected)+ self.assertEqual(len(evaluations) > 1, resumes)+ assert final.checkpoint is not None+ self.assertEqual(datetime.fromisoformat(str(final.checkpoint["counted_through"])), first_now - INGESTION_LAG)+@parameterized.expand([("named_event", "$autocapture"), ("any_event", None)])def test_conversions_returns_best_goal_conversion_count(self, _name: str, event: str | None) -> None:self._pay_action(event)diff --git a/products/web_analytics/backend/test/test_achievements_query_concurrency.py b/products/web_analytics/backend/test/test_achievements_query_concurrency.pyindex 73515082ea37..91bc09464067 100644--- a/products/web_analytics/backend/test/test_achievements_query_concurrency.py+++ b/products/web_analytics/backend/test/test_achievements_query_concurrency.py@@ -69,6 +69,18 @@ class TestAchievementQueryConcurrency(SimpleTestCase):self.assertEqual(evaluators.evaluate_cumulative_pageviews(self.ctx, self.prior).value, 10)use.assert_called_once()+ def test_traffic_bootstrap_saves_progress_once_its_time_budget_runs_out(self) -> None:+ with (+ patch.object(evaluators, "TRAFFIC_BOOTSTRAP_TIME_BUDGET_SECONDS", 0),+ patch.object(evaluators, "_project_environment_teams", return_value=[self.team]),+ patch.object(evaluators, "execute_hogql_query", return_value=HogQLQueryResponse(results=[[0]])) as execute,+ ):+ evaluation = evaluators.evaluate_cumulative_pageviews(self.ctx, self.prior)+ execute.assert_called_once()+ self.assertFalse(evaluation.complete)+ assert evaluation.checkpoint is not None+ self.assertIn("bootstrap", evaluation.checkpoint)+def test_query_exception_releases_its_slot_for_the_next_project(self) -> None:with (patch.object(evaluators, "_project_environment_teams", return_value=[self.team]),@@ -111,6 +123,8 @@ class TestAchievementQueryConcurrency(SimpleTestCase):progress = Mock(current_stage=0, last_computed_at=None, progress_value=0, state={})with (patch.object(tasks, "get_or_create_progress", return_value=progress),+ patch.object(tasks, "_claim_attempt", return_value=True),+ patch.object(tasks, "_release_attempt"),patch.dict(tasks.INCREMENTAL_EVALUATORS, {"cumulative_pageviews": evaluator}),patch.object(tasks, "_apply_progress") as apply_progress,):diff --git a/products/web_analytics/backend/test/test_achievements_tasks.py b/products/web_analytics/backend/test/test_achievements_tasks.pyindex c9940cbcbfdd..fe94a46aca67 100644--- a/products/web_analytics/backend/test/test_achievements_tasks.py+++ b/products/web_analytics/backend/test/test_achievements_tasks.py@@ -1,6 +1,7 @@from collections.abc import Callablefrom datetime import datetime, timedelta+import time_machinefrom posthog.test.base import BaseTestfrom unittest.mock import patch@@ -10,6 +11,7 @@ from django.utils import timezonefrom parameterized import parameterizedfrom posthog.clickhouse.client.execute import KillSwitchLevel+from posthog.clickhouse.client.limit import ConcurrencyLimitExceededfrom posthog.models.team.team import Teamfrom products.web_analytics.backend.achievements import tasks@@ -89,6 +91,37 @@ class TestRecomputeTask(BaseTest):self._run_team({**make_incremental_evaluators(), "cumulative_pageviews": pageviews})self.assertEqual(calls["count"], 0)+ @parameterized.expand(+ [+ ("query_failure", ValueError("scan failed"), True),+ ("contention", ConcurrencyLimitExceeded(), False),+ ]+ )+ def test_failed_team_track_backs_off_until_a_successful_run(+ self, _name: str, failure: Exception, backs_off: bool+ ) -> None:+ calls = 0++ def pageviews(_ctx: EvalContext, _prior: PriorProgress) -> TrackEvaluation:+ nonlocal calls+ calls += 1+ if calls == 1:+ raise failure+ return TrackEvaluation(value=10_000, checkpoint={})++ evaluators = {**make_incremental_evaluators(), "cumulative_pageviews": pageviews}+ self._run_team(evaluators)+ self._run_team(evaluators)+ self.assertEqual(calls, 1 if backs_off else 2)++ with time_machine.travel(timezone.now() + tasks.RETRY_BACKOFF_BASE + timedelta(minutes=1), tick=False):+ self._run_team(evaluators)+ progress = self._team_progress("traffic")+ self.assertEqual(calls, 2)+ self.assertEqual(progress.current_stage, 1)+ self.assertEqual(progress.failed_attempts, 0)+ self.assertIsNone(progress.retry_after)+def test_team_track_stores_the_evaluator_checkpoint(self) -> None:checkpoint: dict[str, object] = {"counted_through": "2026-01-02T00:00:00+00:00"}self._run_team(@@ -245,7 +278,12 @@ class TestRecomputeTask(BaseTest):class TestSweep(BaseTest):def _team_with_traffic_progress(- self, name: str, last_computed_at: datetime | None, current_stage: int = 0, visit_days_ago: int = 0+ self,+ name: str,+ last_computed_at: datetime | None,+ current_stage: int = 0,+ visit_days_ago: int = 0,+ retry_after: datetime | None = None,) -> int:team = Team.objects.create(organization=self.organization, name=name)WebAnalyticsVisit(@@ -259,6 +297,7 @@ class TestSweep(BaseTest):progress_value=0,state={},last_computed_at=last_computed_at,+ retry_after=retry_after,).save()return team.id@@ -290,5 +329,8 @@ class TestSweep(BaseTest):"fresh": self._team_with_traffic_progress("fresh", now - timedelta(hours=1)),"maxed": self._team_with_traffic_progress("maxed", now - timedelta(days=2), current_stage=5),"inactive": self._team_with_traffic_progress("inactive", now - timedelta(days=2), visit_days_ago=30),+ "backing_off": self._team_with_traffic_progress(+ "backing off", None, retry_after=now + timedelta(minutes=5)+ ),}self.assertEqual(self._sweep(batch_size, kill_switch), [team_ids[name] for name in expected])