Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(web-analytics): bound the traffic achievement's first pageview scan (#119731)

Jordan Mryyandf4c58eb23c7
diff --git a/products/web_analytics/backend/achievements/evaluators.py b/products/web_analytics/backend/achievements/evaluators.py
index 38c9b65de054..ab6f26ed432b 100644
--- a/products/web_analytics/backend/achievements/evaluators.py
+++ b/products/web_analytics/backend/achievements/evaluators.py
@@ -1,8 +1,9 @@
import json
import hashlib
from collections.abc import Callable
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
from datetime import UTC, date, datetime, time, timedelta
+from time import monotonic
from django.utils import timezone
@@ -20,7 +21,7 @@ from posthog.models.team.team import Team
from posthog.models.user import User
from products.actions.backend.models.action import Action
-from products.web_analytics.backend.achievements.definitions import STREAK_ARM_WEEKLY
+from products.web_analytics.backend.achievements.definitions import STREAK_ARM_WEEKLY, TRACKS, TrackKey
from products.web_analytics.backend.achievements.query_concurrency import achievement_query_scope
from products.web_analytics.backend.hogql_queries.web_lazy_precompute_common import test_account_filter_expr
from products.web_analytics.backend.models import WebAnalyticsInteraction, WebAnalyticsVisit
@@ -164,41 +165,147 @@ def _ingestion_window_expr(
return ast.And(exprs=exprs)
+ACHIEVEMENT_SCAN_LIMITS = (
+ CHQueryErrorTooManyBytes,
+ ClickHouseQueryTimeOut,
+ ClickHouseEstimatedQueryExecutionTimeTooLong,
+)
+ACHIEVEMENT_SCAN_SETTINGS = HogQLGlobalSettings(timeout_overflow_mode="throw", read_overflow_mode="throw")
+ACHIEVEMENT_MAX_EMPTY_CHUNKS = 20
+TRAFFIC_TARGET_PAGEVIEWS = TRACKS[TrackKey.TRAFFIC].stages[-1].threshold
+TRAFFIC_BOOTSTRAP_CHUNK_HOURS = 24
+TRAFFIC_BOOTSTRAP_TARGET_ROWS = 50_000_000
+TRAFFIC_BOOTSTRAP_MAX_CHUNK_HOURS = 24 * 365 * 60
+TRAFFIC_BOOTSTRAP_TIME_BUDGET_SECONDS = 30
+TRAFFIC_MIN_TIMESTAMP = datetime(1970, 1, 1, tzinfo=UTC)
+
+
+@frozen
+class TrafficBootstrap:
+ created_until: datetime
+ counted: int = 0
+ before: datetime | None = None
+ chunk_hours: int = TRAFFIC_BOOTSTRAP_CHUNK_HOURS
+
+ @property
+ def done(self) -> bool:
+ return self.counted >= TRAFFIC_TARGET_PAGEVIEWS or self.before == TRAFFIC_MIN_TIMESTAMP
+
+ @classmethod
+ def from_checkpoint(cls, checkpoint: dict[str, object]) -> "TrafficBootstrap | None":
+ bootstrap = checkpoint.get("bootstrap")
+ if not isinstance(bootstrap, dict):
+ return None
+ created_until = parse_zoned_datetime_string(bootstrap.get("created_until"))
+ before = parse_zoned_datetime_string(bootstrap.get("before"))
+ counted = bootstrap.get("counted")
+ chunk_hours = bootstrap.get("chunk_hours")
+ if (
+ created_until is None
+ or (bootstrap.get("before") is not None and before is None)
+ or not isinstance(counted, int)
+ or counted < 0
+ or not isinstance(chunk_hours, int)
+ or not 1 <= chunk_hours <= TRAFFIC_BOOTSTRAP_MAX_CHUNK_HOURS
+ ):
+ return None
+ return cls(created_until=created_until, counted=counted, before=before, chunk_hours=chunk_hours)
+
+ def to_checkpoint(self) -> dict[str, object]:
+ if self.done:
+ return {"counted_through": self.created_until.isoformat()}
+ return {
+ "bootstrap": {
+ "created_until": self.created_until.isoformat(),
+ "counted": self.counted,
+ "before": self.before.isoformat() if self.before is not None else None,
+ "chunk_hours": self.chunk_hours,
+ }
+ }
+
+
+def _count_pageviews(
+ teams: list[Team],
+ since: datetime | None,
+ until: datetime,
+ earliest_timestamp: datetime | None,
+ latest_timestamp: datetime | None = None,
+) -> int:
+ total = 0
+ for team in teams:
+ query = parse_select(
+ "SELECT count() FROM events WHERE and(event IN ('$pageview', '$screen'), {window}, {test})",
+ placeholders={
+ "window": _ingestion_window_expr(since, until, earliest_timestamp, latest_timestamp),
+ "test": _test_account_filter_expr(team),
+ },
+ )
+ response = execute_hogql_query(
+ query=query,
+ team=team,
+ query_type="web_achievements_pageviews",
+ # The team-scoped sweep has no acting user, and only an aggregate count leaves the query.
+ bypass_warehouse_access_control=True,
+ settings=ACHIEVEMENT_SCAN_SETTINGS,
+ )
+ if response.results:
+ total += int(response.results[0][0] or 0)
+ return total
+
+
+def _next_traffic_chunk_hours(chunk_hours: int, rows: int) -> int:
+ if rows == 0:
+ return min(chunk_hours * 2, TRAFFIC_BOOTSTRAP_MAX_CHUNK_HOURS)
+ return max(1, min(chunk_hours * TRAFFIC_BOOTSTRAP_TARGET_ROWS // rows, TRAFFIC_BOOTSTRAP_MAX_CHUNK_HOURS))
+
+
+def _advance_traffic_bootstrap(teams: list[Team], bootstrap: TrafficBootstrap) -> TrafficBootstrap:
+ deadline = monotonic() + TRAFFIC_BOOTSTRAP_TIME_BUDGET_SECONDS
+ for _ in range(ACHIEVEMENT_MAX_EMPTY_CHUNKS):
+ upper = bootstrap.before or bootstrap.created_until
+ lower = max(TRAFFIC_MIN_TIMESTAMP, upper - timedelta(hours=bootstrap.chunk_hours))
+ try:
+ rows = _count_pageviews(
+ teams,
+ since=None,
+ until=bootstrap.created_until,
+ earliest_timestamp=None if lower == TRAFFIC_MIN_TIMESTAMP else lower,
+ latest_timestamp=bootstrap.before,
+ )
+ except ACHIEVEMENT_SCAN_LIMITS:
+ if bootstrap.chunk_hours <= 1:
+ raise
+ return replace(bootstrap, chunk_hours=max(1, bootstrap.chunk_hours // 2))
+ bootstrap = replace(
+ bootstrap,
+ counted=bootstrap.counted + rows,
+ before=lower,
+ chunk_hours=_next_traffic_chunk_hours(bootstrap.chunk_hours, rows),
+ )
+ if bootstrap.done or rows or monotonic() >= deadline:
+ break
+ return bootstrap
+
+
def evaluate_cumulative_pageviews(ctx: EvalContext, prior: PriorProgress) -> TrackEvaluation:
until = timezone.now() - INGESTION_LAG
+ bootstrap = TrafficBootstrap.from_checkpoint(prior.checkpoint)
since = parse_zoned_datetime_string(prior.checkpoint.get("counted_through")) or prior.last_computed_at
- earliest_timestamp = since - LATE_ARRIVAL_LOOKBACK if since is not None else None
- total = prior.value if since is not None else 0
+ teams = _project_environment_teams(ctx.team)
+ if bootstrap is None and since is not None:
+ with achievement_query_scope(ctx.team.id):
+ added = _count_pageviews(teams, since, until, since - LATE_ARRIVAL_LOOKBACK)
+ return TrackEvaluation(value=prior.value + added, checkpoint={"counted_through": until.isoformat()})
+
with achievement_query_scope(ctx.team.id):
- for team in _project_environment_teams(ctx.team):
- query = parse_select(
- "SELECT count() FROM events WHERE and(event IN ('$pageview', '$screen'), {window}, {test})",
- placeholders={
- "window": _ingestion_window_expr(since, until, earliest_timestamp),
- "test": _test_account_filter_expr(team),
- },
- )
- response = execute_hogql_query(
- query=query,
- team=team,
- query_type="web_achievements_pageviews",
- # The team-scoped sweep has no acting user, and only an aggregate count leaves the query.
- bypass_warehouse_access_control=True,
- )
- if response.results:
- total += int(response.results[0][0] or 0)
- return TrackEvaluation(value=total, checkpoint={"counted_through": until.isoformat()})
+ bootstrap = _advance_traffic_bootstrap(teams, bootstrap or TrafficBootstrap(created_until=until))
+ return TrackEvaluation(value=bootstrap.counted, checkpoint=bootstrap.to_checkpoint(), complete=bootstrap.done)
CONVERSIONS_LOOKBACK_DAYS = 90
CONVERSIONS_BOOTSTRAP_CHUNK_DAYS = 7
CONVERSIONS_MAX_TIMESTAMP = datetime(2299, 12, 31, 23, 59, 59, 999999, tzinfo=UTC)
CONVERSIONS_FUTURE_CHUNK_MAX_HOURS = 24 * 365 * 50
-CONVERSIONS_SCAN_LIMITS = (
- CHQueryErrorTooManyBytes,
- ClickHouseQueryTimeOut,
- ClickHouseEstimatedQueryExecutionTimeTooLong,
-)
@frozen
@@ -318,7 +425,7 @@ def _add_conversion_counts(
query_type="web_achievements_conversions",
# The team-scoped sweep has no acting user, and only per-day counts leave the query.
bypass_warehouse_access_control=True,
- settings=HogQLGlobalSettings(timeout_overflow_mode="throw", read_overflow_mode="throw"),
+ settings=ACHIEVEMENT_SCAN_SETTINGS,
)
for row in response.results or []:
day_counts = chunk_daily.setdefault(row[0].isoformat(), [0] * len(actions))
@@ -343,7 +450,7 @@ def _advance_conversion_future(
chunk_hours = bootstrap.chunk_hours
# Empty timestamp ranges are cheap to skip with the event timestamp index. Grow those
# ranges so a scan with no future-dated events does not take hundreds of daily sweeps.
- for _ in range(20):
+ for _ in range(ACHIEVEMENT_MAX_EMPTY_CHUNKS):
chunk_end = min(next_start + timedelta(hours=chunk_hours), CONVERSIONS_MAX_TIMESTAMP)
try:
has_rows = _add_conversion_counts(
@@ -356,7 +463,7 @@ def _advance_conversion_future(
chunk_end,
latest_inclusive=chunk_end == CONVERSIONS_MAX_TIMESTAMP,
)
- except CONVERSIONS_SCAN_LIMITS:
+ except ACHIEVEMENT_SCAN_LIMITS:
if chunk_hours <= 1:
raise
return ConversionBootstrap(
@@ -402,7 +509,7 @@ def _advance_conversion_bootstrap(
if bootstrap.phase == "tail":
try:
_add_conversion_counts(ctx, actions, daily, bootstrap.created_since, until, window_start)
- except CONVERSIONS_SCAN_LIMITS:
+ except ACHIEVEMENT_SCAN_LIMITS:
return ConversionBootstrap(
next_start=window_start,
end=bootstrap.end,
@@ -446,7 +553,7 @@ def _advance_conversion_bootstrap(
next_start,
chunk_end,
)
- except CONVERSIONS_SCAN_LIMITS:
+ except ACHIEVEMENT_SCAN_LIMITS:
if chunk_hours <= 1:
raise
return ConversionBootstrap(
@@ -527,7 +634,7 @@ def evaluate_conversions(ctx: EvalContext, prior: PriorProgress) -> TrackEvaluat
earliest_timestamp = max(window_start, since - LATE_ARRIVAL_LOOKBACK) if since is not None else window_start
try:
_add_conversion_counts(ctx, actions, daily, since, until, earliest_timestamp)
- except CONVERSIONS_SCAN_LIMITS:
+ except ACHIEVEMENT_SCAN_LIMITS:
if since is not None:
raise
daily = {}
diff --git a/products/web_analytics/backend/achievements/tasks.py b/products/web_analytics/backend/achievements/tasks.py
index a8dfa4718c43..999bc2598d7a 100644
--- a/products/web_analytics/backend/achievements/tasks.py
+++ b/products/web_analytics/backend/achievements/tasks.py
@@ -55,6 +55,8 @@ ACHIEVEMENTS_FLAG = "web-analytics-achievements"
SWEEP_ACTIVE_WINDOW_DAYS = 7
RECOMPUTE_INTERVAL = timedelta(hours=20)
RECOMPUTE_EXPIRES_SECONDS = 5 * 60
+RETRY_BACKOFF_BASE = timedelta(minutes=15)
+RETRY_BACKOFF_MAX = timedelta(hours=24)
TRANSIENT_ACHIEVEMENT_ERRORS = (ConcurrencyLimitExceeded, RedisError, *CH_TRANSIENT_ERRORS)
TEAM_QUERY_TRACK_KEYS = [str(track.key) for track in TRACKS.values() if track.evaluator_key in INCREMENTAL_EVALUATORS]
@@ -185,6 +187,8 @@ def persist_progress(
if bump_last_computed_at:
progress.last_computed_at = timezone.now()
progress.state = state
+ progress.failed_attempts = 0
+ progress.retry_after = None
progress.save()
@@ -216,14 +220,45 @@ def evaluate_track(
return incremental_evaluator(ctx, prior)
+def _retry_backoff(failed_attempts: int) -> timedelta:
+ return min(RETRY_BACKOFF_BASE * 2 ** min(failed_attempts - 1, 10), RETRY_BACKOFF_MAX)
+
+
+def _due_q(now: datetime) -> Q:
+ return (Q(last_computed_at__isnull=True) | Q(last_computed_at__lte=now - RECOMPUTE_INTERVAL)) & (
+ Q(retry_after__isnull=True) | Q(retry_after__lte=now)
+ )
+
+
+def _claim_attempt(progress: WebAnalyticsAchievementProgress) -> bool:
+ now = timezone.now()
+ failed_attempts = progress.failed_attempts + 1
+ return bool(
+ WebAnalyticsAchievementProgress.objects.for_team(progress.team_id, canonical=True)
+ .filter(_due_q(now), pk=progress.pk, failed_attempts=progress.failed_attempts)
+ .update(failed_attempts=failed_attempts, retry_after=now + _retry_backoff(failed_attempts))
+ )
+
+
+def _release_attempt(progress: WebAnalyticsAchievementProgress) -> None:
+ WebAnalyticsAchievementProgress.objects.for_team(progress.team_id, canonical=True).filter(pk=progress.pk).update(
+ failed_attempts=progress.failed_attempts, retry_after=progress.retry_after
+ )
+
+
def _recompute_track(ctx: EvalContext, track: TrackDefinition) -> None:
progress = get_or_create_progress(ctx, track)
if progress.current_stage >= len(track.stages):
return
- if track.evaluator_key in INCREMENTAL_EVALUATORS and not is_due(progress):
+ incremental = track.evaluator_key in INCREMENTAL_EVALUATORS
+ if incremental and (not is_due(progress) or not _claim_attempt(progress)):
return
try:
evaluation = evaluate_track(ctx, track, progress)
+ except ConcurrencyLimitExceeded:
+ if incremental:
+ _release_attempt(progress)
+ raise
except TRANSIENT_ACHIEVEMENT_ERRORS:
raise
except Exception as e:
@@ -330,7 +365,8 @@ def _send_unlock_notification(ctx: EvalContext, track: TrackDefinition, stage: i
def due_team_ids(limit: int) -> list[int]:
- active_since = timezone.now().date() - timedelta(days=SWEEP_ACTIVE_WINDOW_DAYS)
+ now = timezone.now()
+ active_since = now.date() - timedelta(days=SWEEP_ACTIVE_WINDOW_DAYS)
active_team_ids = (
WebAnalyticsVisit.objects.unscoped().filter(visit_date__gte=active_since).values("team_id").distinct()
)
@@ -342,7 +378,7 @@ def due_team_ids(limit: int) -> list[int]:
current_stage__lt=STAGE_COUNT,
team_id__in=active_team_ids,
)
- .filter(Q(last_computed_at__isnull=True) | Q(last_computed_at__lte=timezone.now() - RECOMPUTE_INTERVAL))
+ .filter(_due_q(now))
.values("team_id")
.annotate(oldest_computed_at=Min("last_computed_at"))
.order_by(F("oldest_computed_at").asc(nulls_first=True), "team_id")
diff --git a/products/web_analytics/backend/migrations/0017_webanalyticsachievementprogress_retry_backoff.py b/products/web_analytics/backend/migrations/0017_webanalyticsachievementprogress_retry_backoff.py
new file mode 100644
index 000000000000..bbad0697ab61
--- /dev/null
+++ b/products/web_analytics/backend/migrations/0017_webanalyticsachievementprogress_retry_backoff.py
@@ -0,0 +1,22 @@
+# Generated by Django 5.2.18 on 2026-10-19 15:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("web_analytics", "0016_savedheatmap_last_viewed_idx"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="webanalyticsachievementprogress",
+ name="failed_attempts",
+ field=models.PositiveSmallIntegerField(db_default=0, default=0),
+ ),
+ migrations.AddField(
+ model_name="webanalyticsachievementprogress",
+ name="retry_after",
+ field=models.DateTimeField(blank=True, null=True),
+ ),
+ ]
diff --git a/products/web_analytics/backend/migrations/max_migration.txt b/products/web_analytics/backend/migrations/max_migration.txt
index d8bfa7bbe71a..096e6c656cc8 100644
--- a/products/web_analytics/backend/migrations/max_migration.txt
+++ b/products/web_analytics/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0016_savedheatmap_last_viewed_idx
+0017_webanalyticsachievementprogress_retry_backoff
diff --git a/products/web_analytics/backend/models/web_analytics_achievement_progress.py b/products/web_analytics/backend/models/web_analytics_achievement_progress.py
index ab25724e89d4..6e6fadec0c31 100644
--- a/products/web_analytics/backend/models/web_analytics_achievement_progress.py
+++ b/products/web_analytics/backend/models/web_analytics_achievement_progress.py
@@ -18,6 +18,8 @@ class WebAnalyticsAchievementProgress(TeamScopedRootMixin, UUIDModel):
progress_value = models.BigIntegerField(default=0)
state = models.JSONField(default=dict)
last_computed_at = models.DateTimeField(null=True, blank=True)
+ failed_attempts = models.PositiveSmallIntegerField(default=0, db_default=0)
+ retry_after = models.DateTimeField(null=True, blank=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
diff --git a/products/web_analytics/backend/test/test_achievements_evaluators.py b/products/web_analytics/backend/test/test_achievements_evaluators.py
index 3215cf8e98ba..ef5fd5948de0 100644
--- a/products/web_analytics/backend/test/test_achievements_evaluators.py
+++ b/products/web_analytics/backend/test/test_achievements_evaluators.py
@@ -19,6 +19,7 @@ from posthog.test.warehouse_access import WAREHOUSE_ACCESS_CONTROL_FLAG, filter_
from products.actions.backend.models.action import Action
from products.web_analytics.backend.achievements.definitions import STREAK_ARM_DAILY, STREAK_ARM_WEEKLY
from products.web_analytics.backend.achievements.evaluators import (
+ INGESTION_LAG,
EvalContext,
PriorProgress,
TrackEvaluation,
@@ -147,6 +148,68 @@ class TestTeamEvaluators(ClickhouseTestMixin, APIBaseTest):
assert evaluation.checkpoint is not None
self.assertGreater(datetime.fromisoformat(str(evaluation.checkpoint["counted_through"])), watermark)
+ @parameterized.expand(
+ [
+ ("walks_back_after_a_scan_limit", 100_000_000, True, 4, True),
+ ("stops_at_the_top_stage", 2, False, 2, False),
+ ]
+ )
+ def test_cumulative_pageviews_bootstraps_history_in_bounded_chunks(
+ self, _name: str, target: int, fail_first_chunk: bool, expected: int, resumes: bool
+ ) -> None:
+ first_now = timezone.now()
+ for timestamp in (
+ first_now + timedelta(days=1),
+ first_now - timedelta(hours=2),
+ first_now - timedelta(days=5),
+ first_now - timedelta(days=400),
+ ):
+ _create_event(
+ team=self.team,
+ event="$pageview",
+ distinct_id="d1",
+ timestamp=timestamp,
+ created_at=first_now - timedelta(hours=2),
+ )
+ flush_persons_and_events()
+
+ attempts = 0
+
+ def fail_first_attempt(*args: object, **kwargs: object):
+ nonlocal attempts
+ attempts += 1
+ if fail_first_chunk and attempts == 1:
+ raise CHQueryErrorTooManyBytes("read limit", code=307)
+ return execute_hogql_query(*args, **kwargs)
+
+ evaluations: list[TrackEvaluation] = []
+ prior = EMPTY_PRIOR
+ with (
+ patch("products.web_analytics.backend.achievements.evaluators.TRAFFIC_TARGET_PAGEVIEWS", target),
+ patch(
+ "products.web_analytics.backend.achievements.evaluators.execute_hogql_query",
+ side_effect=fail_first_attempt,
+ ),
+ ):
+ for sweep in range(20):
+ with patch(
+ "products.web_analytics.backend.achievements.evaluators.timezone.now",
+ return_value=first_now + timedelta(minutes=5 * sweep),
+ ):
+ evaluation = evaluate_cumulative_pageviews(self._ctx(), prior)
+ evaluations.append(evaluation)
+ if evaluation.complete:
+ break
+ assert evaluation.checkpoint is not None
+ prior = PriorProgress(value=0, last_computed_at=None, checkpoint=evaluation.checkpoint)
+
+ final = evaluations[-1]
+ self.assertTrue(final.complete)
+ self.assertEqual(final.value, expected)
+ self.assertEqual(len(evaluations) > 1, resumes)
+ assert final.checkpoint is not None
+ self.assertEqual(datetime.fromisoformat(str(final.checkpoint["counted_through"])), first_now - INGESTION_LAG)
+
@parameterized.expand([("named_event", "$autocapture"), ("any_event", None)])
def test_conversions_returns_best_goal_conversion_count(self, _name: str, event: str | None) -> None:
self._pay_action(event)
diff --git a/products/web_analytics/backend/test/test_achievements_query_concurrency.py b/products/web_analytics/backend/test/test_achievements_query_concurrency.py
index 73515082ea37..91bc09464067 100644
--- a/products/web_analytics/backend/test/test_achievements_query_concurrency.py
+++ b/products/web_analytics/backend/test/test_achievements_query_concurrency.py
@@ -69,6 +69,18 @@ class TestAchievementQueryConcurrency(SimpleTestCase):
self.assertEqual(evaluators.evaluate_cumulative_pageviews(self.ctx, self.prior).value, 10)
use.assert_called_once()
+ def test_traffic_bootstrap_saves_progress_once_its_time_budget_runs_out(self) -> None:
+ with (
+ patch.object(evaluators, "TRAFFIC_BOOTSTRAP_TIME_BUDGET_SECONDS", 0),
+ patch.object(evaluators, "_project_environment_teams", return_value=[self.team]),
+ patch.object(evaluators, "execute_hogql_query", return_value=HogQLQueryResponse(results=[[0]])) as execute,
+ ):
+ evaluation = evaluators.evaluate_cumulative_pageviews(self.ctx, self.prior)
+ execute.assert_called_once()
+ self.assertFalse(evaluation.complete)
+ assert evaluation.checkpoint is not None
+ self.assertIn("bootstrap", evaluation.checkpoint)
+
def test_query_exception_releases_its_slot_for_the_next_project(self) -> None:
with (
patch.object(evaluators, "_project_environment_teams", return_value=[self.team]),
@@ -111,6 +123,8 @@ class TestAchievementQueryConcurrency(SimpleTestCase):
progress = Mock(current_stage=0, last_computed_at=None, progress_value=0, state={})
with (
patch.object(tasks, "get_or_create_progress", return_value=progress),
+ patch.object(tasks, "_claim_attempt", return_value=True),
+ patch.object(tasks, "_release_attempt"),
patch.dict(tasks.INCREMENTAL_EVALUATORS, {"cumulative_pageviews": evaluator}),
patch.object(tasks, "_apply_progress") as apply_progress,
):
diff --git a/products/web_analytics/backend/test/test_achievements_tasks.py b/products/web_analytics/backend/test/test_achievements_tasks.py
index c9940cbcbfdd..fe94a46aca67 100644
--- a/products/web_analytics/backend/test/test_achievements_tasks.py
+++ b/products/web_analytics/backend/test/test_achievements_tasks.py
@@ -1,6 +1,7 @@
from collections.abc import Callable
from datetime import datetime, timedelta
+import time_machine
from posthog.test.base import BaseTest
from unittest.mock import patch
@@ -10,6 +11,7 @@ from django.utils import timezone
from parameterized import parameterized
from posthog.clickhouse.client.execute import KillSwitchLevel
+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
from posthog.models.team.team import Team
from products.web_analytics.backend.achievements import tasks
@@ -89,6 +91,37 @@ class TestRecomputeTask(BaseTest):
self._run_team({**make_incremental_evaluators(), "cumulative_pageviews": pageviews})
self.assertEqual(calls["count"], 0)
+ @parameterized.expand(
+ [
+ ("query_failure", ValueError("scan failed"), True),
+ ("contention", ConcurrencyLimitExceeded(), False),
+ ]
+ )
+ def test_failed_team_track_backs_off_until_a_successful_run(
+ self, _name: str, failure: Exception, backs_off: bool
+ ) -> None:
+ calls = 0
+
+ def pageviews(_ctx: EvalContext, _prior: PriorProgress) -> TrackEvaluation:
+ nonlocal calls
+ calls += 1
+ if calls == 1:
+ raise failure
+ return TrackEvaluation(value=10_000, checkpoint={})
+
+ evaluators = {**make_incremental_evaluators(), "cumulative_pageviews": pageviews}
+ self._run_team(evaluators)
+ self._run_team(evaluators)
+ self.assertEqual(calls, 1 if backs_off else 2)
+
+ with time_machine.travel(timezone.now() + tasks.RETRY_BACKOFF_BASE + timedelta(minutes=1), tick=False):
+ self._run_team(evaluators)
+ progress = self._team_progress("traffic")
+ self.assertEqual(calls, 2)
+ self.assertEqual(progress.current_stage, 1)
+ self.assertEqual(progress.failed_attempts, 0)
+ self.assertIsNone(progress.retry_after)
+
def test_team_track_stores_the_evaluator_checkpoint(self) -> None:
checkpoint: dict[str, object] = {"counted_through": "2026-01-02T00:00:00+00:00"}
self._run_team(
@@ -245,7 +278,12 @@ class TestRecomputeTask(BaseTest):
class TestSweep(BaseTest):
def _team_with_traffic_progress(
- self, name: str, last_computed_at: datetime | None, current_stage: int = 0, visit_days_ago: int = 0
+ self,
+ name: str,
+ last_computed_at: datetime | None,
+ current_stage: int = 0,
+ visit_days_ago: int = 0,
+ retry_after: datetime | None = None,
) -> int:
team = Team.objects.create(organization=self.organization, name=name)
WebAnalyticsVisit(
@@ -259,6 +297,7 @@ class TestSweep(BaseTest):
progress_value=0,
state={},
last_computed_at=last_computed_at,
+ retry_after=retry_after,
).save()
return team.id
@@ -290,5 +329,8 @@ class TestSweep(BaseTest):
"fresh": self._team_with_traffic_progress("fresh", now - timedelta(hours=1)),
"maxed": self._team_with_traffic_progress("maxed", now - timedelta(days=2), current_stage=5),
"inactive": self._team_with_traffic_progress("inactive", now - timedelta(days=2), visit_days_ago=30),
+ "backing_off": self._team_with_traffic_progress(
+ "backing off", None, retry_after=now + timedelta(minutes=5)
+ ),
}
self.assertEqual(self._sweep(batch_size, kill_switch), [team_ids[name] for name in expected])