Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(replay-vision): turn off unrunnable scanners that hit their credit limit (#127041)

Tue Haulund88df7bb9432c
diff --git a/products/replay_vision/backend/scanner_access.py b/products/replay_vision/backend/scanner_access.py
index 59d1fe5036dd..4865d9f6f9ea 100644
--- a/products/replay_vision/backend/scanner_access.py
+++ b/products/replay_vision/backend/scanner_access.py
@@ -57,6 +57,15 @@ def creator_can_scan(scanner: ReplayScanner) -> bool:
return scanner.created_by is None or recording_viewer_access(scanner.created_by, scanner.team) is not None
+def unrunnable_reason(scanner: ReplayScanner) -> Literal["no_ai_consent", "creator_lost_access"] | None:
+ """Why the sweep can never spend for this scanner, or None when it can."""
+ if not scanner.team.organization.is_ai_data_processing_approved:
+ return "no_ai_consent"
+ if not creator_can_scan(scanner):
+ return "creator_lost_access"
+ return None
+
+
def scanners_for_reading_observations(team_id: int) -> "QuerySet[ReplayScanner]":
"""Every scanner whose observations can be read, inline scans included.
diff --git a/products/replay_vision/backend/temporal/activities/check_scanner_budget.py b/products/replay_vision/backend/temporal/activities/check_scanner_budget.py
index 008d1086415e..e797f3252da3 100644
--- a/products/replay_vision/backend/temporal/activities/check_scanner_budget.py
+++ b/products/replay_vision/backend/temporal/activities/check_scanner_budget.py
@@ -3,6 +3,7 @@ from temporalio import activity
from products.replay_vision.backend.models.replay_scanner import ReplayScanner, initial_watermark
from products.replay_vision.backend.models.replay_scanner_backfill import BackfillStatus, ReplayScannerBackfill
from products.replay_vision.backend.quota import compute_scanner_budget, current_period_bounds
+from products.replay_vision.backend.scanner_access import unrunnable_reason
from products.replay_vision.backend.temporal.constants import CHECK_SCANNER_BUDGET_TIMEOUT
from products.replay_vision.backend.temporal.decorators import track_activity
from products.replay_vision.backend.temporal.metrics import record_sweep_outcome
@@ -107,12 +108,19 @@ def check_scanner_budget_activity(inputs: CheckScannerBudgetInputs) -> CheckScan
the watermark: in-flight reservations can release without settling and must not permanently skip
a window the scanner could still afford.
"""
- scanner = ReplayScanner.objects.filter(pk=inputs.scanner_id, team_id=inputs.team_id).select_related("team").first()
+ scanner = (
+ ReplayScanner.objects.filter(pk=inputs.scanner_id, team_id=inputs.team_id)
+ .select_related("team__organization", "created_by")
+ .first()
+ )
if scanner is None:
# The reconciler removes schedules for deleted scanners. A racing tick just stops here.
return CheckScannerBudgetOutput(capped=False)
if not scanner.has_spend_ceiling:
return CheckScannerBudgetOutput(capped=False)
+ # Not capped, so the sweep reaches the step that turns an unrunnable scanner off.
+ if unrunnable_reason(scanner):
+ return CheckScannerBudgetOutput(capped=False)
# Resolve the period once so the cap decision and the notification stamp cannot straddle a rollover.
period = current_period_bounds(scanner.team.organization_id)
with bounded_queries(CHECK_SCANNER_BUDGET_TIMEOUT):
diff --git a/products/replay_vision/backend/temporal/activities/find_scanner_candidates.py b/products/replay_vision/backend/temporal/activities/find_scanner_candidates.py
index 853d47634cc2..8213cb54c92a 100644
--- a/products/replay_vision/backend/temporal/activities/find_scanner_candidates.py
+++ b/products/replay_vision/backend/temporal/activities/find_scanner_candidates.py
@@ -42,7 +42,7 @@ from products.replay_vision.backend.queries.scanner_candidate_query import (
build_candidate_batch,
)
from products.replay_vision.backend.queries.variant_sampling import variant_sampling_plan_for_scope
-from products.replay_vision.backend.scanner_access import creator_can_scan
+from products.replay_vision.backend.scanner_access import unrunnable_reason
from products.replay_vision.backend.temporal.constants import (
DEEP_SWEEP_INTERVAL,
DEEP_SWEEP_MAX_EXECUTION_SECONDS,
@@ -191,10 +191,8 @@ def find_scanner_candidates_activity(inputs: FindScannerCandidatesInputs) -> Fin
return FindScannerCandidatesOutput(candidates=[], saturated=False)
# Without AI consent or the creator's recording access a scanner can never spend, so turn it off instead of leaving it silently idle.
- if not scanner.team.organization.is_ai_data_processing_approved:
- return _disable_unrunnable_scanner(scanner, "no_ai_consent")
- if not creator_can_scan(scanner):
- return _disable_unrunnable_scanner(scanner, "creator_lost_access")
+ if reason := unrunnable_reason(scanner):
+ return _disable_unrunnable_scanner(scanner, reason)
lifecycle_block = _experiment_lifecycle_block(scanner)
sweep_until: dt.datetime | None = None
diff --git a/products/replay_vision/backend/tests/test_sweep.py b/products/replay_vision/backend/tests/test_sweep.py
index 711cdf6cc573..861b964e5a30 100644
--- a/products/replay_vision/backend/tests/test_sweep.py
+++ b/products/replay_vision/backend/tests/test_sweep.py
@@ -1294,6 +1294,24 @@ def test_check_scanner_budget_activity_caps_and_advances_the_watermark(
assert scanner.deep_swept_through == stale
[email protected]("case", ["creator_left_org", "no_ai_consent"])
[email protected]_db(transaction=True)
+def test_check_scanner_budget_activity_lets_an_unrunnable_scanner_through_to_be_turned_off(case: str) -> None:
+ scanner = _make_scanner(credit_limit=20 * _OBSERVATION_CREDITS)
+ if case == "creator_left_org":
+ creator = User.objects.create_user(email="[email protected]", password="x", first_name="c")
+ scanner.created_by = creator
+ scanner.save(update_fields=["created_by"])
+ else:
+ Organization.objects.filter(pk=scanner.team.organization_id).update(is_ai_data_processing_approved=False)
+ seed_scanner_spend(scanner, _OBSERVATION_CREDITS, observations=20)
+
+ with patch("products.replay_vision.backend.quota.MONTHLY_CREDIT_QUOTA", 1_000_000):
+ output = check_scanner_budget_activity(CheckScannerBudgetInputs(scanner_id=scanner.id, team_id=scanner.team_id))
+
+ assert output.capped is False
+
+
@pytest.mark.django_db(transaction=True)
def test_check_scanner_budget_activity_capped_by_in_flight_alone_does_not_advance_the_watermark() -> None:
# Only adding in-flight reservations pushes this over the limit: a transient spike must skip