:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(signals): back off anomaly scoring insights that keep timing out (#127023)
posthog[bot]f6da983e1ec0
diff --git a/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py b/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.pyindex 17df6f95d867..f262cd9be4c8 100644--- a/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py+++ b/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py@@ -19,7 +19,7 @@ from posthog.test.base import (flush_persons_and_events,materialized,)-from unittest.mock import MagicMock, patch+from unittest.mock import MagicMock, call, patchfrom django.test import override_settings@@ -71,6 +71,7 @@ from posthog.hogql.query import execute_hogql_queryfrom posthog.clickhouse.client.connection import ClickHouseUserfrom posthog.clickhouse.client.execute import sync_execute+from posthog.exceptions import ClickHouseQueryTimeOutfrom posthog.hogql_queries.utils.breakdowns import (BREAKDOWN_NULL_DISPLAY,BREAKDOWN_NULL_STRING_LABEL,@@ -3556,6 +3557,26 @@ class TestTrendsQueryRunner(ClickhouseTestMixin, APIBaseTest):"Error thrown inside thread",)+ @parameterized.expand([("without_callback", False, 3), ("with_callback", True, 1)])+ @patch("products.product_analytics.backend.hogql_queries.trends.trends_query_runner.execute_hogql_query")+ def test_on_series_error_stops_the_queued_series_queries(+ self, _name: str, with_callback: bool, expected_queries: int, patch_execute: MagicMock+ ) -> None:+ error = ClickHouseQueryTimeOut()+ patch_execute.side_effect = error+ runner = self._create_query_runner(+ "2020-01-09", "2020-01-20", IntervalType.DAY, [EventsNode(event=event) for event in ("a", "b", "c")]+ )+ callback = MagicMock()+ if with_callback:+ runner.on_series_error = callback++ with self.assertRaises(ClickHouseQueryTimeOut):+ runner.calculate()++ assert patch_execute.call_count == expected_queries+ assert callback.call_args_list == ([call(error)] if with_callback else [])+def test_to_actors_query_options(self):self._create_test_events()flush_persons_and_events()diff --git a/products/product_analytics/backend/hogql_queries/trends/trends_query_runner.py b/products/product_analytics/backend/hogql_queries/trends/trends_query_runner.pyindex a7528969732f..26899742b163 100644--- a/products/product_analytics/backend/hogql_queries/trends/trends_query_runner.py+++ b/products/product_analytics/backend/hogql_queries/trends/trends_query_runner.py@@ -1,4 +1,4 @@-from collections.abc import Sequence+from collections.abc import Callable, Sequencefrom copy import deepcopyfrom datetime import date, datetime, timedeltafrom functools import cached_property, partial@@ -105,6 +105,9 @@ class TrendsQueryRunner(AnalyticsQueryRunner[TrendsQueryResponse]):cached_response: CachedTrendsQueryResponseseries: list[SeriesWithExtras]hogql_settings: HogQLGlobalSettings | None+ # For a caller that cannot use a partial result. When set, the first failed series query calls it once,+ # and no queued series query starts after that. The callback can cancel the series queries in flight.+ on_series_error: Callable[[Exception], None] | None = Nonedef __init__(self,@@ -459,6 +462,8 @@ class TrendsQueryRunner(AnalyticsQueryRunner[TrendsQueryResponse]):timings: HogQLTimings,is_parallel: bool,):+ if self.on_series_error is not None and errors:+ returntry:series_with_extra = self.series[index]@@ -482,6 +487,9 @@ class TrendsQueryRunner(AnalyticsQueryRunner[TrendsQueryResponse]):debug_errors.append(response.error)except Exception as e:errors.append(e)+ # Only the first error calls back. A list append is atomic, so two failing threads cannot both see it first.+ if self.on_series_error is not None and errors[0] is e:+ self.on_series_error(e)finally:if is_parallel:from django.db import connectiondiff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.pyindex b45c1da801c6..1ef85b41bfe0 100644--- a/products/signals/backend/anomaly_scores/queue.py+++ b/products/signals/backend/anomaly_scores/queue.py@@ -86,8 +86,18 @@ _MAX_SCORED_SERIES = 50_000# An insight is not picked again in this time after an attempt. Events take a moment to reach# ClickHouse, and an insight that writes nothing would otherwise stay at the head of the queue.ATTEMPT_COOLDOWN = datetime.timedelta(hours=1)-# For a failure that the next hour does not fix: an unsupported shape, a query that is too expensive, or no access.+# For a failure that the next hour does not fix: an unsupported shape, a broken query, or no access. It is also the+# first step of the too-expensive cooldowns.SKIP_COOLDOWN = datetime.timedelta(days=1)+# Each consecutive too-expensive visit of an insight takes the next step. The last step repeats.+TOO_EXPENSIVE_COOLDOWNS = (+ SKIP_COOLDOWN,+ datetime.timedelta(days=3),+ datetime.timedelta(days=7),+ datetime.timedelta(days=30),+)+# The count outlives the longest step, so the visit after it still knows the count.+_TOO_EXPENSIVE_COUNT_TTL = TOO_EXPENSIVE_COOLDOWNS[-1] + datetime.timedelta(days=7)# A query counter lives a little longer than its clock hour, so a tick at the end of the hour still reads it.# A lost counter lets the scorer spend at most one more hour of budget._BUDGET_TTL = datetime.timedelta(minutes=70)@@ -164,6 +174,8 @@ class InsightPick:query_cost: int = 1# None when the train and score split is off.run: ScoringRun | None = None+ # Consecutive too-expensive visits before this pick.+ too_expensive: int = 0@frozen@@ -202,6 +214,8 @@ class Selection:# Eligible teams, and their organizations.enrolled_teams: int = 0enrolled_organizations: int = 0+ # Insights in a cooldown after two or more consecutive too-expensive visits.+ long_cooldown: int = 0# Due rows of the stored due list, the time since the oldest of them became due, and the time since the# oldest refresh of a row. None when the tick does not plan from the due list.due_insights: int | None = None@@ -468,6 +482,42 @@ def _cooling_down(insight_ids: list[int]) -> set[int]:return {insight_id for insight_id in insight_ids if _cooldown_key(insight_id) in found}+def _too_expensive_key(insight_id: int) -> str:+ return f"signals:anomaly_scores:too_expensive:{insight_id}"+++def too_expensive_counts(insight_ids: list[int]) -> dict[int, int]:+ """Consecutive too-expensive visits of each insight. An insight with no such visit is not in the result."""+ if not insight_ids:+ return {}+ found = cache.get_many([_too_expensive_key(insight_id) for insight_id in insight_ids])+ return {+ insight_id: int(found[_too_expensive_key(insight_id)])+ for insight_id in insight_ids+ if _too_expensive_key(insight_id) in found+ }+++def too_expensive_cooldown(count: int) -> datetime.timedelta:+ return TOO_EXPENSIVE_COOLDOWNS[min(count, len(TOO_EXPENSIVE_COOLDOWNS)) - 1]+++def mark_too_expensive(insight_id: int, previous: int) -> int:+ """Count one more too-expensive visit, and start the cooldown of the new count. Returns the new count.++ ``previous`` comes from the plan. The cooldown lets one visit of an insight run at a time, so no other visit+ changes the count in between.+ """+ count = previous + 1+ cache.set(_too_expensive_key(insight_id), count, timeout=int(_TOO_EXPENSIVE_COUNT_TTL.total_seconds()))+ mark_attempted(insight_id, too_expensive_cooldown(count))+ return count+++def reset_too_expensive(insight_id: int) -> None:+ cache.delete(_too_expensive_key(insight_id))++def _team_loaded_key(team_id: int) -> str:return f"signals:anomaly_scores:team_loaded:{team_id}"@@ -714,6 +764,7 @@ def select_insights(insight.insight_id: max(state.get(insight.insight_id, {}).values(), default=None) for insight in insights}cooling = _cooling_down([insight.insight_id for insight in insights])+ too_expensive = too_expensive_counts([insight.insight_id for insight in insights])active_by_id = {insight.insight_id: insight for insight in insights if insight.insight_id not in cooling}team_ids = {insight_id: insight.team_id for insight_id, insight in active_by_id.items()}@@ -840,6 +891,7 @@ def select_insights(interval=active_by_id[insight_id].interval,query_cost=plan.queries,run=plan.run,+ too_expensive=too_expensive.get(insight_id, 0),)for insight_id, plan in [*score_picked.items(), *train_picked.items()]]@@ -859,6 +911,7 @@ def select_insights(teams_skipped=len(teams_with_insights) - len(loaded_teams),enrolled_teams=len(teams),enrolled_organizations=len({team.organization_id for team in teams.values()}),+ long_cooldown=sum(too_expensive.get(insight_id, 0) > 1 for insight_id in cooling),due_insights=None if due_list is None else len(due_list.insights),oldest_due_seconds=None if due_list is None else due_list.oldest_due_seconds,oldest_refresh_seconds=None if due_list is None else due_list.oldest_refresh_seconds,diff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.pyindex c612b7a18108..662c582d7d0b 100644--- a/products/signals/backend/anomaly_scores/scoring.py+++ b/products/signals/backend/anomaly_scores/scoring.py@@ -1,5 +1,6 @@import jsonimport math+import uuidimport hashlibimport dataclassesfrom collections.abc import Sequence@@ -15,8 +16,9 @@ from posthog.schema import BreakdownFilter, ChartDisplayType, DateRange, Intervafrom posthog.hogql.constants import INSIGHT_QUERY_FANOUT_CONCURRENCY, HogQLGlobalSettings, LimitContextfrom posthog.api.services.query import ExecutionMode-from posthog.clickhouse.client.connection import ClickHouseUser-from posthog.clickhouse.query_tagging import Feature, Product, get_team_query_tags, tag_queries+from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.client.connection import ClickHouseUser, default_client+from posthog.clickhouse.query_tagging import Feature, Product, get_team_query_tags, tag_queries, tags_contextfrom posthog.clickhouse.workload import Workloadfrom posthog.dataclasses import frozenfrom posthog.event_usage import EventSource@@ -55,6 +57,8 @@ _MAX_QUERY_SECONDS = 150# Trends runs one query per series in a pool of INSIGHT_QUERY_FANOUT_CONCURRENCY threads, so more series run in waves.# All the waves of one insight share this budget. The insight activity then has time to classify a timeout.SCORING_QUERY_BUDGET_SECONDS = 210+# A kill is cleanup with its own time limit, so a silent node must not hold the scoring thread.+_CANCEL_SEND_RECEIVE_TIMEOUT_SECONDS = 15class UnsupportedInsightError(Exception):@@ -297,6 +301,43 @@ def scoring_query_settings(queries: int) -> HogQLGlobalSettings:)+def cancel_series_queries(team_id: int, client_query_id: str) -> None:+ """Kill the scoring queries of one insight that still run. A failed kill lets them run to their own time limit."""+ # Without this, the kill queries get ids with the same prefix as the queries that they kill.+ with tags_context(client_query_id=None):+ try:+ with default_client(send_receive_timeout=_CANCEL_SEND_RECEIVE_TIMEOUT_SECONDS) as client:+ rows = sync_execute(+ """+ SELECT FQDN(), query_id+ FROM distributed_system_processes+ WHERE query_id LIKE %(prefix)s+ SETTINGS max_execution_time = 2+ """,+ {"prefix": f"{team_id}_{client_query_id}_%"},+ sync_client=client,+ )+ query_ids_by_host: dict[str, list[str]] = {}+ for host, query_id in rows:+ query_ids_by_host.setdefault(host, []).append(query_id)+ for host, query_ids in query_ids_by_host.items():+ with default_client(host=host, send_receive_timeout=_CANCEL_SEND_RECEIVE_TIMEOUT_SECONDS) as client:+ sync_execute(+ "KILL QUERY WHERE query_id IN %(query_ids)s ASYNC SETTINGS max_execution_time = 5",+ {"query_ids": query_ids},+ sync_client=client,+ )+ except Exception as error:+ logger.warning(+ "anomaly_scores.cancel_series_queries_failed",+ team_id=team_id,+ error_class=type(error).__name__,+ error=str(error)[:200],+ )+ return+ logger.info("anomaly_scores.series_queries_cancelled", team_id=team_id, cancelled=len(rows))++def supports_score_step(query: TrendsQuery) -> bool:"""A score step reads the buckets before a scored bucket as its context, so the buckets must have no gaps."""# Trends drops the buckets of the days that the filter leaves out.@@ -629,6 +670,8 @@ def _fetch(tag_queries(**get_team_query_tags(team), insight_id=insight.pk)tag_queries(product=Product.SIGNALS, feature=Feature.ALERTING)+ # One prefix for all the series queries of this fetch, so that the first failure can kill the others.+ client_query_id = f"anomaly_scoring_{uuid.uuid4().hex}"# Splits the scoring queries in `query_log`, so train and score cost can be compared, and lets `spend.py`# find them. A visit with the split off fetches the full history, as a train step does.tag_queries(name=f"anomaly_scoring_{(run or ScoringRun.TRAIN).value}")@@ -644,11 +687,16 @@ def _fetch()# The runner expands breakdown cohorts and compare periods into the series, so its count is the query count.runner.hogql_settings = scoring_query_settings(len(runner.series))- response = runner.run(- execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS,- insight_id=insight.pk,- analytics_props={"source": EventSource.SELF_DRIVING},- )+ if len(runner.series) > 1:+ # With `timeout_overflow_mode="throw"` no partial result is usable, so one failed series query fails the insight.+ runner.on_series_error = lambda _error: cancel_series_queries(team.id, client_query_id)+ # The worker thread runs other queries after this fetch, and those must not carry this prefix.+ with tags_context(client_query_id=client_query_id):+ response = runner.run(+ execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS,+ insight_id=insight.pk,+ analytics_props={"source": EventSource.SELF_DRIVING},+ )results = getattr(response, "results", None)if not isinstance(results, list):raise RuntimeError(f"No results for insight {insight.pk}")diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.pyindex c6b79046a762..f27415fd2a4b 100644--- a/products/signals/backend/anomaly_scores/tick.py+++ b/products/signals/backend/anomaly_scores/tick.py@@ -59,7 +59,10 @@ from products.signals.backend.anomaly_scores.queue import (InsightPick,TeamCensus,mark_attempted,+ mark_too_expensive,+ reset_too_expensive,select_insights,+ too_expensive_cooldown,)from products.signals.backend.anomaly_scores.scoring import (SCORING_QUERY_BUDGET_SECONDS,@@ -80,7 +83,8 @@ STATUS_SCORED = "scored"STATUS_UP_TO_DATE = "up_to_date"STATUS_UNSUPPORTED = "unsupported"STATUS_MISSING = "missing"-# The scoring query hit a ClickHouse time or memory limit. A retry in an hour hits it again.+# The scoring query hit a ClickHouse time or memory limit. A retry in an hour hits it again, so the cooldown+# grows with each consecutive one.STATUS_TOO_EXPENSIVE = "too_expensive"# The insight's creator cannot read a table of the query.STATUS_ACCESS_DENIED = "access_denied"@@ -98,7 +102,7 @@ _PLAN_TIMEOUT = datetime.timedelta(minutes=2)# A timed-out attempt does not stop its thread, so a plan that ends in time also keeps its retry from overlapping it._PLAN_LOAD_BUDGET = _PLAN_TIMEOUT / 2# All the scoring queries of one insight stop within their budget. The rest is detection, the capture request,-# and the failure classification. A query timeout must reach the classification, so that it gets the 1 day cooldown.+# and the failure classification. A query timeout must reach the classification, so that its cooldown grows._INSIGHT_TIMEOUT = datetime.timedelta(seconds=SCORING_QUERY_BUDGET_SECONDS) + datetime.timedelta(seconds=90)_INSIGHT_HEARTBEAT_TIMEOUT = datetime.timedelta(minutes=1)_CAPTURE_TIMEOUT = datetime.timedelta(minutes=1)@@ -137,6 +141,7 @@ class AnomalyScoringTickPlan:enrolled_organizations: int = 0teams_planned: int = 0teams_skipped: int = 0+ long_cooldown: int = 0self_driving_organization_percentage: float = 0.0due_insights: int | None = Noneoldest_due_seconds: int | None = None@@ -202,6 +207,8 @@ class AnomalyScoringTickResult:# The plan activity failed, and the tick scored nothing. `plan_error` is the error class.plan_failed: bool = Falseplan_error: str | None = None+ # Insights in a cooldown after two or more consecutive too-expensive visits.+ long_cooldown: int = 0# Due rows that the plan read from the stored due list, the time since the oldest of them became due, and# the time since the oldest row refresh. None when the plan does not read the due list.due_insights: int | None = None@@ -256,6 +263,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlenrolled_organizations=selection.enrolled_organizations,teams_planned=selection.teams_planned,teams_skipped=selection.teams_skipped,+ long_cooldown=selection.long_cooldown,self_driving_organization_percentage=config.self_driving_organization_percentage,due_insights=selection.due_insights,oldest_due_seconds=selection.oldest_due_seconds,@@ -332,17 +340,33 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigherror=str(error)[:200],duration_ms=duration_ms,)- record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0, run=pick.run)- capture_pick(pick, status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, run=pick.run)+ too_expensive = pick.too_expensive+ cooldown = ATTEMPT_COOLDOWN+ if status == STATUS_TOO_EXPENSIVE:+ too_expensive = mark_too_expensive(pick.insight_id, pick.too_expensive)+ cooldown = too_expensive_cooldown(too_expensive)# A full cluster is not a fault of the insight, so it keeps the attempt cooldown.- cooldown = SKIP_COOLDOWN if status not in (None, STATUS_AT_CAPACITY) else ATTEMPT_COOLDOWN+ elif status is not None and status != STATUS_AT_CAPACITY:+ cooldown = SKIP_COOLDOWN+ mark_attempted(pick.insight_id, cooldown)+ record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0, run=pick.run)+ capture_pick(+ pick,+ status=status or STATUS_ERRORED,+ interval=interval,+ duration_ms=duration_ms,+ run=pick.run,+ too_expensive=too_expensive,+ )record_failed(pick.team_id, pick.insight_id, now, cooldown)if status is None:raise- if cooldown == SKIP_COOLDOWN:- mark_attempted(pick.insight_id, SKIP_COOLDOWN)return ScoreInsightResult(status=status, events=0)duration_ms = round((time.monotonic() - started) * 1000)+ too_expensive = pick.too_expensive+ if too_expensive and result.status in (STATUS_SCORED, STATUS_UP_TO_DATE):+ reset_too_expensive(pick.insight_id)+ too_expensive = 0logger.info("anomaly_scoring_insight_scored",team_id=pick.team_id,@@ -365,6 +389,7 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsighrun=result.run,series=result.series,events=result.events,+ too_expensive=too_expensive,)return result@@ -378,6 +403,7 @@ def capture_pick(run: ScoringRun | None,series: int = 0,events: int = 0,+ too_expensive: int = 0,) -> None:"""One event per pick, with the ids that the SDK metrics leave out, so cost and outcome break down per project."""ph_background_capture()(@@ -394,6 +420,7 @@ def capture_pick("series": series,"events": events,"queries": pick.query_cost,+ "too_expensive_streak": too_expensive,},)@@ -537,6 +564,7 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:self_driving_organization_percentage=result.self_driving_organization_percentage,plan_failed=result.plan_failed,plan_error=result.plan_error,+ long_cooldown=result.long_cooldown,due_insights=result.due_insights,oldest_due_seconds=result.oldest_due_seconds,oldest_refresh_seconds=result.oldest_refresh_seconds,@@ -578,6 +606,7 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:"self_driving_organization_percentage": result.self_driving_organization_percentage,"plan_failed": result.plan_failed,"plan_error": result.plan_error,+ "long_cooldown": result.long_cooldown,"due_insights": result.due_insights,"oldest_due_seconds": result.oldest_due_seconds,"oldest_refresh_seconds": result.oldest_refresh_seconds,@@ -683,6 +712,7 @@ class AnomalyScoringTickWorkflow:plan_ms=plan_ms,teams_planned=plan.teams_planned,teams_skipped=plan.teams_skipped,+ long_cooldown=plan.long_cooldown,self_driving_organization_percentage=plan.self_driving_organization_percentage,due_insights=plan.due_insights,oldest_due_seconds=plan.oldest_due_seconds,diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.pyindex a4dfb80e7874..418e53a75557 100644--- a/products/signals/backend/test/test_anomaly_scoring_tick.py+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py@@ -27,7 +27,12 @@ from posthog.schema import IntervalTypefrom posthog.hogql.errors import QueryError, ResolutionError, TableAccessDeniedErrorfrom posthog.clickhouse.client import sync_execute-from posthog.exceptions import ClickHouseAtCapacity, ClickHouseClusterMemoryLimitExceeded, ClickHouseQueryTimeOut+from posthog.exceptions import (+ ClickHouseAtCapacity,+ ClickHouseClusterMemoryLimitExceeded,+ ClickHouseQueryMemoryLimitExceeded,+ ClickHouseQueryTimeOut,+)from posthog.models import Organization, Team, Userfrom posthog.tasks.alerts.detectors.base import ARTIFACT_SCHEMA_VERSION@@ -50,6 +55,7 @@ from products.signals.backend.anomaly_scores.config import (from products.signals.backend.anomaly_scores.due_list import record_scored, refresh_due_listfrom products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_eventsfrom products.signals.backend.anomaly_scores.queue import (+ ATTEMPT_COOLDOWN,SKIP_COOLDOWN,ActiveInsight,ActivityReason,@@ -60,6 +66,7 @@ from products.signals.backend.anomaly_scores.queue import (_fleet_budget_key,_project_budget_key,_team_loaded_key,+ _too_expensive_key,active_insights,eligible_teams,is_due,@@ -67,6 +74,7 @@ from products.signals.backend.anomaly_scores.queue import (reserve_queries,scored_buckets,select_insights,+ too_expensive_counts,within_budget,)from products.signals.backend.anomaly_scores.scoring import (@@ -670,10 +678,16 @@ class TestScoreEvents(SimpleTestCase):class TestRunInsight(SimpleTestCase):- def _run(self, error: Exception) -> tuple[str, list[Any], MagicMock, dict[str, Any]]:+ def setUp(self) -> None:+ super().setUp()+ cache.delete(_too_expensive_key(7))+ self.addCleanup(cache.delete, _too_expensive_key(7))++ def _run(self, error: Exception, too_expensive: int = 0) -> tuple[str, list[Any], MagicMock, dict[str, Any]]:tick = "products.signals.backend.anomaly_scores.tick"client = MagicMock()capture = MagicMock()+ mark_attempted = MagicMock()with (patch(f"{tick}.ph_background_capture", return_value=capture),patch(@@ -682,11 +696,14 @@ class TestRunInsight(SimpleTestCase):),patch(f"{tick}.Team"),patch(f"{tick}.score_insight", side_effect=error),- patch(f"{tick}.mark_attempted") as mark_attempted,+ patch(f"{tick}.mark_attempted", mark_attempted),+ patch("products.signals.backend.anomaly_scores.queue.mark_attempted", mark_attempted),patch(f"{tick}.record_failed") as record_failed,patch("products.signals.backend.anomaly_scores.metrics.posthoganalytics.default_client", client),):- pick = InsightPick(team_id=1, insight_id=7, scored_through={}, interval=IntervalType.DAY)+ pick = InsightPick(+ team_id=1, insight_id=7, scored_through={}, interval=IntervalType.DAY, too_expensive=too_expensive+ )try:status = run_insight(ScoreInsightInput(pick=pick), NOW).statusexcept Exception:@@ -726,6 +743,47 @@ class TestRunInsight(SimpleTestCase):"queries": 1,}+ def test_consecutive_too_expensive_visits_grow_the_cooldown_and_a_success_resets_it(self) -> None:+ cooldowns = []+ streaks = []+ for error in [+ ClickHouseQueryTimeOut(),+ ClickHouseAtCapacity(),+ ClickHouseClusterMemoryLimitExceeded(),+ ClickHouseQueryMemoryLimitExceeded(),+ ClickHouseQueryTimeOut(),+ ClickHouseQueryTimeOut(),+ ClickHouseQueryTimeOut(),+ ]:+ _status, visit_cooldowns, _metrics, pick_event = self._run(error, too_expensive_counts([7]).get(7, 0))+ cooldowns.append(visit_cooldowns[-1])+ streaks.append(pick_event["too_expensive_streak"])++ assert cooldowns == [+ SKIP_COOLDOWN,+ ATTEMPT_COOLDOWN,+ ATTEMPT_COOLDOWN,+ timedelta(days=3),+ timedelta(days=7),+ timedelta(days=30),+ timedelta(days=30),+ ]+ assert streaks == [1, 1, 1, 2, 3, 4, 5]++ capture = MagicMock()+ with (+ patch("products.signals.backend.anomaly_scores.tick.ph_background_capture", return_value=capture),+ patch(+ "products.signals.backend.anomaly_scores.tick._score_pick",+ return_value=ScoreInsightResult(status="up_to_date", events=0),+ ),+ ):+ pick = InsightPick(team_id=1, insight_id=7, scored_through={}, too_expensive=5)+ run_insight(ScoreInsightInput(pick=pick), NOW)++ assert too_expensive_counts([7]) == {}+ assert capture.call_args.kwargs["properties"]["too_expensive_streak"] == 0+def test_a_full_cluster_returns_at_capacity_and_waits_an_hour(self) -> None:status, cooldowns, _metrics, _pick_event = self._run(ClickHouseAtCapacity())@@ -1368,7 +1426,9 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):small_dashboard = Dashboard.objects.create(team=small_team, name="Main")DashboardTile.objects.create(dashboard=small_dashboard, insight=small_insight)for insight in [*large_insights, small_insight]:- cache.delete(_cooldown_key(insight.id))+ cache.delete_many([_cooldown_key(insight.id), _too_expensive_key(insight.id)])+ cache.set(_too_expensive_key(small_insight.id), 2)+ self.addCleanup(cache.delete, _too_expensive_key(small_insight.id))config = AnomalyScoringConfig(enabled=True,organization_ids=(str(self.organization.id),),@@ -1381,6 +1441,7 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):assert [pick.team_id for pick in selection.picks] == [self.team.id, small_team.id]assert selection.picks[0].insight_id in {insight.id for insight in large_insights}assert selection.picks[1].insight_id == small_insight.id+ assert [pick.too_expensive for pick in selection.picks] == [0, 2]assert selection.candidates == 4assert [(team.team_id, team.active, team.active_intervals) for team in selection.census] == [(self.team.id, 3, {"day": 3}),