Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): back off anomaly scoring insights that keep timing out (#127023)

posthog[bot]f6da983e1ec0
diff --git a/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py b/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py
index 17df6f95d867..f262cd9be4c8 100644
--- a/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py
+++ b/products/product_analytics/backend/hogql_queries/trends/test/test_trends_query_runner.py
@@ -19,7 +19,7 @@ from posthog.test.base import (
flush_persons_and_events,
materialized,
)
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock, call, patch
from django.test import override_settings
@@ -71,6 +71,7 @@ from posthog.hogql.query import execute_hogql_query
from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.client.execute import sync_execute
+from posthog.exceptions import ClickHouseQueryTimeOut
from posthog.hogql_queries.utils.breakdowns import (
BREAKDOWN_NULL_DISPLAY,
BREAKDOWN_NULL_STRING_LABEL,
@@ -3556,6 +3557,26 @@ class TestTrendsQueryRunner(ClickhouseTestMixin, APIBaseTest):
"Error thrown inside thread",
)
+ @parameterized.expand([("without_callback", False, 3), ("with_callback", True, 1)])
+ @patch("products.product_analytics.backend.hogql_queries.trends.trends_query_runner.execute_hogql_query")
+ def test_on_series_error_stops_the_queued_series_queries(
+ self, _name: str, with_callback: bool, expected_queries: int, patch_execute: MagicMock
+ ) -> None:
+ error = ClickHouseQueryTimeOut()
+ patch_execute.side_effect = error
+ runner = self._create_query_runner(
+ "2020-01-09", "2020-01-20", IntervalType.DAY, [EventsNode(event=event) for event in ("a", "b", "c")]
+ )
+ callback = MagicMock()
+ if with_callback:
+ runner.on_series_error = callback
+
+ with self.assertRaises(ClickHouseQueryTimeOut):
+ runner.calculate()
+
+ assert patch_execute.call_count == expected_queries
+ assert callback.call_args_list == ([call(error)] if with_callback else [])
+
def test_to_actors_query_options(self):
self._create_test_events()
flush_persons_and_events()
diff --git a/products/product_analytics/backend/hogql_queries/trends/trends_query_runner.py b/products/product_analytics/backend/hogql_queries/trends/trends_query_runner.py
index a7528969732f..26899742b163 100644
--- a/products/product_analytics/backend/hogql_queries/trends/trends_query_runner.py
+++ b/products/product_analytics/backend/hogql_queries/trends/trends_query_runner.py
@@ -1,4 +1,4 @@
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
from copy import deepcopy
from datetime import date, datetime, timedelta
from functools import cached_property, partial
@@ -105,6 +105,9 @@ class TrendsQueryRunner(AnalyticsQueryRunner[TrendsQueryResponse]):
cached_response: CachedTrendsQueryResponse
series: list[SeriesWithExtras]
hogql_settings: HogQLGlobalSettings | None
+ # For a caller that cannot use a partial result. When set, the first failed series query calls it once,
+ # and no queued series query starts after that. The callback can cancel the series queries in flight.
+ on_series_error: Callable[[Exception], None] | None = None
def __init__(
self,
@@ -459,6 +462,8 @@ class TrendsQueryRunner(AnalyticsQueryRunner[TrendsQueryResponse]):
timings: HogQLTimings,
is_parallel: bool,
):
+ if self.on_series_error is not None and errors:
+ return
try:
series_with_extra = self.series[index]
@@ -482,6 +487,9 @@ class TrendsQueryRunner(AnalyticsQueryRunner[TrendsQueryResponse]):
debug_errors.append(response.error)
except Exception as e:
errors.append(e)
+ # Only the first error calls back. A list append is atomic, so two failing threads cannot both see it first.
+ if self.on_series_error is not None and errors[0] is e:
+ self.on_series_error(e)
finally:
if is_parallel:
from django.db import connection
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index b45c1da801c6..1ef85b41bfe0 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -86,8 +86,18 @@ _MAX_SCORED_SERIES = 50_000
# An insight is not picked again in this time after an attempt. Events take a moment to reach
# ClickHouse, and an insight that writes nothing would otherwise stay at the head of the queue.
ATTEMPT_COOLDOWN = datetime.timedelta(hours=1)
-# For a failure that the next hour does not fix: an unsupported shape, a query that is too expensive, or no access.
+# For a failure that the next hour does not fix: an unsupported shape, a broken query, or no access. It is also the
+# first step of the too-expensive cooldowns.
SKIP_COOLDOWN = datetime.timedelta(days=1)
+# Each consecutive too-expensive visit of an insight takes the next step. The last step repeats.
+TOO_EXPENSIVE_COOLDOWNS = (
+ SKIP_COOLDOWN,
+ datetime.timedelta(days=3),
+ datetime.timedelta(days=7),
+ datetime.timedelta(days=30),
+)
+# The count outlives the longest step, so the visit after it still knows the count.
+_TOO_EXPENSIVE_COUNT_TTL = TOO_EXPENSIVE_COOLDOWNS[-1] + datetime.timedelta(days=7)
# A query counter lives a little longer than its clock hour, so a tick at the end of the hour still reads it.
# A lost counter lets the scorer spend at most one more hour of budget.
_BUDGET_TTL = datetime.timedelta(minutes=70)
@@ -164,6 +174,8 @@ class InsightPick:
query_cost: int = 1
# None when the train and score split is off.
run: ScoringRun | None = None
+ # Consecutive too-expensive visits before this pick.
+ too_expensive: int = 0
@frozen
@@ -202,6 +214,8 @@ class Selection:
# Eligible teams, and their organizations.
enrolled_teams: int = 0
enrolled_organizations: int = 0
+ # Insights in a cooldown after two or more consecutive too-expensive visits.
+ long_cooldown: int = 0
# Due rows of the stored due list, the time since the oldest of them became due, and the time since the
# oldest refresh of a row. None when the tick does not plan from the due list.
due_insights: int | None = None
@@ -468,6 +482,42 @@ def _cooling_down(insight_ids: list[int]) -> set[int]:
return {insight_id for insight_id in insight_ids if _cooldown_key(insight_id) in found}
+def _too_expensive_key(insight_id: int) -> str:
+ return f"signals:anomaly_scores:too_expensive:{insight_id}"
+
+
+def too_expensive_counts(insight_ids: list[int]) -> dict[int, int]:
+ """Consecutive too-expensive visits of each insight. An insight with no such visit is not in the result."""
+ if not insight_ids:
+ return {}
+ found = cache.get_many([_too_expensive_key(insight_id) for insight_id in insight_ids])
+ return {
+ insight_id: int(found[_too_expensive_key(insight_id)])
+ for insight_id in insight_ids
+ if _too_expensive_key(insight_id) in found
+ }
+
+
+def too_expensive_cooldown(count: int) -> datetime.timedelta:
+ return TOO_EXPENSIVE_COOLDOWNS[min(count, len(TOO_EXPENSIVE_COOLDOWNS)) - 1]
+
+
+def mark_too_expensive(insight_id: int, previous: int) -> int:
+ """Count one more too-expensive visit, and start the cooldown of the new count. Returns the new count.
+
+ ``previous`` comes from the plan. The cooldown lets one visit of an insight run at a time, so no other visit
+ changes the count in between.
+ """
+ count = previous + 1
+ cache.set(_too_expensive_key(insight_id), count, timeout=int(_TOO_EXPENSIVE_COUNT_TTL.total_seconds()))
+ mark_attempted(insight_id, too_expensive_cooldown(count))
+ return count
+
+
+def reset_too_expensive(insight_id: int) -> None:
+ cache.delete(_too_expensive_key(insight_id))
+
+
def _team_loaded_key(team_id: int) -> str:
return f"signals:anomaly_scores:team_loaded:{team_id}"
@@ -714,6 +764,7 @@ def select_insights(
insight.insight_id: max(state.get(insight.insight_id, {}).values(), default=None) for insight in insights
}
cooling = _cooling_down([insight.insight_id for insight in insights])
+ too_expensive = too_expensive_counts([insight.insight_id for insight in insights])
active_by_id = {insight.insight_id: insight for insight in insights if insight.insight_id not in cooling}
team_ids = {insight_id: insight.team_id for insight_id, insight in active_by_id.items()}
@@ -840,6 +891,7 @@ def select_insights(
interval=active_by_id[insight_id].interval,
query_cost=plan.queries,
run=plan.run,
+ too_expensive=too_expensive.get(insight_id, 0),
)
for insight_id, plan in [*score_picked.items(), *train_picked.items()]
]
@@ -859,6 +911,7 @@ def select_insights(
teams_skipped=len(teams_with_insights) - len(loaded_teams),
enrolled_teams=len(teams),
enrolled_organizations=len({team.organization_id for team in teams.values()}),
+ long_cooldown=sum(too_expensive.get(insight_id, 0) > 1 for insight_id in cooling),
due_insights=None if due_list is None else len(due_list.insights),
oldest_due_seconds=None if due_list is None else due_list.oldest_due_seconds,
oldest_refresh_seconds=None if due_list is None else due_list.oldest_refresh_seconds,
diff --git a/products/signals/backend/anomaly_scores/scoring.py b/products/signals/backend/anomaly_scores/scoring.py
index c612b7a18108..662c582d7d0b 100644
--- a/products/signals/backend/anomaly_scores/scoring.py
+++ b/products/signals/backend/anomaly_scores/scoring.py
@@ -1,5 +1,6 @@
import json
import math
+import uuid
import hashlib
import dataclasses
from collections.abc import Sequence
@@ -15,8 +16,9 @@ from posthog.schema import BreakdownFilter, ChartDisplayType, DateRange, Interva
from posthog.hogql.constants import INSIGHT_QUERY_FANOUT_CONCURRENCY, HogQLGlobalSettings, LimitContext
from posthog.api.services.query import ExecutionMode
-from posthog.clickhouse.client.connection import ClickHouseUser
-from posthog.clickhouse.query_tagging import Feature, Product, get_team_query_tags, tag_queries
+from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.client.connection import ClickHouseUser, default_client
+from posthog.clickhouse.query_tagging import Feature, Product, get_team_query_tags, tag_queries, tags_context
from posthog.clickhouse.workload import Workload
from posthog.dataclasses import frozen
from posthog.event_usage import EventSource
@@ -55,6 +57,8 @@ _MAX_QUERY_SECONDS = 150
# Trends runs one query per series in a pool of INSIGHT_QUERY_FANOUT_CONCURRENCY threads, so more series run in waves.
# All the waves of one insight share this budget. The insight activity then has time to classify a timeout.
SCORING_QUERY_BUDGET_SECONDS = 210
+# A kill is cleanup with its own time limit, so a silent node must not hold the scoring thread.
+_CANCEL_SEND_RECEIVE_TIMEOUT_SECONDS = 15
class UnsupportedInsightError(Exception):
@@ -297,6 +301,43 @@ def scoring_query_settings(queries: int) -> HogQLGlobalSettings:
)
+def cancel_series_queries(team_id: int, client_query_id: str) -> None:
+ """Kill the scoring queries of one insight that still run. A failed kill lets them run to their own time limit."""
+ # Without this, the kill queries get ids with the same prefix as the queries that they kill.
+ with tags_context(client_query_id=None):
+ try:
+ with default_client(send_receive_timeout=_CANCEL_SEND_RECEIVE_TIMEOUT_SECONDS) as client:
+ rows = sync_execute(
+ """
+ SELECT FQDN(), query_id
+ FROM distributed_system_processes
+ WHERE query_id LIKE %(prefix)s
+ SETTINGS max_execution_time = 2
+ """,
+ {"prefix": f"{team_id}_{client_query_id}_%"},
+ sync_client=client,
+ )
+ query_ids_by_host: dict[str, list[str]] = {}
+ for host, query_id in rows:
+ query_ids_by_host.setdefault(host, []).append(query_id)
+ for host, query_ids in query_ids_by_host.items():
+ with default_client(host=host, send_receive_timeout=_CANCEL_SEND_RECEIVE_TIMEOUT_SECONDS) as client:
+ sync_execute(
+ "KILL QUERY WHERE query_id IN %(query_ids)s ASYNC SETTINGS max_execution_time = 5",
+ {"query_ids": query_ids},
+ sync_client=client,
+ )
+ except Exception as error:
+ logger.warning(
+ "anomaly_scores.cancel_series_queries_failed",
+ team_id=team_id,
+ error_class=type(error).__name__,
+ error=str(error)[:200],
+ )
+ return
+ logger.info("anomaly_scores.series_queries_cancelled", team_id=team_id, cancelled=len(rows))
+
+
def supports_score_step(query: TrendsQuery) -> bool:
"""A score step reads the buckets before a scored bucket as its context, so the buckets must have no gaps."""
# Trends drops the buckets of the days that the filter leaves out.
@@ -629,6 +670,8 @@ def _fetch(
tag_queries(**get_team_query_tags(team), insight_id=insight.pk)
tag_queries(product=Product.SIGNALS, feature=Feature.ALERTING)
+ # One prefix for all the series queries of this fetch, so that the first failure can kill the others.
+ client_query_id = f"anomaly_scoring_{uuid.uuid4().hex}"
# Splits the scoring queries in `query_log`, so train and score cost can be compared, and lets `spend.py`
# find them. A visit with the split off fetches the full history, as a train step does.
tag_queries(name=f"anomaly_scoring_{(run or ScoringRun.TRAIN).value}")
@@ -644,11 +687,16 @@ def _fetch(
)
# The runner expands breakdown cohorts and compare periods into the series, so its count is the query count.
runner.hogql_settings = scoring_query_settings(len(runner.series))
- response = runner.run(
- execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS,
- insight_id=insight.pk,
- analytics_props={"source": EventSource.SELF_DRIVING},
- )
+ if len(runner.series) > 1:
+ # With `timeout_overflow_mode="throw"` no partial result is usable, so one failed series query fails the insight.
+ runner.on_series_error = lambda _error: cancel_series_queries(team.id, client_query_id)
+ # The worker thread runs other queries after this fetch, and those must not carry this prefix.
+ with tags_context(client_query_id=client_query_id):
+ response = runner.run(
+ execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS,
+ insight_id=insight.pk,
+ analytics_props={"source": EventSource.SELF_DRIVING},
+ )
results = getattr(response, "results", None)
if not isinstance(results, list):
raise RuntimeError(f"No results for insight {insight.pk}")
diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.py
index c6b79046a762..f27415fd2a4b 100644
--- a/products/signals/backend/anomaly_scores/tick.py
+++ b/products/signals/backend/anomaly_scores/tick.py
@@ -59,7 +59,10 @@ from products.signals.backend.anomaly_scores.queue import (
InsightPick,
TeamCensus,
mark_attempted,
+ mark_too_expensive,
+ reset_too_expensive,
select_insights,
+ too_expensive_cooldown,
)
from products.signals.backend.anomaly_scores.scoring import (
SCORING_QUERY_BUDGET_SECONDS,
@@ -80,7 +83,8 @@ STATUS_SCORED = "scored"
STATUS_UP_TO_DATE = "up_to_date"
STATUS_UNSUPPORTED = "unsupported"
STATUS_MISSING = "missing"
-# The scoring query hit a ClickHouse time or memory limit. A retry in an hour hits it again.
+# The scoring query hit a ClickHouse time or memory limit. A retry in an hour hits it again, so the cooldown
+# grows with each consecutive one.
STATUS_TOO_EXPENSIVE = "too_expensive"
# The insight's creator cannot read a table of the query.
STATUS_ACCESS_DENIED = "access_denied"
@@ -98,7 +102,7 @@ _PLAN_TIMEOUT = datetime.timedelta(minutes=2)
# A timed-out attempt does not stop its thread, so a plan that ends in time also keeps its retry from overlapping it.
_PLAN_LOAD_BUDGET = _PLAN_TIMEOUT / 2
# All the scoring queries of one insight stop within their budget. The rest is detection, the capture request,
-# and the failure classification. A query timeout must reach the classification, so that it gets the 1 day cooldown.
+# and the failure classification. A query timeout must reach the classification, so that its cooldown grows.
_INSIGHT_TIMEOUT = datetime.timedelta(seconds=SCORING_QUERY_BUDGET_SECONDS) + datetime.timedelta(seconds=90)
_INSIGHT_HEARTBEAT_TIMEOUT = datetime.timedelta(minutes=1)
_CAPTURE_TIMEOUT = datetime.timedelta(minutes=1)
@@ -137,6 +141,7 @@ class AnomalyScoringTickPlan:
enrolled_organizations: int = 0
teams_planned: int = 0
teams_skipped: int = 0
+ long_cooldown: int = 0
self_driving_organization_percentage: float = 0.0
due_insights: int | None = None
oldest_due_seconds: int | None = None
@@ -202,6 +207,8 @@ class AnomalyScoringTickResult:
# The plan activity failed, and the tick scored nothing. `plan_error` is the error class.
plan_failed: bool = False
plan_error: str | None = None
+ # Insights in a cooldown after two or more consecutive too-expensive visits.
+ long_cooldown: int = 0
# Due rows that the plan read from the stored due list, the time since the oldest of them became due, and
# the time since the oldest row refresh. None when the plan does not read the due list.
due_insights: int | None = None
@@ -256,6 +263,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
enrolled_organizations=selection.enrolled_organizations,
teams_planned=selection.teams_planned,
teams_skipped=selection.teams_skipped,
+ long_cooldown=selection.long_cooldown,
self_driving_organization_percentage=config.self_driving_organization_percentage,
due_insights=selection.due_insights,
oldest_due_seconds=selection.oldest_due_seconds,
@@ -332,17 +340,33 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
error=str(error)[:200],
duration_ms=duration_ms,
)
- record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0, run=pick.run)
- capture_pick(pick, status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, run=pick.run)
+ too_expensive = pick.too_expensive
+ cooldown = ATTEMPT_COOLDOWN
+ if status == STATUS_TOO_EXPENSIVE:
+ too_expensive = mark_too_expensive(pick.insight_id, pick.too_expensive)
+ cooldown = too_expensive_cooldown(too_expensive)
# A full cluster is not a fault of the insight, so it keeps the attempt cooldown.
- cooldown = SKIP_COOLDOWN if status not in (None, STATUS_AT_CAPACITY) else ATTEMPT_COOLDOWN
+ elif status is not None and status != STATUS_AT_CAPACITY:
+ cooldown = SKIP_COOLDOWN
+ mark_attempted(pick.insight_id, cooldown)
+ record_pick(status=status or STATUS_ERRORED, interval=interval, duration_ms=duration_ms, events=0, run=pick.run)
+ capture_pick(
+ pick,
+ status=status or STATUS_ERRORED,
+ interval=interval,
+ duration_ms=duration_ms,
+ run=pick.run,
+ too_expensive=too_expensive,
+ )
record_failed(pick.team_id, pick.insight_id, now, cooldown)
if status is None:
raise
- if cooldown == SKIP_COOLDOWN:
- mark_attempted(pick.insight_id, SKIP_COOLDOWN)
return ScoreInsightResult(status=status, events=0)
duration_ms = round((time.monotonic() - started) * 1000)
+ too_expensive = pick.too_expensive
+ if too_expensive and result.status in (STATUS_SCORED, STATUS_UP_TO_DATE):
+ reset_too_expensive(pick.insight_id)
+ too_expensive = 0
logger.info(
"anomaly_scoring_insight_scored",
team_id=pick.team_id,
@@ -365,6 +389,7 @@ def run_insight(input: ScoreInsightInput, now: datetime.datetime) -> ScoreInsigh
run=result.run,
series=result.series,
events=result.events,
+ too_expensive=too_expensive,
)
return result
@@ -378,6 +403,7 @@ def capture_pick(
run: ScoringRun | None,
series: int = 0,
events: int = 0,
+ too_expensive: int = 0,
) -> None:
"""One event per pick, with the ids that the SDK metrics leave out, so cost and outcome break down per project."""
ph_background_capture()(
@@ -394,6 +420,7 @@ def capture_pick(
"series": series,
"events": events,
"queries": pick.query_cost,
+ "too_expensive_streak": too_expensive,
},
)
@@ -537,6 +564,7 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
self_driving_organization_percentage=result.self_driving_organization_percentage,
plan_failed=result.plan_failed,
plan_error=result.plan_error,
+ long_cooldown=result.long_cooldown,
due_insights=result.due_insights,
oldest_due_seconds=result.oldest_due_seconds,
oldest_refresh_seconds=result.oldest_refresh_seconds,
@@ -578,6 +606,7 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
"self_driving_organization_percentage": result.self_driving_organization_percentage,
"plan_failed": result.plan_failed,
"plan_error": result.plan_error,
+ "long_cooldown": result.long_cooldown,
"due_insights": result.due_insights,
"oldest_due_seconds": result.oldest_due_seconds,
"oldest_refresh_seconds": result.oldest_refresh_seconds,
@@ -683,6 +712,7 @@ class AnomalyScoringTickWorkflow:
plan_ms=plan_ms,
teams_planned=plan.teams_planned,
teams_skipped=plan.teams_skipped,
+ long_cooldown=plan.long_cooldown,
self_driving_organization_percentage=plan.self_driving_organization_percentage,
due_insights=plan.due_insights,
oldest_due_seconds=plan.oldest_due_seconds,
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index a4dfb80e7874..418e53a75557 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -27,7 +27,12 @@ from posthog.schema import IntervalType
from posthog.hogql.errors import QueryError, ResolutionError, TableAccessDeniedError
from posthog.clickhouse.client import sync_execute
-from posthog.exceptions import ClickHouseAtCapacity, ClickHouseClusterMemoryLimitExceeded, ClickHouseQueryTimeOut
+from posthog.exceptions import (
+ ClickHouseAtCapacity,
+ ClickHouseClusterMemoryLimitExceeded,
+ ClickHouseQueryMemoryLimitExceeded,
+ ClickHouseQueryTimeOut,
+)
from posthog.models import Organization, Team, User
from posthog.tasks.alerts.detectors.base import ARTIFACT_SCHEMA_VERSION
@@ -50,6 +55,7 @@ from products.signals.backend.anomaly_scores.config import (
from products.signals.backend.anomaly_scores.due_list import record_scored, refresh_due_list
from products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_events
from products.signals.backend.anomaly_scores.queue import (
+ ATTEMPT_COOLDOWN,
SKIP_COOLDOWN,
ActiveInsight,
ActivityReason,
@@ -60,6 +66,7 @@ from products.signals.backend.anomaly_scores.queue import (
_fleet_budget_key,
_project_budget_key,
_team_loaded_key,
+ _too_expensive_key,
active_insights,
eligible_teams,
is_due,
@@ -67,6 +74,7 @@ from products.signals.backend.anomaly_scores.queue import (
reserve_queries,
scored_buckets,
select_insights,
+ too_expensive_counts,
within_budget,
)
from products.signals.backend.anomaly_scores.scoring import (
@@ -670,10 +678,16 @@ class TestScoreEvents(SimpleTestCase):
class TestRunInsight(SimpleTestCase):
- def _run(self, error: Exception) -> tuple[str, list[Any], MagicMock, dict[str, Any]]:
+ def setUp(self) -> None:
+ super().setUp()
+ cache.delete(_too_expensive_key(7))
+ self.addCleanup(cache.delete, _too_expensive_key(7))
+
+ def _run(self, error: Exception, too_expensive: int = 0) -> tuple[str, list[Any], MagicMock, dict[str, Any]]:
tick = "products.signals.backend.anomaly_scores.tick"
client = MagicMock()
capture = MagicMock()
+ mark_attempted = MagicMock()
with (
patch(f"{tick}.ph_background_capture", return_value=capture),
patch(
@@ -682,11 +696,14 @@ class TestRunInsight(SimpleTestCase):
),
patch(f"{tick}.Team"),
patch(f"{tick}.score_insight", side_effect=error),
- patch(f"{tick}.mark_attempted") as mark_attempted,
+ patch(f"{tick}.mark_attempted", mark_attempted),
+ patch("products.signals.backend.anomaly_scores.queue.mark_attempted", mark_attempted),
patch(f"{tick}.record_failed") as record_failed,
patch("products.signals.backend.anomaly_scores.metrics.posthoganalytics.default_client", client),
):
- pick = InsightPick(team_id=1, insight_id=7, scored_through={}, interval=IntervalType.DAY)
+ pick = InsightPick(
+ team_id=1, insight_id=7, scored_through={}, interval=IntervalType.DAY, too_expensive=too_expensive
+ )
try:
status = run_insight(ScoreInsightInput(pick=pick), NOW).status
except Exception:
@@ -726,6 +743,47 @@ class TestRunInsight(SimpleTestCase):
"queries": 1,
}
+ def test_consecutive_too_expensive_visits_grow_the_cooldown_and_a_success_resets_it(self) -> None:
+ cooldowns = []
+ streaks = []
+ for error in [
+ ClickHouseQueryTimeOut(),
+ ClickHouseAtCapacity(),
+ ClickHouseClusterMemoryLimitExceeded(),
+ ClickHouseQueryMemoryLimitExceeded(),
+ ClickHouseQueryTimeOut(),
+ ClickHouseQueryTimeOut(),
+ ClickHouseQueryTimeOut(),
+ ]:
+ _status, visit_cooldowns, _metrics, pick_event = self._run(error, too_expensive_counts([7]).get(7, 0))
+ cooldowns.append(visit_cooldowns[-1])
+ streaks.append(pick_event["too_expensive_streak"])
+
+ assert cooldowns == [
+ SKIP_COOLDOWN,
+ ATTEMPT_COOLDOWN,
+ ATTEMPT_COOLDOWN,
+ timedelta(days=3),
+ timedelta(days=7),
+ timedelta(days=30),
+ timedelta(days=30),
+ ]
+ assert streaks == [1, 1, 1, 2, 3, 4, 5]
+
+ capture = MagicMock()
+ with (
+ patch("products.signals.backend.anomaly_scores.tick.ph_background_capture", return_value=capture),
+ patch(
+ "products.signals.backend.anomaly_scores.tick._score_pick",
+ return_value=ScoreInsightResult(status="up_to_date", events=0),
+ ),
+ ):
+ pick = InsightPick(team_id=1, insight_id=7, scored_through={}, too_expensive=5)
+ run_insight(ScoreInsightInput(pick=pick), NOW)
+
+ assert too_expensive_counts([7]) == {}
+ assert capture.call_args.kwargs["properties"]["too_expensive_streak"] == 0
+
def test_a_full_cluster_returns_at_capacity_and_waits_an_hour(self) -> None:
status, cooldowns, _metrics, _pick_event = self._run(ClickHouseAtCapacity())
@@ -1368,7 +1426,9 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
small_dashboard = Dashboard.objects.create(team=small_team, name="Main")
DashboardTile.objects.create(dashboard=small_dashboard, insight=small_insight)
for insight in [*large_insights, small_insight]:
- cache.delete(_cooldown_key(insight.id))
+ cache.delete_many([_cooldown_key(insight.id), _too_expensive_key(insight.id)])
+ cache.set(_too_expensive_key(small_insight.id), 2)
+ self.addCleanup(cache.delete, _too_expensive_key(small_insight.id))
config = AnomalyScoringConfig(
enabled=True,
organization_ids=(str(self.organization.id),),
@@ -1381,6 +1441,7 @@ class TestScoredBuckets(ClickhouseTestMixin, BaseTest):
assert [pick.team_id for pick in selection.picks] == [self.team.id, small_team.id]
assert selection.picks[0].insight_id in {insight.id for insight in large_insights}
assert selection.picks[1].insight_id == small_insight.id
+ assert [pick.too_expensive for pick in selection.picks] == [0, 2]
assert selection.candidates == 4
assert [(team.team_id, team.active, team.active_intervals) for team in selection.census] == [
(self.team.id, 3, {"day": 3}),