Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(alerts): render an insight firing's chart after its check frees its slot (#123599)

Jimmy Zhu3bdb5d1ed9ec
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py
index 105c2e94eef2..6b07a41cd7b2 100644
--- a/posthog/settings/temporal.py
+++ b/posthog/settings/temporal.py
@@ -411,6 +411,16 @@ if ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS <= 0:
raise ImproperlyConfigured(
"ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS must be a positive integer, or no platform insight check starts"
)
+# Chart renders for platform insight firings allowed to run at once, across every worker. A render
+# blocks a worker thread for up to 90 seconds, so this keeps renders from taking the slots that
+# planning and recording need.
+ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_CHART_RENDERS: int = get_from_env(
+ "ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_CHART_RENDERS", 3, type_cast=int
+)
+if ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_CHART_RENDERS <= 0:
+ raise ImproperlyConfigured(
+ "ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_CHART_RENDERS must be a positive integer, or no platform insight chart renders"
+ )
# The platform's parallel logs batches allowed to run at once, across every team. A batch runs its
# cohort queries one after another, so this bounds the parallel run's concurrent logs queries.
ALERTS_PLATFORM_LOGS_MAX_INFLIGHT_BATCHES: int = get_from_env(
diff --git a/products/alerts/backend/platform_chart_detail.py b/products/alerts/backend/platform_chart_detail.py
new file mode 100644
index 000000000000..311a8d4f438a
--- /dev/null
+++ b/products/alerts/backend/platform_chart_detail.py
@@ -0,0 +1,68 @@
+"""The chart an insight firing carries in its `source_detail`, rendered after the check.
+
+A render takes up to `RENDER_TIMEOUT`, so the check only marks the firings that want one, and the
+evaluation workflow renders once the check has given back its pool slot. This module has no Django
+imports, so the workflow can call it.
+"""
+
+from dataclasses import replace
+from typing import Any
+
+from posthog.dataclasses import frozen
+
+from products.alerts_platform.backend.facade.contracts import SourceBatchEvaluation
+
+# Never recorded: the workflow replaces it with the chart's ids, or drops it when the render fails.
+CHART_PENDING_KEY = "insight_chart_pending_alert_id"
+# The ids rather than a URL, because the URL opens the chart without a login and history keeps
+# them for its whole retention. A delivery mints the URL when it sends.
+CHART_ASSET_ID_KEY = "insight_chart_asset_id"
+CHART_RENDER_USER_ID_KEY = "insight_chart_render_user_id"
+
+
+@frozen
+class RenderedChart:
+ asset_id: int
+ render_user_id: int
+
+
+def pending_chart_alert_id(evaluation: SourceBatchEvaluation) -> str | None:
+ """The legacy alert whose chart this check's firings wait for, or None when none wants one."""
+ for outcome in evaluation.outcomes:
+ for group in outcome.groups:
+ alert_id = group.source_detail.get(CHART_PENDING_KEY)
+ if alert_id is not None:
+ return str(alert_id)
+ return None
+
+
+def with_chart(evaluation: SourceBatchEvaluation, chart: RenderedChart | None) -> SourceBatchEvaluation:
+ """The evaluation with every pending firing given the chart, or left without one when it is None.
+
+ One render serves every firing of a check, because they all show the same insight.
+ """
+ if pending_chart_alert_id(evaluation) is None:
+ return evaluation
+ return replace(
+ evaluation,
+ outcomes=tuple(
+ replace(
+ outcome,
+ groups=tuple(
+ replace(group, source_detail=_resolved(group.source_detail, chart))
+ if CHART_PENDING_KEY in group.source_detail
+ else group
+ for group in outcome.groups
+ ),
+ )
+ for outcome in evaluation.outcomes
+ ),
+ )
+
+
+def _resolved(source_detail: dict[str, Any], chart: RenderedChart | None) -> dict[str, Any]:
+ detail = {key: value for key, value in source_detail.items() if key != CHART_PENDING_KEY}
+ if chart is not None:
+ detail[CHART_ASSET_ID_KEY] = chart.asset_id
+ detail[CHART_RENDER_USER_ID_KEY] = chart.render_user_id
+ return detail
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index c645cca03f83..62912e98b7bf 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -32,12 +32,7 @@ from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
from posthog.errors import CH_TRANSIENT_ERRORS, QueryErrorCategory, classify_query_error
from posthog.schema_enums import AlertCalculationInterval
from posthog.tasks.alerts.schedule_restriction import is_utc_datetime_blocked
-from posthog.tasks.alerts.utils import (
- AlertEvaluationResult,
- RenderedInsightChart,
- render_alert_insight_chart,
- skip_because_of_weekend,
-)
+from posthog.tasks.alerts.utils import AlertEvaluationResult, render_alert_insight_chart, skip_because_of_weekend
from posthog.temporal.alerts.admission import admit_evaluation_slots, release_evaluation_slot
from products.alerts.backend.evaluation import check_alert_for_insight
@@ -54,6 +49,7 @@ from products.alerts.backend.llm_detector_limits import is_llm_detector_config
from products.alerts.backend.logic.alert_email import INSIGHT_ALERT_ERRORED_EVENT_ID
from products.alerts.backend.models.alert import AlertConfiguration
from products.alerts.backend.platform_alert_wording import insight_alert_url
+from products.alerts.backend.platform_chart_detail import CHART_PENDING_KEY, RenderedChart
from products.alerts_platform.backend.facade.api import due_check_ids, due_checks, refresh_settings, slot_of
from products.alerts_platform.backend.facade.contracts import (
SOURCE_CONDITION_KEY,
@@ -106,6 +102,12 @@ from products.cdp.backend.internal_events import LEGACY_INSIGHT_ALERT_EVENT
logger = structlog.get_logger(__name__)
INFLIGHT_KEY = "alerts:platform:insight:evaluations:inflight"
+CHART_RENDER_INFLIGHT_KEY = "alerts:platform:insight:chart-renders:inflight"
+
+
+class ChartRenderPoolFull(Exception):
+ """Every chart render slot is taken. The activity retries later, holding no worker slot meanwhile."""
+
# How many threads of the shared AI pool the parallel run may hold, so a stalled platform check
# never takes more than this from production. A thread counts until it ends, not until Temporal
@@ -682,8 +684,7 @@ def _decide_grouped(
key, found = bounded_grouping_key(series.identity), series.result.breaches or []
if key not in breaches or (found and not breaches[key]):
breaches[key] = found
- # One render serves every value that fires, because they all show the same insight.
- chart = _chart(check, alert)
+ wants_chart = _wants_chart(check, alert)
decision = replace(
decision,
verdicts=tuple(
@@ -691,7 +692,9 @@ def _decide_grouped(
verdict,
group=replace(
verdict.group,
- source_detail=_detail(alert, breaches.get(verdict.group.grouping_key, []), chart),
+ source_detail=_detail(
+ alert, breaches.get(verdict.group.grouping_key, []), wants_chart=wants_chart
+ ),
),
)
if verdict.outcome.notification == NotificationAction.FIRE
@@ -712,15 +715,38 @@ def _firing_detail(
check: PlatformAlertCheckInput, alert: AlertConfiguration, breaches: Sequence[str]
) -> dict[str, Any]:
"""What production's `$insight_alert_firing` carries beside the verdict, which only the check can know."""
- return _detail(alert, breaches, _chart(check, alert))
+ return _detail(alert, breaches, wants_chart=_wants_chart(check, alert))
-def _chart(check: PlatformAlertCheckInput, alert: AlertConfiguration) -> RenderedInsightChart | None:
+def _wants_chart(check: PlatformAlertCheckInput, alert: AlertConfiguration) -> bool:
# A render takes up to a minute and a half, so only an alert the platform delivers for pays
# for one. Production renders no chart for a real-time alert either.
- if check.owned_by_platform and alert.calculation_interval != AlertCalculationInterval.REAL_TIME:
- return render_alert_insight_chart(alert)
- return None
+ return check.owned_by_platform and alert.calculation_interval != AlertCalculationInterval.REAL_TIME
+
+
+def render_firing_chart(team_id: int, alert_id: str, *, held_until: float) -> RenderedChart | None:
+ """Render the chart a firing of this alert sends, or None when none applies or the render fails.
+
+ The evaluation workflow calls this after the check, so the render never holds the check's slot.
+ Raises `ChartRenderPoolFull` when the render pool has no room.
+ """
+ if not admit_evaluation_slots(
+ [alert_id],
+ limit=settings.ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_CHART_RENDERS,
+ expires_at=held_until,
+ key=CHART_RENDER_INFLIGHT_KEY,
+ ):
+ raise ChartRenderPoolFull()
+ try:
+ alert = AlertConfiguration.objects.filter(team_id=team_id, id=alert_id).select_related("team").first()
+ if alert is None:
+ return None
+ chart = render_alert_insight_chart(alert)
+ return (
+ RenderedChart(asset_id=chart.asset_id, render_user_id=chart.render_user_id) if chart is not None else None
+ )
+ finally:
+ release_evaluation_slot(alert_id, held_until=held_until, key=CHART_RENDER_INFLIGHT_KEY)
def _context(alert: AlertConfiguration) -> dict[str, Any]:
@@ -732,7 +758,7 @@ def _clipped(text: str, limit: int = MAX_BREACH_CHARS) -> str:
return text if len(text) <= limit else text[: limit - 1] + "…"
-def _detail(alert: AlertConfiguration, breaches: Sequence[str], chart: RenderedInsightChart | None) -> dict[str, Any]:
+def _detail(alert: AlertConfiguration, breaches: Sequence[str], *, wants_chart: bool) -> dict[str, Any]:
# Bounded, because the outcome crosses into Temporal and a breach names user-written labels.
detail: dict[str, Any] = {
"breaches": [_clipped(breach) for breach in breaches[:MAX_DETAIL_BREACHES]],
@@ -740,11 +766,8 @@ def _detail(alert: AlertConfiguration, breaches: Sequence[str], chart: RenderedI
# The legacy alert's id, which a native message's snooze menu snoozes.
"alert_id": str(alert.id),
}
- if chart is not None:
- # The ids rather than a URL, because the URL opens the chart without a login and history keeps
- # this for its whole retention. A delivery mints the URL when it sends.
- detail["insight_chart_asset_id"] = chart.asset_id
- detail["insight_chart_render_user_id"] = chart.render_user_id
+ if wants_chart:
+ detail[CHART_PENDING_KEY] = str(alert.id)
return detail
diff --git a/products/alerts/backend/temporal/platform_evaluate.py b/products/alerts/backend/temporal/platform_evaluate.py
index ef990d0cdc93..e5158cef1f5a 100644
--- a/products/alerts/backend/temporal/platform_evaluate.py
+++ b/products/alerts/backend/temporal/platform_evaluate.py
@@ -8,6 +8,7 @@ minute, and daily alerts in one timezone share their minute, so evaluating a key
another would put the last check minutes behind the first.
"""
+import time
import asyncio
import datetime as dt
from collections.abc import Callable
@@ -27,6 +28,7 @@ with workflow.unsafe.imports_passed_through():
from posthog.temporal.alerts.retry_policy import alert_timeouts
from posthog.temporal.common.utils import close_db_connections
+ from products.alerts.backend.platform_chart_detail import RenderedChart, pending_chart_alert_id, with_chart
from products.alerts_platform.backend.facade.contracts import (
RECORD_OUTCOMES_ACTIVITY,
AlertDeliveryRequest,
@@ -57,6 +59,16 @@ REAL_TIME_CHECK_RETRY_POLICY = _REAL_TIME_TIMEOUTS.evaluate_retry_policy
# Production's retries for an AI detector check, inside the default budget. A judge that reaches
# no verdict raises for another attempt, and the last attempt records the failure.
LLM_CHECK_RETRY_POLICY = alert_timeouts(None).evaluate_retry_policy
+# A firing's chart renders in its own activity, after the check has released its pool slot. It
+# outlasts the export render's own 90-second limit, so the render, not the activity, gives up first.
+CHART_START_TO_CLOSE = dt.timedelta(minutes=2)
+# Room to wait for a render slot. A render that gets none in time sends its firing without a chart.
+CHART_SCHEDULE_TO_CLOSE = CHART_START_TO_CLOSE + dt.timedelta(minutes=2)
+CHART_RETRY_POLICY = RetryPolicy(
+ initial_interval=dt.timedelta(seconds=5),
+ backoff_coefficient=2.0,
+ maximum_interval=dt.timedelta(seconds=30),
+)
RECORD_START_TO_CLOSE = dt.timedelta(seconds=8)
RECORD_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=12)
# The pool admits once per round, so a batch that admits only once evaluates at most one pool of
@@ -70,6 +82,7 @@ EVALUATION_BUDGET = (
PLAN_SCHEDULE_TO_CLOSE
+ ADMISSION_RUN_BUDGET
+ max(CHECK_SCHEDULE_TO_CLOSE, REAL_TIME_CHECK_SCHEDULE_TO_CLOSE)
+ + CHART_SCHEDULE_TO_CLOSE
+ RECORD_SCHEDULE_TO_CLOSE
)
_PATCH_READMISSION = "platform-insight-readmission-2026-10"
@@ -80,6 +93,8 @@ _PATCH_CHECK_BUDGETS = "platform-insight-check-budgets-2026-10"
# Runs started before it evaluated every check on this queue. A plan can return an AI detector
# check into such a run during a deploy, and replay must schedule the activity that run recorded.
_PATCH_AI_DETECTOR_QUEUE = "platform-insight-ai-detector-queue-2026-10"
+# Runs started before it rendered charts inside the check, and replay must not schedule a render.
+_PATCH_CHART_AFTER_CHECK = "platform-insight-chart-after-check-2026-10"
@frozen
@@ -100,6 +115,12 @@ class AdmittedInsightCheck:
owned_by_platform: bool = False
+@frozen
+class InsightChartInputs:
+ team_id: int
+ alert_id: str
+
+
@frozen
class InsightCheckInputs:
evaluation: SourceEvaluationInputs
@@ -249,6 +270,27 @@ def _release_platform_llm_check(thread: "asyncio.Future[SourceBatchEvaluation |
thread.exception()
+@close_db_connections
+def render_platform_insight_chart_activity(inputs: InsightChartInputs) -> RenderedChart | None:
+ """Sync, because the render blocks for its whole duration. Only a full render pool raises, for a
+ retry. Any other failure sends the firing without a chart, as production does."""
+ from products.alerts.backend.platform_source_cycle import ChartRenderPoolFull, render_firing_chart # noqa: PLC0415
+
+ try:
+ # A slot of its own per attempt, so a timed-out attempt cannot release its retry's slot.
+ return render_firing_chart(
+ inputs.team_id, inputs.alert_id, held_until=time.time() + CHART_START_TO_CLOSE.total_seconds()
+ )
+ except ChartRenderPoolFull:
+ raise
+ except Exception:
+ activity.logger.exception(
+ "Could not render a platform insight firing's chart", extra={"alert_id": inputs.alert_id}
+ )
+ return None
+
+
@workflow.defn(name="insight-alert-platform-evaluate")
class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
"""Evaluates one batch key, records what it decided, and starts the deliveries its checks asked
@@ -283,6 +325,10 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
if isinstance(result, BaseException) and not isinstance(result, ActivityError):
raise result
evaluations = [result for result in settled if isinstance(result, SourceBatchEvaluation)]
+ if workflow.patched(_PATCH_CHART_AFTER_CHECK):
+ evaluations = await self._with_charts(inputs, evaluations)
+ # A check that ran on new code under a run that started before the patch leaves its marker.
+ evaluations = [with_chart(evaluation, None) for evaluation in evaluations]
outcomes: tuple[PlatformAlertOutcome, ...] = tuple(o for e in evaluations for o in e.outcomes)
deliveries: tuple[AlertDeliveryRequest, ...] = tuple(d for e in evaluations for d in e.deliveries)
if len(outcomes) < len(settled):
@@ -309,6 +355,38 @@ class InsightAlertPlatformEvaluateWorkflow(PostHogWorkflow):
await start_deliveries([delivery for delivery in deliveries if delivery.configuration_id in recorded])
return len(outcomes)
+ async def _with_charts(
+ self, inputs: SourceEvaluationInputs, evaluations: list[SourceBatchEvaluation]
+ ) -> list[SourceBatchEvaluation]:
+ """Renders each firing's chart side by side. A render that fails leaves its firing without one,
+ as production sends a firing without its chart."""
+ pending = [pending_chart_alert_id(evaluation) for evaluation in evaluations]
+ rendered = await asyncio.gather(
+ *(
+ workflow.execute_activity(
+ render_platform_insight_chart_activity,
+ InsightChartInputs(team_id=inputs.batch_key.team_id, alert_id=alert_id),
+ start_to_close_timeout=CHART_START_TO_CLOSE,
+ schedule_to_close_timeout=CHART_SCHEDULE_TO_CLOSE,
+ retry_policy=CHART_RETRY_POLICY,
+ )
+ for alert_id in pending
+ if alert_id is not None
+ ),
+ return_exceptions=True,
+ )
+ charts = iter(rendered)
+ result: list[SourceBatchEvaluation] = []
+ for evaluation, alert_id in zip(evaluations, pending):
+ if alert_id is None:
+ result.append(evaluation)
+ continue
+ chart = next(charts)
+ if isinstance(chart, BaseException) and not isinstance(chart, ActivityError):
+ raise chart
+ result.append(with_chart(evaluation, chart if isinstance(chart, RenderedChart) else None))
+ return result
+
async def _plan(
self,
inputs: SourceEvaluationInputs,
@@ -413,4 +491,5 @@ PLATFORM_EVALUATION_ACTIVITIES = [
plan_platform_insight_round_activity,
plan_platform_insight_checks_activity,
evaluate_platform_insight_check_activity,
+ render_platform_insight_chart_activity,
]
diff --git a/products/alerts/backend/test/test_platform_evaluate_workflow.py b/products/alerts/backend/test/test_platform_evaluate_workflow.py
index ae55843f015f..ae99755c490e 100644
--- a/products/alerts/backend/test/test_platform_evaluate_workflow.py
+++ b/products/alerts/backend/test/test_platform_evaluate_workflow.py
@@ -15,12 +15,19 @@ from temporalio.exceptions import ApplicationError
from temporalio.testing import ActivityEnvironment, WorkflowEnvironment
from temporalio.worker import UnsandboxedWorkflowRunner, Worker
+from products.alerts.backend.platform_chart_detail import (
+ CHART_ASSET_ID_KEY,
+ CHART_PENDING_KEY,
+ CHART_RENDER_USER_ID_KEY,
+ RenderedChart,
+)
from products.alerts.backend.platform_source_cycle import PLATFORM_LLM_CHECKS
from products.alerts.backend.temporal.platform_evaluate import (
EVALUATION_BUDGET,
AdmittedInsightCheck,
InsightAlertPlatformEvaluateWorkflow,
InsightBatchPlanInputs,
+ InsightChartInputs,
InsightCheckInputs,
evaluate_platform_insight_llm_check_activity,
)
@@ -293,6 +300,81 @@ async def test_a_batch_refills_the_slots_its_finished_checks_free(
assert [sorted(str(o.configuration_id) for o in batch.outcomes) for batch in recorded] == [expected]
+ "failed_attempts, expected_detail",
+ [
+ (0, {"alert_id": "legacy-alert", CHART_ASSET_ID_KEY: 7, CHART_RENDER_USER_ID_KEY: 3}),
+ # The render pool was full at first, so the render waited for a slot.
+ (1, {"alert_id": "legacy-alert", CHART_ASSET_ID_KEY: 7, CHART_RENDER_USER_ID_KEY: 3}),
+ (None, {"alert_id": "legacy-alert"}),
+ ],
+)
+async def test_a_firing_records_the_chart_rendered_after_its_check(
+ environment: WorkflowEnvironment, failed_attempts: int | None, expected_detail: dict[str, object]
+) -> None:
+ events: list[str] = []
+ recorded: list[SourceOutcomeInputs] = []
+
+ @activity.defn(name="plan_platform_insight_checks_activity")
+ async def plan(inputs: InsightBatchPlanInputs) -> list[AdmittedInsightCheck]:
+ return [AdmittedInsightCheck(configuration_id=ADMITTED[0])] if not inputs.exclude else []
+
+ @activity.defn(name="evaluate_platform_insight_check_activity")
+ async def evaluate(inputs: InsightCheckInputs) -> SourceBatchEvaluation | None:
+ events.append("check")
+ outcome = PlatformAlertOutcome(
+ configuration_id=uuid.UUID(inputs.configuration_id),
+ evaluation_key="slot:2026-09-16T10:00:00+00:00",
+ consecutive_failures=0,
+ groups=(
+ GroupOutcome(
+ grouping_key="",
+ kind=AlertEventKind.FIRING,
+ new_state="firing",
+ notified=True,
+ source_detail={"alert_id": "legacy-alert", CHART_PENDING_KEY: "legacy-alert"},
+ ),
+ ),
+ )
+ return SourceBatchEvaluation(outcomes=(outcome,), deliveries=())
+
+ @activity.defn(name="render_platform_insight_chart_activity")
+ async def render(inputs: InsightChartInputs) -> RenderedChart | None:
+ if failed_attempts is None or activity.info().attempt <= failed_attempts:
+ raise ApplicationError("render pool full")
+ events.append(f"render:{inputs.alert_id}")
+ return RenderedChart(asset_id=7, render_user_id=3)
+
+ @activity.defn(name=RECORD_OUTCOMES_ACTIVITY)
+ async def record(inputs: SourceOutcomeInputs) -> int:
+ events.append("record")
+ recorded.append(inputs)
+ return len(inputs.outcomes)
+
+ async with Worker(
+ environment.client,
+ task_queue=QUEUE,
+ workflows=[InsightAlertPlatformEvaluateWorkflow],
+ activities=[plan, evaluate, render, record],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ await environment.client.execute_workflow(
+ InsightAlertPlatformEvaluateWorkflow.run,
+ SourceEvaluationInputs(
+ source=SourceKind.INSIGHT,
+ cutoff="2026-09-16T10:00:00+00:00",
+ batch_key=AlertBatchKey(team_id=1, slot="2026-09-16T10:00:00+00:00"),
+ ),
+ id=f"test-{uuid.uuid4()}",
+ task_queue=QUEUE,
+ )
+
+ assert events == (["check", "record"] if failed_attempts is None else ["check", "render:legacy-alert", "record"])
+ assert [group.source_detail for batch in recorded for o in batch.outcomes for group in o.groups] == [
+ expected_detail
+ ]
+
+
def test_the_binding_holds_the_whole_batch() -> None:
assert source_evaluation_timeout(SourceKind.INSIGHT) > EVALUATION_BUDGET
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index 70988b3edc28..46b763cab000 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -20,7 +20,7 @@ from posthog.models.scoping import team_scope
from posthog.redis import get_client
from posthog.schema_enums import AlertCalculationInterval
from posthog.tasks.alerts.utils import AlertEvaluationResult, RenderedInsightChart, next_check_time
-from posthog.temporal.alerts.admission import inflight_alert_ids, release_evaluation_slot
+from posthog.temporal.alerts.admission import admit_evaluation_slots, inflight_alert_ids, release_evaluation_slot
from products.alerts.backend.evaluation.contract import AlertExtractionError
from products.alerts.backend.evaluation.dispatcher import PerSeriesEvaluation, SeriesEvaluation
@@ -33,15 +33,19 @@ from products.alerts.backend.judge.contract import (
from products.alerts.backend.logic.alert_email import INSIGHT_ALERT_ERRORED_EVENT_ID
from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration
from products.alerts.backend.platform_alert_wording import insight_alert_url
+from products.alerts.backend.platform_chart_detail import CHART_PENDING_KEY, RenderedChart
from products.alerts.backend.platform_source_cycle import (
CAPACITY_REJECTED,
+ CHART_RENDER_INFLIGHT_KEY,
INFLIGHT_KEY,
MAX_BREACH_CHARS,
MAX_DETECTOR_POINTS,
MAX_RATIONALE_CHARS,
+ ChartRenderPoolFull,
evaluate_insight_check,
plan_insight_batch,
readmit_retried_check,
+ render_firing_chart,
)
from products.alerts.backend.temporal.platform_evaluate import (
AdmittedInsightCheck,
@@ -71,8 +75,8 @@ CUTOFF = datetime(2026, 9, 16, 10, tzinfo=UTC)
class TestPlatformInsightEvaluation(APIBaseTest):
def setUp(self) -> None:
super().setUp()
- get_client().delete(INFLIGHT_KEY)
- self.addCleanup(get_client().delete, INFLIGHT_KEY)
+ get_client().delete(INFLIGHT_KEY, CHART_RENDER_INFLIGHT_KEY)
+ self.addCleanup(get_client().delete, INFLIGHT_KEY, CHART_RENDER_INFLIGHT_KEY)
self.organization.available_product_features = [
{"key": AvailableFeature.HIGH_FREQUENCY_ALERTS, "name": "High-frequency alerts"},
{"key": AvailableFeature.REAL_TIME_ALERTS, "name": "Real-time alerts"},
@@ -196,13 +200,11 @@ class TestPlatformInsightEvaluation(APIBaseTest):
@parameterized.expand(
[
- ("source_owned", "source", {}),
- ("platform_owned", "platform", {"insight_chart_asset_id": 7, "insight_chart_render_user_id": 3}),
+ ("source_owned", "source", False),
+ ("platform_owned", "platform", True),
]
)
- def test_a_firing_records_what_production_sends_beside_it(
- self, _name: str, owner: str, chart: dict[str, Any]
- ) -> None:
+ def test_a_firing_records_what_production_sends_beside_it(self, _name: str, owner: str, wants_chart: bool) -> None:
alert = self._alert()
configuration = self._copy(alert, owner=owner)
expected = {
@@ -210,21 +212,37 @@ class TestPlatformInsightEvaluation(APIBaseTest):
"insight_name": alert.insight.name,
"insight_short_id": alert.insight.short_id,
"alert_id": str(alert.id),
- **chart,
+ **({CHART_PENDING_KEY: str(alert.id)} if wants_chart else {}),
}
- with patch(
- f"{_MODULE}.render_alert_insight_chart",
- return_value=RenderedInsightChart(asset_id=7, render_user_id=3),
- ) as render:
+ with patch(f"{_MODULE}.render_alert_insight_chart") as render:
outcome, _ = self._evaluate(
configuration, result=AlertEvaluationResult(value=150.0, breaches=["above 100"])
)
assert outcome is not None
assert outcome.groups[0].source_detail == expected
- # A render costs up to a minute and a half, so the parallel run must never pay for one.
- assert render.called is (owner == "platform")
+ # A render takes up to a minute and a half, so it never runs while the check holds its pool slot.
+ render.assert_not_called()
+
+ @override_settings(ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_CHART_RENDERS=1)
+ def test_a_chart_render_holds_a_render_slot_only_while_it_runs(self) -> None:
+ alert = self._alert()
+ held_until = time.time() + 60
+
+ with patch(
+ f"{_MODULE}.render_alert_insight_chart", return_value=RenderedInsightChart(asset_id=7, render_user_id=3)
+ ) as render:
+ # Back to back on a pool of one, so the second render needs the slot the first gave back.
+ for _ in range(2):
+ assert render_firing_chart(self.team.id, str(alert.id), held_until=held_until) == RenderedChart(
+ asset_id=7, render_user_id=3
+ )
+ admit_evaluation_slots(["another-render"], limit=1, expires_at=held_until, key=CHART_RENDER_INFLIGHT_KEY)
+ with pytest.raises(ChartRenderPoolFull):
+ render_firing_chart(self.team.id, str(alert.id), held_until=held_until)
+
+ assert render.call_count == 2
@parameterized.expand(
[
diff --git a/products/alerts_platform/backend/temporal/sources.py b/products/alerts_platform/backend/temporal/sources.py
index de6a0576cd6b..fb6e941c69c3 100644
--- a/products/alerts_platform/backend/temporal/sources.py
+++ b/products/alerts_platform/backend/temporal/sources.py
@@ -51,7 +51,7 @@ SOURCE_BINDINGS: dict[SourceKind, SourceBinding] = {
SourceKind.INSIGHT: SourceBinding(
workflow="insight-alert-platform-evaluate",
task_queue=settings.ALERTS_PLATFORM_EVALUATION_TASK_QUEUE,
- evaluation_timeout=dt.timedelta(minutes=15),
+ evaluation_timeout=dt.timedelta(minutes=20),
# A key holds at least one check and the pool admits this many, so more keys than this
# only start workflows that find the pool full.
discovery_limit=settings.ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS,