Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(alerts): start a dispatcher page's evaluations together (#124364)

Jimmy Zhu6f68a3ad3f35
diff --git a/products/alerts_platform/backend/temporal/workflows.py b/products/alerts_platform/backend/temporal/workflows.py
index fe0be5d202ab..4273a8016396 100644
--- a/products/alerts_platform/backend/temporal/workflows.py
+++ b/products/alerts_platform/backend/temporal/workflows.py
@@ -40,6 +40,7 @@ with workflow.unsafe.imports_passed_through():
from products.alerts_platform.backend.delivery.evaluation import deliver_evaluation
from products.alerts_platform.backend.delivery.thread_store import PENDING_CLAIM_TTL, ThreadBusy
from products.alerts_platform.backend.facade.contracts import (
+ AlertBatchKey,
AlertDeliveryRequest,
AlertDemand,
DemandDiscoveryInputs,
@@ -93,6 +94,10 @@ BUSY_RETRY_DELAY = dt.timedelta(seconds=20)
# `workflow.deprecate_patch(NO_TRANSITION_EVENTS_PATCH)`: every newer run carries that marker.
TRANSITION_EVENTS_PATCH = "alerts-platform-transition-events"
NO_TRANSITION_EVENTS_PATCH = "alerts-platform-no-transition-events"
+# A dispatcher that started its page one child at a time replays that order. A dispatcher runs for
+# at most SOURCE_DISPATCH_TIMEOUT, so a deploy later the sequential branch can go and the check can
+# become `workflow.deprecate_patch(CONCURRENT_EVALUATION_STARTS_PATCH)`.
+CONCURRENT_EVALUATION_STARTS_PATCH = "alerts-platform-concurrent-evaluation-starts"
# From the first busy result to the start of a later attempt. An attempt that starts this late
# finds every claim the first one saw already stale, so a thread still held is a newer send's.
BUSY_WAIT_LIMIT = PENDING_CLAIM_TTL + BUSY_RETRY_DELAY
@@ -455,40 +460,13 @@ class AlertsPlatformSourceDispatchWorkflow(PostHogWorkflow):
async def run(self, inputs: SourceDispatchInputs) -> SourceDispatchReport:
taken = inputs.batch_keys[:MAX_EVALUATIONS_PER_DISPATCH]
remaining = inputs.batch_keys[MAX_EVALUATIONS_PER_DISPATCH:]
- binding = SOURCE_BINDINGS.get(inputs.source)
- started_ids: list[str] = []
- already_running = 0
- for key in taken:
- evaluation_workflow_id = f"alerts-eval-{inputs.source.value}-{key.team_id}-{key.slot}"
- try:
- if binding is None:
- # No adapter yet. The key is not passed to the noop, and the probe path stays as is.
- await workflow.start_child_workflow(
- AlertsPlatformEvaluateWorkflow.run,
- AlertsPlatformInputs(),
- id=evaluation_workflow_id,
- task_queue=settings.ALERTS_PLATFORM_EVALUATION_TASK_QUEUE,
- parent_close_policy=workflow.ParentClosePolicy.ABANDON,
- execution_timeout=NOOP_EVALUATION_TIMEOUT,
- retry_policy=RetryPolicy(maximum_attempts=1),
- )
- else:
- await workflow.start_child_workflow(
- binding.workflow,
- SourceEvaluationInputs(source=inputs.source, cutoff=inputs.cutoff, batch_key=key),
- id=evaluation_workflow_id,
- task_queue=binding.task_queue,
- parent_close_policy=workflow.ParentClosePolicy.ABANDON,
- execution_timeout=binding.evaluation_timeout,
- retry_policy=RetryPolicy(maximum_attempts=1),
- )
- except WorkflowAlreadyStartedError:
- # Skipping is how a slow evaluation blocks its own re-dispatch. Letting the error
- # escape would fail this page, and the orchestrator awaits its pages, so the tick.
- already_running += 1
- continue
- started_ids.append(evaluation_workflow_id)
+ if workflow.patched(CONCURRENT_EVALUATION_STARTS_PATCH):
+ outcomes = await asyncio.gather(*(self._start_evaluation(inputs, key) for key in taken))
+ else:
+ outcomes = [await self._start_evaluation(inputs, key) for key in taken]
+ started_ids = [workflow_id for workflow_id in outcomes if workflow_id is not None]
+ already_running = len(outcomes) - len(started_ids)
if already_running:
workflow.logger.info(
@@ -503,6 +481,38 @@ class AlertsPlatformSourceDispatchWorkflow(PostHogWorkflow):
already_running=already_running,
)
+ async def _start_evaluation(self, inputs: SourceDispatchInputs, key: AlertBatchKey) -> str | None:
+ """The started evaluation's id, or None when that key's evaluation still runs from an earlier tick."""
+ evaluation_workflow_id = f"alerts-eval-{inputs.source.value}-{key.team_id}-{key.slot}"
+ binding = SOURCE_BINDINGS.get(inputs.source)
+ try:
+ if binding is None:
+ # No adapter yet. The key is not passed to the noop, and the probe path stays as is.
+ await workflow.start_child_workflow(
+ AlertsPlatformEvaluateWorkflow.run,
+ AlertsPlatformInputs(),
+ id=evaluation_workflow_id,
+ task_queue=settings.ALERTS_PLATFORM_EVALUATION_TASK_QUEUE,
+ parent_close_policy=workflow.ParentClosePolicy.ABANDON,
+ execution_timeout=NOOP_EVALUATION_TIMEOUT,
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+ else:
+ await workflow.start_child_workflow(
+ binding.workflow,
+ SourceEvaluationInputs(source=inputs.source, cutoff=inputs.cutoff, batch_key=key),
+ id=evaluation_workflow_id,
+ task_queue=binding.task_queue,
+ parent_close_policy=workflow.ParentClosePolicy.ABANDON,
+ execution_timeout=binding.evaluation_timeout,
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+ except WorkflowAlreadyStartedError:
+ # Skipping is how a slow evaluation blocks its own re-dispatch. Letting the error
+ # escape would fail this page, and the orchestrator awaits its pages, so the tick.
+ return None
+ return evaluation_workflow_id
+
@workflow.defn(name="alerts-platform-record-inventory")
class AlertsPlatformRecordInventoryWorkflow(PostHogWorkflow):
diff --git a/products/alerts_platform/backend/tests/test_temporal_dispatch.py b/products/alerts_platform/backend/tests/test_temporal_dispatch.py
index 13da43a98a31..f7cfb0f096ed 100644
--- a/products/alerts_platform/backend/tests/test_temporal_dispatch.py
+++ b/products/alerts_platform/backend/tests/test_temporal_dispatch.py
@@ -263,10 +263,12 @@ async def test_real_dispatcher_takes_everything_and_abandons_one_evaluation(envi
assert child.workflow_type.name == "alerts-platform-evaluate"
assert child.parent_close_policy == ParentClosePolicy.PARENT_CLOSE_POLICY_ABANDON
assert child.workflow_execution_timeout.ToTimedelta() == workflows.NOOP_EVALUATION_TIMEOUT
- # One task per child start, plus the one that completed the run. A dispatcher that awaited an
- # evaluation would need a further task to resume on its result. Counting tasks rather than
- # forbidding a child-completion event keeps this off the race with an abandoned evaluation.
- assert len(events_of(history, EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED)) == len(initiated) + 1
+ # A single sending task means the page's starts went out together, not one round trip each.
+ sending_tasks = {
+ event.start_child_workflow_execution_initiated_event_attributes.workflow_task_completed_event_id
+ for event in initiated
+ }
+ assert len(sending_tasks) == 1
evaluation = client.get_workflow_handle(f"alerts-eval-logs-{key('a').team_id}-{key('a').slot}")
assert await evaluation.result() is None
assert (await evaluation.describe()).status == WorkflowExecutionStatus.COMPLETED