:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(alerts): start a dispatcher page's evaluations together (#124364)
Jimmy Zhu6f68a3ad3f35
diff --git a/products/alerts_platform/backend/temporal/workflows.py b/products/alerts_platform/backend/temporal/workflows.pyindex fe0be5d202ab..4273a8016396 100644--- a/products/alerts_platform/backend/temporal/workflows.py+++ b/products/alerts_platform/backend/temporal/workflows.py@@ -40,6 +40,7 @@ with workflow.unsafe.imports_passed_through():from products.alerts_platform.backend.delivery.evaluation import deliver_evaluationfrom products.alerts_platform.backend.delivery.thread_store import PENDING_CLAIM_TTL, ThreadBusyfrom products.alerts_platform.backend.facade.contracts import (+ AlertBatchKey,AlertDeliveryRequest,AlertDemand,DemandDiscoveryInputs,@@ -93,6 +94,10 @@ BUSY_RETRY_DELAY = dt.timedelta(seconds=20)# `workflow.deprecate_patch(NO_TRANSITION_EVENTS_PATCH)`: every newer run carries that marker.TRANSITION_EVENTS_PATCH = "alerts-platform-transition-events"NO_TRANSITION_EVENTS_PATCH = "alerts-platform-no-transition-events"+# A dispatcher that started its page one child at a time replays that order. A dispatcher runs for+# at most SOURCE_DISPATCH_TIMEOUT, so a deploy later the sequential branch can go and the check can+# become `workflow.deprecate_patch(CONCURRENT_EVALUATION_STARTS_PATCH)`.+CONCURRENT_EVALUATION_STARTS_PATCH = "alerts-platform-concurrent-evaluation-starts"# From the first busy result to the start of a later attempt. An attempt that starts this late# finds every claim the first one saw already stale, so a thread still held is a newer send's.BUSY_WAIT_LIMIT = PENDING_CLAIM_TTL + BUSY_RETRY_DELAY@@ -455,40 +460,13 @@ class AlertsPlatformSourceDispatchWorkflow(PostHogWorkflow):async def run(self, inputs: SourceDispatchInputs) -> SourceDispatchReport:taken = inputs.batch_keys[:MAX_EVALUATIONS_PER_DISPATCH]remaining = inputs.batch_keys[MAX_EVALUATIONS_PER_DISPATCH:]- binding = SOURCE_BINDINGS.get(inputs.source)- started_ids: list[str] = []- already_running = 0- for key in taken:- evaluation_workflow_id = f"alerts-eval-{inputs.source.value}-{key.team_id}-{key.slot}"- try:- if binding is None:- # No adapter yet. The key is not passed to the noop, and the probe path stays as is.- await workflow.start_child_workflow(- AlertsPlatformEvaluateWorkflow.run,- AlertsPlatformInputs(),- id=evaluation_workflow_id,- task_queue=settings.ALERTS_PLATFORM_EVALUATION_TASK_QUEUE,- parent_close_policy=workflow.ParentClosePolicy.ABANDON,- execution_timeout=NOOP_EVALUATION_TIMEOUT,- retry_policy=RetryPolicy(maximum_attempts=1),- )- else:- await workflow.start_child_workflow(- binding.workflow,- SourceEvaluationInputs(source=inputs.source, cutoff=inputs.cutoff, batch_key=key),- id=evaluation_workflow_id,- task_queue=binding.task_queue,- parent_close_policy=workflow.ParentClosePolicy.ABANDON,- execution_timeout=binding.evaluation_timeout,- retry_policy=RetryPolicy(maximum_attempts=1),- )- except WorkflowAlreadyStartedError:- # Skipping is how a slow evaluation blocks its own re-dispatch. Letting the error- # escape would fail this page, and the orchestrator awaits its pages, so the tick.- already_running += 1- continue- started_ids.append(evaluation_workflow_id)+ if workflow.patched(CONCURRENT_EVALUATION_STARTS_PATCH):+ outcomes = await asyncio.gather(*(self._start_evaluation(inputs, key) for key in taken))+ else:+ outcomes = [await self._start_evaluation(inputs, key) for key in taken]+ started_ids = [workflow_id for workflow_id in outcomes if workflow_id is not None]+ already_running = len(outcomes) - len(started_ids)if already_running:workflow.logger.info(@@ -503,6 +481,38 @@ class AlertsPlatformSourceDispatchWorkflow(PostHogWorkflow):already_running=already_running,)+ async def _start_evaluation(self, inputs: SourceDispatchInputs, key: AlertBatchKey) -> str | None:+ """The started evaluation's id, or None when that key's evaluation still runs from an earlier tick."""+ evaluation_workflow_id = f"alerts-eval-{inputs.source.value}-{key.team_id}-{key.slot}"+ binding = SOURCE_BINDINGS.get(inputs.source)+ try:+ if binding is None:+ # No adapter yet. The key is not passed to the noop, and the probe path stays as is.+ await workflow.start_child_workflow(+ AlertsPlatformEvaluateWorkflow.run,+ AlertsPlatformInputs(),+ id=evaluation_workflow_id,+ task_queue=settings.ALERTS_PLATFORM_EVALUATION_TASK_QUEUE,+ parent_close_policy=workflow.ParentClosePolicy.ABANDON,+ execution_timeout=NOOP_EVALUATION_TIMEOUT,+ retry_policy=RetryPolicy(maximum_attempts=1),+ )+ else:+ await workflow.start_child_workflow(+ binding.workflow,+ SourceEvaluationInputs(source=inputs.source, cutoff=inputs.cutoff, batch_key=key),+ id=evaluation_workflow_id,+ task_queue=binding.task_queue,+ parent_close_policy=workflow.ParentClosePolicy.ABANDON,+ execution_timeout=binding.evaluation_timeout,+ retry_policy=RetryPolicy(maximum_attempts=1),+ )+ except WorkflowAlreadyStartedError:+ # Skipping is how a slow evaluation blocks its own re-dispatch. Letting the error+ # escape would fail this page, and the orchestrator awaits its pages, so the tick.+ return None+ return evaluation_workflow_id+@workflow.defn(name="alerts-platform-record-inventory")class AlertsPlatformRecordInventoryWorkflow(PostHogWorkflow):diff --git a/products/alerts_platform/backend/tests/test_temporal_dispatch.py b/products/alerts_platform/backend/tests/test_temporal_dispatch.pyindex 13da43a98a31..f7cfb0f096ed 100644--- a/products/alerts_platform/backend/tests/test_temporal_dispatch.py+++ b/products/alerts_platform/backend/tests/test_temporal_dispatch.py@@ -263,10 +263,12 @@ async def test_real_dispatcher_takes_everything_and_abandons_one_evaluation(enviassert child.workflow_type.name == "alerts-platform-evaluate"assert child.parent_close_policy == ParentClosePolicy.PARENT_CLOSE_POLICY_ABANDONassert child.workflow_execution_timeout.ToTimedelta() == workflows.NOOP_EVALUATION_TIMEOUT- # One task per child start, plus the one that completed the run. A dispatcher that awaited an- # evaluation would need a further task to resume on its result. Counting tasks rather than- # forbidding a child-completion event keeps this off the race with an abandoned evaluation.- assert len(events_of(history, EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED)) == len(initiated) + 1+ # A single sending task means the page's starts went out together, not one round trip each.+ sending_tasks = {+ event.start_child_workflow_execution_initiated_event_attributes.workflow_task_completed_event_id+ for event in initiated+ }+ assert len(sending_tasks) == 1evaluation = client.get_workflow_handle(f"alerts-eval-logs-{key('a').team_id}-{key('a').slot}")assert await evaluation.result() is Noneassert (await evaluation.describe()).status == WorkflowExecutionStatus.COMPLETED