:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(temporal): let a worker poll only workflows or only activities (#125038)
Jimmy Zhu19b013a510e2
diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.pyindex 5d6f92120e00..c5f1da4cacf3 100644--- a/posthog/management/commands/start_temporal_worker.py+++ b/posthog/management/commands/start_temporal_worker.py@@ -8,6 +8,7 @@ import threadingimport faulthandlerimport collections.abcfrom collections import defaultdict+from enum import StrEnumimport structlogfrom temporalio import workflow@@ -719,6 +720,12 @@ else:LOGGER = get_logger(__name__)+class WorkerRole(StrEnum):+ ALL = "all"+ WORKFLOWS = "workflows"+ ACTIVITIES = "activities"++def apply_hogql_shadow_sample_rate_override() -> None:"""Sets HogQL's cpp shadow parse rate for this worker from `HOGQL_PARSER_SHADOW_SAMPLE_RATE`.@@ -802,6 +809,12 @@ class Command(BaseCommand):default=settings.MAX_CONCURRENT_ACTIVITIES,help="Maximum number of concurrent activity tasks for this worker",)+ parser.add_argument(+ "--worker-role",+ choices=[role.value for role in WorkerRole],+ default=settings.TEMPORAL_WORKER_ROLE,+ help="Poll only the workflow tasks or only the activity tasks of the task queue, or both",+ )parser.add_argument("--use-pydantic-converter",action="store_true",@@ -880,6 +893,15 @@ class Command(BaseCommand):if workflows_include_data_import_syncs(workflows):load_all_sources()+ # Temporal keeps workflow tasks and activity tasks apart even on one queue name, so a workflows+ # worker and an activities worker on the same queue split its work between them. This runs after+ # the source preload, because an activities-only data-sync worker still runs the syncs.+ worker_role = WorkerRole(options["worker_role"])+ if worker_role == WorkerRole.WORKFLOWS:+ activities = []+ elif worker_role == WorkerRole.ACTIVITIES:+ workflows = []+if options["client_key"]:options["client_key"] = "--SECRET--"@@ -983,6 +1005,7 @@ class Command(BaseCommand):port=temporal_port,namespace=namespace,task_queue=task_queue,+ worker_role=worker_role,graceful_shutdown_timeout_seconds=graceful_shutdown_timeout_seconds,max_concurrent_workflow_tasks=max_concurrent_workflow_tasks,max_concurrent_activities=max_concurrent_activities,diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.pyindex 536ae569164a..37635fba6a92 100644--- a/posthog/settings/temporal.py+++ b/posthog/settings/temporal.py@@ -36,6 +36,10 @@ MAX_CONCURRENT_WORKFLOW_TASKS: int | None = get_from_env("MAX_CONCURRENT_WORKFLOW_TASKS", None, optional=True, type_cast=int)MAX_CONCURRENT_ACTIVITIES: int | None = get_from_env("MAX_CONCURRENT_ACTIVITIES", None, optional=True, type_cast=int)+# "workflows" or "activities" makes the worker poll only that half of its task queue, so that a second+# deployment on the same queue can serve the other half. Workflow tasks then do not compete with+# CPU-heavy activities for the GIL. "all", the default, polls both.+TEMPORAL_WORKER_ROLE: str = get_from_env("TEMPORAL_WORKER_ROLE", "all")# Caps the @asyncify pool. An asyncify thread can hold a Django connection for its whole call, so the# pool is a pgbouncer client-connection multiplier: worker replicas x pool size must stay under the# pooler's max_client_conn at its minimum replica count. Raise only with that arithmetic redone.diff --git a/posthog/temporal/common/liveness_tracker.py b/posthog/temporal/common/liveness_tracker.pyindex ecf804812cda..1338b011600f 100644--- a/posthog/temporal/common/liveness_tracker.py+++ b/posthog/temporal/common/liveness_tracker.py@@ -6,15 +6,19 @@ import dataclassesfrom collections.abc import Callablefrom typing import Any-from temporalio import activity+from temporalio import activity, workflowfrom temporalio.worker import (ActivityInboundInterceptor,ActivityOutboundInterceptor,ExecuteActivityInput,ExecuteWorkflowInput,Interceptor,+ StartActivityInput,+ StartChildWorkflowInput,+ StartLocalActivityInput,WorkflowInboundInterceptor,WorkflowInterceptorClassInput,+ WorkflowOutboundInterceptor,)from posthog.dataclasses import frozen@@ -217,6 +221,31 @@ class _LivenessActivityInboundInterceptor(ActivityInboundInterceptor):)+class _LivenessWorkflowOutboundInterceptor(WorkflowOutboundInterceptor):+ """Records liveness each time workflow code schedules work.++ A worker that polls only workflow tasks runs no activities, and a long workflow can run for longer+ than the health check's idle limit before it returns. Each scheduled activity or child workflow+ shows that the worker still completes workflow tasks.+ """++ def __init__(self, next: WorkflowOutboundInterceptor, tracker: LivenessTracker):+ super().__init__(next)+ self._tracker = tracker++ def start_activity(self, input: StartActivityInput) -> workflow.ActivityHandle[Any]:+ self._tracker.record_workflow_execution()+ return super().start_activity(input)++ def start_local_activity(self, input: StartLocalActivityInput) -> workflow.ActivityHandle[Any]:+ self._tracker.record_workflow_execution()+ return super().start_local_activity(input)++ async def start_child_workflow(self, input: StartChildWorkflowInput) -> workflow.ChildWorkflowHandle[Any, Any]:+ self._tracker.record_workflow_execution()+ return await super().start_child_workflow(input)++class _LivenessWorkflowInterceptor(WorkflowInboundInterceptor):_tracker: LivenessTracker@@ -224,6 +253,9 @@ class _LivenessWorkflowInterceptor(WorkflowInboundInterceptor):super().__init__(next)self._tracker = get_liveness_tracker()+ def init(self, outbound: WorkflowOutboundInterceptor) -> None:+ super().init(_LivenessWorkflowOutboundInterceptor(outbound, self._tracker))+async def execute_workflow(self, input: ExecuteWorkflowInput) -> Any:try:result = await super().execute_workflow(input)diff --git a/posthog/temporal/tests/common/test_liveness_tracker.py b/posthog/temporal/tests/common/test_liveness_tracker.pyindex 49c4508f0bef..c706e1256a55 100644--- a/posthog/temporal/tests/common/test_liveness_tracker.py+++ b/posthog/temporal/tests/common/test_liveness_tracker.py@@ -7,7 +7,7 @@ from unittest.mock import AsyncMock, MagicMock, patchfrom django.conf import settings-from temporalio.worker import ExecuteActivityInput, ExecuteWorkflowInput+from temporalio.worker import ExecuteActivityInput, ExecuteWorkflowInput, StartActivityInput, WorkflowInboundInterceptorfrom posthog.temporal.common.interceptor import is_task_queue_supportedfrom posthog.temporal.common.liveness_tracker import (@@ -16,6 +16,7 @@ from posthog.temporal.common.liveness_tracker import (RunningActivity,_LivenessActivityInboundInterceptor,_LivenessWorkflowInterceptor,+ _LivenessWorkflowOutboundInterceptor,get_liveness_tracker,)@@ -347,3 +348,21 @@ class TestLivenessWorkflowInterceptor:# Should still have recorded executionassert tracker.get_idle_time() < 1.0++ async def test_records_progress_when_a_running_workflow_schedules_an_activity(self):+ tracker = LivenessTracker()+ tracker._last_activity_time = time.time() - 10.0+ tracker._last_workflow_time = time.time() - 10.0+ handle = MagicMock()+ next_outbound = MagicMock()+ next_outbound.start_activity.return_value = handle++ interceptor = _LivenessWorkflowInterceptor(AsyncMock())+ interceptor._tracker = tracker+ with patch.object(WorkflowInboundInterceptor, "init") as inbound_init:+ interceptor.init(next_outbound)+ outbound = inbound_init.call_args.args[0]++ assert isinstance(outbound, _LivenessWorkflowOutboundInterceptor)+ assert outbound.start_activity(MagicMock(spec=StartActivityInput)) is handle+ assert tracker.get_idle_time() < 1.0