Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(temporal): let a worker poll only workflows or only activities (#125038)

Jimmy Zhu19b013a510e2
diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.py
index 5d6f92120e00..c5f1da4cacf3 100644
--- a/posthog/management/commands/start_temporal_worker.py
+++ b/posthog/management/commands/start_temporal_worker.py
@@ -8,6 +8,7 @@ import threading
import faulthandler
import collections.abc
from collections import defaultdict
+from enum import StrEnum
import structlog
from temporalio import workflow
@@ -719,6 +720,12 @@ else:
LOGGER = get_logger(__name__)
+class WorkerRole(StrEnum):
+ ALL = "all"
+ WORKFLOWS = "workflows"
+ ACTIVITIES = "activities"
+
+
def apply_hogql_shadow_sample_rate_override() -> None:
"""Sets HogQL's cpp shadow parse rate for this worker from `HOGQL_PARSER_SHADOW_SAMPLE_RATE`.
@@ -802,6 +809,12 @@ class Command(BaseCommand):
default=settings.MAX_CONCURRENT_ACTIVITIES,
help="Maximum number of concurrent activity tasks for this worker",
)
+ parser.add_argument(
+ "--worker-role",
+ choices=[role.value for role in WorkerRole],
+ default=settings.TEMPORAL_WORKER_ROLE,
+ help="Poll only the workflow tasks or only the activity tasks of the task queue, or both",
+ )
parser.add_argument(
"--use-pydantic-converter",
action="store_true",
@@ -880,6 +893,15 @@ class Command(BaseCommand):
if workflows_include_data_import_syncs(workflows):
load_all_sources()
+ # Temporal keeps workflow tasks and activity tasks apart even on one queue name, so a workflows
+ # worker and an activities worker on the same queue split its work between them. This runs after
+ # the source preload, because an activities-only data-sync worker still runs the syncs.
+ worker_role = WorkerRole(options["worker_role"])
+ if worker_role == WorkerRole.WORKFLOWS:
+ activities = []
+ elif worker_role == WorkerRole.ACTIVITIES:
+ workflows = []
+
if options["client_key"]:
options["client_key"] = "--SECRET--"
@@ -983,6 +1005,7 @@ class Command(BaseCommand):
port=temporal_port,
namespace=namespace,
task_queue=task_queue,
+ worker_role=worker_role,
graceful_shutdown_timeout_seconds=graceful_shutdown_timeout_seconds,
max_concurrent_workflow_tasks=max_concurrent_workflow_tasks,
max_concurrent_activities=max_concurrent_activities,
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py
index 536ae569164a..37635fba6a92 100644
--- a/posthog/settings/temporal.py
+++ b/posthog/settings/temporal.py
@@ -36,6 +36,10 @@ MAX_CONCURRENT_WORKFLOW_TASKS: int | None = get_from_env(
"MAX_CONCURRENT_WORKFLOW_TASKS", None, optional=True, type_cast=int
)
MAX_CONCURRENT_ACTIVITIES: int | None = get_from_env("MAX_CONCURRENT_ACTIVITIES", None, optional=True, type_cast=int)
+# "workflows" or "activities" makes the worker poll only that half of its task queue, so that a second
+# deployment on the same queue can serve the other half. Workflow tasks then do not compete with
+# CPU-heavy activities for the GIL. "all", the default, polls both.
+TEMPORAL_WORKER_ROLE: str = get_from_env("TEMPORAL_WORKER_ROLE", "all")
# Caps the @asyncify pool. An asyncify thread can hold a Django connection for its whole call, so the
# pool is a pgbouncer client-connection multiplier: worker replicas x pool size must stay under the
# pooler's max_client_conn at its minimum replica count. Raise only with that arithmetic redone.
diff --git a/posthog/temporal/common/liveness_tracker.py b/posthog/temporal/common/liveness_tracker.py
index ecf804812cda..1338b011600f 100644
--- a/posthog/temporal/common/liveness_tracker.py
+++ b/posthog/temporal/common/liveness_tracker.py
@@ -6,15 +6,19 @@ import dataclasses
from collections.abc import Callable
from typing import Any
-from temporalio import activity
+from temporalio import activity, workflow
from temporalio.worker import (
ActivityInboundInterceptor,
ActivityOutboundInterceptor,
ExecuteActivityInput,
ExecuteWorkflowInput,
Interceptor,
+ StartActivityInput,
+ StartChildWorkflowInput,
+ StartLocalActivityInput,
WorkflowInboundInterceptor,
WorkflowInterceptorClassInput,
+ WorkflowOutboundInterceptor,
)
from posthog.dataclasses import frozen
@@ -217,6 +221,31 @@ class _LivenessActivityInboundInterceptor(ActivityInboundInterceptor):
)
+class _LivenessWorkflowOutboundInterceptor(WorkflowOutboundInterceptor):
+ """Records liveness each time workflow code schedules work.
+
+ A worker that polls only workflow tasks runs no activities, and a long workflow can run for longer
+ than the health check's idle limit before it returns. Each scheduled activity or child workflow
+ shows that the worker still completes workflow tasks.
+ """
+
+ def __init__(self, next: WorkflowOutboundInterceptor, tracker: LivenessTracker):
+ super().__init__(next)
+ self._tracker = tracker
+
+ def start_activity(self, input: StartActivityInput) -> workflow.ActivityHandle[Any]:
+ self._tracker.record_workflow_execution()
+ return super().start_activity(input)
+
+ def start_local_activity(self, input: StartLocalActivityInput) -> workflow.ActivityHandle[Any]:
+ self._tracker.record_workflow_execution()
+ return super().start_local_activity(input)
+
+ async def start_child_workflow(self, input: StartChildWorkflowInput) -> workflow.ChildWorkflowHandle[Any, Any]:
+ self._tracker.record_workflow_execution()
+ return await super().start_child_workflow(input)
+
+
class _LivenessWorkflowInterceptor(WorkflowInboundInterceptor):
_tracker: LivenessTracker
@@ -224,6 +253,9 @@ class _LivenessWorkflowInterceptor(WorkflowInboundInterceptor):
super().__init__(next)
self._tracker = get_liveness_tracker()
+ def init(self, outbound: WorkflowOutboundInterceptor) -> None:
+ super().init(_LivenessWorkflowOutboundInterceptor(outbound, self._tracker))
+
async def execute_workflow(self, input: ExecuteWorkflowInput) -> Any:
try:
result = await super().execute_workflow(input)
diff --git a/posthog/temporal/tests/common/test_liveness_tracker.py b/posthog/temporal/tests/common/test_liveness_tracker.py
index 49c4508f0bef..c706e1256a55 100644
--- a/posthog/temporal/tests/common/test_liveness_tracker.py
+++ b/posthog/temporal/tests/common/test_liveness_tracker.py
@@ -7,7 +7,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
from django.conf import settings
-from temporalio.worker import ExecuteActivityInput, ExecuteWorkflowInput
+from temporalio.worker import ExecuteActivityInput, ExecuteWorkflowInput, StartActivityInput, WorkflowInboundInterceptor
from posthog.temporal.common.interceptor import is_task_queue_supported
from posthog.temporal.common.liveness_tracker import (
@@ -16,6 +16,7 @@ from posthog.temporal.common.liveness_tracker import (
RunningActivity,
_LivenessActivityInboundInterceptor,
_LivenessWorkflowInterceptor,
+ _LivenessWorkflowOutboundInterceptor,
get_liveness_tracker,
)
@@ -347,3 +348,21 @@ class TestLivenessWorkflowInterceptor:
# Should still have recorded execution
assert tracker.get_idle_time() < 1.0
+
+ async def test_records_progress_when_a_running_workflow_schedules_an_activity(self):
+ tracker = LivenessTracker()
+ tracker._last_activity_time = time.time() - 10.0
+ tracker._last_workflow_time = time.time() - 10.0
+ handle = MagicMock()
+ next_outbound = MagicMock()
+ next_outbound.start_activity.return_value = handle
+
+ interceptor = _LivenessWorkflowInterceptor(AsyncMock())
+ interceptor._tracker = tracker
+ with patch.object(WorkflowInboundInterceptor, "init") as inbound_init:
+ interceptor.init(next_outbound)
+ outbound = inbound_init.call_args.args[0]
+
+ assert isinstance(outbound, _LivenessWorkflowOutboundInterceptor)
+ assert outbound.start_activity(MagicMock(spec=StartActivityInput)) is handle
+ assert tracker.get_idle_time() < 1.0