:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(temporal): let a single worker service every task queue on hobby (#123069)
Paul D'Ambra0662577188e9
diff --git a/docker-compose.hobby.yml b/docker-compose.hobby.ymlindex 55779d970747..69e69b305080 100644--- a/docker-compose.hobby.yml+++ b/docker-compose.hobby.yml@@ -23,6 +23,12 @@ x-hypercache-s3: &hypercache-s3AWS_ACCESS_KEY_ID: 'object_storage_root_user'AWS_SECRET_ACCESS_KEY: 'object_storage_root_password'+# Hobby runs one temporal-django-worker, and a worker polls one task queue. The services that+# start workflows and the worker that runs them collapse every Temporal task queue onto that+# one queue, so workflows on other queues (exports, PostHog AI, replay, batch exports) still run.+x-single-temporal-queue: &single-temporal-queue+ TEMPORAL_USE_SINGLE_TASK_QUEUE: 'true'+services:db:extends:@@ -120,6 +126,7 @@ services:HEATMAP_BROWSERLESS_URL: 'http://browserless:3000'HEATMAP_BROWSERLESS_TOKEN: ${BROWSERLESS_SECRET:-$POSTHOG_SECRET}HEATMAP_BROWSERLESS_SCREENSHOT_COOKIES_ENABLED: ${HEATMAP_BROWSERLESS_SCREENSHOT_COOKIES_ENABLED:-false}+ <<: *single-temporal-queue# Point Django at the Rust feature-flags container; the setting# otherwise defaults to localhost:3001.FEATURE_FLAGS_SERVICE_URL: 'http://feature-flags:3001'@@ -189,6 +196,7 @@ services:DEPLOYMENT: 'hobby'PERSONHOG_ADDR: 'personhog-router:50052'PERSONHOG_ENABLED: 'true'+ <<: *single-temporal-queue# Point Django at the Rust feature-flags container; the setting# otherwise defaults to localhost:3001.FEATURE_FLAGS_SERVICE_URL: 'http://feature-flags:3001'@@ -531,6 +539,7 @@ services:RECORDING_API_JWT_SECRET: ${RECORDING_API_JWT_SECRET:-}PERSONHOG_ADDR: 'personhog-router:50052'PERSONHOG_ENABLED: 'true'+ <<: *single-temporal-queue# Point Django at the Rust feature-flags container; the setting# otherwise defaults to localhost:3001.FEATURE_FLAGS_SERVICE_URL: 'http://feature-flags:3001'diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.pyindex d8f5c8b2410d..6cbc480138cd 100644--- a/posthog/management/commands/start_temporal_worker.py+++ b/posthog/management/commands/start_temporal_worker.py@@ -680,7 +680,7 @@ def workflows_include_data_import_syncs(workflows: collections.abc.Iterable[typereturn any(wf in DATA_SYNC_WORKFLOWS for wf in workflows)-if settings.DEBUG:+if settings.DEBUG or settings.TEMPORAL_USE_SINGLE_TASK_QUEUE:TASK_QUEUE_METRIC_PREFIXES = {}else:TASK_QUEUE_METRIC_PREFIXES = {diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.pyindex 4b52f0104dda..0da3b903822c 100644--- a/posthog/settings/temporal.py+++ b/posthog/settings/temporal.py@@ -268,13 +268,27 @@ CLICKHOUSE_MAX_BLOCK_SIZE_OVERRIDES: dict[int, int] = dict(# Temporal has a limitation where a worker can only listen to a single queue.# To avoid running multiple workers, when running locally (DEBUG=True), we use a single queue for all tasks.# In production (DEBUG=False), we use separate queues for each worker type.+# Self-hosted stacks that run one temporal-django-worker (hobby) opt into the same collapse with+# TEMPORAL_USE_SINGLE_TASK_QUEUE, so every workflow is scheduled on the queue that worker polls+# instead of on queues nothing services.+TEMPORAL_USE_SINGLE_TASK_QUEUE: bool = get_from_env("TEMPORAL_USE_SINGLE_TASK_QUEUE", False, type_cast=str_to_bool)+default_task_queue = os.getenv("TEMPORAL_TASK_QUEUE", "general-purpose-task-queue")+++# Use for queues that stay separate even in local development. A single-worker stack has no+# second worker to poll them, so it still collapses them.+def _set_isolated_temporal_task_queue(task_queue: str) -> str:+ if TEMPORAL_USE_SINGLE_TASK_QUEUE and not DEBUG:+ return default_task_queue+ return task_queue++def _set_temporal_task_queue(task_queue: str) -> str:if DEBUG:return "development-task-queue"- return task_queue+ return _set_isolated_temporal_task_queue(task_queue)-default_task_queue = os.getenv("TEMPORAL_TASK_QUEUE", "general-purpose-task-queue")TEMPORAL_TASK_QUEUE: str = _set_temporal_task_queue(default_task_queue)DATA_WAREHOUSE_TASK_QUEUE = _set_temporal_task_queue("data-warehouse-task-queue")DATA_WAREHOUSE_CDP_PRODUCER_TASK_QUEUE = _set_temporal_task_queue("data-warehouse-cdp-producer-task-queue")@@ -377,9 +391,11 @@ USAGE_REPORT_AGGREGATION_TASK_QUEUE = _set_temporal_task_queue("billing-task-queVIDEO_EXPORT_TASK_QUEUE = _set_temporal_task_queue("video-export-task-queue")ANALYTICS_PLATFORM_TASK_QUEUE = _set_temporal_task_queue("analytics-platform-task-queue")# Keep the smoke fleets separate in local development as well as deployed environments.-ALERTS_PLATFORM_SHARED_ORCHESTRATION_TASK_QUEUE = "alerts-platform-shared-orchestration-task-queue"-ALERTS_PLATFORM_EVALUATION_TASK_QUEUE = "alerts-platform-evaluation-task-queue"-ALERTS_PLATFORM_DELIVERY_TASK_QUEUE = "alerts-platform-delivery-task-queue"+ALERTS_PLATFORM_SHARED_ORCHESTRATION_TASK_QUEUE = _set_isolated_temporal_task_queue(+ "alerts-platform-shared-orchestration-task-queue"+)+ALERTS_PLATFORM_EVALUATION_TASK_QUEUE = _set_isolated_temporal_task_queue("alerts-platform-evaluation-task-queue")+ALERTS_PLATFORM_DELIVERY_TASK_QUEUE = _set_isolated_temporal_task_queue("alerts-platform-delivery-task-queue")# The platform's parallel insight checks allowed to run at once, across every team. A pool of its# own, so the parallel run never takes a slot from `ALERTS_MAX_INFLIGHT_EVALUATIONS`.ALERTS_PLATFORM_INSIGHT_MAX_INFLIGHT_EVALUATIONS: int = get_from_env(diff --git a/posthog/settings/test_temporal.py b/posthog/settings/test_temporal.pynew file mode 100644index 000000000000..79876db6fe15--- /dev/null+++ b/posthog/settings/test_temporal.py@@ -0,0 +1,25 @@+import pytest+from unittest import mock++from posthog.settings import temporal+++ "debug,single_task_queue,expected_shared,expected_isolated",+ [+ (False, False, "max-ai-task-queue", "alerts-platform-delivery-task-queue"),+ (False, True, "general-purpose-task-queue", "general-purpose-task-queue"),+ (True, False, "development-task-queue", "alerts-platform-delivery-task-queue"),+ (True, True, "development-task-queue", "alerts-platform-delivery-task-queue"),+ ],+)+def test_task_queue_collapse(+ debug: bool, single_task_queue: bool, expected_shared: str, expected_isolated: str+) -> None:+ with (+ mock.patch.object(temporal, "DEBUG", debug),+ mock.patch.object(temporal, "TEMPORAL_USE_SINGLE_TASK_QUEUE", single_task_queue),+ mock.patch.object(temporal, "default_task_queue", "general-purpose-task-queue"),+ ):+ assert temporal._set_temporal_task_queue("max-ai-task-queue") == expected_shared+ assert temporal._set_isolated_temporal_task_queue("alerts-platform-delivery-task-queue") == expected_isolated