Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(workflows): give batch audience page queries their own clickhouse cap (#125935)

Daniel Marchuk96d9fdf1b64e
diff --git a/posthog/settings/web.py b/posthog/settings/web.py
index a83b733a11bc..8983f72b859b 100644
--- a/posthog/settings/web.py
+++ b/posthog/settings/web.py
@@ -1130,6 +1130,15 @@ WORKFLOWS_PERSON_BATCH_SIZE = int(get_from_env("WORKFLOWS_PERSON_BATCH_SIZE", 50
if WORKFLOWS_PERSON_BATCH_SIZE < 1:
# An empty page reports has_more, so the resolver would refetch it forever.
raise ImproperlyConfigured("WORKFLOWS_PERSON_BATCH_SIZE must be at least 1")
+# ClickHouse execution cap for one audience page of the batch resolver. Interactive HogQL runs under the
+# 60s default; a page query is background work for one caller, so it gets more time. The resolver's
+# HTTP budget (CDP_HOG_FLOW_BATCH_AUDIENCE_FETCH_TIMEOUT_MS) must stay above this value, so ClickHouse
+# always ends a slow query before the client gives up and a retry never overlaps a query that is still running.
+WORKFLOWS_BATCH_AUDIENCE_QUERY_TIMEOUT_SECONDS = int(
+ get_from_env("WORKFLOWS_BATCH_AUDIENCE_QUERY_TIMEOUT_SECONDS", 180)
+)
+if WORKFLOWS_BATCH_AUDIENCE_QUERY_TIMEOUT_SECONDS < 1:
+ raise ImproperlyConfigured("WORKFLOWS_BATCH_AUDIENCE_QUERY_TIMEOUT_SECONDS must be at least 1")
# Elevated maximum audience size, returned for teams listed in HOGFLOW_BATCH_TRIGGER_ELEVATED_TEAM_IDS.
HOGFLOW_BATCH_TRIGGER_LIMIT_ELEVATED = int(get_from_env("HOGFLOW_BATCH_TRIGGER_LIMIT_ELEVATED", 1000000))
# Comma-separated list of team IDs that get the elevated batch trigger limit instead of the default.
diff --git a/products/workflows/backend/services/blast_radius.py b/products/workflows/backend/services/blast_radius.py
index 9001b2c0d1c9..8880d0f09f17 100644
--- a/products/workflows/backend/services/blast_radius.py
+++ b/products/workflows/backend/services/blast_radius.py
@@ -1,6 +1,7 @@
"""Batch audience sizing and paging: the preview counts the editor shows, and the pages the Node
batch resolver walks when it sends."""
+from django.conf import settings
from django.core.cache import cache
from posthog.models.property import GroupTypeIndex
@@ -130,7 +131,10 @@ def get_audience_person_page(
dedupe_key: str | None,
) -> AudiencePage:
team = Team.objects.get(id=team_id)
- enumeration_settings = bounded_memory_settings() if group_type_index is None else None
+ enumeration_settings = None
+ if group_type_index is None:
+ enumeration_settings = bounded_memory_settings()
+ enumeration_settings.max_execution_time = settings.WORKFLOWS_BATCH_AUDIENCE_QUERY_TIMEOUT_SECONDS
ids = get_batch_audience_person_ids(
team, filters, group_type_index, cursor, dedupe_key=dedupe_key, settings=enumeration_settings
)
diff --git a/products/workflows/backend/tests/api/test_hog_flow.py b/products/workflows/backend/tests/api/test_hog_flow.py
index 259e3e897eb4..6d9c1bb37e57 100644
--- a/products/workflows/backend/tests/api/test_hog_flow.py
+++ b/products/workflows/backend/tests/api/test_hog_flow.py
@@ -4971,7 +4971,7 @@ class TestHogFlowAPI(APIBaseTest):
assert "Feature flags can't be used as a batch audience condition" in response.json().get("error", "")
mock_get_batch_audience_person_ids.assert_not_called()
- @override_settings(INTERNAL_API_SECRET="test-secret-123")
+ @override_settings(INTERNAL_API_SECRET="test-secret-123", WORKFLOWS_BATCH_AUDIENCE_QUERY_TIMEOUT_SECONDS=123)
def test_internal_user_blast_radius_persons_uses_workflows_query(self):
with patch(
"products.workflows.backend.services.blast_radius.get_batch_audience_person_ids",
@@ -4993,6 +4993,8 @@ class TestHogFlowAPI(APIBaseTest):
# the end of the audience, so the batch send skips every recipient after it.
passed_settings = mock_workflows_query.call_args.kwargs["settings"]
assert getattr(passed_settings, "timeout_overflow_mode", None) == "throw"
+ # The page query runs under the resolver's own ClickHouse cap, not the interactive 60s default.
+ assert passed_settings.max_execution_time == 123
@parameterized.expand(
[