Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(clickhouse): run celery cache warming on the online cluster (#120044)

Rory Shanksb8c79eb3af2a
diff --git a/posthog/clickhouse/client/execute.py b/posthog/clickhouse/client/execute.py
index 46d15eaace78..fd7db465e41a 100644
--- a/posthog/clickhouse/client/execute.py
+++ b/posthog/clickhouse/client/execute.py
@@ -446,10 +446,16 @@ def sync_execute(
# traffic stays on the online cluster. See is_api_key_access_method for the exact set.
is_api_key_auth = is_api_key_access_method(tags.access_method)
- # When someone uses an API key, always put their query to the offline cluster
- # Execute all celery tasks not directly set to be online on the offline cluster
- if workload == Workload.DEFAULT and (is_api_key_auth or tags.kind == "celery"):
- workload = Workload.OFFLINE
+ if workload == Workload.DEFAULT:
+ # When someone uses an API key, always put their query to the offline cluster
+ if is_api_key_auth:
+ workload = Workload.OFFLINE
+ # Cache warming runs on the online cluster
+ elif tags.kind == "celery" and tags.feature == Feature.CACHE_WARMUP:
+ workload = Workload.ONLINE
+ # Execute all other celery tasks not directly set to be online on the offline cluster
+ elif tags.kind == "celery":
+ workload = Workload.OFFLINE
# Make sure we always have app traffic through process_query_task on the online cluster.
# API-key traffic stays offline here too, so an async query lands on the same cluster its
@@ -569,7 +575,7 @@ def sync_execute(
**core_settings,
"log_comment": query_log_tags.to_json(),
}
- if workload == Workload.OFFLINE:
+ if workload == Workload.OFFLINE or ch_user == ClickHouseUser.CACHE_WARMUP:
# disabling hedged requests for offline queries reduces the likelihood of these queries bleeding over into the
# online resource pool when the offline resource pool is under heavy load. this comes at the cost of higher and
# more variable latency and a higher likelihood of query failures - but offline workloads should be tolerant to
diff --git a/posthog/clickhouse/client/test/test_execute.py b/posthog/clickhouse/client/test/test_execute.py
index e1896d81fbed..74cdfba45d8c 100644
--- a/posthog/clickhouse/client/test/test_execute.py
+++ b/posthog/clickhouse/client/test/test_execute.py
@@ -16,7 +16,7 @@ from posthog.clickhouse.client.connection import (
)
from posthog.clickhouse.client.execute import query_with_columns, sync_execute
from posthog.clickhouse.client.limit import ConcurrencySlot, RateLimit, get_llm_analytics_rate_limiter
-from posthog.clickhouse.query_tagging import AccessMethod, Product, tags_context
+from posthog.clickhouse.query_tagging import AccessMethod, Feature, Product, tags_context
@pytest.fixture
@@ -64,6 +64,22 @@ def test_process_query_task_workload_routing(
assert called_ch_user == expected_ch_user
+ "access_method,expected_workload",
+ [
+ (None, Workload.ONLINE),
+ (AccessMethod.PERSONAL_API_KEY, Workload.OFFLINE),
+ ],
+)
+def test_celery_cache_warming_workload_routing(client_from_pool, access_method, expected_workload):
+ with tags_context(kind="celery", feature=Feature.CACHE_WARMUP, access_method=access_method):
+ sync_execute("SELECT 1", flush=False)
+
+ assert client_from_pool.call_args[0][0] == expected_workload
+ client = client_from_pool.return_value.__enter__.return_value
+ assert client.execute.call_args.kwargs["settings"].get("use_hedged_requests") == "0"
+
+
@pytest.mark.parametrize(
"workload,expected_workload",
[