:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(clickhouse): run celery cache warming on the online cluster (#120044)
Rory Shanksb8c79eb3af2a
diff --git a/posthog/clickhouse/client/execute.py b/posthog/clickhouse/client/execute.pyindex 46d15eaace78..fd7db465e41a 100644--- a/posthog/clickhouse/client/execute.py+++ b/posthog/clickhouse/client/execute.py@@ -446,10 +446,16 @@ def sync_execute(# traffic stays on the online cluster. See is_api_key_access_method for the exact set.is_api_key_auth = is_api_key_access_method(tags.access_method)- # When someone uses an API key, always put their query to the offline cluster- # Execute all celery tasks not directly set to be online on the offline cluster- if workload == Workload.DEFAULT and (is_api_key_auth or tags.kind == "celery"):- workload = Workload.OFFLINE+ if workload == Workload.DEFAULT:+ # When someone uses an API key, always put their query to the offline cluster+ if is_api_key_auth:+ workload = Workload.OFFLINE+ # Cache warming runs on the online cluster+ elif tags.kind == "celery" and tags.feature == Feature.CACHE_WARMUP:+ workload = Workload.ONLINE+ # Execute all other celery tasks not directly set to be online on the offline cluster+ elif tags.kind == "celery":+ workload = Workload.OFFLINE# Make sure we always have app traffic through process_query_task on the online cluster.# API-key traffic stays offline here too, so an async query lands on the same cluster its@@ -569,7 +575,7 @@ def sync_execute(**core_settings,"log_comment": query_log_tags.to_json(),}- if workload == Workload.OFFLINE:+ if workload == Workload.OFFLINE or ch_user == ClickHouseUser.CACHE_WARMUP:# disabling hedged requests for offline queries reduces the likelihood of these queries bleeding over into the# online resource pool when the offline resource pool is under heavy load. this comes at the cost of higher and# more variable latency and a higher likelihood of query failures - but offline workloads should be tolerant todiff --git a/posthog/clickhouse/client/test/test_execute.py b/posthog/clickhouse/client/test/test_execute.pyindex e1896d81fbed..74cdfba45d8c 100644--- a/posthog/clickhouse/client/test/test_execute.py+++ b/posthog/clickhouse/client/test/test_execute.py@@ -16,7 +16,7 @@ from posthog.clickhouse.client.connection import ()from posthog.clickhouse.client.execute import query_with_columns, sync_executefrom posthog.clickhouse.client.limit import ConcurrencySlot, RateLimit, get_llm_analytics_rate_limiter-from posthog.clickhouse.query_tagging import AccessMethod, Product, tags_context+from posthog.clickhouse.query_tagging import AccessMethod, Feature, Product, tags_context@pytest.fixture@@ -64,6 +64,22 @@ def test_process_query_task_workload_routing(assert called_ch_user == expected_ch_user+ "access_method,expected_workload",+ [+ (None, Workload.ONLINE),+ (AccessMethod.PERSONAL_API_KEY, Workload.OFFLINE),+ ],+)+def test_celery_cache_warming_workload_routing(client_from_pool, access_method, expected_workload):+ with tags_context(kind="celery", feature=Feature.CACHE_WARMUP, access_method=access_method):+ sync_execute("SELECT 1", flush=False)++ assert client_from_pool.call_args[0][0] == expected_workload+ client = client_from_pool.return_value.__enter__.return_value+ assert client.execute.call_args.kwargs["settings"].get("use_hedged_requests") == "0"++@pytest.mark.parametrize("workload,expected_workload",[