Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(clickhouse): stop setting use_hedged_requests (#120054)

Rory Shanks435b130a6147
diff --git a/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts b/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts
index a2f765f91cc0..f02b38c269d0 100644
--- a/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts
+++ b/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts
@@ -63,7 +63,6 @@ const EDITABLE_INSTANCE_SETTINGS = [
'CLICKHOUSE_KILL_SWITCH',
'CLICKHOUSE_KILL_SWITCH_LIGHT_TEAMS',
'CLICKHOUSE_KILL_SWITCH_FULL_TEAMS',
- 'CLICKHOUSE_HEDGED_APP_QUERIES',
'HOGQL_SHARED_INSIGHT_DATABASE_ENABLED',
'WEB_ANALYTICS_WARMING_SHARD_THREADS',
'WEB_ANALYTICS_WARMING_SHARDS',
diff --git a/posthog/clickhouse/client/execute.py b/posthog/clickhouse/client/execute.py
index fd7db465e41a..b07477421496 100644
--- a/posthog/clickhouse/client/execute.py
+++ b/posthog/clickhouse/client/execute.py
@@ -147,20 +147,6 @@ def get_team_kill_switch_level(team_id: int) -> KillSwitchLevel:
return KillSwitchLevel.OFF
-def get_hedged_app_queries_enabled() -> bool:
- return _get_hedged_app_queries_enabled(round(time.time() / 60))
-
-
-@lru_cache(maxsize=1)
-def _get_hedged_app_queries_enabled(_ttl: int) -> bool:
- from posthog.models.instance_setting import get_instance_setting
-
- try:
- return get_instance_setting("CLICKHOUSE_HEDGED_APP_QUERIES")
- except Exception:
- return False
-
-
@lru_cache(maxsize=1)
def _get_kill_switch_level(_ttl: int) -> KillSwitchLevel:
from posthog.models.instance_setting import get_instance_setting
@@ -575,17 +561,6 @@ def sync_execute(
**core_settings,
"log_comment": query_log_tags.to_json(),
}
- if workload == Workload.OFFLINE or ch_user == ClickHouseUser.CACHE_WARMUP:
- # disabling hedged requests for offline queries reduces the likelihood of these queries bleeding over into the
- # online resource pool when the offline resource pool is under heavy load. this comes at the cost of higher and
- # more variable latency and a higher likelihood of query failures - but offline workloads should be tolerant to
- # these disruptions
- settings["use_hedged_requests"] = "0"
- elif workload == Workload.ONLINE and ch_user == ClickHouseUser.APP:
- if kill_switch_level != KillSwitchLevel.OFF:
- settings["use_hedged_requests"] = "0"
- else:
- settings["use_hedged_requests"] = "1" if get_hedged_app_queries_enabled() else "0"
start_time = perf_counter()
budgeted_query_info: Optional[Any] = None
diff --git a/posthog/clickhouse/client/test/test_execute.py b/posthog/clickhouse/client/test/test_execute.py
index 74cdfba45d8c..0f3fa35935ee 100644
--- a/posthog/clickhouse/client/test/test_execute.py
+++ b/posthog/clickhouse/client/test/test_execute.py
@@ -76,8 +76,6 @@ def test_celery_cache_warming_workload_routing(client_from_pool, access_method,
sync_execute("SELECT 1", flush=False)
assert client_from_pool.call_args[0][0] == expected_workload
- client = client_from_pool.return_value.__enter__.return_value
- assert client.execute.call_args.kwargs["settings"].get("use_hedged_requests") == "0"
@pytest.mark.parametrize(
diff --git a/posthog/settings/dynamic_settings.py b/posthog/settings/dynamic_settings.py
index 177f7fe51fca..23a4fe782da3 100644
--- a/posthog/settings/dynamic_settings.py
+++ b/posthog/settings/dynamic_settings.py
@@ -338,11 +338,6 @@ CONSTANCE_CONFIG = {
"Comma-separated team IDs always subject to the 'full' ClickHouse kill switch, even when the global kill switch is 'off'. Use this to restrict heavy or abusive teams without degrading the rest of the cluster.",
list[int],
),
- "CLICKHOUSE_HEDGED_APP_QUERIES": (
- get_from_env("CLICKHOUSE_HEDGED_APP_QUERIES", False, type_cast=str_to_bool),
- "Enable hedged requests for online APP queries to ClickHouse.",
- bool,
- ),
"RATE_LIMITING_ALLOW_LIST_TEAMS": (
get_from_env("RATE_LIMITING_ALLOW_LIST_TEAMS", ""),
"Whether teams are on an allow list to bypass rate limiting. Comma separated list of team-ids",
@@ -492,7 +487,6 @@ SETTINGS_ALLOWING_API_OVERRIDE = (
"CLICKHOUSE_KILL_SWITCH",
"CLICKHOUSE_KILL_SWITCH_LIGHT_TEAMS",
"CLICKHOUSE_KILL_SWITCH_FULL_TEAMS",
- "CLICKHOUSE_HEDGED_APP_QUERIES",
"REDIRECT_APP_TO_US",
"WEB_ANALYTICS_WARMING_DAYS",
"WEB_ANALYTICS_WARMING_SELECTION_TTL_SECONDS",
diff --git a/products/notebooks/backend/temporal/frame_materialize.py b/products/notebooks/backend/temporal/frame_materialize.py
index b0eab08801cf..9eb383f9cadd 100644
--- a/products/notebooks/backend/temporal/frame_materialize.py
+++ b/products/notebooks/backend/temporal/frame_materialize.py
@@ -681,7 +681,7 @@ def _execute_insert_to_s3(team: Team, user: User | None, query: str, key: str) -
"""Materialize by having ClickHouse write the object itself.
Runs through the pooled native clients (sync_execute) with a bounded socket timeout:
- offline workload and hedging hygiene come from the workload routing, errors arrive
+ the offline workload comes from the workload routing, errors arrive
in-band and typed, and no result bytes transit the worker — so the streaming path's
EOS-marker check and query_log recovery have no equivalent here.
"""
@@ -837,7 +837,7 @@ def materialize_frame(inputs: FrameMaterializeInputs) -> str:
"""
# The new flow rides one flag, resolved per user in the web process and carried on the
# job. Off, this run is the path that has been serving frames all along: interactive
- # pool, default credentials, hedging left alone.
+ # pool, default credentials.
offline = inputs.ch_writes
# Dedicated `notebooks` CH user (server-side profile/quota backstop no application
# bug can exceed); falls back to the default credentials where not provisioned.
@@ -931,14 +931,6 @@ def materialize_frame(inputs: FrameMaterializeInputs) -> str:
max_result_bytes=_MAX_RESULT_BYTES,
result_overflow_mode="throw",
)
- if offline:
- # sync_execute's offline hygiene, and only meaningful off the
- # interactive pool: without it, distributed subqueries of a saturated
- # offline query hedge onto online replicas, bleeding the whale back
- # into the pool this move protects. Set after construction rather than
- # as a kwarg so the online path sends no such setting at all, exactly
- # as it did before this flag existed.
- client.params["use_hedged_requests"] = "0"
printed = _print_clickhouse_sql(
lambda sql, sql_values: _describe_columns(client, sql, sql_values, ch_query_id),
team,