Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(temporal): let a worker set the HogQL shadow parse rate (#124571)

Jimmy Zhu24b0459f04c3
diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.py
index 84748d535cf7..7a0641f45718 100644
--- a/posthog/management/commands/start_temporal_worker.py
+++ b/posthog/management/commands/start_temporal_worker.py
@@ -19,6 +19,8 @@ with workflow.unsafe.imports_passed_through():
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
+from posthog.hogql import parser as hogql_parser
+
from posthog.clickhouse.query_tagging import tag_queries
from posthog.temporal.ai import AI_ACTIVITIES, AI_WORKFLOWS
from posthog.temporal.ai_observability import (
@@ -705,6 +707,27 @@ else:
LOGGER = get_logger(__name__)
+def apply_hogql_shadow_sample_rate_override() -> None:
+ """Sets HogQL's cpp shadow parse rate for this worker from `HOGQL_PARSER_SHADOW_SAMPLE_RATE`.
+
+ The cpp shadow parse holds the GIL for the whole parse, and on a large expression that stalls
+ workflow tasks past Temporal's 2s deadlock detector. Remove this once the parser reads the
+ variable itself.
+ """
+ raw = os.environ.get("HOGQL_PARSER_SHADOW_SAMPLE_RATE")
+ if raw is None:
+ return
+ try:
+ rate = float(raw)
+ except ValueError:
+ LOGGER.warning("Ignoring invalid HOGQL_PARSER_SHADOW_SAMPLE_RATE", value=raw)
+ return
+ if not 0.0 <= rate <= 1.0:
+ LOGGER.warning("Ignoring invalid HOGQL_PARSER_SHADOW_SAMPLE_RATE", value=raw)
+ return
+ hogql_parser._SHADOW_SAMPLE_RATE = rate
+
+
class Command(BaseCommand):
help = "Start Temporal Python Django-aware Worker"
@@ -855,6 +878,7 @@ class Command(BaseCommand):
# `kill -USR1 <pid>` dumps every thread's stack to stderr and leaves the worker running. It
# works while a thread holds the GIL, and in containers where py-spy cannot ptrace.
faulthandler.register(signal.SIGUSR1)
+ apply_hogql_shadow_sample_rate_override()
metrics_port = int(options["metrics_port"])
diff --git a/posthog/management/commands/test/test_start_temporal_worker.py b/posthog/management/commands/test/test_start_temporal_worker.py
index 836a77a075b2..b81f55abff39 100644
--- a/posthog/management/commands/test/test_start_temporal_worker.py
+++ b/posthog/management/commands/test/test_start_temporal_worker.py
@@ -4,6 +4,8 @@ import pytest
from django.conf import settings
+from posthog.hogql import parser as hogql_parser
+
from posthog.management.commands.start_temporal_worker import (
ACTIVITIES_DICT,
AI_WORKFLOWS,
@@ -15,6 +17,7 @@ from posthog.management.commands.start_temporal_worker import (
WEEKLY_DIGEST_WORKFLOWS,
WORKFLOWS_DICT,
_task_queue_specs,
+ apply_hogql_shadow_sample_rate_override,
workflows_include_data_import_syncs,
)
from posthog.temporal.usage_report.activities import aggregate_and_chunk_org_reports
@@ -141,3 +144,24 @@ def test_ai_queue_registers_the_alert_evaluate_activity() -> None:
assert ALERT_AI_QUEUE_ACTIVITIES
assert set(ALERT_AI_QUEUE_ACTIVITIES) <= set(entries[0])
assert set(INSIGHT_PLATFORM_AI_QUEUE_ACTIVITIES) <= set(entries[0])
+
+
+ "env_value, expected",
+ [
+ (None, 0.001),
+ ("0", 0.0),
+ ("0.25", 0.25),
+ ("off", 0.001),
+ ("2", 0.001),
+ ],
+)
+def test_hogql_shadow_sample_rate_override(
+ monkeypatch: pytest.MonkeyPatch, env_value: str | None, expected: float
+) -> None:
+ monkeypatch.delenv("HOGQL_PARSER_SHADOW_SAMPLE_RATE", raising=False)
+ if env_value is not None:
+ monkeypatch.setenv("HOGQL_PARSER_SHADOW_SAMPLE_RATE", env_value)
+ monkeypatch.setattr(hogql_parser, "_SHADOW_SAMPLE_RATE", 0.001)
+ apply_hogql_shadow_sample_rate_override()
+ assert hogql_parser._SHADOW_SAMPLE_RATE == expected