:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(temporal): cache otel tracers to stop per-span filterwarnings calls (#122837)
Jimmy Zhu647cffb251f9
diff --git a/posthog/temporal/common/open_telemetry.py b/posthog/temporal/common/open_telemetry.pyindex 109cb47508b2..cf3dfe17180a 100644--- a/posthog/temporal/common/open_telemetry.py+++ b/posthog/temporal/common/open_telemetry.py@@ -1,13 +1,59 @@import collections.abc+from typing import TYPE_CHECKINGfrom opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter+from opentelemetry.sdk import trace as trace_sdkfrom opentelemetry.sdk.resources import Resourcefrom opentelemetry.sdk.trace.export import BatchSpanProcessor-from opentelemetry.trace import set_tracer_provider-from temporalio.contrib.opentelemetry import create_tracer_provider+from opentelemetry.trace import Tracer, set_tracer_provider+from opentelemetry.util import types+from temporalio.contrib.opentelemetry import ReplaySafeTracerProvider, create_tracer_providerfrom posthog.otel_instrumentation import INSTRUMENTORS+if TYPE_CHECKING:+ from temporalio.contrib.opentelemetry._id_generator import TemporalIdGenerator+++class CachingReplaySafeTracerProvider(ReplaySafeTracerProvider):+ """Replay-safe tracer provider that returns one cached tracer for each instrumentation scope.++ The OpenTelemetry SDK calls `warnings.filterwarnings` in every `get_tracer` call, and+ Temporal's `OpenTelemetryPlugin` calls `get_tracer` for every span that it starts in workflow code.+ Each `filterwarnings` call takes the process-wide warnings lock and clears every warning registry,+ so each later warning is printed again while a thread holds that lock.+ When the Temporal deadlock detector interrupts a workflow thread that waits for that lock,+ the interrupt can arrive after the thread gets the lock and before the `with` block starts.+ The lock then stays held, and every thread in the process that warns or changes a filter blocks.+ """++ def __init__(self, tracer_provider: trace_sdk.TracerProvider, id_generator: "TemporalIdGenerator") -> None:+ super().__init__(tracer_provider, id_generator)+ self._tracers: dict[tuple[str, str | None, str | None], Tracer] = {}++ @classmethod+ def create(cls, resource: Resource) -> "CachingReplaySafeTracerProvider":+ # create_tracer_provider sets up the TemporalIdGenerator that replay safety needs, so wrap its SDK provider.+ provider = create_tracer_provider(resource=resource)+ return cls(provider._tracer_provider, provider.id_generator())++ def get_tracer(+ self,+ instrumenting_module_name: str,+ instrumenting_library_version: str | None = None,+ schema_url: str | None = None,+ attributes: types.Attributes | None = None,+ ) -> Tracer:+ # A mapping of attributes is not hashable, so a call with attributes does not use the cache.+ if attributes:+ return super().get_tracer(instrumenting_module_name, instrumenting_library_version, schema_url, attributes)++ key = (instrumenting_module_name, instrumenting_library_version, schema_url)+ tracer = self._tracers.get(key)+ if tracer is None:+ tracer = self._tracers.setdefault(key, super().get_tracer(*key))+ return tracer+def initialize_otel(service_name: str, libraries_to_instrument: collections.abc.Iterable[str] = ()) -> None:"""Initialize Open Telemetry for Temporal workers.@@ -16,7 +62,7 @@ def initialize_otel(service_name: str, libraries_to_instrument: collections.abc.Temporal's `OpenTelemetryPlugin`."""resource = Resource.create(attributes={"service.name": service_name})- provider = create_tracer_provider(resource=resource)+ provider = CachingReplaySafeTracerProvider.create(resource)provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))set_tracer_provider(provider)diff --git a/posthog/temporal/tests/common/test_open_telemetry.py b/posthog/temporal/tests/common/test_open_telemetry.pynew file mode 100644index 000000000000..0f8aeeda20aa--- /dev/null+++ b/posthog/temporal/tests/common/test_open_telemetry.py@@ -0,0 +1,21 @@+import warnings++from opentelemetry.sdk.resources import Resource+from temporalio.contrib.opentelemetry import ReplaySafeTracerProvider++from posthog.temporal.common.open_telemetry import CachingReplaySafeTracerProvider+++def test_repeated_get_tracer_does_not_reset_warning_registries() -> None:+ provider = CachingReplaySafeTracerProvider.create(Resource.create(attributes={"service.name": "test"}))+ assert isinstance(provider, ReplaySafeTracerProvider)++ with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("default")+ first_tracer = provider.get_tracer("temporalio.contrib.opentelemetry")++ for _ in range(3):+ assert provider.get_tracer("temporalio.contrib.opentelemetry") is first_tracer+ warnings.warn("probe warning", UserWarning, stacklevel=1)++ assert [str(warning.message) for warning in caught] == ["probe warning"]