Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(temporal): cache otel tracers to stop per-span filterwarnings calls (#122837)

Jimmy Zhu647cffb251f9
diff --git a/posthog/temporal/common/open_telemetry.py b/posthog/temporal/common/open_telemetry.py
index 109cb47508b2..cf3dfe17180a 100644
--- a/posthog/temporal/common/open_telemetry.py
+++ b/posthog/temporal/common/open_telemetry.py
@@ -1,13 +1,59 @@
import collections.abc
+from typing import TYPE_CHECKING
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk import trace as trace_sdk
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace.export import BatchSpanProcessor
-from opentelemetry.trace import set_tracer_provider
-from temporalio.contrib.opentelemetry import create_tracer_provider
+from opentelemetry.trace import Tracer, set_tracer_provider
+from opentelemetry.util import types
+from temporalio.contrib.opentelemetry import ReplaySafeTracerProvider, create_tracer_provider
from posthog.otel_instrumentation import INSTRUMENTORS
+if TYPE_CHECKING:
+ from temporalio.contrib.opentelemetry._id_generator import TemporalIdGenerator
+
+
+class CachingReplaySafeTracerProvider(ReplaySafeTracerProvider):
+ """Replay-safe tracer provider that returns one cached tracer for each instrumentation scope.
+
+ The OpenTelemetry SDK calls `warnings.filterwarnings` in every `get_tracer` call, and
+ Temporal's `OpenTelemetryPlugin` calls `get_tracer` for every span that it starts in workflow code.
+ Each `filterwarnings` call takes the process-wide warnings lock and clears every warning registry,
+ so each later warning is printed again while a thread holds that lock.
+ When the Temporal deadlock detector interrupts a workflow thread that waits for that lock,
+ the interrupt can arrive after the thread gets the lock and before the `with` block starts.
+ The lock then stays held, and every thread in the process that warns or changes a filter blocks.
+ """
+
+ def __init__(self, tracer_provider: trace_sdk.TracerProvider, id_generator: "TemporalIdGenerator") -> None:
+ super().__init__(tracer_provider, id_generator)
+ self._tracers: dict[tuple[str, str | None, str | None], Tracer] = {}
+
+ @classmethod
+ def create(cls, resource: Resource) -> "CachingReplaySafeTracerProvider":
+ # create_tracer_provider sets up the TemporalIdGenerator that replay safety needs, so wrap its SDK provider.
+ provider = create_tracer_provider(resource=resource)
+ return cls(provider._tracer_provider, provider.id_generator())
+
+ def get_tracer(
+ self,
+ instrumenting_module_name: str,
+ instrumenting_library_version: str | None = None,
+ schema_url: str | None = None,
+ attributes: types.Attributes | None = None,
+ ) -> Tracer:
+ # A mapping of attributes is not hashable, so a call with attributes does not use the cache.
+ if attributes:
+ return super().get_tracer(instrumenting_module_name, instrumenting_library_version, schema_url, attributes)
+
+ key = (instrumenting_module_name, instrumenting_library_version, schema_url)
+ tracer = self._tracers.get(key)
+ if tracer is None:
+ tracer = self._tracers.setdefault(key, super().get_tracer(*key))
+ return tracer
+
def initialize_otel(service_name: str, libraries_to_instrument: collections.abc.Iterable[str] = ()) -> None:
"""Initialize Open Telemetry for Temporal workers.
@@ -16,7 +62,7 @@ def initialize_otel(service_name: str, libraries_to_instrument: collections.abc.
Temporal's `OpenTelemetryPlugin`.
"""
resource = Resource.create(attributes={"service.name": service_name})
- provider = create_tracer_provider(resource=resource)
+ provider = CachingReplaySafeTracerProvider.create(resource)
provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
set_tracer_provider(provider)
diff --git a/posthog/temporal/tests/common/test_open_telemetry.py b/posthog/temporal/tests/common/test_open_telemetry.py
new file mode 100644
index 000000000000..0f8aeeda20aa
--- /dev/null
+++ b/posthog/temporal/tests/common/test_open_telemetry.py
@@ -0,0 +1,21 @@
+import warnings
+
+from opentelemetry.sdk.resources import Resource
+from temporalio.contrib.opentelemetry import ReplaySafeTracerProvider
+
+from posthog.temporal.common.open_telemetry import CachingReplaySafeTracerProvider
+
+
+def test_repeated_get_tracer_does_not_reset_warning_registries() -> None:
+ provider = CachingReplaySafeTracerProvider.create(Resource.create(attributes={"service.name": "test"}))
+ assert isinstance(provider, ReplaySafeTracerProvider)
+
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("default")
+ first_tracer = provider.get_tracer("temporalio.contrib.opentelemetry")
+
+ for _ in range(3):
+ assert provider.get_tracer("temporalio.contrib.opentelemetry") is first_tracer
+ warnings.warn("probe warning", UserWarning, stacklevel=1)
+
+ assert [str(warning.message) for warning in caught] == ["probe warning"]