:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(otel): trace outbound http and s3 calls from web workers (#130114)
Thiago Salvatore5cddb7cb83fa
diff --git a/posthog/asgi.py b/posthog/asgi.pyindex 637e45eed1e8..de289e796540 100644--- a/posthog/asgi.py+++ b/posthog/asgi.py@@ -88,7 +88,7 @@ def _ensure_post_fork_init():from posthog.web_memory_sampler import start_web_memory_samplerstart_continuous_profiling()- initialize_otel()+ initialize_otel(instrument_outbound_calls=True)prewarm_query_cache_cluster_in_background()# Production runs ASGI, so the sampler — previously only started from wsgi.py — never# ran in prod, leaving posthog_web_worker_rss_mb empty. Start it here, post-fork.diff --git a/posthog/otel_instrumentation.py b/posthog/otel_instrumentation.pyindex eeac501ad6a2..ccf3614bde24 100644--- a/posthog/otel_instrumentation.py+++ b/posthog/otel_instrumentation.py@@ -2,17 +2,21 @@ import osimport atexitimport typingimport logging-from collections.abc import Sequence+from collections.abc import Iterator, Sequence+from urllib.parse import urlsplit, urlunsplitfrom django.http import HttpRequest, HttpResponseimport structlogfrom opentelemetry import trace+from opentelemetry.context import Contextfrom opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporterfrom opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentorfrom opentelemetry.instrumentation.aiokafka import AIOKafkaInstrumentor+from opentelemetry.instrumentation.botocore import BotocoreInstrumentorfrom opentelemetry.instrumentation.celery import CeleryInstrumentorfrom opentelemetry.instrumentation.django import DjangoInstrumentor+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor, RequestInfofrom opentelemetry.instrumentation.kafka import KafkaInstrumentorfrom opentelemetry.instrumentation.psycopg import PsycopgInstrumentorfrom opentelemetry.instrumentation.redis import RedisInstrumentor@@ -20,12 +24,18 @@ from opentelemetry.instrumentation.requests import RequestsInstrumentorfrom opentelemetry.sdk.resources import Resourcefrom opentelemetry.sdk.trace import TracerProviderfrom opentelemetry.sdk.trace.export import BatchSpanProcessor-from opentelemetry.trace import Span+from opentelemetry.semconv.attributes.error_attributes import ERROR_TYPE+from opentelemetry.trace import Link, Span, SpanKind, StatusCode+from opentelemetry.util._decorator import _agnosticcontextmanagerfrom opentelemetry.util.http import sanitize_method+from opentelemetry.util.types import Attributes+from requests import PreparedRequest# Get a structlog logger for this module's own messageslogger = structlog.get_logger(__name__)+_OUTBOUND_URL_ATTRIBUTE_KEYS = ("http.url", "url.full")+def _otel_django_request_hook(span: Span, request: HttpRequest) -> None:if span and span.is_recording():@@ -88,7 +98,110 @@ def _otel_redis_request_hook(span: Span, instance: object, args: Sequence[objectlogger.debug("otel_redis_request_hook_failed", exc_info=True)-def initialize_otel():+def _url_origin(url: str) -> str:+ try:+ parts = urlsplit(url)+ except ValueError:+ return ""+ return urlunsplit((parts.scheme, parts.netloc.rpartition("@")[2], "", "", ""))+++def _replace_span_url_with_origin(span: Span, url: str) -> None:+ if not span.is_recording():+ return+ origin = _url_origin(url)+ for key in _OUTBOUND_URL_ATTRIBUTE_KEYS:+ span.set_attribute(key, origin)+++def _otel_requests_request_hook(span: Span, request: PreparedRequest) -> None:+ _replace_span_url_with_origin(span, request.url or "")+++def _otel_httpx_request_hook(span: Span, request: RequestInfo) -> None:+ _replace_span_url_with_origin(span, str(request.url))+++async def _otel_httpx_async_request_hook(span: Span, request: RequestInfo) -> None:+ _otel_httpx_request_hook(span, request)+++class _ExceptionTypeOnlyTracer(trace.Tracer):+ def __init__(self, tracer: trace.Tracer) -> None:+ self._tracer = tracer++ def start_span(+ self,+ name: str,+ context: Context | None = None,+ kind: SpanKind = SpanKind.INTERNAL,+ attributes: Attributes = None,+ links: Sequence[Link] | None = None,+ start_time: int | None = None,+ record_exception: bool = True,+ set_status_on_exception: bool = True,+ ) -> Span:+ return self._tracer.start_span(+ name,+ context,+ kind,+ attributes,+ links,+ start_time,+ record_exception=False,+ set_status_on_exception=False,+ )++ @_agnosticcontextmanager+ def start_as_current_span(+ self,+ name: str,+ context: Context | None = None,+ kind: SpanKind = SpanKind.INTERNAL,+ attributes: Attributes = None,+ links: Sequence[Link] | None = None,+ start_time: int | None = None,+ record_exception: bool = True,+ set_status_on_exception: bool = True,+ end_on_exit: bool = True,+ ) -> Iterator[Span]:+ with self._tracer.start_as_current_span(+ name,+ context,+ kind,+ attributes,+ links,+ start_time,+ record_exception=False,+ set_status_on_exception=False,+ end_on_exit=end_on_exit,+ ) as span:+ try:+ yield span+ except Exception as error:+ span.set_attribute(ERROR_TYPE, type(error).__name__)+ span.set_status(StatusCode.ERROR)+ raise+++class _ExceptionTypeOnlyTracerProvider(trace.TracerProvider):+ def __init__(self, provider: trace.TracerProvider) -> None:+ self._provider = provider++ def get_tracer(+ self,+ instrumenting_module_name: str,+ instrumenting_library_version: str | None = None,+ schema_url: str | None = None,+ attributes: Attributes = None,+ ) -> trace.Tracer:+ tracer = self._provider.get_tracer(+ instrumenting_module_name, instrumenting_library_version, schema_url, attributes+ )+ return _ExceptionTypeOnlyTracer(tracer)+++def initialize_otel(*, instrument_outbound_calls: bool = False) -> None:# App tracing targets a real OTLP collector (dev/prod). In tests, booting the app# (e.g. posthog/test/test_asgi_lifespan.py imports posthog.asgi) would otherwise arm a# process-wide span exporter to the SDK default localhost:4317 with no collector there,@@ -163,6 +276,10 @@ def initialize_otel():instrument_psycopg(provider)instrument_kafka(provider)instrument_aiokafka(provider)+ if instrument_outbound_calls:+ instrument_requests_client(provider)+ instrument_httpx_client(provider)+ instrument_botocore(provider)logger.info("otel_manual_init_status_from_instrumentation_module",@@ -246,9 +363,12 @@ def instrument_aiohttp_client(provider: trace.TracerProvider):)-def instrument_requests_client(provider: trace.TracerProvider):+def instrument_requests_client(provider: trace.TracerProvider) -> None:try:- RequestsInstrumentor().instrument(tracer_provider=provider)+ RequestsInstrumentor().instrument(+ tracer_provider=_ExceptionTypeOnlyTracerProvider(provider),+ request_hook=_otel_requests_request_hook,+ )logger.info("otel_instrumentation_attempt", instrumentor="RequestsInstrumentor", status="success")except Exception as e:logger.exception(@@ -256,6 +376,30 @@ def instrument_requests_client(provider: trace.TracerProvider):)+def instrument_httpx_client(provider: trace.TracerProvider) -> None:+ try:+ HTTPXClientInstrumentor().instrument(+ tracer_provider=_ExceptionTypeOnlyTracerProvider(provider),+ request_hook=_otel_httpx_request_hook,+ async_request_hook=_otel_httpx_async_request_hook,+ )+ logger.info("otel_instrumentation_attempt", instrumentor="HTTPXClientInstrumentor", status="success")+ except Exception as e:+ logger.exception(+ "otel_instrumentation_attempt", instrumentor="HTTPXClientInstrumentor", status="error", exc_info=e+ )+++def instrument_botocore(provider: trace.TracerProvider) -> None:+ try:+ BotocoreInstrumentor().instrument(tracer_provider=_ExceptionTypeOnlyTracerProvider(provider))+ logger.info("otel_instrumentation_attempt", instrumentor="BotocoreInstrumentor", status="success")+ except Exception as e:+ logger.exception(+ "otel_instrumentation_attempt", instrumentor="BotocoreInstrumentor", status="error", exc_info=e+ )++INSTRUMENTORS: dict[str, typing.Callable[[trace.TracerProvider], None]] = {"django": instrument_django,"psycopg": instrument_psycopg,diff --git a/posthog/test/test_otel_instrumentation.py b/posthog/test/test_otel_instrumentation.pyindex f9bb1ac8d6c8..f93c39a967e8 100644--- a/posthog/test/test_otel_instrumentation.py+++ b/posthog/test/test_otel_instrumentation.py@@ -1,6 +1,10 @@# posthog/test/test_otel_instrumentation.pyimport os+import re+import asyncioimport logging+import contextlib+from collections.abc import Callablefrom types import SimpleNamespacefrom typing import NoReturn@@ -8,10 +12,22 @@ from unittest import mockfrom django.test import SimpleTestCase+import boto3+import httpx+import httpcore+import requests+import responses+from botocore.config import Config+from botocore.exceptions import EndpointConnectionError+from botocore.httpsession import URLLib3Sessionfrom opentelemetry import trace-from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.instrumentation.botocore import BotocoreInstrumentor+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor+from opentelemetry.instrumentation.requests import RequestsInstrumentor+from opentelemetry.sdk.trace import ReadableSpan, TracerProviderfrom opentelemetry.sdk.trace.export import SimpleSpanProcessorfrom opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter+from opentelemetry.trace import StatusCodefrom opentelemetry.util.types import AttributeValuefrom parameterized import parameterized@@ -20,8 +36,53 @@ from posthog.otel_instrumentation import (_otel_django_response_hook,_otel_redis_request_hook,initialize_otel,+ instrument_botocore,+ instrument_httpx_client,+ instrument_requests_client,)+URL_WITH_CREDENTIALS = "https://user:[email protected]/hooks/secret?token=secret"+++def _get_with_requests(url: str, fail: bool) -> None:+ body = requests.ConnectionError(f"cannot reach {url}") if fail else ""+ with responses.RequestsMock() as mocked_responses, contextlib.suppress(requests.ConnectionError):+ mocked_responses.get(re.compile(".*"), body=body)+ requests.get(url)+++def _get_with_httpx(url: str, fail: bool) -> None:+ error = httpcore.ConnectError(f"cannot reach {url}") if fail else None+ with (+ mock.patch.object(+ httpcore.ConnectionPool, "handle_request", return_value=httpcore.Response(200), side_effect=error+ ),+ contextlib.suppress(httpx.ConnectError),+ ):+ httpx.get(url)+++def _get_with_async_httpx(url: str, fail: bool) -> None:+ async def get() -> None:+ async with httpx.AsyncClient() as client:+ await client.get(url)++ error = httpcore.ConnectError(f"cannot reach {url}") if fail else None+ with (+ mock.patch.object(+ httpcore.AsyncConnectionPool,+ "handle_async_request",+ return_value=httpcore.Response(200),+ side_effect=error,+ ),+ contextlib.suppress(httpx.ConnectError),+ ):+ asyncio.run(get())+++def _everything_recorded(span: ReadableSpan) -> str:+ return str((span.attributes, [(event.name, event.attributes) for event in span.events], span.status.description))+class TestOtelInstrumentation(SimpleTestCase):def setUp(self):@@ -425,3 +486,55 @@ class TestOtelRedisRequestHook(SimpleTestCase):raise RuntimeError("topology unavailable")self.assertEqual(self._span_attributes_after_hook(ClientWithoutTopology()), {})+++class TestOutboundCallSpans(SimpleTestCase):+ def setUp(self) -> None:+ super().setUp()+ self.exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(self.exporter))+ instrument_requests_client(provider)+ self.addCleanup(RequestsInstrumentor().uninstrument)+ instrument_httpx_client(provider)+ self.addCleanup(HTTPXClientInstrumentor().uninstrument)+ instrument_botocore(provider)+ self.addCleanup(BotocoreInstrumentor().uninstrument)++ @parameterized.expand(+ [+ ("requests", _get_with_requests, False),+ ("requests_failed", _get_with_requests, True),+ ("httpx", _get_with_httpx, False),+ ("httpx_failed", _get_with_httpx, True),+ ("httpx_async", _get_with_async_httpx, False),+ ("httpx_async_failed", _get_with_async_httpx, True),+ ]+ )+ def test_span_keeps_only_the_url_origin(self, _case: str, get: Callable[[str, bool], None], fail: bool) -> None:+ get(URL_WITH_CREDENTIALS, fail)++ (span,) = self.exporter.get_finished_spans()+ self.assertEqual(dict(span.attributes or {})["http.url"], "https://api.example.com")+ self.assertEqual(span.status.status_code, StatusCode.ERROR if fail else StatusCode.UNSET)+ self.assertNotIn("secret", _everything_recorded(span))++ def test_failed_s3_call_records_only_the_exception_type(self) -> None:+ client = boto3.client(+ "s3",+ endpoint_url="https://storage.example.com",+ region_name="us-east-1",+ aws_access_key_id="test-key-id",+ aws_secret_access_key="test-key",+ config=Config(retries={"total_max_attempts": 1}),+ )+ connection_error = EndpointConnectionError(endpoint_url="https://storage.example.com/bucket/secret")+ with (+ mock.patch.object(URLLib3Session, "send", side_effect=connection_error),+ contextlib.suppress(EndpointConnectionError),+ ):+ client.get_object(Bucket="bucket", Key="secret")++ (span,) = self.exporter.get_finished_spans()+ self.assertEqual(span.status.status_code, StatusCode.ERROR)+ self.assertNotIn("secret", _everything_recorded(span))diff --git a/posthog/wsgi.py b/posthog/wsgi.pyindex 0ff9dfbb20ee..bee568659dda 100644--- a/posthog/wsgi.py+++ b/posthog/wsgi.py@@ -26,7 +26,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "posthog.settings")os.environ.setdefault("SERVER_GATEWAY_INTERFACE", "WSGI")start_continuous_profiling()-initialize_otel()+initialize_otel(instrument_outbound_calls=True)# Boot allocations are almost all permanent, so cyclic GC during django.setup() only adds# pauses (~300ms). Disable it for the boot, then freeze the survivors so later fulldiff --git a/pyproject.toml b/pyproject.tomlindex 82fa7142edd0..d325e0dce449 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -180,8 +180,10 @@ dependencies = ["opentelemetry-instrumentation-aiokafka==0.56b0","opentelemetry-instrumentation-aiohttp-client==0.56b0","opentelemetry-instrumentation-asgi==0.56b0",+ "opentelemetry-instrumentation-botocore==0.56b0","opentelemetry-instrumentation-celery==0.56b0","opentelemetry-instrumentation-django==0.56b0",+ "opentelemetry-instrumentation-httpx==0.56b0","opentelemetry-instrumentation-kafka-python==0.56b0","opentelemetry-instrumentation-psycopg==0.56b0","opentelemetry-instrumentation-redis==0.56b0",diff --git a/uv.lock b/uv.lockindex ecec694465c1..520b1a310de3 100644--- a/uv.lock+++ b/uv.lock@@ -5284,6 +5284,21 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/c7/dd/655355453ee918455b5e7132f80366f594e0e7433dbcd03d2b0d6033088f/opentelemetry_instrumentation_asgi-0.56b0-py3-none-any.whl", hash = "sha256:7144793c1c601e47db6174d748ab437f7fa92f93a87d13882ad60a1f8147d2cf", size = 16602, upload-time = "2025-07-11T12:25:28.7Z" },]+[[package]]+name = "opentelemetry-instrumentation-botocore"+version = "0.56b0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "opentelemetry-api" },+ { name = "opentelemetry-instrumentation" },+ { name = "opentelemetry-propagator-aws-xray" },+ { name = "opentelemetry-semantic-conventions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/a9/59/c663a613fa3bd9becd94f916095bf82aa6e45a59cd5b7d70eb52dc1ae146/opentelemetry_instrumentation_botocore-0.56b0.tar.gz", hash = "sha256:c2dd342eec24acdac7cb6e59c268f523c1e1165b6c7f2ac1f339623a2e1d6118", size = 118800, upload-time = "2025-07-11T12:26:27.196Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/d3/62/d2cb63c740513cd9f4f635792c276764488c110467f7b39eee247e5594de/opentelemetry_instrumentation_botocore-0.56b0-py3-none-any.whl", hash = "sha256:7ccacdcb5b858d931b0b526b2e3e857c06f89c67847252fd048002b295862bf8", size = 35808, upload-time = "2025-07-11T12:25:35.3Z" },+]+[[package]]name = "opentelemetry-instrumentation-celery"version = "0.56b0"@@ -5329,6 +5344,22 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/da/c7/727f9699c855a52002631f2f0b5be3716d0599419209ba13aeffe5c0419c/opentelemetry_instrumentation_django-0.56b0-py3-none-any.whl", hash = "sha256:c2e05fc1321166791e529aeb3a4b27cfb07fb69a743fe92c6faa3866290dc591", size = 19600, upload-time = "2025-07-11T12:25:40.327Z" },]+[[package]]+name = "opentelemetry-instrumentation-httpx"+version = "0.56b0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "opentelemetry-api" },+ { name = "opentelemetry-instrumentation" },+ { name = "opentelemetry-semantic-conventions" },+ { name = "opentelemetry-util-http" },+ { name = "wrapt" },+]+sdist = { url = "https://files.pythonhosted.org/packages/8b/93/784623262776d90aa07e5cc08f58e1d476ed4b208127c1718db4b57b78c8/opentelemetry_instrumentation_httpx-0.56b0.tar.gz", hash = "sha256:d93c7a0e0ce5cc2170f7cc0d26f19167bc8314b7d2a5204afdf84000d5ca8647", size = 19510, upload-time = "2025-07-11T12:26:37.492Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/b4/0e/fe2f38e0170a22a44b26e9a8f555ac23b2fcf09f89264d718800ec1d5954/opentelemetry_instrumentation_httpx-0.56b0-py3-none-any.whl", hash = "sha256:34d9558413d695ea69fe002efa07c00aa859a8aa4930815726e5813d5f002e1d", size = 15113, upload-time = "2025-07-11T12:25:45.671Z" },+]+[[package]]name = "opentelemetry-instrumentation-kafka-python"version = "0.56b0"@@ -5402,6 +5433,18 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/45/9e/97a89c0def0819d6c8b9556fdf9969a7503a01fd19b1a9f13770c530f19e/opentelemetry_instrumentation_wsgi-0.56b0-py3-none-any.whl", hash = "sha256:ecf29aeac2e945054af2f8e3f3a162bc1f83016b08a0f3e590ad10f2f3c518c8", size = 14449, upload-time = "2025-07-11T12:26:12.714Z" },]+[[package]]+name = "opentelemetry-propagator-aws-xray"+version = "1.0.2"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "opentelemetry-api" },+]+sdist = { url = "https://files.pythonhosted.org/packages/f4/31/40004e9e55b1e5694ef3a7526f0b7637df44196fc68a8b7d248a3684680f/opentelemetry_propagator_aws_xray-1.0.2.tar.gz", hash = "sha256:6b2cee5479d2ef0172307b66ed2ed151f598a0fd29b3c01133ac87ca06326260", size = 10994, upload-time = "2024-08-05T17:45:57.601Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/ea/89/849a0847871fd9745315896ad9e23d6479db84d90b8b36c4c26dc46e92b8/opentelemetry_propagator_aws_xray-1.0.2-py3-none-any.whl", hash = "sha256:1c99181ee228e99bddb638a0c911a297fa21f1c3a0af951f841e79919b5f1934", size = 10856, upload-time = "2024-08-05T17:45:56.492Z" },+]+[[package]]name = "opentelemetry-proto"version = "1.35.0"@@ -6008,8 +6051,10 @@ dependencies = [{ name = "opentelemetry-instrumentation-aiohttp-client" },{ name = "opentelemetry-instrumentation-aiokafka" },{ name = "opentelemetry-instrumentation-asgi" },+ { name = "opentelemetry-instrumentation-botocore" },{ name = "opentelemetry-instrumentation-celery" },{ name = "opentelemetry-instrumentation-django" },+ { name = "opentelemetry-instrumentation-httpx" },{ name = "opentelemetry-instrumentation-kafka-python" },{ name = "opentelemetry-instrumentation-psycopg" },{ name = "opentelemetry-instrumentation-redis" },@@ -6308,8 +6353,10 @@ requires-dist = [{ name = "opentelemetry-instrumentation-aiohttp-client", specifier = "==0.56b0" },{ name = "opentelemetry-instrumentation-aiokafka", specifier = "==0.56b0" },{ name = "opentelemetry-instrumentation-asgi", specifier = "==0.56b0" },+ { name = "opentelemetry-instrumentation-botocore", specifier = "==0.56b0" },{ name = "opentelemetry-instrumentation-celery", specifier = "==0.56b0" },{ name = "opentelemetry-instrumentation-django", specifier = "==0.56b0" },+ { name = "opentelemetry-instrumentation-httpx", specifier = "==0.56b0" },{ name = "opentelemetry-instrumentation-kafka-python", specifier = "==0.56b0" },{ name = "opentelemetry-instrumentation-psycopg", specifier = "==0.56b0" },{ name = "opentelemetry-instrumentation-redis", specifier = "==0.56b0" },