Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(otel): trace outbound http and s3 calls from web workers (#130114)

Thiago Salvatore5cddb7cb83fa
diff --git a/posthog/asgi.py b/posthog/asgi.py
index 637e45eed1e8..de289e796540 100644
--- a/posthog/asgi.py
+++ b/posthog/asgi.py
@@ -88,7 +88,7 @@ def _ensure_post_fork_init():
from posthog.web_memory_sampler import start_web_memory_sampler
start_continuous_profiling()
- initialize_otel()
+ initialize_otel(instrument_outbound_calls=True)
prewarm_query_cache_cluster_in_background()
# Production runs ASGI, so the sampler — previously only started from wsgi.py — never
# ran in prod, leaving posthog_web_worker_rss_mb empty. Start it here, post-fork.
diff --git a/posthog/otel_instrumentation.py b/posthog/otel_instrumentation.py
index eeac501ad6a2..ccf3614bde24 100644
--- a/posthog/otel_instrumentation.py
+++ b/posthog/otel_instrumentation.py
@@ -2,17 +2,21 @@ import os
import atexit
import typing
import logging
-from collections.abc import Sequence
+from collections.abc import Iterator, Sequence
+from urllib.parse import urlsplit, urlunsplit
from django.http import HttpRequest, HttpResponse
import structlog
from opentelemetry import trace
+from opentelemetry.context import Context
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor
from opentelemetry.instrumentation.aiokafka import AIOKafkaInstrumentor
+from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
from opentelemetry.instrumentation.celery import CeleryInstrumentor
from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor, RequestInfo
from opentelemetry.instrumentation.kafka import KafkaInstrumentor
from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
from opentelemetry.instrumentation.redis import RedisInstrumentor
@@ -20,12 +24,18 @@ from opentelemetry.instrumentation.requests import RequestsInstrumentor
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
-from opentelemetry.trace import Span
+from opentelemetry.semconv.attributes.error_attributes import ERROR_TYPE
+from opentelemetry.trace import Link, Span, SpanKind, StatusCode
+from opentelemetry.util._decorator import _agnosticcontextmanager
from opentelemetry.util.http import sanitize_method
+from opentelemetry.util.types import Attributes
+from requests import PreparedRequest
# Get a structlog logger for this module's own messages
logger = structlog.get_logger(__name__)
+_OUTBOUND_URL_ATTRIBUTE_KEYS = ("http.url", "url.full")
+
def _otel_django_request_hook(span: Span, request: HttpRequest) -> None:
if span and span.is_recording():
@@ -88,7 +98,110 @@ def _otel_redis_request_hook(span: Span, instance: object, args: Sequence[object
logger.debug("otel_redis_request_hook_failed", exc_info=True)
-def initialize_otel():
+def _url_origin(url: str) -> str:
+ try:
+ parts = urlsplit(url)
+ except ValueError:
+ return ""
+ return urlunsplit((parts.scheme, parts.netloc.rpartition("@")[2], "", "", ""))
+
+
+def _replace_span_url_with_origin(span: Span, url: str) -> None:
+ if not span.is_recording():
+ return
+ origin = _url_origin(url)
+ for key in _OUTBOUND_URL_ATTRIBUTE_KEYS:
+ span.set_attribute(key, origin)
+
+
+def _otel_requests_request_hook(span: Span, request: PreparedRequest) -> None:
+ _replace_span_url_with_origin(span, request.url or "")
+
+
+def _otel_httpx_request_hook(span: Span, request: RequestInfo) -> None:
+ _replace_span_url_with_origin(span, str(request.url))
+
+
+async def _otel_httpx_async_request_hook(span: Span, request: RequestInfo) -> None:
+ _otel_httpx_request_hook(span, request)
+
+
+class _ExceptionTypeOnlyTracer(trace.Tracer):
+ def __init__(self, tracer: trace.Tracer) -> None:
+ self._tracer = tracer
+
+ def start_span(
+ self,
+ name: str,
+ context: Context | None = None,
+ kind: SpanKind = SpanKind.INTERNAL,
+ attributes: Attributes = None,
+ links: Sequence[Link] | None = None,
+ start_time: int | None = None,
+ record_exception: bool = True,
+ set_status_on_exception: bool = True,
+ ) -> Span:
+ return self._tracer.start_span(
+ name,
+ context,
+ kind,
+ attributes,
+ links,
+ start_time,
+ record_exception=False,
+ set_status_on_exception=False,
+ )
+
+ @_agnosticcontextmanager
+ def start_as_current_span(
+ self,
+ name: str,
+ context: Context | None = None,
+ kind: SpanKind = SpanKind.INTERNAL,
+ attributes: Attributes = None,
+ links: Sequence[Link] | None = None,
+ start_time: int | None = None,
+ record_exception: bool = True,
+ set_status_on_exception: bool = True,
+ end_on_exit: bool = True,
+ ) -> Iterator[Span]:
+ with self._tracer.start_as_current_span(
+ name,
+ context,
+ kind,
+ attributes,
+ links,
+ start_time,
+ record_exception=False,
+ set_status_on_exception=False,
+ end_on_exit=end_on_exit,
+ ) as span:
+ try:
+ yield span
+ except Exception as error:
+ span.set_attribute(ERROR_TYPE, type(error).__name__)
+ span.set_status(StatusCode.ERROR)
+ raise
+
+
+class _ExceptionTypeOnlyTracerProvider(trace.TracerProvider):
+ def __init__(self, provider: trace.TracerProvider) -> None:
+ self._provider = provider
+
+ def get_tracer(
+ self,
+ instrumenting_module_name: str,
+ instrumenting_library_version: str | None = None,
+ schema_url: str | None = None,
+ attributes: Attributes = None,
+ ) -> trace.Tracer:
+ tracer = self._provider.get_tracer(
+ instrumenting_module_name, instrumenting_library_version, schema_url, attributes
+ )
+ return _ExceptionTypeOnlyTracer(tracer)
+
+
+def initialize_otel(*, instrument_outbound_calls: bool = False) -> None:
# App tracing targets a real OTLP collector (dev/prod). In tests, booting the app
# (e.g. posthog/test/test_asgi_lifespan.py imports posthog.asgi) would otherwise arm a
# process-wide span exporter to the SDK default localhost:4317 with no collector there,
@@ -163,6 +276,10 @@ def initialize_otel():
instrument_psycopg(provider)
instrument_kafka(provider)
instrument_aiokafka(provider)
+ if instrument_outbound_calls:
+ instrument_requests_client(provider)
+ instrument_httpx_client(provider)
+ instrument_botocore(provider)
logger.info(
"otel_manual_init_status_from_instrumentation_module",
@@ -246,9 +363,12 @@ def instrument_aiohttp_client(provider: trace.TracerProvider):
)
-def instrument_requests_client(provider: trace.TracerProvider):
+def instrument_requests_client(provider: trace.TracerProvider) -> None:
try:
- RequestsInstrumentor().instrument(tracer_provider=provider)
+ RequestsInstrumentor().instrument(
+ tracer_provider=_ExceptionTypeOnlyTracerProvider(provider),
+ request_hook=_otel_requests_request_hook,
+ )
logger.info("otel_instrumentation_attempt", instrumentor="RequestsInstrumentor", status="success")
except Exception as e:
logger.exception(
@@ -256,6 +376,30 @@ def instrument_requests_client(provider: trace.TracerProvider):
)
+def instrument_httpx_client(provider: trace.TracerProvider) -> None:
+ try:
+ HTTPXClientInstrumentor().instrument(
+ tracer_provider=_ExceptionTypeOnlyTracerProvider(provider),
+ request_hook=_otel_httpx_request_hook,
+ async_request_hook=_otel_httpx_async_request_hook,
+ )
+ logger.info("otel_instrumentation_attempt", instrumentor="HTTPXClientInstrumentor", status="success")
+ except Exception as e:
+ logger.exception(
+ "otel_instrumentation_attempt", instrumentor="HTTPXClientInstrumentor", status="error", exc_info=e
+ )
+
+
+def instrument_botocore(provider: trace.TracerProvider) -> None:
+ try:
+ BotocoreInstrumentor().instrument(tracer_provider=_ExceptionTypeOnlyTracerProvider(provider))
+ logger.info("otel_instrumentation_attempt", instrumentor="BotocoreInstrumentor", status="success")
+ except Exception as e:
+ logger.exception(
+ "otel_instrumentation_attempt", instrumentor="BotocoreInstrumentor", status="error", exc_info=e
+ )
+
+
INSTRUMENTORS: dict[str, typing.Callable[[trace.TracerProvider], None]] = {
"django": instrument_django,
"psycopg": instrument_psycopg,
diff --git a/posthog/test/test_otel_instrumentation.py b/posthog/test/test_otel_instrumentation.py
index f9bb1ac8d6c8..f93c39a967e8 100644
--- a/posthog/test/test_otel_instrumentation.py
+++ b/posthog/test/test_otel_instrumentation.py
@@ -1,6 +1,10 @@
# posthog/test/test_otel_instrumentation.py
import os
+import re
+import asyncio
import logging
+import contextlib
+from collections.abc import Callable
from types import SimpleNamespace
from typing import NoReturn
@@ -8,10 +12,22 @@ from unittest import mock
from django.test import SimpleTestCase
+import boto3
+import httpx
+import httpcore
+import requests
+import responses
+from botocore.config import Config
+from botocore.exceptions import EndpointConnectionError
+from botocore.httpsession import URLLib3Session
from opentelemetry import trace
-from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from opentelemetry.trace import StatusCode
from opentelemetry.util.types import AttributeValue
from parameterized import parameterized
@@ -20,8 +36,53 @@ from posthog.otel_instrumentation import (
_otel_django_response_hook,
_otel_redis_request_hook,
initialize_otel,
+ instrument_botocore,
+ instrument_httpx_client,
+ instrument_requests_client,
)
+URL_WITH_CREDENTIALS = "https://user:[email protected]/hooks/secret?token=secret"
+
+
+def _get_with_requests(url: str, fail: bool) -> None:
+ body = requests.ConnectionError(f"cannot reach {url}") if fail else ""
+ with responses.RequestsMock() as mocked_responses, contextlib.suppress(requests.ConnectionError):
+ mocked_responses.get(re.compile(".*"), body=body)
+ requests.get(url)
+
+
+def _get_with_httpx(url: str, fail: bool) -> None:
+ error = httpcore.ConnectError(f"cannot reach {url}") if fail else None
+ with (
+ mock.patch.object(
+ httpcore.ConnectionPool, "handle_request", return_value=httpcore.Response(200), side_effect=error
+ ),
+ contextlib.suppress(httpx.ConnectError),
+ ):
+ httpx.get(url)
+
+
+def _get_with_async_httpx(url: str, fail: bool) -> None:
+ async def get() -> None:
+ async with httpx.AsyncClient() as client:
+ await client.get(url)
+
+ error = httpcore.ConnectError(f"cannot reach {url}") if fail else None
+ with (
+ mock.patch.object(
+ httpcore.AsyncConnectionPool,
+ "handle_async_request",
+ return_value=httpcore.Response(200),
+ side_effect=error,
+ ),
+ contextlib.suppress(httpx.ConnectError),
+ ):
+ asyncio.run(get())
+
+
+def _everything_recorded(span: ReadableSpan) -> str:
+ return str((span.attributes, [(event.name, event.attributes) for event in span.events], span.status.description))
+
class TestOtelInstrumentation(SimpleTestCase):
def setUp(self):
@@ -425,3 +486,55 @@ class TestOtelRedisRequestHook(SimpleTestCase):
raise RuntimeError("topology unavailable")
self.assertEqual(self._span_attributes_after_hook(ClientWithoutTopology()), {})
+
+
+class TestOutboundCallSpans(SimpleTestCase):
+ def setUp(self) -> None:
+ super().setUp()
+ self.exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(self.exporter))
+ instrument_requests_client(provider)
+ self.addCleanup(RequestsInstrumentor().uninstrument)
+ instrument_httpx_client(provider)
+ self.addCleanup(HTTPXClientInstrumentor().uninstrument)
+ instrument_botocore(provider)
+ self.addCleanup(BotocoreInstrumentor().uninstrument)
+
+ @parameterized.expand(
+ [
+ ("requests", _get_with_requests, False),
+ ("requests_failed", _get_with_requests, True),
+ ("httpx", _get_with_httpx, False),
+ ("httpx_failed", _get_with_httpx, True),
+ ("httpx_async", _get_with_async_httpx, False),
+ ("httpx_async_failed", _get_with_async_httpx, True),
+ ]
+ )
+ def test_span_keeps_only_the_url_origin(self, _case: str, get: Callable[[str, bool], None], fail: bool) -> None:
+ get(URL_WITH_CREDENTIALS, fail)
+
+ (span,) = self.exporter.get_finished_spans()
+ self.assertEqual(dict(span.attributes or {})["http.url"], "https://api.example.com")
+ self.assertEqual(span.status.status_code, StatusCode.ERROR if fail else StatusCode.UNSET)
+ self.assertNotIn("secret", _everything_recorded(span))
+
+ def test_failed_s3_call_records_only_the_exception_type(self) -> None:
+ client = boto3.client(
+ "s3",
+ endpoint_url="https://storage.example.com",
+ region_name="us-east-1",
+ aws_access_key_id="test-key-id",
+ aws_secret_access_key="test-key",
+ config=Config(retries={"total_max_attempts": 1}),
+ )
+ connection_error = EndpointConnectionError(endpoint_url="https://storage.example.com/bucket/secret")
+ with (
+ mock.patch.object(URLLib3Session, "send", side_effect=connection_error),
+ contextlib.suppress(EndpointConnectionError),
+ ):
+ client.get_object(Bucket="bucket", Key="secret")
+
+ (span,) = self.exporter.get_finished_spans()
+ self.assertEqual(span.status.status_code, StatusCode.ERROR)
+ self.assertNotIn("secret", _everything_recorded(span))
diff --git a/posthog/wsgi.py b/posthog/wsgi.py
index 0ff9dfbb20ee..bee568659dda 100644
--- a/posthog/wsgi.py
+++ b/posthog/wsgi.py
@@ -26,7 +26,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "posthog.settings")
os.environ.setdefault("SERVER_GATEWAY_INTERFACE", "WSGI")
start_continuous_profiling()
-initialize_otel()
+initialize_otel(instrument_outbound_calls=True)
# Boot allocations are almost all permanent, so cyclic GC during django.setup() only adds
# pauses (~300ms). Disable it for the boot, then freeze the survivors so later full
diff --git a/pyproject.toml b/pyproject.toml
index 82fa7142edd0..d325e0dce449 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -180,8 +180,10 @@ dependencies = [
"opentelemetry-instrumentation-aiokafka==0.56b0",
"opentelemetry-instrumentation-aiohttp-client==0.56b0",
"opentelemetry-instrumentation-asgi==0.56b0",
+ "opentelemetry-instrumentation-botocore==0.56b0",
"opentelemetry-instrumentation-celery==0.56b0",
"opentelemetry-instrumentation-django==0.56b0",
+ "opentelemetry-instrumentation-httpx==0.56b0",
"opentelemetry-instrumentation-kafka-python==0.56b0",
"opentelemetry-instrumentation-psycopg==0.56b0",
"opentelemetry-instrumentation-redis==0.56b0",
diff --git a/uv.lock b/uv.lock
index ecec694465c1..520b1a310de3 100644
--- a/uv.lock
+++ b/uv.lock
@@ -5284,6 +5284,21 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c7/dd/655355453ee918455b5e7132f80366f594e0e7433dbcd03d2b0d6033088f/opentelemetry_instrumentation_asgi-0.56b0-py3-none-any.whl", hash = "sha256:7144793c1c601e47db6174d748ab437f7fa92f93a87d13882ad60a1f8147d2cf", size = 16602, upload-time = "2025-07-11T12:25:28.7Z" },
]
+[[package]]
+name = "opentelemetry-instrumentation-botocore"
+version = "0.56b0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "opentelemetry-api" },
+ { name = "opentelemetry-instrumentation" },
+ { name = "opentelemetry-propagator-aws-xray" },
+ { name = "opentelemetry-semantic-conventions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a9/59/c663a613fa3bd9becd94f916095bf82aa6e45a59cd5b7d70eb52dc1ae146/opentelemetry_instrumentation_botocore-0.56b0.tar.gz", hash = "sha256:c2dd342eec24acdac7cb6e59c268f523c1e1165b6c7f2ac1f339623a2e1d6118", size = 118800, upload-time = "2025-07-11T12:26:27.196Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d3/62/d2cb63c740513cd9f4f635792c276764488c110467f7b39eee247e5594de/opentelemetry_instrumentation_botocore-0.56b0-py3-none-any.whl", hash = "sha256:7ccacdcb5b858d931b0b526b2e3e857c06f89c67847252fd048002b295862bf8", size = 35808, upload-time = "2025-07-11T12:25:35.3Z" },
+]
+
[[package]]
name = "opentelemetry-instrumentation-celery"
version = "0.56b0"
@@ -5329,6 +5344,22 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/da/c7/727f9699c855a52002631f2f0b5be3716d0599419209ba13aeffe5c0419c/opentelemetry_instrumentation_django-0.56b0-py3-none-any.whl", hash = "sha256:c2e05fc1321166791e529aeb3a4b27cfb07fb69a743fe92c6faa3866290dc591", size = 19600, upload-time = "2025-07-11T12:25:40.327Z" },
]
+[[package]]
+name = "opentelemetry-instrumentation-httpx"
+version = "0.56b0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "opentelemetry-api" },
+ { name = "opentelemetry-instrumentation" },
+ { name = "opentelemetry-semantic-conventions" },
+ { name = "opentelemetry-util-http" },
+ { name = "wrapt" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8b/93/784623262776d90aa07e5cc08f58e1d476ed4b208127c1718db4b57b78c8/opentelemetry_instrumentation_httpx-0.56b0.tar.gz", hash = "sha256:d93c7a0e0ce5cc2170f7cc0d26f19167bc8314b7d2a5204afdf84000d5ca8647", size = 19510, upload-time = "2025-07-11T12:26:37.492Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b4/0e/fe2f38e0170a22a44b26e9a8f555ac23b2fcf09f89264d718800ec1d5954/opentelemetry_instrumentation_httpx-0.56b0-py3-none-any.whl", hash = "sha256:34d9558413d695ea69fe002efa07c00aa859a8aa4930815726e5813d5f002e1d", size = 15113, upload-time = "2025-07-11T12:25:45.671Z" },
+]
+
[[package]]
name = "opentelemetry-instrumentation-kafka-python"
version = "0.56b0"
@@ -5402,6 +5433,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/45/9e/97a89c0def0819d6c8b9556fdf9969a7503a01fd19b1a9f13770c530f19e/opentelemetry_instrumentation_wsgi-0.56b0-py3-none-any.whl", hash = "sha256:ecf29aeac2e945054af2f8e3f3a162bc1f83016b08a0f3e590ad10f2f3c518c8", size = 14449, upload-time = "2025-07-11T12:26:12.714Z" },
]
+[[package]]
+name = "opentelemetry-propagator-aws-xray"
+version = "1.0.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "opentelemetry-api" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/f4/31/40004e9e55b1e5694ef3a7526f0b7637df44196fc68a8b7d248a3684680f/opentelemetry_propagator_aws_xray-1.0.2.tar.gz", hash = "sha256:6b2cee5479d2ef0172307b66ed2ed151f598a0fd29b3c01133ac87ca06326260", size = 10994, upload-time = "2024-08-05T17:45:57.601Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ea/89/849a0847871fd9745315896ad9e23d6479db84d90b8b36c4c26dc46e92b8/opentelemetry_propagator_aws_xray-1.0.2-py3-none-any.whl", hash = "sha256:1c99181ee228e99bddb638a0c911a297fa21f1c3a0af951f841e79919b5f1934", size = 10856, upload-time = "2024-08-05T17:45:56.492Z" },
+]
+
[[package]]
name = "opentelemetry-proto"
version = "1.35.0"
@@ -6008,8 +6051,10 @@ dependencies = [
{ name = "opentelemetry-instrumentation-aiohttp-client" },
{ name = "opentelemetry-instrumentation-aiokafka" },
{ name = "opentelemetry-instrumentation-asgi" },
+ { name = "opentelemetry-instrumentation-botocore" },
{ name = "opentelemetry-instrumentation-celery" },
{ name = "opentelemetry-instrumentation-django" },
+ { name = "opentelemetry-instrumentation-httpx" },
{ name = "opentelemetry-instrumentation-kafka-python" },
{ name = "opentelemetry-instrumentation-psycopg" },
{ name = "opentelemetry-instrumentation-redis" },
@@ -6308,8 +6353,10 @@ requires-dist = [
{ name = "opentelemetry-instrumentation-aiohttp-client", specifier = "==0.56b0" },
{ name = "opentelemetry-instrumentation-aiokafka", specifier = "==0.56b0" },
{ name = "opentelemetry-instrumentation-asgi", specifier = "==0.56b0" },
+ { name = "opentelemetry-instrumentation-botocore", specifier = "==0.56b0" },
{ name = "opentelemetry-instrumentation-celery", specifier = "==0.56b0" },
{ name = "opentelemetry-instrumentation-django", specifier = "==0.56b0" },
+ { name = "opentelemetry-instrumentation-httpx", specifier = "==0.56b0" },
{ name = "opentelemetry-instrumentation-kafka-python", specifier = "==0.56b0" },
{ name = "opentelemetry-instrumentation-psycopg", specifier = "==0.56b0" },
{ name = "opentelemetry-instrumentation-redis", specifier = "==0.56b0" },