:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(logs): emit otel exception attributes from python services (#119482)
Hugues Pouillot991e71ada986
diff --git a/posthog/logging/exceptions.py b/posthog/logging/exceptions.pynew file mode 100644index 000000000000..518f66bb84df--- /dev/null+++ b/posthog/logging/exceptions.py@@ -0,0 +1,44 @@+import sys+import logging+import traceback+from types import TracebackType++import structlog+++def _resolve_exc_info(exc_info: object) -> tuple[BaseException, TracebackType | None] | None:+ if exc_info is True:+ exc_info = sys.exc_info()+ if isinstance(exc_info, BaseException):+ return exc_info, exc_info.__traceback__+ # The caller can capture the tuple before the exception's `__traceback__` changes, so keep its traceback.+ if isinstance(exc_info, tuple) and len(exc_info) == 3 and isinstance(exc_info[1], BaseException):+ return exc_info[1], exc_info[2]+ return None+++def _exception_message(exc: BaseException) -> str:+ try:+ return str(exc)+ except Exception:+ # Same fallback as `traceback.format_exception`. A raising processor drops the record, and in the+ # Temporal chain the error escapes into the caller's error handling.+ return "<exception str() failed>"+++def add_otel_exception_attributes(+ logger: logging.Logger, method_name: str, event_dict: structlog.types.EventDict+) -> structlog.types.EventDict:+ # https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/+ # The collector stores each top-level JSON key as one log attribute, so the keys keep their dots.+ resolved = _resolve_exc_info(event_dict.pop("exc_info", None))+ if resolved is None:+ return event_dict+ exc, tb = resolved++ # The bare class name matches what the OTel Python SDK `LoggingHandler` writes, although the spec asks+ # for a qualified name. Services that log through the SDK then group with these logs.+ event_dict["exception.type"] = type(exc).__name__+ event_dict["exception.message"] = _exception_message(exc)+ event_dict["exception.stacktrace"] = "".join(traceback.format_exception(type(exc), exc, tb))+ return event_dictdiff --git a/posthog/logging/test/test_exceptions.py b/posthog/logging/test/test_exceptions.pynew file mode 100644index 000000000000..414366eab625--- /dev/null+++ b/posthog/logging/test/test_exceptions.py@@ -0,0 +1,72 @@+import sys+import json+import logging+from typing import Any++import pytest++import structlog++from posthog.settings.logs import LOGGING+++class PaymentDeclined(Exception):+ pass+++def _json_formatter() -> structlog.stdlib.ProcessorFormatter:+ config = {key: value for key, value in LOGGING["formatters"]["json"].items() if key != "()"}+ return structlog.stdlib.ProcessorFormatter(**config)+++def _record(msg: Any, exc_info: Any) -> logging.LogRecord:+ return logging.LogRecord("test", logging.ERROR, __file__, 1, msg, None, exc_info)+++def _raise_chained() -> None:+ try:+ raise KeyError("card")+ except KeyError as e:+ raise PaymentDeclined("card declined") from e++[email protected]("from_structlog", [False, True])+def test_json_formatter_renders_otel_exception_attributes(from_structlog: bool) -> None:+ try:+ _raise_chained()+ except PaymentDeclined:+ exc_info = sys.exc_info()++ if from_structlog:+ record = _record({"event": "payment failed", "exc_info": exc_info}, None)+ record._logger = logging.getLogger("test")+ record._name = "error"+ else:+ record = _record("payment failed", exc_info)++ rendered = json.loads(_json_formatter().format(record))++ assert "exc_info" not in rendered+ assert "exception" not in rendered+ assert rendered["exception.type"] == "PaymentDeclined"+ assert rendered["exception.message"] == "card declined"+ stacktrace = rendered["exception.stacktrace"]+ assert "KeyError: 'card'" in stacktrace+ assert "The above exception was the direct cause of the following exception" in stacktrace+ assert stacktrace.endswith(f"{__name__}.PaymentDeclined: card declined\n")+++class UnprintableError(Exception):+ def __str__(self) -> str:+ raise RuntimeError("no str")+++def test_json_formatter_survives_exception_with_raising_str() -> None:+ try:+ raise UnprintableError()+ except UnprintableError:+ rendered = json.loads(_json_formatter().format(_record("payment failed", sys.exc_info())))++ assert rendered["exception.type"] == "UnprintableError"+ assert rendered["exception.message"] == "<exception str() failed>"+ assert "Traceback (most recent call last):" in rendered["exception.stacktrace"]diff --git a/posthog/settings/logs.py b/posthog/settings/logs.pyindex c33867d9d03b..8e0b2df41d9a 100644--- a/posthog/settings/logs.py+++ b/posthog/settings/logs.py@@ -7,6 +7,7 @@ from typing import Anyimport structlogfrom opentelemetry import trace+from posthog.logging.exceptions import add_otel_exception_attributesfrom posthog.settings.base_variables import DEBUG, IS_INTERACTIVE_SHELL, TEST# Setup logging@@ -74,7 +75,6 @@ foreign_pre_chain: list[structlog.types.Processor] = [add_otel_trace_context,structlog.stdlib.PositionalArgumentsFormatter(),structlog.processors.StackInfoRenderer(),- structlog.processors.format_exc_info,structlog.processors.UnicodeDecoder(),]@@ -98,12 +98,18 @@ LOGGING: dict[str, Any] = {"formatters": {"default": {"()": structlog.stdlib.ProcessorFormatter,- "processor": structlog.dev.ConsoleRenderer(colors=DEBUG),+ # The default rich formatter prints local variables, which can contain secrets.+ "processor": structlog.dev.ConsoleRenderer(colors=DEBUG, exception_formatter=structlog.dev.plain_traceback),"foreign_pre_chain": foreign_pre_chain,},+ # Each formatter renders `exc_info` itself, so the shared chain must leave it in place."json": {"()": structlog.stdlib.ProcessorFormatter,- "processor": structlog.processors.JSONRenderer(),+ "processors": [+ structlog.stdlib.ProcessorFormatter.remove_processors_meta,+ add_otel_exception_attributes,+ structlog.processors.JSONRenderer(),+ ],"foreign_pre_chain": foreign_pre_chain,},},diff --git a/posthog/temporal/common/logger.py b/posthog/temporal/common/logger.pyindex f73f921dae5f..aa27cdf304f3 100644--- a/posthog/temporal/common/logger.py+++ b/posthog/temporal/common/logger.py@@ -43,6 +43,7 @@ from structlog._log_levels import LEVEL_TO_NAME, NAME_TO_LEVELfrom structlog.processors import EventRenamerfrom posthog.kafka_client.topics import KAFKA_LOG_ENTRIES+from posthog.logging.exceptions import add_otel_exception_attributesif typing.TYPE_CHECKING:from posthog.kafka_client.client import _AsyncKafkaProducer@@ -580,9 +581,7 @@ def configure_logger()base_processors += [- # show_locals=False prevents sensitive data from being logged in exception tracebacks.- # The default (True) dumps all local variables at each stack frame.- structlog.processors.ExceptionRenderer(structlog.tracebacks.ExceptionDictTransformer(show_locals=False)),+ add_otel_exception_attributes,EventRenamer("msg"),LogMessagesRenderer(event_key="msg"),]diff --git a/posthog/temporal/tests/common/test_logger.py b/posthog/temporal/tests/common/test_logger.pyindex 3a689e13cc33..da4d6d044eab 100644--- a/posthog/temporal/tests/common/test_logger.py+++ b/posthog/temporal/tests/common/test_logger.py@@ -266,17 +266,11 @@ async def test_logger_renders_tracebacks(log_capture):for entry in log_capture.write_entries:entry_dict = json.loads(entry)- assert "exception" in entry_dict- assert isinstance(entry_dict["exception"], list)-- exceptions = entry_dict["exception"]-- assert len(exceptions) == 1-- exception = exceptions[0]-- assert exception["exc_type"] == "ValueError"- assert exception["exc_value"] == "Oh no!"+ assert "exc_info" not in entry_dict+ assert entry_dict["exception.type"] == "ValueError"+ assert entry_dict["exception.message"] == "Oh no!"+ assert entry_dict["exception.stacktrace"].startswith("Traceback (most recent call last):\n")+ assert 'raise ValueError("Oh no!")' in entry_dict["exception.stacktrace"]async def test_logger_formats_positional_args(log_capture):