Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(logs): emit otel exception attributes from python services (#119482)

Hugues Pouillot991e71ada986
diff --git a/posthog/logging/exceptions.py b/posthog/logging/exceptions.py
new file mode 100644
index 000000000000..518f66bb84df
--- /dev/null
+++ b/posthog/logging/exceptions.py
@@ -0,0 +1,44 @@
+import sys
+import logging
+import traceback
+from types import TracebackType
+
+import structlog
+
+
+def _resolve_exc_info(exc_info: object) -> tuple[BaseException, TracebackType | None] | None:
+ if exc_info is True:
+ exc_info = sys.exc_info()
+ if isinstance(exc_info, BaseException):
+ return exc_info, exc_info.__traceback__
+ # The caller can capture the tuple before the exception's `__traceback__` changes, so keep its traceback.
+ if isinstance(exc_info, tuple) and len(exc_info) == 3 and isinstance(exc_info[1], BaseException):
+ return exc_info[1], exc_info[2]
+ return None
+
+
+def _exception_message(exc: BaseException) -> str:
+ try:
+ return str(exc)
+ except Exception:
+ # Same fallback as `traceback.format_exception`. A raising processor drops the record, and in the
+ # Temporal chain the error escapes into the caller's error handling.
+ return "<exception str() failed>"
+
+
+def add_otel_exception_attributes(
+ logger: logging.Logger, method_name: str, event_dict: structlog.types.EventDict
+) -> structlog.types.EventDict:
+ # https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/
+ # The collector stores each top-level JSON key as one log attribute, so the keys keep their dots.
+ resolved = _resolve_exc_info(event_dict.pop("exc_info", None))
+ if resolved is None:
+ return event_dict
+ exc, tb = resolved
+
+ # The bare class name matches what the OTel Python SDK `LoggingHandler` writes, although the spec asks
+ # for a qualified name. Services that log through the SDK then group with these logs.
+ event_dict["exception.type"] = type(exc).__name__
+ event_dict["exception.message"] = _exception_message(exc)
+ event_dict["exception.stacktrace"] = "".join(traceback.format_exception(type(exc), exc, tb))
+ return event_dict
diff --git a/posthog/logging/test/test_exceptions.py b/posthog/logging/test/test_exceptions.py
new file mode 100644
index 000000000000..414366eab625
--- /dev/null
+++ b/posthog/logging/test/test_exceptions.py
@@ -0,0 +1,72 @@
+import sys
+import json
+import logging
+from typing import Any
+
+import pytest
+
+import structlog
+
+from posthog.settings.logs import LOGGING
+
+
+class PaymentDeclined(Exception):
+ pass
+
+
+def _json_formatter() -> structlog.stdlib.ProcessorFormatter:
+ config = {key: value for key, value in LOGGING["formatters"]["json"].items() if key != "()"}
+ return structlog.stdlib.ProcessorFormatter(**config)
+
+
+def _record(msg: Any, exc_info: Any) -> logging.LogRecord:
+ return logging.LogRecord("test", logging.ERROR, __file__, 1, msg, None, exc_info)
+
+
+def _raise_chained() -> None:
+ try:
+ raise KeyError("card")
+ except KeyError as e:
+ raise PaymentDeclined("card declined") from e
+
+
[email protected]("from_structlog", [False, True])
+def test_json_formatter_renders_otel_exception_attributes(from_structlog: bool) -> None:
+ try:
+ _raise_chained()
+ except PaymentDeclined:
+ exc_info = sys.exc_info()
+
+ if from_structlog:
+ record = _record({"event": "payment failed", "exc_info": exc_info}, None)
+ record._logger = logging.getLogger("test")
+ record._name = "error"
+ else:
+ record = _record("payment failed", exc_info)
+
+ rendered = json.loads(_json_formatter().format(record))
+
+ assert "exc_info" not in rendered
+ assert "exception" not in rendered
+ assert rendered["exception.type"] == "PaymentDeclined"
+ assert rendered["exception.message"] == "card declined"
+ stacktrace = rendered["exception.stacktrace"]
+ assert "KeyError: 'card'" in stacktrace
+ assert "The above exception was the direct cause of the following exception" in stacktrace
+ assert stacktrace.endswith(f"{__name__}.PaymentDeclined: card declined\n")
+
+
+class UnprintableError(Exception):
+ def __str__(self) -> str:
+ raise RuntimeError("no str")
+
+
+def test_json_formatter_survives_exception_with_raising_str() -> None:
+ try:
+ raise UnprintableError()
+ except UnprintableError:
+ rendered = json.loads(_json_formatter().format(_record("payment failed", sys.exc_info())))
+
+ assert rendered["exception.type"] == "UnprintableError"
+ assert rendered["exception.message"] == "<exception str() failed>"
+ assert "Traceback (most recent call last):" in rendered["exception.stacktrace"]
diff --git a/posthog/settings/logs.py b/posthog/settings/logs.py
index c33867d9d03b..8e0b2df41d9a 100644
--- a/posthog/settings/logs.py
+++ b/posthog/settings/logs.py
@@ -7,6 +7,7 @@ from typing import Any
import structlog
from opentelemetry import trace
+from posthog.logging.exceptions import add_otel_exception_attributes
from posthog.settings.base_variables import DEBUG, IS_INTERACTIVE_SHELL, TEST
# Setup logging
@@ -74,7 +75,6 @@ foreign_pre_chain: list[structlog.types.Processor] = [
add_otel_trace_context,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.StackInfoRenderer(),
- structlog.processors.format_exc_info,
structlog.processors.UnicodeDecoder(),
]
@@ -98,12 +98,18 @@ LOGGING: dict[str, Any] = {
"formatters": {
"default": {
"()": structlog.stdlib.ProcessorFormatter,
- "processor": structlog.dev.ConsoleRenderer(colors=DEBUG),
+ # The default rich formatter prints local variables, which can contain secrets.
+ "processor": structlog.dev.ConsoleRenderer(colors=DEBUG, exception_formatter=structlog.dev.plain_traceback),
"foreign_pre_chain": foreign_pre_chain,
},
+ # Each formatter renders `exc_info` itself, so the shared chain must leave it in place.
"json": {
"()": structlog.stdlib.ProcessorFormatter,
- "processor": structlog.processors.JSONRenderer(),
+ "processors": [
+ structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+ add_otel_exception_attributes,
+ structlog.processors.JSONRenderer(),
+ ],
"foreign_pre_chain": foreign_pre_chain,
},
},
diff --git a/posthog/temporal/common/logger.py b/posthog/temporal/common/logger.py
index f73f921dae5f..aa27cdf304f3 100644
--- a/posthog/temporal/common/logger.py
+++ b/posthog/temporal/common/logger.py
@@ -43,6 +43,7 @@ from structlog._log_levels import LEVEL_TO_NAME, NAME_TO_LEVEL
from structlog.processors import EventRenamer
from posthog.kafka_client.topics import KAFKA_LOG_ENTRIES
+from posthog.logging.exceptions import add_otel_exception_attributes
if typing.TYPE_CHECKING:
from posthog.kafka_client.client import _AsyncKafkaProducer
@@ -580,9 +581,7 @@ def configure_logger(
)
base_processors += [
- # show_locals=False prevents sensitive data from being logged in exception tracebacks.
- # The default (True) dumps all local variables at each stack frame.
- structlog.processors.ExceptionRenderer(structlog.tracebacks.ExceptionDictTransformer(show_locals=False)),
+ add_otel_exception_attributes,
EventRenamer("msg"),
LogMessagesRenderer(event_key="msg"),
]
diff --git a/posthog/temporal/tests/common/test_logger.py b/posthog/temporal/tests/common/test_logger.py
index 3a689e13cc33..da4d6d044eab 100644
--- a/posthog/temporal/tests/common/test_logger.py
+++ b/posthog/temporal/tests/common/test_logger.py
@@ -266,17 +266,11 @@ async def test_logger_renders_tracebacks(log_capture):
for entry in log_capture.write_entries:
entry_dict = json.loads(entry)
- assert "exception" in entry_dict
- assert isinstance(entry_dict["exception"], list)
-
- exceptions = entry_dict["exception"]
-
- assert len(exceptions) == 1
-
- exception = exceptions[0]
-
- assert exception["exc_type"] == "ValueError"
- assert exception["exc_value"] == "Oh no!"
+ assert "exc_info" not in entry_dict
+ assert entry_dict["exception.type"] == "ValueError"
+ assert entry_dict["exception.message"] == "Oh no!"
+ assert entry_dict["exception.stacktrace"].startswith("Traceback (most recent call last):\n")
+ assert 'raise ValueError("Oh no!")' in entry_dict["exception.stacktrace"]
async def test_logger_formats_positional_args(log_capture):