Skip to content

Automate browser based workflows with AI

Prevent oversized logs from splitting into error fragments (#8213)

Aaron Perez2dd95060ce5b
diff --git a/skyvern/forge/sdk/forge_log.py b/skyvern/forge/sdk/forge_log.py
index e24fa1023..f1414e7f9 100644
--- a/skyvern/forge/sdk/forge_log.py
+++ b/skyvern/forge/sdk/forge_log.py
@@ -36,6 +36,53 @@ _ORPHANED_TASK_MESSAGE = "Task exception was never retrieved"
_TARGET_CLOSED_ERROR_TYPE = "TargetClosedError"
_CHANNEL_COLLECTED_ERROR = "The object has been collected to prevent unbounded heap growth"
+# Production collection splits records at the observed 75 KiB boundary. Keep JSON
+# below that boundary so one application event remains one Datadog event.
+MAX_JSON_LOG_BYTES = 64 * 1024
+_OVERSIZED_LOG_VALUE_CHARS = 4 * 1024
+_OVERSIZED_LOG_METADATA_CHARS = 512
+_OVERSIZED_LOG_FIELDS = (
+ "msg",
+ "timestamp",
+ "level",
+ "logger",
+ "entrypoint",
+ "env",
+ "version",
+ "event_status",
+ "event_severity",
+ "event_message",
+ "event_host",
+ "event_hostname",
+ "event_service",
+ "event_source",
+ "pathname",
+ "filename",
+ "module",
+ "func_name",
+ "lineno",
+ "request_id",
+ "organization_id",
+ "organization_name",
+ "step_id",
+ "task_id",
+ "run_id",
+ "workflow_id",
+ "workflow_run_id",
+ "workflow_permanent_id",
+ "task_v2_id",
+ "browser_session_id",
+ "copilot_session_id",
+ "browser_container_ip",
+ "browser_container_task_arn",
+ "error",
+ "error_type",
+ "error_category",
+ "exception_hash",
+ "exception",
+)
+_JSON_RENDERER = structlog.processors.JSONRenderer()
+
class _DriverPipeNoiseFilter(logging.Filter):
"""Drop asyncio's orphaned-task/future noise from a torn-down Playwright driver/target.
@@ -116,7 +163,7 @@ def _add_entrypoint(logger: logging.Logger, method_name: str, event_dict: EventD
# Datadog intake preprocessing derives the log's severity/message/host/service/source from
# these reserved attribute names, so a domain kwarg like status="completed" becomes the
# log's severity (`c*` → critical). Renamed at the render seam only: `context.log` (the S3
-# run artifact), the folded `msg` text, and the console renderer keep the authored names.
+# run artifact) and the console renderer keep the authored names.
# `severity` is here because it sits between `status` and `level` in Datadog's status-attribute
# list, so stripping only `status` would promote it to the severity source. `msg` is
# deliberately absent: this runs after EventRenamer has moved the real message there, so
@@ -140,10 +187,8 @@ def escape_reserved_log_keys(logger: logging.Logger, method_name: str, event_dic
return event_dict
-def add_kv_pairs_to_msg(logger: logging.Logger, method_name: str, event_dict: EventDict) -> EventDict:
- """
- A custom processor to add key-value pairs to the 'msg' field.
- """
+def add_log_context(logger: logging.Logger, method_name: str, event_dict: EventDict) -> EventDict:
+ """Add request and process context without copying structured values into ``msg``."""
# Add context to the log
context = skyvern_context.current()
if context:
@@ -180,22 +225,80 @@ def add_kv_pairs_to_msg(logger: logging.Logger, method_name: str, event_dict: Ev
event_dict["env"] = settings.ENV
event_dict["version"] = __version__
- if method_name not in ["info", "warning", "error", "critical", "exception"]:
- # Only modify the log for these log levels
- return event_dict
-
- # Assuming 'event' or 'msg' is the field to update
- msg_field = event_dict.get("msg", "")
-
- # Add key-value pairs
- kv_pairs = {k: v for k, v in event_dict.items() if k not in ["msg", "timestamp", "level", "sampling"]}
- if kv_pairs:
- additional_info = ", ".join(f"{k}={v}" for k, v in kv_pairs.items())
- msg_field += f" | {additional_info}"
+ return event_dict
- event_dict["msg"] = msg_field
- return event_dict
+def _truncate_log_value(value: Any, max_chars: int) -> Any:
+ if value is None or isinstance(value, (bool, float)):
+ return value
+ if isinstance(value, int):
+ value = int(value)
+ text = value if isinstance(value, str) else str(value)
+ if isinstance(value, int) and len(text) <= max_chars:
+ return value
+ if len(text) <= max_chars:
+ return text
+ return f"{text[:max_chars]}... [truncated]"
+
+
+def render_bounded_json(logger: logging.Logger, method_name: str, event_dict: EventDict) -> str:
+ """Render one valid JSON record below the collector's observed split boundary."""
+ rendered = _JSON_RENDERER(logger, method_name, event_dict)
+ # JSONRenderer's default serializer escapes non-ASCII characters, so character
+ # length is also the emitted UTF-8 byte length without allocating another copy.
+ original_size_bytes = len(rendered)
+ if original_size_bytes <= MAX_JSON_LOG_BYTES:
+ return rendered
+
+ bounded = {
+ key: _truncate_log_value(
+ event_dict[key],
+ _OVERSIZED_LOG_VALUE_CHARS if key in {"msg", "exception"} else _OVERSIZED_LOG_METADATA_CHARS,
+ )
+ for key in _OVERSIZED_LOG_FIELDS
+ if key in event_dict
+ }
+ omitted_fields = sorted(str(key)[:128] for key in event_dict if key not in bounded)
+ bounded.update(
+ {
+ "log_truncated": True,
+ "original_size_bytes": original_size_bytes,
+ "omitted_field_count": len(omitted_fields),
+ "omitted_fields": omitted_fields[:50],
+ }
+ )
+ rendered = _JSON_RENDERER(logger, method_name, bounded)
+ if len(rendered) <= MAX_JSON_LOG_BYTES:
+ return rendered
+
+ # Unusual escaped/control-heavy metadata can expand during JSON encoding. Keep a
+ # minimal correlated record rather than emitting another line the collector splits.
+ minimal = {
+ key: _truncate_log_value(bounded[key], 256 if key == "msg" else 128)
+ for key in _OVERSIZED_LOG_FIELDS
+ if key in bounded and key != "exception"
+ }
+ minimal.update(
+ {
+ "log_truncated": True,
+ "original_size_bytes": original_size_bytes,
+ "omitted_field_count": sum(1 for key in event_dict if key not in minimal),
+ }
+ )
+ rendered = _JSON_RENDERER(logger, method_name, minimal)
+ if len(rendered) <= MAX_JSON_LOG_BYTES:
+ return rendered
+ return _JSON_RENDERER(
+ logger,
+ method_name,
+ {
+ "msg": "Oversized log record",
+ "level": _truncate_log_value(event_dict.get("level"), 32),
+ "log_truncated": True,
+ "original_size_bytes": original_size_bytes,
+ "omitted_field_count": len(event_dict),
+ },
+ )
def redact_registered_secrets(logger: logging.Logger, method_name: str, event_dict: EventDict) -> EventDict:
@@ -210,8 +313,7 @@ def redact_registered_secrets(logger: logging.Logger, method_name: str, event_di
return event_dict
def scrub(node: Any) -> Any:
- # Nested values are folded into `msg` by add_kv_pairs_to_msg further down the chain, so a
- # secret carried only inside a nested kwarg reaches the log line unless we recurse here.
+ # Structured nested values are rendered in full, so recurse before JSON serialization.
if isinstance(node, str):
for secret in secrets:
if secret in node:
@@ -249,7 +351,7 @@ def redact_sensitive_event_fields(logger: logging.Logger, method_name: str, even
Reuses the shared field redactor so structured kwargs such as
``headers={"Authorization": ...}``, ``payload={...}``, or ``response_body={...}``
- are masked before ``add_kv_pairs_to_msg`` folds them into the rendered line.
+ are masked before JSON serialization.
Top-level keys whose name is sensitive are masked outright; every other
non-string value is redacted recursively — models, tuples and sets included,
since the formatter renders those in full too. Plain string values are left to
@@ -562,7 +664,7 @@ def setup_logger() -> None:
_entrypoint = _get_entrypoint()
# logging.config.dictConfig(logging_config)
- renderer = structlog.processors.JSONRenderer() if settings.JSON_LOGGING else CustomConsoleRenderer()
+ renderer = render_bounded_json if settings.JSON_LOGGING else CustomConsoleRenderer()
additional_processors = (
[
redact_bearer_tokens,
@@ -573,7 +675,7 @@ def setup_logger() -> None:
compact_action_objects,
redact_sensitive_event_fields,
structlog.processors.EventRenamer("msg"),
- add_kv_pairs_to_msg,
+ add_log_context,
structlog.processors.CallsiteParameterAdder(
{
structlog.processors.CallsiteParameter.PATHNAME,
@@ -616,9 +718,9 @@ def setup_logger() -> None:
)
# Foreign stdlib records never run the structlog chain above, so without these two a
# record reaches Datadog with an empty message (its remapper reads `msg`, not `event`)
- # and no `organization_id` to group on. Order matters: `add_kv_pairs_to_msg` reads `msg`.
+ # and no `organization_id` to group on.
foreign_msg_chain: list[Processor] = (
- [structlog.processors.EventRenamer("msg"), add_kv_pairs_to_msg] if settings.JSON_LOGGING else []
+ [structlog.processors.EventRenamer("msg"), add_log_context] if settings.JSON_LOGGING else []
)
handler = logging.StreamHandler()
diff --git a/tests/unit/test_forge_log_foreign_tracebacks.py b/tests/unit/test_forge_log_foreign_tracebacks.py
index 14931665e..76e7ac1d1 100644
--- a/tests/unit/test_forge_log_foreign_tracebacks.py
+++ b/tests/unit/test_forge_log_foreign_tracebacks.py
@@ -8,12 +8,14 @@ from collections.abc import Iterator
import pytest
import structlog
+import skyvern.forge.sdk.forge_log as forge_log
from skyvern.config import settings
from skyvern.forge.sdk.copilot import secret_scrub
from skyvern.forge.sdk.copilot.secret_scrub import REDACTED_SECRET_PLACEHOLDER
from skyvern.forge.sdk.forge_log import setup_logger
_REGISTERED_CREDENTIAL = "fake-registered-pa55w0rd-9f3c1a"
+_MAX_EMITTED_JSON_BYTES = 64 * 1024
@pytest.fixture
@@ -60,6 +62,67 @@ def test_foreign_stdlib_exception_renders_single_structured_line(json_stream: io
assert record["exception_hash"]
+def test_oversized_structured_payload_is_bounded_to_one_log_record(json_stream: io.StringIO) -> None:
+ structlog.get_logger("oversized-test").error(
+ "oversized_payload", payload="x" * 100_000, status="failed", error="payload rejected"
+ )
+
+ lines = json_stream.getvalue().strip().splitlines()
+ assert len(lines) == 1
+ assert len(lines[0].encode()) <= _MAX_EMITTED_JSON_BYTES
+
+ record = json.loads(lines[0])
+ assert record["msg"] == "oversized_payload"
+ assert record["event_status"] == "failed"
+ assert record["error"] == "payload rejected"
+ assert record["log_truncated"] is True
+ assert record["original_size_bytes"] > _MAX_EMITTED_JSON_BYTES
+ assert "payload" in record["omitted_fields"]
+
+
+def test_control_heavy_oversized_log_keeps_correlation_fields() -> None:
+ expanded = "\0" * 10_000
+ rendered = forge_log.render_bounded_json(
+ None, # type: ignore[arg-type]
+ "error",
+ {
+ "msg": expanded,
+ "exception": expanded,
+ "logger": expanded,
+ "entrypoint": expanded,
+ "env": expanded,
+ "version": expanded,
+ "pathname": expanded,
+ "filename": expanded,
+ "request_id": "req_test",
+ "task_id": "tsk_test",
+ },
+ )
+
+ assert len(rendered.encode()) <= _MAX_EMITTED_JSON_BYTES
+ record = json.loads(rendered)
+ assert record["request_id"] == "req_test"
+ assert record["task_id"] == "tsk_test"
+ assert record["log_truncated"] is True
+ assert record["omitted_field_count"] == 1
+
+
+def test_oversized_numeric_fields_are_bounded() -> None:
+ class ShortStringInteger(int):
+ def __str__(self) -> str:
+ return "1"
+
+ huge_number = ShortStringInteger(10**3999)
+ rendered = forge_log.render_bounded_json(
+ None, # type: ignore[arg-type]
+ "error",
+ {key: huge_number for key in forge_log._OVERSIZED_LOG_FIELDS},
+ )
+
+ assert len(rendered.encode()) <= _MAX_EMITTED_JSON_BYTES
+ assert json.loads(rendered)["log_truncated"] is True
+
+
def test_exception_log_fields_matches_processor_output_for_raised_exception() -> None:
"""exception_log_fields lets a warning carry the same dashboard fields add_error_processor
derives from exc_info, so downgraded lines stay groupable without rendering a traceback."""
diff --git a/tests/unit/test_forge_log_redaction.py b/tests/unit/test_forge_log_redaction.py
index ec6ec3dd9..a739ead89 100644
--- a/tests/unit/test_forge_log_redaction.py
+++ b/tests/unit/test_forge_log_redaction.py
@@ -120,7 +120,7 @@ def test_credential_redaction_tolerates_non_string_values() -> None:
def test_redacts_a_credential_nested_inside_a_kwarg() -> None:
- """add_kv_pairs_to_msg folds nested kwargs into the rendered line, so recursion is required."""
+ """Nested kwargs are serialized, so registered secrets must be redacted recursively."""
_register_credential(_FAKE_CREDENTIAL)
event = {
"event": "tool call",
diff --git a/tests/unit/test_forge_log_reserved_keys.py b/tests/unit/test_forge_log_reserved_keys.py
index 1e15680f9..fd420bd7c 100644
--- a/tests/unit/test_forge_log_reserved_keys.py
+++ b/tests/unit/test_forge_log_reserved_keys.py
@@ -111,15 +111,13 @@ def test_msg_and_error_kwargs_are_deliberately_not_renamed(
assert "event_error" not in payload
-def test_native_json_log_keeps_authored_names_in_rendered_msg(
+def test_native_json_log_does_not_duplicate_structured_fields_in_msg(
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
- # The human-readable line keeps the authored kwarg name; only the queryable
- # JSON attribute is escaped.
monkeypatch.setattr(settings, "JSON_LOGGING", True)
payload = _native_json_payload(capsys, status="completed")
- assert "status=completed" in payload["msg"]
+ assert payload["msg"] == "Wrote the final state"
def test_foreign_record_extra_reserved_keys_renamed(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -139,14 +137,13 @@ def test_explicit_escaped_kwarg_wins_over_renamed_reserved_key(
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
# If a call site ever passes both, the explicit `event_status` is the intentional one;
- # the reserved key is still stripped so it cannot reach intake. Both values remain
- # visible in the folded `msg` text.
+ # the reserved key is still stripped so it cannot reach intake.
monkeypatch.setattr(settings, "JSON_LOGGING", True)
payload = _native_json_payload(capsys, status="completed", event_status="explicit")
assert payload["event_status"] == "explicit"
assert "status" not in payload
- assert "status=completed" in payload["msg"]
+ assert payload["msg"] == "Wrote the final state"
def test_console_mode_keeps_authored_keys(monkeypatch: pytest.MonkeyPatch) -> None:
diff --git a/tests/unit/test_forge_log_sampling.py b/tests/unit/test_forge_log_sampling.py
index e6c00a9cd..d332bc7f1 100644
--- a/tests/unit/test_forge_log_sampling.py
+++ b/tests/unit/test_forge_log_sampling.py
@@ -5,7 +5,7 @@ import skyvern.forge.sdk.forge_log as forge_log
from skyvern.config import settings
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
-from skyvern.forge.sdk.forge_log import add_kv_pairs_to_msg, sample_logs_processor
+from skyvern.forge.sdk.forge_log import add_log_context, sample_logs_processor
SAMPLED_ORG = "o_sampled"
@@ -72,16 +72,17 @@ def test_empty_org_list_disables_sampling(monkeypatch: pytest.MonkeyPatch) -> No
assert "sampling" not in result
-def test_sampling_marker_excluded_from_rendered_message() -> None:
+def test_structured_fields_are_not_duplicated_into_message() -> None:
event = {"msg": "Handling action", "sampling": True, "action_type": "click"}
- result = add_kv_pairs_to_msg(None, "info", event) # type: ignore[arg-type]
- assert "sampling" not in result["msg"]
- assert "action_type=click" in result["msg"]
+ result = add_log_context(None, "info", event) # type: ignore[arg-type]
+ assert result["msg"] == "Handling action"
+ assert result["sampling"] is True
+ assert result["action_type"] == "click"
-def test_add_kv_pairs_includes_copilot_session_id(monkeypatch: pytest.MonkeyPatch) -> None:
+def test_add_log_context_includes_copilot_session_id(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(skyvern_context, "current", lambda: SkyvernContext(copilot_session_id="chat_1"))
- result = add_kv_pairs_to_msg(None, "info", {"msg": "Handling action"}) # type: ignore[arg-type]
+ result = add_log_context(None, "info", {"msg": "Handling action"}) # type: ignore[arg-type]
assert result["copilot_session_id"] == "chat_1"