Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Prevent oversized logs from splitting into error fragments (#8213)
Aaron Perez2dd95060ce5b
diff --git a/skyvern/forge/sdk/forge_log.py b/skyvern/forge/sdk/forge_log.pyindex e24fa1023..f1414e7f9 100644--- a/skyvern/forge/sdk/forge_log.py+++ b/skyvern/forge/sdk/forge_log.py@@ -36,6 +36,53 @@ _ORPHANED_TASK_MESSAGE = "Task exception was never retrieved"_TARGET_CLOSED_ERROR_TYPE = "TargetClosedError"_CHANNEL_COLLECTED_ERROR = "The object has been collected to prevent unbounded heap growth"+# Production collection splits records at the observed 75 KiB boundary. Keep JSON+# below that boundary so one application event remains one Datadog event.+MAX_JSON_LOG_BYTES = 64 * 1024+_OVERSIZED_LOG_VALUE_CHARS = 4 * 1024+_OVERSIZED_LOG_METADATA_CHARS = 512+_OVERSIZED_LOG_FIELDS = (+ "msg",+ "timestamp",+ "level",+ "logger",+ "entrypoint",+ "env",+ "version",+ "event_status",+ "event_severity",+ "event_message",+ "event_host",+ "event_hostname",+ "event_service",+ "event_source",+ "pathname",+ "filename",+ "module",+ "func_name",+ "lineno",+ "request_id",+ "organization_id",+ "organization_name",+ "step_id",+ "task_id",+ "run_id",+ "workflow_id",+ "workflow_run_id",+ "workflow_permanent_id",+ "task_v2_id",+ "browser_session_id",+ "copilot_session_id",+ "browser_container_ip",+ "browser_container_task_arn",+ "error",+ "error_type",+ "error_category",+ "exception_hash",+ "exception",+)+_JSON_RENDERER = structlog.processors.JSONRenderer()+class _DriverPipeNoiseFilter(logging.Filter):"""Drop asyncio's orphaned-task/future noise from a torn-down Playwright driver/target.@@ -116,7 +163,7 @@ def _add_entrypoint(logger: logging.Logger, method_name: str, event_dict: EventD# Datadog intake preprocessing derives the log's severity/message/host/service/source from# these reserved attribute names, so a domain kwarg like status="completed" becomes the# log's severity (`c*` → critical). Renamed at the render seam only: `context.log` (the S3-# run artifact), the folded `msg` text, and the console renderer keep the authored names.+# run artifact) and the console renderer keep the authored names.# `severity` is here because it sits between `status` and `level` in Datadog's status-attribute# list, so stripping only `status` would promote it to the severity source. `msg` is# deliberately absent: this runs after EventRenamer has moved the real message there, so@@ -140,10 +187,8 @@ def escape_reserved_log_keys(logger: logging.Logger, method_name: str, event_dicreturn event_dict-def add_kv_pairs_to_msg(logger: logging.Logger, method_name: str, event_dict: EventDict) -> EventDict:- """- A custom processor to add key-value pairs to the 'msg' field.- """+def add_log_context(logger: logging.Logger, method_name: str, event_dict: EventDict) -> EventDict:+ """Add request and process context without copying structured values into ``msg``."""# Add context to the logcontext = skyvern_context.current()if context:@@ -180,22 +225,80 @@ def add_kv_pairs_to_msg(logger: logging.Logger, method_name: str, event_dict: Evevent_dict["env"] = settings.ENVevent_dict["version"] = __version__- if method_name not in ["info", "warning", "error", "critical", "exception"]:- # Only modify the log for these log levels- return event_dict-- # Assuming 'event' or 'msg' is the field to update- msg_field = event_dict.get("msg", "")-- # Add key-value pairs- kv_pairs = {k: v for k, v in event_dict.items() if k not in ["msg", "timestamp", "level", "sampling"]}- if kv_pairs:- additional_info = ", ".join(f"{k}={v}" for k, v in kv_pairs.items())- msg_field += f" | {additional_info}"+ return event_dict- event_dict["msg"] = msg_field- return event_dict+def _truncate_log_value(value: Any, max_chars: int) -> Any:+ if value is None or isinstance(value, (bool, float)):+ return value+ if isinstance(value, int):+ value = int(value)+ text = value if isinstance(value, str) else str(value)+ if isinstance(value, int) and len(text) <= max_chars:+ return value+ if len(text) <= max_chars:+ return text+ return f"{text[:max_chars]}... [truncated]"+++def render_bounded_json(logger: logging.Logger, method_name: str, event_dict: EventDict) -> str:+ """Render one valid JSON record below the collector's observed split boundary."""+ rendered = _JSON_RENDERER(logger, method_name, event_dict)+ # JSONRenderer's default serializer escapes non-ASCII characters, so character+ # length is also the emitted UTF-8 byte length without allocating another copy.+ original_size_bytes = len(rendered)+ if original_size_bytes <= MAX_JSON_LOG_BYTES:+ return rendered++ bounded = {+ key: _truncate_log_value(+ event_dict[key],+ _OVERSIZED_LOG_VALUE_CHARS if key in {"msg", "exception"} else _OVERSIZED_LOG_METADATA_CHARS,+ )+ for key in _OVERSIZED_LOG_FIELDS+ if key in event_dict+ }+ omitted_fields = sorted(str(key)[:128] for key in event_dict if key not in bounded)+ bounded.update(+ {+ "log_truncated": True,+ "original_size_bytes": original_size_bytes,+ "omitted_field_count": len(omitted_fields),+ "omitted_fields": omitted_fields[:50],+ }+ )+ rendered = _JSON_RENDERER(logger, method_name, bounded)+ if len(rendered) <= MAX_JSON_LOG_BYTES:+ return rendered++ # Unusual escaped/control-heavy metadata can expand during JSON encoding. Keep a+ # minimal correlated record rather than emitting another line the collector splits.+ minimal = {+ key: _truncate_log_value(bounded[key], 256 if key == "msg" else 128)+ for key in _OVERSIZED_LOG_FIELDS+ if key in bounded and key != "exception"+ }+ minimal.update(+ {+ "log_truncated": True,+ "original_size_bytes": original_size_bytes,+ "omitted_field_count": sum(1 for key in event_dict if key not in minimal),+ }+ )+ rendered = _JSON_RENDERER(logger, method_name, minimal)+ if len(rendered) <= MAX_JSON_LOG_BYTES:+ return rendered+ return _JSON_RENDERER(+ logger,+ method_name,+ {+ "msg": "Oversized log record",+ "level": _truncate_log_value(event_dict.get("level"), 32),+ "log_truncated": True,+ "original_size_bytes": original_size_bytes,+ "omitted_field_count": len(event_dict),+ },+ )def redact_registered_secrets(logger: logging.Logger, method_name: str, event_dict: EventDict) -> EventDict:@@ -210,8 +313,7 @@ def redact_registered_secrets(logger: logging.Logger, method_name: str, event_direturn event_dictdef scrub(node: Any) -> Any:- # Nested values are folded into `msg` by add_kv_pairs_to_msg further down the chain, so a- # secret carried only inside a nested kwarg reaches the log line unless we recurse here.+ # Structured nested values are rendered in full, so recurse before JSON serialization.if isinstance(node, str):for secret in secrets:if secret in node:@@ -249,7 +351,7 @@ def redact_sensitive_event_fields(logger: logging.Logger, method_name: str, evenReuses the shared field redactor so structured kwargs such as``headers={"Authorization": ...}``, ``payload={...}``, or ``response_body={...}``- are masked before ``add_kv_pairs_to_msg`` folds them into the rendered line.+ are masked before JSON serialization.Top-level keys whose name is sensitive are masked outright; every othernon-string value is redacted recursively — models, tuples and sets included,since the formatter renders those in full too. Plain string values are left to@@ -562,7 +664,7 @@ def setup_logger() -> None:_entrypoint = _get_entrypoint()# logging.config.dictConfig(logging_config)- renderer = structlog.processors.JSONRenderer() if settings.JSON_LOGGING else CustomConsoleRenderer()+ renderer = render_bounded_json if settings.JSON_LOGGING else CustomConsoleRenderer()additional_processors = ([redact_bearer_tokens,@@ -573,7 +675,7 @@ def setup_logger() -> None:compact_action_objects,redact_sensitive_event_fields,structlog.processors.EventRenamer("msg"),- add_kv_pairs_to_msg,+ add_log_context,structlog.processors.CallsiteParameterAdder({structlog.processors.CallsiteParameter.PATHNAME,@@ -616,9 +718,9 @@ def setup_logger() -> None:)# Foreign stdlib records never run the structlog chain above, so without these two a# record reaches Datadog with an empty message (its remapper reads `msg`, not `event`)- # and no `organization_id` to group on. Order matters: `add_kv_pairs_to_msg` reads `msg`.+ # and no `organization_id` to group on.foreign_msg_chain: list[Processor] = (- [structlog.processors.EventRenamer("msg"), add_kv_pairs_to_msg] if settings.JSON_LOGGING else []+ [structlog.processors.EventRenamer("msg"), add_log_context] if settings.JSON_LOGGING else [])handler = logging.StreamHandler()diff --git a/tests/unit/test_forge_log_foreign_tracebacks.py b/tests/unit/test_forge_log_foreign_tracebacks.pyindex 14931665e..76e7ac1d1 100644--- a/tests/unit/test_forge_log_foreign_tracebacks.py+++ b/tests/unit/test_forge_log_foreign_tracebacks.py@@ -8,12 +8,14 @@ from collections.abc import Iteratorimport pytestimport structlog+import skyvern.forge.sdk.forge_log as forge_logfrom skyvern.config import settingsfrom skyvern.forge.sdk.copilot import secret_scrubfrom skyvern.forge.sdk.copilot.secret_scrub import REDACTED_SECRET_PLACEHOLDERfrom skyvern.forge.sdk.forge_log import setup_logger_REGISTERED_CREDENTIAL = "fake-registered-pa55w0rd-9f3c1a"+_MAX_EMITTED_JSON_BYTES = 64 * 1024@pytest.fixture@@ -60,6 +62,67 @@ def test_foreign_stdlib_exception_renders_single_structured_line(json_stream: ioassert record["exception_hash"]+def test_oversized_structured_payload_is_bounded_to_one_log_record(json_stream: io.StringIO) -> None:+ structlog.get_logger("oversized-test").error(+ "oversized_payload", payload="x" * 100_000, status="failed", error="payload rejected"+ )++ lines = json_stream.getvalue().strip().splitlines()+ assert len(lines) == 1+ assert len(lines[0].encode()) <= _MAX_EMITTED_JSON_BYTES++ record = json.loads(lines[0])+ assert record["msg"] == "oversized_payload"+ assert record["event_status"] == "failed"+ assert record["error"] == "payload rejected"+ assert record["log_truncated"] is True+ assert record["original_size_bytes"] > _MAX_EMITTED_JSON_BYTES+ assert "payload" in record["omitted_fields"]+++def test_control_heavy_oversized_log_keeps_correlation_fields() -> None:+ expanded = "\0" * 10_000+ rendered = forge_log.render_bounded_json(+ None, # type: ignore[arg-type]+ "error",+ {+ "msg": expanded,+ "exception": expanded,+ "logger": expanded,+ "entrypoint": expanded,+ "env": expanded,+ "version": expanded,+ "pathname": expanded,+ "filename": expanded,+ "request_id": "req_test",+ "task_id": "tsk_test",+ },+ )++ assert len(rendered.encode()) <= _MAX_EMITTED_JSON_BYTES+ record = json.loads(rendered)+ assert record["request_id"] == "req_test"+ assert record["task_id"] == "tsk_test"+ assert record["log_truncated"] is True+ assert record["omitted_field_count"] == 1+++def test_oversized_numeric_fields_are_bounded() -> None:+ class ShortStringInteger(int):+ def __str__(self) -> str:+ return "1"++ huge_number = ShortStringInteger(10**3999)+ rendered = forge_log.render_bounded_json(+ None, # type: ignore[arg-type]+ "error",+ {key: huge_number for key in forge_log._OVERSIZED_LOG_FIELDS},+ )++ assert len(rendered.encode()) <= _MAX_EMITTED_JSON_BYTES+ assert json.loads(rendered)["log_truncated"] is True++def test_exception_log_fields_matches_processor_output_for_raised_exception() -> None:"""exception_log_fields lets a warning carry the same dashboard fields add_error_processorderives from exc_info, so downgraded lines stay groupable without rendering a traceback."""diff --git a/tests/unit/test_forge_log_redaction.py b/tests/unit/test_forge_log_redaction.pyindex ec6ec3dd9..a739ead89 100644--- a/tests/unit/test_forge_log_redaction.py+++ b/tests/unit/test_forge_log_redaction.py@@ -120,7 +120,7 @@ def test_credential_redaction_tolerates_non_string_values() -> None:def test_redacts_a_credential_nested_inside_a_kwarg() -> None:- """add_kv_pairs_to_msg folds nested kwargs into the rendered line, so recursion is required."""+ """Nested kwargs are serialized, so registered secrets must be redacted recursively."""_register_credential(_FAKE_CREDENTIAL)event = {"event": "tool call",diff --git a/tests/unit/test_forge_log_reserved_keys.py b/tests/unit/test_forge_log_reserved_keys.pyindex 1e15680f9..fd420bd7c 100644--- a/tests/unit/test_forge_log_reserved_keys.py+++ b/tests/unit/test_forge_log_reserved_keys.py@@ -111,15 +111,13 @@ def test_msg_and_error_kwargs_are_deliberately_not_renamed(assert "event_error" not in payload-def test_native_json_log_keeps_authored_names_in_rendered_msg(+def test_native_json_log_does_not_duplicate_structured_fields_in_msg(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:- # The human-readable line keeps the authored kwarg name; only the queryable- # JSON attribute is escaped.monkeypatch.setattr(settings, "JSON_LOGGING", True)payload = _native_json_payload(capsys, status="completed")- assert "status=completed" in payload["msg"]+ assert payload["msg"] == "Wrote the final state"def test_foreign_record_extra_reserved_keys_renamed(monkeypatch: pytest.MonkeyPatch) -> None:@@ -139,14 +137,13 @@ def test_explicit_escaped_kwarg_wins_over_renamed_reserved_key(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:# If a call site ever passes both, the explicit `event_status` is the intentional one;- # the reserved key is still stripped so it cannot reach intake. Both values remain- # visible in the folded `msg` text.+ # the reserved key is still stripped so it cannot reach intake.monkeypatch.setattr(settings, "JSON_LOGGING", True)payload = _native_json_payload(capsys, status="completed", event_status="explicit")assert payload["event_status"] == "explicit"assert "status" not in payload- assert "status=completed" in payload["msg"]+ assert payload["msg"] == "Wrote the final state"def test_console_mode_keeps_authored_keys(monkeypatch: pytest.MonkeyPatch) -> None:diff --git a/tests/unit/test_forge_log_sampling.py b/tests/unit/test_forge_log_sampling.pyindex e6c00a9cd..d332bc7f1 100644--- a/tests/unit/test_forge_log_sampling.py+++ b/tests/unit/test_forge_log_sampling.py@@ -5,7 +5,7 @@ import skyvern.forge.sdk.forge_log as forge_logfrom skyvern.config import settingsfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext-from skyvern.forge.sdk.forge_log import add_kv_pairs_to_msg, sample_logs_processor+from skyvern.forge.sdk.forge_log import add_log_context, sample_logs_processorSAMPLED_ORG = "o_sampled"@@ -72,16 +72,17 @@ def test_empty_org_list_disables_sampling(monkeypatch: pytest.MonkeyPatch) -> Noassert "sampling" not in result-def test_sampling_marker_excluded_from_rendered_message() -> None:+def test_structured_fields_are_not_duplicated_into_message() -> None:event = {"msg": "Handling action", "sampling": True, "action_type": "click"}- result = add_kv_pairs_to_msg(None, "info", event) # type: ignore[arg-type]- assert "sampling" not in result["msg"]- assert "action_type=click" in result["msg"]+ result = add_log_context(None, "info", event) # type: ignore[arg-type]+ assert result["msg"] == "Handling action"+ assert result["sampling"] is True+ assert result["action_type"] == "click"-def test_add_kv_pairs_includes_copilot_session_id(monkeypatch: pytest.MonkeyPatch) -> None:+def test_add_log_context_includes_copilot_session_id(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr(skyvern_context, "current", lambda: SkyvernContext(copilot_session_id="chat_1"))- result = add_kv_pairs_to_msg(None, "info", {"msg": "Handling action"}) # type: ignore[arg-type]+ result = add_log_context(None, "info", {"msg": "Handling action"}) # type: ignore[arg-type]assert result["copilot_session_id"] == "chat_1"