Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Redact secret values from exception text at the reporting seams (SKY-12626) (#7832)
Aaron Perez6bc77b8a28cb
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex b32e0f5d1..18047643c 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -122,7 +122,7 @@ from skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.tasks import Task, TaskRequest, TaskResponse, TaskStatusfrom skyvern.forge.sdk.schemas.totp_codes import OTPTypefrom skyvern.forge.sdk.submission import shadow as submission_shadow-from skyvern.forge.sdk.trace import VerificationTrigger, apply_context_attrs, traced+from skyvern.forge.sdk.trace import VerificationTrigger, apply_context_attrs, traced, traced_spanfrom skyvern.forge.sdk.workflow.context_manager import WorkflowRunContextfrom skyvern.forge.sdk.workflow.models.block import (ActionBlock,@@ -3471,7 +3471,7 @@ class ForgeAgent:# step=None and speculative steps force the non-bundled path._bundled = bool(step and not step.is_speculative and _ctx and _ctx.use_artifact_bundling)_tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.agent.artifact.screenshot_action") as _ss_art_span:+ with traced_span(_tracer, "skyvern.agent.artifact.screenshot_action") as _ss_art_span:apply_context_attrs(_ss_art_span)_ss_art_span.set_attribute("screenshot_bytes", len(screenshot))_ss_art_span.set_attribute("bundled", _bundled)@@ -3517,7 +3517,7 @@ class ForgeAgent:html_bytes = html.encode("utf-8")_bundled = bool(step and not step.is_speculative and _ctx and _ctx.use_artifact_bundling)_tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.agent.artifact.html_action") as _html_art_span:+ with traced_span(_tracer, "skyvern.agent.artifact.html_action") as _html_art_span:apply_context_attrs(_html_art_span)_html_art_span.set_attribute("html_bytes", len(html_bytes))_html_art_span.set_attribute("bundled", _bundled)@@ -3536,7 +3536,7 @@ class ForgeAgent:if artifacts:_tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.agent.artifact.bulk_create") as _bulk_span:+ with traced_span(_tracer, "skyvern.agent.artifact.bulk_create") as _bulk_span:apply_context_attrs(_bulk_span)# Count underlying artifacts (main + screenshots per request), not request wrappers._bulk_span.set_attribute("artifact_count", sum(len(a.artifacts) for a in artifacts if a is not None))@@ -3559,7 +3559,7 @@ class ForgeAgent:)else:_tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.agent.artifact.update_action_screenshot_fk") as _fk_span:+ with traced_span(_tracer, "skyvern.agent.artifact.update_action_screenshot_fk") as _fk_span:apply_context_attrs(_fk_span)try:await app.DATABASE.artifacts.update_action_screenshot_artifact_id(@@ -4883,7 +4883,7 @@ class ForgeAgent:if task.organization_id:_tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.agent.cleanup.save_downloaded_files") as _cl_save_span:+ with traced_span(_tracer, "skyvern.agent.cleanup.save_downloaded_files") as _cl_save_span:apply_context_attrs(_cl_save_span)try:# Keep both finalize and save inside a single timeout budget so a hung@@ -4966,19 +4966,19 @@ class ForgeAgent:await self.async_operation_pool.remove_task(task.task_id)_tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.agent.cleanup.browser_and_artifacts") as _cl_br_span:+ with traced_span(_tracer, "skyvern.agent.cleanup.browser_and_artifacts") as _cl_br_span:apply_context_attrs(_cl_br_span)await self.cleanup_browser_and_create_artifacts(close_browser_on_completion, last_step, task, browser_session_id=browser_session_id)# Wait for all tasks to complete before generating the links for the artifacts- with _tracer.start_as_current_span("skyvern.agent.cleanup.wait_for_upload") as _cl_wait_span:+ with traced_span(_tracer, "skyvern.agent.cleanup.wait_for_upload") as _cl_wait_span:apply_context_attrs(_cl_wait_span)await app.ARTIFACT_MANAGER.wait_for_upload_aiotasks([task.task_id])if need_call_webhook:- with _tracer.start_as_current_span("skyvern.agent.cleanup.webhook") as _cl_wh_span:+ with traced_span(_tracer, "skyvern.agent.cleanup.webhook") as _cl_wh_span:apply_context_attrs(_cl_wh_span)await self.execute_task_webhook(task=task, api_key=api_key)diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex c4bf1ea20..939344a0b 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -55,7 +55,7 @@ from skyvern.forge.sdk.models import SpeculativeLLMMetadata, Stepfrom skyvern.forge.sdk.schemas.ai_suggestions import AISuggestionfrom skyvern.forge.sdk.schemas.task_v2 import TaskV2, Thoughtfrom skyvern.forge.sdk.settings_manager import SettingsManager-from skyvern.forge.sdk.trace import apply_context_attrs, traced+from skyvern.forge.sdk.trace import apply_context_attrs, traced, traced_spanfrom skyvern.schemas.llm import (LLMAllowedFailsPolicy,LLMConfig,@@ -1302,7 +1302,7 @@ class LLMAPIHandlerFactory:# staging before the LLM call. The hot cost is inside the non-bundled# branch (prepare_llm_artifact → S3 upload)._tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.llm.artifact.pre_request") as _pre_span:+ with traced_span(_tracer, "skyvern.llm.artifact.pre_request") as _pre_span:apply_context_attrs(_pre_span)_pre_span.set_attribute("bundled", bool(_should_bundle))_pre_span.set_attribute("persist", bool(should_persist_llm_artifacts))@@ -1945,7 +1945,7 @@ class LLMAPIHandlerFactory:# does the S3 uploads + DB rows for every LLM artifact queued during# this request; the bundled-path archive accumulate is fast in-memory._tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.llm.artifact.post_response") as _post_span:+ with traced_span(_tracer, "skyvern.llm.artifact.post_response") as _post_span:apply_context_attrs(_post_span)_post_span.set_attribute("bundled", bool(_should_bundle))# Count underlying artifacts (main + screenshots per request), not request wrappers.@@ -2098,7 +2098,7 @@ class LLMAPIHandlerFactory:# staging before the LLM call. The hot cost is inside the non-bundled# branch (prepare_llm_artifact → S3 upload)._tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.llm.artifact.pre_request") as _pre_span:+ with traced_span(_tracer, "skyvern.llm.artifact.pre_request") as _pre_span:apply_context_attrs(_pre_span)_pre_span.set_attribute("bundled", bool(_should_bundle))_pre_span.set_attribute("persist", bool(should_persist_llm_artifacts))@@ -2512,7 +2512,7 @@ class LLMAPIHandlerFactory:# does the S3 uploads + DB rows for every LLM artifact queued during# this request; the bundled-path archive accumulate is fast in-memory._tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.llm.artifact.post_response") as _post_span:+ with traced_span(_tracer, "skyvern.llm.artifact.post_response") as _post_span:apply_context_attrs(_post_span)_post_span.set_attribute("bundled", bool(_should_bundle))# Count underlying artifacts (main + screenshots per request), not request wrappers.@@ -2753,7 +2753,7 @@ class LLMCaller:if "display_width_px" in tool:tool["display_width_px"] = target_dimension["width"]_tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.llm.screenshot_resize") as _resize_span:+ with traced_span(_tracer, "skyvern.llm.screenshot_resize") as _resize_span:apply_context_attrs(_resize_span)_resize_span.set_attribute("input_count", len(screenshots))_resize_span.set_attribute("input_bytes", sum(len(s) for s in screenshots))@@ -2767,7 +2767,7 @@ class LLMCaller:# staging before the LLM call. The hot cost is inside the non-bundled# branch (prepare_llm_artifact → S3 upload)._tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.llm.artifact.pre_request") as _pre_span:+ with traced_span(_tracer, "skyvern.llm.artifact.pre_request") as _pre_span:apply_context_attrs(_pre_span)_pre_span.set_attribute("bundled", bool(_should_bundle))_pre_span.set_attribute("persist", bool(prompt and should_persist_llm_artifacts))@@ -3055,7 +3055,7 @@ class LLMCaller:# the S3 uploads + DB rows for every LLM artifact queued during this# request; the bundled-path archive accumulate is fast in-memory._tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.llm.artifact.post_response") as _post_span:+ with traced_span(_tracer, "skyvern.llm.artifact.post_response") as _post_span:apply_context_attrs(_post_span)_post_span.set_attribute("bundled", bool(_should_bundle))# Count underlying artifacts (main + screenshots per request), not request wrappers.diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 4eb92e3af..17bbcf05e 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -239,7 +239,7 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotChatHistoryMessage,WorkflowCopilotChatSender,)-from skyvern.forge.sdk.trace import apply_context_attrs+from skyvern.forge.sdk.trace import apply_context_attrs, record_span_exception, traced_spanfrom skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPExceptionfrom skyvern.utils.strings import escape_code_fencesfrom skyvern.utils.yaml_loader import safe_load_no_dates@@ -305,7 +305,7 @@ def _copilot_turn_span(turn_id: str | None = None,) -> Iterator[Any]:tracer = otel_trace.get_tracer("skyvern")- with tracer.start_as_current_span(_COPILOT_TURN_SPAN_NAME) as span:+ with traced_span(tracer, _COPILOT_TURN_SPAN_NAME) as span:span.set_attribute("skyvern.span.role", "wrapper")span.set_attribute("copilot.turn_index", _derive_turn_index(chat_history, turn_index))if turn_id is not None:@@ -5332,7 +5332,7 @@ async def run_copilot_agent(workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,exc_info=True,)- turn_span.record_exception(exc)+ record_span_exception(turn_span, exc, set_error_status=False)ctx = CopilotContext(organization_id=organization_id,workflow_id=chat_request.workflow_id,diff --git a/skyvern/forge/sdk/copilot/secret_scrub.py b/skyvern/forge/sdk/copilot/secret_scrub.pyindex 6789d3d8c..7182099e9 100644--- a/skyvern/forge/sdk/copilot/secret_scrub.py+++ b/skyvern/forge/sdk/copilot/secret_scrub.py@@ -11,8 +11,6 @@ from typing import TYPE_CHECKING, Anyimport structlog-from skyvern.forge.sdk.copilot.output_utils import is_valid_image_base64-if TYPE_CHECKING:from skyvern.forge.sdk.copilot.runtime import AgentContext@@ -29,18 +27,21 @@ _MAX_SCRUB_SESSIONS = 1024# not, because over-redacting a log line is cheap and leaking a secret into one is not.MIN_PERSISTED_REDACTION_LENGTH = 8-_ALL_VALUES_CACHE: tuple[tuple[int, int], list[str]] | None = None+_ALL_VALUES_CACHE: tuple[tuple[tuple[str, int], ...], list[str]] | None = None-def _registry_fingerprint() -> tuple[int, int]:+def _registry_fingerprint() -> tuple[tuple[str, int], ...]:"""Cheap key that changes on any registry mutation, including a direct one.- Values are append-only and deduped per session, so the total count moves whenever a value is- added and the session count moves whenever one is added or dropped. Deriving the key from the- data rather than a hand-maintained counter means a caller that mutates the dict directly cannot- be served a stale list.+ Keyed per session rather than on totals: aggregate counts alone collide whenever one session is+ dropped and another registers the same number of values, which served the dropped session's+ values as the live list and left the new session's credential unscrubbed. Values are+ append-only within a session, so a per-session count moves on every addition.++ Deriving the key from the data rather than a hand-maintained counter means a caller that+ mutates the dict directly cannot be served a stale list."""- return len(_SESSION_SCRUB_VALUES), sum(len(values) for values in _SESSION_SCRUB_VALUES.values())+ return tuple(sorted((session_id, len(values)) for session_id, values in _SESSION_SCRUB_VALUES.items()))def _session_id(ctx: AgentContext) -> str | None:@@ -114,7 +115,22 @@ def scrub_secrets_from_text(ctx: AgentContext, text: str) -> str:return text+def scrub_all_registered_from_text(text: str) -> str:+ """Session-agnostic counterpart to ``scrub_secrets_from_text`` for reporting seams.++ Exception text is serialized where no ``AgentContext`` is in scope, so this scrubs against+ every session's values for the same reason ``all_registered_secret_values`` does.+ """+ for value in all_registered_secret_values():+ text = text.replace(value, REDACTED_SECRET_PLACEHOLDER)+ return text++def scrub_secrets_from_structure(ctx: AgentContext, obj: Any) -> Any:+ # Lazy: output_utils costs ~7.7s to import, and this module is on the logging and span+ # exception paths, where that would be paid by whichever request raises first.+ from skyvern.forge.sdk.copilot.output_utils import is_valid_image_base64 # noqa: PLC0415+values = _registered_scrub_values(ctx)if not values:return objdiff --git a/skyvern/forge/sdk/forge_log.py b/skyvern/forge/sdk/forge_log.pyindex 23658b094..6c5c7678f 100644--- a/skyvern/forge/sdk/forge_log.py+++ b/skyvern/forge/sdk/forge_log.py@@ -549,6 +549,14 @@ def setup_logger() -> None:structlog.stdlib.add_logger_name,structlog.stdlib.ProcessorFormatter.remove_processors_meta,structlog.processors.TimeStamper(fmt="iso"),+ # Every record on this handler — native structlog AND foreign stdlib (temporal,+ # asyncio, sqlalchemy, uvicorn) — is serialized here, so this is the one seam that+ # covers both. `format_exc_info` in `foreign_pre_chain` has already rendered+ # exc_info to a string by now, so a secret in the exception text is reachable.+ # These stay duplicated in the structlog chain above on purpose: that pass also+ # guards `context.log`, which is persisted to the per-run S3 log artifact.+ redact_bearer_tokens,+ redact_registered_secrets,renderer,],)diff --git a/skyvern/forge/sdk/trace/__init__.py b/skyvern/forge/sdk/trace/__init__.pyindex 157f68dbe..484ba1a0f 100644--- a/skyvern/forge/sdk/trace/__init__.py+++ b/skyvern/forge/sdk/trace/__init__.py@@ -1,4 +1,7 @@import asyncio+import traceback+from collections.abc import Iterator+from contextlib import contextmanagerfrom functools import wrapsfrom typing import Any, Callable, Literal@@ -36,6 +39,49 @@ _CONTEXT_SPAN_ATTRS: tuple[str, ...] = ()+def record_span_exception(span: Any, exc: BaseException, *, set_error_status: bool = True) -> None:+ """Record an exception on a span with credential values scrubbed from its text.++ Prefer this over ``span.record_exception`` everywhere: span events are exported with no+ logging processors in front of them, so ``str(exc)`` and the traceback reach the collector+ verbatim. Caller-supplied attributes override the ones ``record_exception`` derives, which is+ what lets the scrubbed text win.+ """+ # Lazy: this module is imported far earlier in boot than the copilot package.+ from skyvern.forge.sdk.copilot.secret_scrub import scrub_all_registered_from_text # noqa: PLC0415++ stacktrace = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))+ message = scrub_all_registered_from_text(str(exc))+ span.record_exception(+ exc,+ attributes={+ "exception.message": message,+ "exception.stacktrace": scrub_all_registered_from_text(stacktrace),+ },+ )+ if set_error_status:+ # Same shape OTel's use_span would have written, minus the credential.+ span.set_status(trace.Status(trace.StatusCode.ERROR, f"{type(exc).__name__}: {message}"))+++@contextmanager+def traced_span(tracer: Any, name: str, **kwargs: Any) -> Iterator[Any]:+ """Start a span that cannot record an unscrubbed exception.++ OTel's ``use_span`` records the exception and derives the status from ``str(exc)`` on the way+ out, both unredacted, whenever one propagates out of the ``with`` block. Scrubbing at the+ ``record_exception`` call alone is not enough — ``use_span`` then appends a *second*, raw+ exception event and overwrites the scrubbed status. Turning both off and recording here makes+ this the only recorder.+ """+ with tracer.start_as_current_span(name, record_exception=False, set_status_on_exception=False, **kwargs) as span:+ try:+ yield span+ except Exception as exc:+ record_span_exception(span, exc)+ raise++def apply_context_attrs(span: Any) -> None:"""Copy non-None IDs from the active SkyvernContext onto the current span.@@ -98,7 +144,7 @@ def traced(@wraps(func)async def async_wrapper(*args: Any, **kw: Any) -> Any:- with trace.get_tracer("skyvern").start_as_current_span(span_name) as span:+ with traced_span(trace.get_tracer("skyvern"), span_name) as span:span.set_attribute("code.function", code_function)span.set_attribute("code.namespace", code_namespace)if role is not None:@@ -106,19 +152,14 @@ def traced(apply_context_attrs(span)if tags:span.set_attribute("tags", tags)- try:- return await func(*args, **kw)- except Exception as e:- span.record_exception(e)- span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))- raise+ return await func(*args, **kw)return async_wrapperelse:@wraps(func)def sync_wrapper(*args: Any, **kw: Any) -> Any:- with trace.get_tracer("skyvern").start_as_current_span(span_name) as span:+ with traced_span(trace.get_tracer("skyvern"), span_name) as span:span.set_attribute("code.function", code_function)span.set_attribute("code.namespace", code_namespace)if role is not None:@@ -126,12 +167,7 @@ def traced(apply_context_attrs(span)if tags:span.set_attribute("tags", tags)- try:- return func(*args, **kw)- except Exception as e:- span.record_exception(e)- span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))- raise+ return func(*args, **kw)return sync_wrapperdiff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 012f7f76e..194906d2c 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -112,7 +112,7 @@ from skyvern.forge.sdk.services.credentials import (parse_totp_config,)from skyvern.forge.sdk.settings_manager import SettingsManager-from skyvern.forge.sdk.trace import apply_context_attrs, traced+from skyvern.forge.sdk.trace import apply_context_attrs, traced, traced_spanfrom skyvern.services import service_utilsfrom skyvern.services.action_service import get_action_historyfrom skyvern.utils.lean_html import apply_lean_to_tree@@ -2523,7 +2523,7 @@ class ActionHandler:and settings.FILE_DOWNLOAD_FALSE_CLICK_POPUP_GRACE_SECONDS > 0)if not observe_false_click:- with _tracer.start_as_current_span("skyvern.agent.action.handle_inner") as _hi_span:+ with traced_span(_tracer, "skyvern.agent.action.handle_inner") as _hi_span:apply_context_attrs(_hi_span)results = await ActionHandler._handle_action(scraped_page=scraped_page,@@ -2535,7 +2535,7 @@ class ActionHandler:else:assert browser_state is not Nonepage_url_before_download = page.url- with _tracer.start_as_current_span("skyvern.agent.action.false_click_download"):+ with traced_span(_tracer, "skyvern.agent.action.false_click_download"):false_click_download_event: asyncio.Future[tuple[Download, Page]] = (asyncio.get_running_loop().create_future())@@ -2627,7 +2627,7 @@ class ActionHandler:)try:- with _tracer.start_as_current_span("skyvern.agent.action.handle_inner") as _hi_span:+ with traced_span(_tracer, "skyvern.agent.action.handle_inner") as _hi_span:apply_context_attrs(_hi_span)try:results = await ActionHandler._handle_action(@@ -2771,7 +2771,7 @@ class ActionHandler:try:await transient_text_observer.start(scan_initial_visible_state=False)xhr_capture.enable()- with _tracer.start_as_current_span("skyvern.agent.action.handle_inner") as _hi_span:+ with traced_span(_tracer, "skyvern.agent.action.handle_inner") as _hi_span:apply_context_attrs(_hi_span)results = await ActionHandler._handle_action(scraped_page=scraped_page,@@ -2803,7 +2803,7 @@ class ActionHandler:_download_event_grace_seconds = min(DOWNLOAD_EVENT_ACTIVE_DIR_GRACE_SECONDS, download_wait_hard_timeout_seconds)- with _tracer.start_as_current_span("skyvern.agent.action.download_wait") as _dl_wait_span:+ with traced_span(_tracer, "skyvern.agent.action.download_wait") as _dl_wait_span:apply_context_attrs(_dl_wait_span)_dl_wait_span.set_attribute("timeout_seconds", download_wait_hard_timeout_seconds)_dl_wait_span.set_attribute("download_event_grace_seconds", _download_event_grace_seconds)diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 357205f90..de748d4f6 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -25,7 +25,7 @@ from skyvern.forge.sdk.api.crypto import calculate_sha256from skyvern.forge.sdk.browser_action_preflight import advance_observation_epochfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.settings_manager import SettingsManager-from skyvern.forge.sdk.trace import apply_context_attrs, traced+from skyvern.forge.sdk.trace import apply_context_attrs, traced, traced_spanfrom skyvern.utils.image_resizer import Resolutionfrom skyvern.utils.token_counter import approx_count_tokensfrom skyvern.utils.url_validators import strip_query_params@@ -506,7 +506,7 @@ async def scrape_web_unsafe(LOG.warning("Failed to get current x, y position of the page", exc_info=True)_tracer = otel_trace.get_tracer("skyvern")- with _tracer.start_as_current_span("skyvern.browser.scrape_screenshot") as _ss_span:+ with traced_span(_tracer, "skyvern.browser.scrape_screenshot") as _ss_span:apply_context_attrs(_ss_span)# Hardcoded since this is an inline span, not a @traced method.# Update if scrape_web_unsafe is renamed.@@ -521,20 +521,15 @@ async def scrape_web_unsafe(_ss_span.set_attribute("scrape_trigger", _scrape_ctx.scrape_trigger)if _scrape_ctx.scrape_screenshots_consumed is not None:_ss_span.set_attribute("screenshots_consumed", _scrape_ctx.scrape_screenshots_consumed)- try:- screenshots = await SkyvernFrame.take_split_screenshots(- page=page,- url=url,- draw_boxes=draw_boxes,- max_number=max_screenshot_number,- scroll=scroll,- )- _ss_span.set_attribute("screenshot_count", len(screenshots))- _ss_span.set_attribute("screenshot_bytes", sum(len(s) for s in screenshots))- except Exception as e:- _ss_span.record_exception(e)- _ss_span.set_status(otel_trace.Status(otel_trace.StatusCode.ERROR, str(e)))- raise+ screenshots = await SkyvernFrame.take_split_screenshots(+ page=page,+ url=url,+ draw_boxes=draw_boxes,+ max_number=max_screenshot_number,+ scroll=scroll,+ )+ _ss_span.set_attribute("screenshot_count", len(screenshots))+ _ss_span.set_attribute("screenshot_bytes", sum(len(s) for s in screenshots))# scroll back to the original x, y position of the pageif x is not None and y is not None:diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex 09c26f205..cfa03bd05 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -23,7 +23,7 @@ from skyvern.exceptions import FailedToTakeScreenshot, ScreenshotTargetClosed, Sfrom skyvern.forge.sdk.browser_action_preflight import policy_observation_enabled, record_observed_tabsfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.settings_manager import SettingsManager-from skyvern.forge.sdk.trace import apply_context_attrs, traced+from skyvern.forge.sdk.trace import apply_context_attrs, traced, traced_spanfrom skyvern.webeye.browser_engine import BrowserEngineSelectionfrom skyvern.webeye.browser_object_predicates import is_page_likefrom skyvern.webeye.main_world_eval import evaluate_in_main_world, get_main_world_prefix@@ -1448,7 +1448,7 @@ class SkyvernFrame:# 1. Wait for loading indicators to disappear (longest timeout first)loading_indicator_result = "success"- with _tracer.start_as_current_span("skyvern.browser.page_ready.loading_indicators") as _li_span:+ with traced_span(_tracer, "skyvern.browser.page_ready.loading_indicators") as _li_span:apply_context_attrs(_li_span)_li_span.set_attribute("timeout_ms", loading_indicator_timeout_ms)try:@@ -1464,7 +1464,7 @@ class SkyvernFrame:# 2. Wait for network idle (with short timeout - some pages never go idle)network_idle_result = "success"- with _tracer.start_as_current_span("skyvern.browser.page_ready.network_idle") as _ni_span:+ with traced_span(_tracer, "skyvern.browser.page_ready.network_idle") as _ni_span:apply_context_attrs(_ni_span)_ni_span.set_attribute("timeout_ms", network_idle_timeout_ms)try:@@ -1480,7 +1480,7 @@ class SkyvernFrame:# 3. Wait for DOM to stabilizedom_stability_result = "success"- with _tracer.start_as_current_span("skyvern.browser.page_ready.dom_stability") as _ds_span:+ with traced_span(_tracer, "skyvern.browser.page_ready.dom_stability") as _ds_span:apply_context_attrs(_ds_span)_ds_span.set_attribute("timeout_ms", dom_stability_timeout_ms)_ds_span.set_attribute("stable_ms", dom_stable_ms)diff --git a/tests/unit/test_copilot_secret_scrub.py b/tests/unit/test_copilot_secret_scrub.pyindex cb75d576e..40c80bcf0 100644--- a/tests/unit/test_copilot_secret_scrub.py+++ b/tests/unit/test_copilot_secret_scrub.py@@ -372,6 +372,49 @@ class TestPersistenceSeam:assert set(all_registered_secret_values()) == {_FAKE_PASSWORD, _FAKE_OTP}+ def test_importing_this_module_stays_cheap(self) -> None:+ """This module sits on the logging and span-exception paths.++ `output_utils` costs seconds to import. While it was a module-level import here, the first+ exception to reach `record_span_exception` in a process paid it inline -- enough to blow+ wall-clock budgets in whichever request happened to raise first. Keep it lazy.+ """+ import ast+ from pathlib import Path++ source = Path(secret_scrub.__file__).read_text(encoding="utf-8")+ module_level = {+ node.module+ for node in ast.parse(source).body+ if isinstance(node, ast.ImportFrom) and node.module and node.col_offset == 0+ }++ assert "skyvern.forge.sdk.copilot.output_utils" not in module_level++ def test_a_dropped_session_does_not_serve_its_values_to_the_next_one(self) -> None:+ """The cache key must track WHICH session holds values, not just how many there are.++ Aggregate counts collide: one session holding one value looks identical to a different+ session holding one value, so the dropped session's list was served as live and the new+ session's credential went unscrubbed.+ """+ register_secret_scrub_value(_agent_ctx("pbs_a"), _FAKE_PASSWORD)+ assert all_registered_secret_values() == [_FAKE_PASSWORD]++ clear_session_scrub_values("pbs_a")+ register_secret_scrub_value(_agent_ctx("pbs_b"), _FAKE_OTP)++ assert all_registered_secret_values() == [_FAKE_OTP]++ def test_appending_to_a_session_invalidates_the_cache(self) -> None:+ ctx = _agent_ctx("pbs_a")+ register_secret_scrub_value(ctx, _FAKE_PASSWORD)+ assert all_registered_secret_values() == [_FAKE_PASSWORD]++ register_secret_scrub_value(ctx, _FAKE_OTP)++ assert set(all_registered_secret_values()) == {_FAKE_PASSWORD, _FAKE_OTP}+def test_no_registered_values_leaves_the_workflow_alone(self) -> None:workflow_yaml = f'code: await page.fill("#password", "{_FAKE_PASSWORD}")\n'diff --git a/tests/unit/test_forge_log_foreign_tracebacks.py b/tests/unit/test_forge_log_foreign_tracebacks.pyindex 41c29ef06..9c68cfc01 100644--- a/tests/unit/test_forge_log_foreign_tracebacks.py+++ b/tests/unit/test_forge_log_foreign_tracebacks.py@@ -9,8 +9,12 @@ import pytestimport structlogfrom skyvern.config import settings+from skyvern.forge.sdk.copilot import secret_scrub+from skyvern.forge.sdk.copilot.secret_scrub import REDACTED_SECRET_PLACEHOLDERfrom skyvern.forge.sdk.forge_log import setup_logger+_REGISTERED_CREDENTIAL = "fake-registered-pa55w0rd-9f3c1a"+@pytest.fixturedef json_stream(monkeypatch: pytest.MonkeyPatch) -> Iterator[io.StringIO]:@@ -56,6 +60,75 @@ def test_foreign_stdlib_exception_renders_single_structured_line(json_stream: ioassert record["exception_hash"]+def registered_credential() -> Iterator[str]:+ secret_scrub._SESSION_SCRUB_VALUES.clear()+ secret_scrub._SESSION_SCRUB_VALUES["pbs_foreign_traceback"] = [_REGISTERED_CREDENTIAL]+ try:+ yield _REGISTERED_CREDENTIAL+ finally:+ secret_scrub._SESSION_SCRUB_VALUES.clear()+++def _raise_with_credential_in_message(credential: str) -> None:+ """A driver renders a bound parameter — including a credential — into the exception message."""+ raise RuntimeError(f"(psycopg.errors.UniqueViolation) INSERT failed [parameters: ('{credential}',)]")++[email protected]("logger_name", ["temporalio.activity", "asyncio", "sqlalchemy.engine.Engine"])+def test_foreign_record_exception_text_is_redacted(+ json_stream: io.StringIO, registered_credential: str, logger_name: str+) -> None:+ """Foreign stdlib records reach the same serializer as native ones and must be scrubbed there.++ The redaction processors used to run only in the structlog chain, so anything logged through a+ stdlib logger (temporal, asyncio, sqlalchemy, uvicorn) shipped its exception text unredacted.+ """+ logger = logging.getLogger(logger_name)+ logger.setLevel(logging.INFO)+ try:+ _raise_with_credential_in_message(registered_credential)+ except RuntimeError:+ logger.error("statement failed", exc_info=True)++ payload = json_stream.getvalue()+ assert registered_credential not in payload+ assert REDACTED_SECRET_PLACEHOLDER in payload+++def test_redaction_does_not_blind_the_foreign_traceback(json_stream: io.StringIO, registered_credential: str) -> None:+ """Only the credential is removed — type, frame, and traceback structure survive."""+ logger = logging.getLogger("temporalio.activity")+ logger.setLevel(logging.INFO)+ try:+ _raise_with_credential_in_message(registered_credential)+ except RuntimeError:+ logger.error("statement failed", exc_info=True)++ record = json.loads(json_stream.getvalue().strip().splitlines()[0])+ assert "Traceback (most recent call last)" in record["exception"]+ assert "RuntimeError" in record["exception"]+ assert "_raise_with_credential_in_message" in record["exception"]+ assert "psycopg.errors.UniqueViolation" in record["exception"]+ assert record["error_type"] == "builtins.RuntimeError"+++def test_native_structlog_exception_text_stays_redacted(json_stream: io.StringIO, registered_credential: str) -> None:+ """Guards the additive fix: the structlog chain must keep its own redaction pass.++ ``skyvern_logs_processor`` copies the event dict into ``context.log`` (persisted to the per-run+ S3 log artifact) BEFORE the formatter runs, so moving the redactors instead of adding them+ would close the stdout leak and open one into run artifacts.+ """+ logger = structlog.get_logger("skyvern.native_redaction_test")+ try:+ _raise_with_credential_in_message(registered_credential)+ except RuntimeError:+ logger.exception("statement failed")++ assert registered_credential not in json_stream.getvalue()++def test_native_structlog_exception_not_double_processed(json_stream: io.StringIO) -> None:logger = structlog.get_logger("skyvern.foreign_traceback_test")try:diff --git a/tests/unit/test_span_exception_redaction.py b/tests/unit/test_span_exception_redaction.pynew file mode 100644index 000000000..13ad45a05--- /dev/null+++ b/tests/unit/test_span_exception_redaction.py@@ -0,0 +1,202 @@+"""Span events are a reporting path with no logging processors in front of them.++``Span.record_exception`` renders ``str(exc)`` and the traceback into ``exception.message`` /+``exception.stacktrace`` and ships them as span attributes, so a credential inside the exception+text reaches the exporter with nothing in its way.+"""++from __future__ import annotations++import ast+from collections.abc import Iterator+from pathlib import Path++import pytest+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter++from skyvern.forge.sdk.copilot import secret_scrub+from skyvern.forge.sdk.copilot.secret_scrub import REDACTED_SECRET_PLACEHOLDER+from skyvern.forge.sdk.trace import record_span_exception, traced_span++_REGISTERED_CREDENTIAL = "fake-span-pa55w0rd-4b7e2d"+_REPO_ROOT = Path(__file__).resolve().parents[2]+++def registered_credential() -> Iterator[str]:+ secret_scrub._SESSION_SCRUB_VALUES.clear()+ secret_scrub._SESSION_SCRUB_VALUES["pbs_span"] = [_REGISTERED_CREDENTIAL]+ try:+ yield _REGISTERED_CREDENTIAL+ finally:+ secret_scrub._SESSION_SCRUB_VALUES.clear()+++def _drain(exporter: InMemorySpanExporter) -> tuple[str, list[str | None]]:+ finished = exporter.get_finished_spans()+ events_blob = "".join(str(ev.attributes) for s in finished for ev in s.events)+ return events_blob, [s.status.description for s in finished]+++def _record_and_export(credential: str, *, set_error_status: bool = True) -> tuple[str, list[str | None]]:+ """The helper on its own, for a span the caller handles without re-raising."""+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ with traced_span(provider.get_tracer("test"), "probe") as span:+ try:+ raise RuntimeError(f"(psycopg.errors) INSERT failed [parameters: ('{credential}',)]")+ except RuntimeError as exc:+ record_span_exception(span, exc, set_error_status=set_error_status)+ provider.force_flush()+ return _drain(exporter)+++def _raise_through_span(credential: str) -> tuple[str, list[str | None]]:+ """The production shape: the exception PROPAGATES out of the span context.++ This is what `@traced` and the inline `traced_span` sites do. OTel's `use_span` records the+ exception on the way out, so a span left at the default `record_exception=True` appends a+ second, unscrubbed event and overwrites the scrubbed status.+ """+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ with pytest.raises(RuntimeError):+ with traced_span(provider.get_tracer("test"), "probe"):+ raise RuntimeError(f"(psycopg.errors) INSERT failed [parameters: ('{credential}',)]")+ provider.force_flush()+ return _drain(exporter)+++def test_credential_is_redacted_from_the_span_exception_event(registered_credential: str) -> None:+ events, _ = _record_and_export(registered_credential)+ assert registered_credential not in events+ assert REDACTED_SECRET_PLACEHOLDER in events+++def test_credential_is_redacted_from_the_span_error_status(registered_credential: str) -> None:+ _, descriptions = _record_and_export(registered_credential)+ assert descriptions and all(d is not None for d in descriptions)+ blob = "".join(d or "" for d in descriptions)+ assert registered_credential not in blob+ assert REDACTED_SECRET_PLACEHOLDER in blob+++def test_span_exception_stays_useful_for_debugging(registered_credential: str) -> None:+ events, _ = _record_and_export(registered_credential)+ assert "RuntimeError" in events+ assert "psycopg.errors" in events+ assert "Traceback (most recent call last)" in events+ assert "_record_and_export" in events+++def test_status_is_left_alone_when_not_requested(registered_credential: str) -> None:+ """The copilot turn span records the exception without claiming the span failed."""+ _, descriptions = _record_and_export(registered_credential, set_error_status=False)+ assert descriptions == [None]+++def test_propagating_exception_is_redacted_and_recorded_once(registered_credential: str) -> None:+ """Regression for the use_span double-record: exactly one event, and it is scrubbed."""+ events, descriptions = _raise_through_span(registered_credential)++ assert events.count("exception.stacktrace") == 1, "use_span appended a second exception event"+ assert registered_credential not in events+ assert registered_credential not in "".join(d or "" for d in descriptions)+ assert REDACTED_SECRET_PLACEHOLDER in events+++def test_propagating_exception_stays_useful_for_debugging(registered_credential: str) -> None:+ events, descriptions = _raise_through_span(registered_credential)++ assert "RuntimeError" in events+ assert "psycopg.errors" in events+ assert "Traceback (most recent call last)" in events+ assert any(d and d.startswith("RuntimeError: ") for d in descriptions)+++def _bare_span_start_sites() -> list[str]:+ """Every bare ``*.start_as_current_span(...)`` outside the helper, found via AST not grep."""+ offenders: list[str] = []+ helper = _REPO_ROOT / "skyvern" / "forge" / "sdk" / "trace" / "__init__.py"+ for package in ("skyvern", "cloud"):+ root = _REPO_ROOT / package+ if not root.is_dir():+ # This file is synced to the OSS repo, where `cloud/` does not exist. Skipping+ # explicitly rather than relying on rglob() silently yielding nothing on a missing dir.+ continue+ for path in root.rglob("*.py"):+ if path == helper:+ continue+ try:+ tree = ast.parse(path.read_text(encoding="utf-8"))+ except (SyntaxError, UnicodeDecodeError):+ continue+ for node in ast.walk(tree):+ if (+ isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr == "start_as_current_span"+ ):+ offenders.append(f"{path.relative_to(_REPO_ROOT)}:{node.lineno}")+ return sorted(offenders)+++def test_no_bare_start_as_current_span_calls_remain() -> None:+ """A bare span start re-opens the leak: use_span records the raw exception on the way out.++ Use ``traced_span`` instead, which disables that and records the scrubbed exception itself.+ """+ assert _bare_span_start_sites() == []+++def _bare_record_exception_sites() -> list[str]:+ """Every ``*.record_exception(...)`` call outside the helper, found via AST not grep."""+ offenders: list[str] = []+ helper = _REPO_ROOT / "skyvern" / "forge" / "sdk" / "trace" / "__init__.py"+ for package in ("skyvern", "cloud"):+ root = _REPO_ROOT / package+ if not root.is_dir():+ # This file is synced to the OSS repo, where `cloud/` does not exist. Skipping+ # explicitly rather than relying on rglob() silently yielding nothing on a missing dir.+ continue+ for path in root.rglob("*.py"):+ if path == helper:+ continue+ try:+ tree = ast.parse(path.read_text(encoding="utf-8"))+ except (SyntaxError, UnicodeDecodeError):+ continue+ for node in ast.walk(tree):+ if (+ isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr == "record_exception"+ ):+ offenders.append(f"{path.relative_to(_REPO_ROOT)}:{node.lineno}")+ return sorted(offenders)+++def test_no_bare_record_exception_calls_remain() -> None:+ """Remove the capability to err: a bare ``record_exception`` re-opens the leak silently.++ Use ``record_span_exception`` instead. If a call genuinely must bypass scrubbing, widen the+ helper rather than this guard.+ """+ assert _bare_record_exception_sites() == []+++def test_the_guard_can_actually_fail(tmp_path: Path) -> None:+ """Non-vacuity: the AST walk must flag a bare call, not pass because it found nothing."""+ offending = tmp_path / "offender.py"+ offending.write_text("def f(span, exc):\n span.record_exception(exc)\n", encoding="utf-8")+ tree = ast.parse(offending.read_text(encoding="utf-8"))+ found = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "record_exception"+ ]+ assert len(found) == 1