Skip to content

Automate browser based workflows with AI

Redact secret values from exception text at the reporting seams (SKY-12626) (#7832)

Aaron Perez6bc77b8a28cb
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index b32e0f5d1..18047643c 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -122,7 +122,7 @@ from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.tasks import Task, TaskRequest, TaskResponse, TaskStatus
from skyvern.forge.sdk.schemas.totp_codes import OTPType
from skyvern.forge.sdk.submission import shadow as submission_shadow
-from skyvern.forge.sdk.trace import VerificationTrigger, apply_context_attrs, traced
+from skyvern.forge.sdk.trace import VerificationTrigger, apply_context_attrs, traced, traced_span
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.models.block import (
ActionBlock,
@@ -3471,7 +3471,7 @@ class ForgeAgent:
# step=None and speculative steps force the non-bundled path.
_bundled = bool(step and not step.is_speculative and _ctx and _ctx.use_artifact_bundling)
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.agent.artifact.screenshot_action") as _ss_art_span:
+ with traced_span(_tracer, "skyvern.agent.artifact.screenshot_action") as _ss_art_span:
apply_context_attrs(_ss_art_span)
_ss_art_span.set_attribute("screenshot_bytes", len(screenshot))
_ss_art_span.set_attribute("bundled", _bundled)
@@ -3517,7 +3517,7 @@ class ForgeAgent:
html_bytes = html.encode("utf-8")
_bundled = bool(step and not step.is_speculative and _ctx and _ctx.use_artifact_bundling)
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.agent.artifact.html_action") as _html_art_span:
+ with traced_span(_tracer, "skyvern.agent.artifact.html_action") as _html_art_span:
apply_context_attrs(_html_art_span)
_html_art_span.set_attribute("html_bytes", len(html_bytes))
_html_art_span.set_attribute("bundled", _bundled)
@@ -3536,7 +3536,7 @@ class ForgeAgent:
if artifacts:
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.agent.artifact.bulk_create") as _bulk_span:
+ with traced_span(_tracer, "skyvern.agent.artifact.bulk_create") as _bulk_span:
apply_context_attrs(_bulk_span)
# Count underlying artifacts (main + screenshots per request), not request wrappers.
_bulk_span.set_attribute("artifact_count", sum(len(a.artifacts) for a in artifacts if a is not None))
@@ -3559,7 +3559,7 @@ class ForgeAgent:
)
else:
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.agent.artifact.update_action_screenshot_fk") as _fk_span:
+ with traced_span(_tracer, "skyvern.agent.artifact.update_action_screenshot_fk") as _fk_span:
apply_context_attrs(_fk_span)
try:
await app.DATABASE.artifacts.update_action_screenshot_artifact_id(
@@ -4883,7 +4883,7 @@ class ForgeAgent:
if task.organization_id:
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.agent.cleanup.save_downloaded_files") as _cl_save_span:
+ with traced_span(_tracer, "skyvern.agent.cleanup.save_downloaded_files") as _cl_save_span:
apply_context_attrs(_cl_save_span)
try:
# Keep both finalize and save inside a single timeout budget so a hung
@@ -4966,19 +4966,19 @@ class ForgeAgent:
await self.async_operation_pool.remove_task(task.task_id)
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.agent.cleanup.browser_and_artifacts") as _cl_br_span:
+ with traced_span(_tracer, "skyvern.agent.cleanup.browser_and_artifacts") as _cl_br_span:
apply_context_attrs(_cl_br_span)
await self.cleanup_browser_and_create_artifacts(
close_browser_on_completion, last_step, task, browser_session_id=browser_session_id
)
# Wait for all tasks to complete before generating the links for the artifacts
- with _tracer.start_as_current_span("skyvern.agent.cleanup.wait_for_upload") as _cl_wait_span:
+ with traced_span(_tracer, "skyvern.agent.cleanup.wait_for_upload") as _cl_wait_span:
apply_context_attrs(_cl_wait_span)
await app.ARTIFACT_MANAGER.wait_for_upload_aiotasks([task.task_id])
if need_call_webhook:
- with _tracer.start_as_current_span("skyvern.agent.cleanup.webhook") as _cl_wh_span:
+ with traced_span(_tracer, "skyvern.agent.cleanup.webhook") as _cl_wh_span:
apply_context_attrs(_cl_wh_span)
await self.execute_task_webhook(task=task, api_key=api_key)
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index c4bf1ea20..939344a0b 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -55,7 +55,7 @@ from skyvern.forge.sdk.models import SpeculativeLLMMetadata, Step
from skyvern.forge.sdk.schemas.ai_suggestions import AISuggestion
from skyvern.forge.sdk.schemas.task_v2 import TaskV2, Thought
from skyvern.forge.sdk.settings_manager import SettingsManager
-from skyvern.forge.sdk.trace import apply_context_attrs, traced
+from skyvern.forge.sdk.trace import apply_context_attrs, traced, traced_span
from skyvern.schemas.llm import (
LLMAllowedFailsPolicy,
LLMConfig,
@@ -1302,7 +1302,7 @@ class LLMAPIHandlerFactory:
# staging before the LLM call. The hot cost is inside the non-bundled
# branch (prepare_llm_artifact → S3 upload).
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.llm.artifact.pre_request") as _pre_span:
+ with traced_span(_tracer, "skyvern.llm.artifact.pre_request") as _pre_span:
apply_context_attrs(_pre_span)
_pre_span.set_attribute("bundled", bool(_should_bundle))
_pre_span.set_attribute("persist", bool(should_persist_llm_artifacts))
@@ -1945,7 +1945,7 @@ class LLMAPIHandlerFactory:
# does the S3 uploads + DB rows for every LLM artifact queued during
# this request; the bundled-path archive accumulate is fast in-memory.
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.llm.artifact.post_response") as _post_span:
+ with traced_span(_tracer, "skyvern.llm.artifact.post_response") as _post_span:
apply_context_attrs(_post_span)
_post_span.set_attribute("bundled", bool(_should_bundle))
# Count underlying artifacts (main + screenshots per request), not request wrappers.
@@ -2098,7 +2098,7 @@ class LLMAPIHandlerFactory:
# staging before the LLM call. The hot cost is inside the non-bundled
# branch (prepare_llm_artifact → S3 upload).
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.llm.artifact.pre_request") as _pre_span:
+ with traced_span(_tracer, "skyvern.llm.artifact.pre_request") as _pre_span:
apply_context_attrs(_pre_span)
_pre_span.set_attribute("bundled", bool(_should_bundle))
_pre_span.set_attribute("persist", bool(should_persist_llm_artifacts))
@@ -2512,7 +2512,7 @@ class LLMAPIHandlerFactory:
# does the S3 uploads + DB rows for every LLM artifact queued during
# this request; the bundled-path archive accumulate is fast in-memory.
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.llm.artifact.post_response") as _post_span:
+ with traced_span(_tracer, "skyvern.llm.artifact.post_response") as _post_span:
apply_context_attrs(_post_span)
_post_span.set_attribute("bundled", bool(_should_bundle))
# Count underlying artifacts (main + screenshots per request), not request wrappers.
@@ -2753,7 +2753,7 @@ class LLMCaller:
if "display_width_px" in tool:
tool["display_width_px"] = target_dimension["width"]
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.llm.screenshot_resize") as _resize_span:
+ with traced_span(_tracer, "skyvern.llm.screenshot_resize") as _resize_span:
apply_context_attrs(_resize_span)
_resize_span.set_attribute("input_count", len(screenshots))
_resize_span.set_attribute("input_bytes", sum(len(s) for s in screenshots))
@@ -2767,7 +2767,7 @@ class LLMCaller:
# staging before the LLM call. The hot cost is inside the non-bundled
# branch (prepare_llm_artifact → S3 upload).
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.llm.artifact.pre_request") as _pre_span:
+ with traced_span(_tracer, "skyvern.llm.artifact.pre_request") as _pre_span:
apply_context_attrs(_pre_span)
_pre_span.set_attribute("bundled", bool(_should_bundle))
_pre_span.set_attribute("persist", bool(prompt and should_persist_llm_artifacts))
@@ -3055,7 +3055,7 @@ class LLMCaller:
# the S3 uploads + DB rows for every LLM artifact queued during this
# request; the bundled-path archive accumulate is fast in-memory.
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.llm.artifact.post_response") as _post_span:
+ with traced_span(_tracer, "skyvern.llm.artifact.post_response") as _post_span:
apply_context_attrs(_post_span)
_post_span.set_attribute("bundled", bool(_should_bundle))
# Count underlying artifacts (main + screenshots per request), not request wrappers.
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 4eb92e3af..17bbcf05e 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -239,7 +239,7 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatHistoryMessage,
WorkflowCopilotChatSender,
)
-from skyvern.forge.sdk.trace import apply_context_attrs
+from skyvern.forge.sdk.trace import apply_context_attrs, record_span_exception, traced_span
from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException
from skyvern.utils.strings import escape_code_fences
from skyvern.utils.yaml_loader import safe_load_no_dates
@@ -305,7 +305,7 @@ def _copilot_turn_span(
turn_id: str | None = None,
) -> Iterator[Any]:
tracer = otel_trace.get_tracer("skyvern")
- with tracer.start_as_current_span(_COPILOT_TURN_SPAN_NAME) as span:
+ with traced_span(tracer, _COPILOT_TURN_SPAN_NAME) as span:
span.set_attribute("skyvern.span.role", "wrapper")
span.set_attribute("copilot.turn_index", _derive_turn_index(chat_history, turn_index))
if turn_id is not None:
@@ -5332,7 +5332,7 @@ async def run_copilot_agent(
workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
exc_info=True,
)
- turn_span.record_exception(exc)
+ record_span_exception(turn_span, exc, set_error_status=False)
ctx = CopilotContext(
organization_id=organization_id,
workflow_id=chat_request.workflow_id,
diff --git a/skyvern/forge/sdk/copilot/secret_scrub.py b/skyvern/forge/sdk/copilot/secret_scrub.py
index 6789d3d8c..7182099e9 100644
--- a/skyvern/forge/sdk/copilot/secret_scrub.py
+++ b/skyvern/forge/sdk/copilot/secret_scrub.py
@@ -11,8 +11,6 @@ from typing import TYPE_CHECKING, Any
import structlog
-from skyvern.forge.sdk.copilot.output_utils import is_valid_image_base64
-
if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.runtime import AgentContext
@@ -29,18 +27,21 @@ _MAX_SCRUB_SESSIONS = 1024
# not, because over-redacting a log line is cheap and leaking a secret into one is not.
MIN_PERSISTED_REDACTION_LENGTH = 8
-_ALL_VALUES_CACHE: tuple[tuple[int, int], list[str]] | None = None
+_ALL_VALUES_CACHE: tuple[tuple[tuple[str, int], ...], list[str]] | None = None
-def _registry_fingerprint() -> tuple[int, int]:
+def _registry_fingerprint() -> tuple[tuple[str, int], ...]:
"""Cheap key that changes on any registry mutation, including a direct one.
- Values are append-only and deduped per session, so the total count moves whenever a value is
- added and the session count moves whenever one is added or dropped. Deriving the key from the
- data rather than a hand-maintained counter means a caller that mutates the dict directly cannot
- be served a stale list.
+ Keyed per session rather than on totals: aggregate counts alone collide whenever one session is
+ dropped and another registers the same number of values, which served the dropped session's
+ values as the live list and left the new session's credential unscrubbed. Values are
+ append-only within a session, so a per-session count moves on every addition.
+
+ Deriving the key from the data rather than a hand-maintained counter means a caller that
+ mutates the dict directly cannot be served a stale list.
"""
- return len(_SESSION_SCRUB_VALUES), sum(len(values) for values in _SESSION_SCRUB_VALUES.values())
+ return tuple(sorted((session_id, len(values)) for session_id, values in _SESSION_SCRUB_VALUES.items()))
def _session_id(ctx: AgentContext) -> str | None:
@@ -114,7 +115,22 @@ def scrub_secrets_from_text(ctx: AgentContext, text: str) -> str:
return text
+def scrub_all_registered_from_text(text: str) -> str:
+ """Session-agnostic counterpart to ``scrub_secrets_from_text`` for reporting seams.
+
+ Exception text is serialized where no ``AgentContext`` is in scope, so this scrubs against
+ every session's values for the same reason ``all_registered_secret_values`` does.
+ """
+ for value in all_registered_secret_values():
+ text = text.replace(value, REDACTED_SECRET_PLACEHOLDER)
+ return text
+
+
def scrub_secrets_from_structure(ctx: AgentContext, obj: Any) -> Any:
+ # Lazy: output_utils costs ~7.7s to import, and this module is on the logging and span
+ # exception paths, where that would be paid by whichever request raises first.
+ from skyvern.forge.sdk.copilot.output_utils import is_valid_image_base64 # noqa: PLC0415
+
values = _registered_scrub_values(ctx)
if not values:
return obj
diff --git a/skyvern/forge/sdk/forge_log.py b/skyvern/forge/sdk/forge_log.py
index 23658b094..6c5c7678f 100644
--- a/skyvern/forge/sdk/forge_log.py
+++ b/skyvern/forge/sdk/forge_log.py
@@ -549,6 +549,14 @@ def setup_logger() -> None:
structlog.stdlib.add_logger_name,
structlog.stdlib.ProcessorFormatter.remove_processors_meta,
structlog.processors.TimeStamper(fmt="iso"),
+ # Every record on this handler — native structlog AND foreign stdlib (temporal,
+ # asyncio, sqlalchemy, uvicorn) — is serialized here, so this is the one seam that
+ # covers both. `format_exc_info` in `foreign_pre_chain` has already rendered
+ # exc_info to a string by now, so a secret in the exception text is reachable.
+ # These stay duplicated in the structlog chain above on purpose: that pass also
+ # guards `context.log`, which is persisted to the per-run S3 log artifact.
+ redact_bearer_tokens,
+ redact_registered_secrets,
renderer,
],
)
diff --git a/skyvern/forge/sdk/trace/__init__.py b/skyvern/forge/sdk/trace/__init__.py
index 157f68dbe..484ba1a0f 100644
--- a/skyvern/forge/sdk/trace/__init__.py
+++ b/skyvern/forge/sdk/trace/__init__.py
@@ -1,4 +1,7 @@
import asyncio
+import traceback
+from collections.abc import Iterator
+from contextlib import contextmanager
from functools import wraps
from typing import Any, Callable, Literal
@@ -36,6 +39,49 @@ _CONTEXT_SPAN_ATTRS: tuple[str, ...] = (
)
+def record_span_exception(span: Any, exc: BaseException, *, set_error_status: bool = True) -> None:
+ """Record an exception on a span with credential values scrubbed from its text.
+
+ Prefer this over ``span.record_exception`` everywhere: span events are exported with no
+ logging processors in front of them, so ``str(exc)`` and the traceback reach the collector
+ verbatim. Caller-supplied attributes override the ones ``record_exception`` derives, which is
+ what lets the scrubbed text win.
+ """
+ # Lazy: this module is imported far earlier in boot than the copilot package.
+ from skyvern.forge.sdk.copilot.secret_scrub import scrub_all_registered_from_text # noqa: PLC0415
+
+ stacktrace = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
+ message = scrub_all_registered_from_text(str(exc))
+ span.record_exception(
+ exc,
+ attributes={
+ "exception.message": message,
+ "exception.stacktrace": scrub_all_registered_from_text(stacktrace),
+ },
+ )
+ if set_error_status:
+ # Same shape OTel's use_span would have written, minus the credential.
+ span.set_status(trace.Status(trace.StatusCode.ERROR, f"{type(exc).__name__}: {message}"))
+
+
+@contextmanager
+def traced_span(tracer: Any, name: str, **kwargs: Any) -> Iterator[Any]:
+ """Start a span that cannot record an unscrubbed exception.
+
+ OTel's ``use_span`` records the exception and derives the status from ``str(exc)`` on the way
+ out, both unredacted, whenever one propagates out of the ``with`` block. Scrubbing at the
+ ``record_exception`` call alone is not enough — ``use_span`` then appends a *second*, raw
+ exception event and overwrites the scrubbed status. Turning both off and recording here makes
+ this the only recorder.
+ """
+ with tracer.start_as_current_span(name, record_exception=False, set_status_on_exception=False, **kwargs) as span:
+ try:
+ yield span
+ except Exception as exc:
+ record_span_exception(span, exc)
+ raise
+
+
def apply_context_attrs(span: Any) -> None:
"""Copy non-None IDs from the active SkyvernContext onto the current span.
@@ -98,7 +144,7 @@ def traced(
@wraps(func)
async def async_wrapper(*args: Any, **kw: Any) -> Any:
- with trace.get_tracer("skyvern").start_as_current_span(span_name) as span:
+ with traced_span(trace.get_tracer("skyvern"), span_name) as span:
span.set_attribute("code.function", code_function)
span.set_attribute("code.namespace", code_namespace)
if role is not None:
@@ -106,19 +152,14 @@ def traced(
apply_context_attrs(span)
if tags:
span.set_attribute("tags", tags)
- try:
- return await func(*args, **kw)
- except Exception as e:
- span.record_exception(e)
- span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
- raise
+ return await func(*args, **kw)
return async_wrapper
else:
@wraps(func)
def sync_wrapper(*args: Any, **kw: Any) -> Any:
- with trace.get_tracer("skyvern").start_as_current_span(span_name) as span:
+ with traced_span(trace.get_tracer("skyvern"), span_name) as span:
span.set_attribute("code.function", code_function)
span.set_attribute("code.namespace", code_namespace)
if role is not None:
@@ -126,12 +167,7 @@ def traced(
apply_context_attrs(span)
if tags:
span.set_attribute("tags", tags)
- try:
- return func(*args, **kw)
- except Exception as e:
- span.record_exception(e)
- span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
- raise
+ return func(*args, **kw)
return sync_wrapper
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 012f7f76e..194906d2c 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -112,7 +112,7 @@ from skyvern.forge.sdk.services.credentials import (
parse_totp_config,
)
from skyvern.forge.sdk.settings_manager import SettingsManager
-from skyvern.forge.sdk.trace import apply_context_attrs, traced
+from skyvern.forge.sdk.trace import apply_context_attrs, traced, traced_span
from skyvern.services import service_utils
from skyvern.services.action_service import get_action_history
from skyvern.utils.lean_html import apply_lean_to_tree
@@ -2523,7 +2523,7 @@ class ActionHandler:
and settings.FILE_DOWNLOAD_FALSE_CLICK_POPUP_GRACE_SECONDS > 0
)
if not observe_false_click:
- with _tracer.start_as_current_span("skyvern.agent.action.handle_inner") as _hi_span:
+ with traced_span(_tracer, "skyvern.agent.action.handle_inner") as _hi_span:
apply_context_attrs(_hi_span)
results = await ActionHandler._handle_action(
scraped_page=scraped_page,
@@ -2535,7 +2535,7 @@ class ActionHandler:
else:
assert browser_state is not None
page_url_before_download = page.url
- with _tracer.start_as_current_span("skyvern.agent.action.false_click_download"):
+ with traced_span(_tracer, "skyvern.agent.action.false_click_download"):
false_click_download_event: asyncio.Future[tuple[Download, Page]] = (
asyncio.get_running_loop().create_future()
)
@@ -2627,7 +2627,7 @@ class ActionHandler:
)
try:
- with _tracer.start_as_current_span("skyvern.agent.action.handle_inner") as _hi_span:
+ with traced_span(_tracer, "skyvern.agent.action.handle_inner") as _hi_span:
apply_context_attrs(_hi_span)
try:
results = await ActionHandler._handle_action(
@@ -2771,7 +2771,7 @@ class ActionHandler:
try:
await transient_text_observer.start(scan_initial_visible_state=False)
xhr_capture.enable()
- with _tracer.start_as_current_span("skyvern.agent.action.handle_inner") as _hi_span:
+ with traced_span(_tracer, "skyvern.agent.action.handle_inner") as _hi_span:
apply_context_attrs(_hi_span)
results = await ActionHandler._handle_action(
scraped_page=scraped_page,
@@ -2803,7 +2803,7 @@ class ActionHandler:
_download_event_grace_seconds = min(
DOWNLOAD_EVENT_ACTIVE_DIR_GRACE_SECONDS, download_wait_hard_timeout_seconds
)
- with _tracer.start_as_current_span("skyvern.agent.action.download_wait") as _dl_wait_span:
+ with traced_span(_tracer, "skyvern.agent.action.download_wait") as _dl_wait_span:
apply_context_attrs(_dl_wait_span)
_dl_wait_span.set_attribute("timeout_seconds", download_wait_hard_timeout_seconds)
_dl_wait_span.set_attribute("download_event_grace_seconds", _download_event_grace_seconds)
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 357205f90..de748d4f6 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -25,7 +25,7 @@ from skyvern.forge.sdk.api.crypto import calculate_sha256
from skyvern.forge.sdk.browser_action_preflight import advance_observation_epoch
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.settings_manager import SettingsManager
-from skyvern.forge.sdk.trace import apply_context_attrs, traced
+from skyvern.forge.sdk.trace import apply_context_attrs, traced, traced_span
from skyvern.utils.image_resizer import Resolution
from skyvern.utils.token_counter import approx_count_tokens
from skyvern.utils.url_validators import strip_query_params
@@ -506,7 +506,7 @@ async def scrape_web_unsafe(
LOG.warning("Failed to get current x, y position of the page", exc_info=True)
_tracer = otel_trace.get_tracer("skyvern")
- with _tracer.start_as_current_span("skyvern.browser.scrape_screenshot") as _ss_span:
+ with traced_span(_tracer, "skyvern.browser.scrape_screenshot") as _ss_span:
apply_context_attrs(_ss_span)
# Hardcoded since this is an inline span, not a @traced method.
# Update if scrape_web_unsafe is renamed.
@@ -521,20 +521,15 @@ async def scrape_web_unsafe(
_ss_span.set_attribute("scrape_trigger", _scrape_ctx.scrape_trigger)
if _scrape_ctx.scrape_screenshots_consumed is not None:
_ss_span.set_attribute("screenshots_consumed", _scrape_ctx.scrape_screenshots_consumed)
- try:
- screenshots = await SkyvernFrame.take_split_screenshots(
- page=page,
- url=url,
- draw_boxes=draw_boxes,
- max_number=max_screenshot_number,
- scroll=scroll,
- )
- _ss_span.set_attribute("screenshot_count", len(screenshots))
- _ss_span.set_attribute("screenshot_bytes", sum(len(s) for s in screenshots))
- except Exception as e:
- _ss_span.record_exception(e)
- _ss_span.set_status(otel_trace.Status(otel_trace.StatusCode.ERROR, str(e)))
- raise
+ screenshots = await SkyvernFrame.take_split_screenshots(
+ page=page,
+ url=url,
+ draw_boxes=draw_boxes,
+ max_number=max_screenshot_number,
+ scroll=scroll,
+ )
+ _ss_span.set_attribute("screenshot_count", len(screenshots))
+ _ss_span.set_attribute("screenshot_bytes", sum(len(s) for s in screenshots))
# scroll back to the original x, y position of the page
if x is not None and y is not None:
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index 09c26f205..cfa03bd05 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -23,7 +23,7 @@ from skyvern.exceptions import FailedToTakeScreenshot, ScreenshotTargetClosed, S
from skyvern.forge.sdk.browser_action_preflight import policy_observation_enabled, record_observed_tabs
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.settings_manager import SettingsManager
-from skyvern.forge.sdk.trace import apply_context_attrs, traced
+from skyvern.forge.sdk.trace import apply_context_attrs, traced, traced_span
from skyvern.webeye.browser_engine import BrowserEngineSelection
from skyvern.webeye.browser_object_predicates import is_page_like
from skyvern.webeye.main_world_eval import evaluate_in_main_world, get_main_world_prefix
@@ -1448,7 +1448,7 @@ class SkyvernFrame:
# 1. Wait for loading indicators to disappear (longest timeout first)
loading_indicator_result = "success"
- with _tracer.start_as_current_span("skyvern.browser.page_ready.loading_indicators") as _li_span:
+ with traced_span(_tracer, "skyvern.browser.page_ready.loading_indicators") as _li_span:
apply_context_attrs(_li_span)
_li_span.set_attribute("timeout_ms", loading_indicator_timeout_ms)
try:
@@ -1464,7 +1464,7 @@ class SkyvernFrame:
# 2. Wait for network idle (with short timeout - some pages never go idle)
network_idle_result = "success"
- with _tracer.start_as_current_span("skyvern.browser.page_ready.network_idle") as _ni_span:
+ with traced_span(_tracer, "skyvern.browser.page_ready.network_idle") as _ni_span:
apply_context_attrs(_ni_span)
_ni_span.set_attribute("timeout_ms", network_idle_timeout_ms)
try:
@@ -1480,7 +1480,7 @@ class SkyvernFrame:
# 3. Wait for DOM to stabilize
dom_stability_result = "success"
- with _tracer.start_as_current_span("skyvern.browser.page_ready.dom_stability") as _ds_span:
+ with traced_span(_tracer, "skyvern.browser.page_ready.dom_stability") as _ds_span:
apply_context_attrs(_ds_span)
_ds_span.set_attribute("timeout_ms", dom_stability_timeout_ms)
_ds_span.set_attribute("stable_ms", dom_stable_ms)
diff --git a/tests/unit/test_copilot_secret_scrub.py b/tests/unit/test_copilot_secret_scrub.py
index cb75d576e..40c80bcf0 100644
--- a/tests/unit/test_copilot_secret_scrub.py
+++ b/tests/unit/test_copilot_secret_scrub.py
@@ -372,6 +372,49 @@ class TestPersistenceSeam:
assert set(all_registered_secret_values()) == {_FAKE_PASSWORD, _FAKE_OTP}
+ def test_importing_this_module_stays_cheap(self) -> None:
+ """This module sits on the logging and span-exception paths.
+
+ `output_utils` costs seconds to import. While it was a module-level import here, the first
+ exception to reach `record_span_exception` in a process paid it inline -- enough to blow
+ wall-clock budgets in whichever request happened to raise first. Keep it lazy.
+ """
+ import ast
+ from pathlib import Path
+
+ source = Path(secret_scrub.__file__).read_text(encoding="utf-8")
+ module_level = {
+ node.module
+ for node in ast.parse(source).body
+ if isinstance(node, ast.ImportFrom) and node.module and node.col_offset == 0
+ }
+
+ assert "skyvern.forge.sdk.copilot.output_utils" not in module_level
+
+ def test_a_dropped_session_does_not_serve_its_values_to_the_next_one(self) -> None:
+ """The cache key must track WHICH session holds values, not just how many there are.
+
+ Aggregate counts collide: one session holding one value looks identical to a different
+ session holding one value, so the dropped session's list was served as live and the new
+ session's credential went unscrubbed.
+ """
+ register_secret_scrub_value(_agent_ctx("pbs_a"), _FAKE_PASSWORD)
+ assert all_registered_secret_values() == [_FAKE_PASSWORD]
+
+ clear_session_scrub_values("pbs_a")
+ register_secret_scrub_value(_agent_ctx("pbs_b"), _FAKE_OTP)
+
+ assert all_registered_secret_values() == [_FAKE_OTP]
+
+ def test_appending_to_a_session_invalidates_the_cache(self) -> None:
+ ctx = _agent_ctx("pbs_a")
+ register_secret_scrub_value(ctx, _FAKE_PASSWORD)
+ assert all_registered_secret_values() == [_FAKE_PASSWORD]
+
+ register_secret_scrub_value(ctx, _FAKE_OTP)
+
+ assert set(all_registered_secret_values()) == {_FAKE_PASSWORD, _FAKE_OTP}
+
def test_no_registered_values_leaves_the_workflow_alone(self) -> None:
workflow_yaml = f'code: await page.fill("#password", "{_FAKE_PASSWORD}")\n'
diff --git a/tests/unit/test_forge_log_foreign_tracebacks.py b/tests/unit/test_forge_log_foreign_tracebacks.py
index 41c29ef06..9c68cfc01 100644
--- a/tests/unit/test_forge_log_foreign_tracebacks.py
+++ b/tests/unit/test_forge_log_foreign_tracebacks.py
@@ -9,8 +9,12 @@ import pytest
import structlog
from skyvern.config import settings
+from skyvern.forge.sdk.copilot import secret_scrub
+from skyvern.forge.sdk.copilot.secret_scrub import REDACTED_SECRET_PLACEHOLDER
from skyvern.forge.sdk.forge_log import setup_logger
+_REGISTERED_CREDENTIAL = "fake-registered-pa55w0rd-9f3c1a"
+
@pytest.fixture
def json_stream(monkeypatch: pytest.MonkeyPatch) -> Iterator[io.StringIO]:
@@ -56,6 +60,75 @@ def test_foreign_stdlib_exception_renders_single_structured_line(json_stream: io
assert record["exception_hash"]
+def registered_credential() -> Iterator[str]:
+ secret_scrub._SESSION_SCRUB_VALUES.clear()
+ secret_scrub._SESSION_SCRUB_VALUES["pbs_foreign_traceback"] = [_REGISTERED_CREDENTIAL]
+ try:
+ yield _REGISTERED_CREDENTIAL
+ finally:
+ secret_scrub._SESSION_SCRUB_VALUES.clear()
+
+
+def _raise_with_credential_in_message(credential: str) -> None:
+ """A driver renders a bound parameter — including a credential — into the exception message."""
+ raise RuntimeError(f"(psycopg.errors.UniqueViolation) INSERT failed [parameters: ('{credential}',)]")
+
+
[email protected]("logger_name", ["temporalio.activity", "asyncio", "sqlalchemy.engine.Engine"])
+def test_foreign_record_exception_text_is_redacted(
+ json_stream: io.StringIO, registered_credential: str, logger_name: str
+) -> None:
+ """Foreign stdlib records reach the same serializer as native ones and must be scrubbed there.
+
+ The redaction processors used to run only in the structlog chain, so anything logged through a
+ stdlib logger (temporal, asyncio, sqlalchemy, uvicorn) shipped its exception text unredacted.
+ """
+ logger = logging.getLogger(logger_name)
+ logger.setLevel(logging.INFO)
+ try:
+ _raise_with_credential_in_message(registered_credential)
+ except RuntimeError:
+ logger.error("statement failed", exc_info=True)
+
+ payload = json_stream.getvalue()
+ assert registered_credential not in payload
+ assert REDACTED_SECRET_PLACEHOLDER in payload
+
+
+def test_redaction_does_not_blind_the_foreign_traceback(json_stream: io.StringIO, registered_credential: str) -> None:
+ """Only the credential is removed — type, frame, and traceback structure survive."""
+ logger = logging.getLogger("temporalio.activity")
+ logger.setLevel(logging.INFO)
+ try:
+ _raise_with_credential_in_message(registered_credential)
+ except RuntimeError:
+ logger.error("statement failed", exc_info=True)
+
+ record = json.loads(json_stream.getvalue().strip().splitlines()[0])
+ assert "Traceback (most recent call last)" in record["exception"]
+ assert "RuntimeError" in record["exception"]
+ assert "_raise_with_credential_in_message" in record["exception"]
+ assert "psycopg.errors.UniqueViolation" in record["exception"]
+ assert record["error_type"] == "builtins.RuntimeError"
+
+
+def test_native_structlog_exception_text_stays_redacted(json_stream: io.StringIO, registered_credential: str) -> None:
+ """Guards the additive fix: the structlog chain must keep its own redaction pass.
+
+ ``skyvern_logs_processor`` copies the event dict into ``context.log`` (persisted to the per-run
+ S3 log artifact) BEFORE the formatter runs, so moving the redactors instead of adding them
+ would close the stdout leak and open one into run artifacts.
+ """
+ logger = structlog.get_logger("skyvern.native_redaction_test")
+ try:
+ _raise_with_credential_in_message(registered_credential)
+ except RuntimeError:
+ logger.exception("statement failed")
+
+ assert registered_credential not in json_stream.getvalue()
+
+
def test_native_structlog_exception_not_double_processed(json_stream: io.StringIO) -> None:
logger = structlog.get_logger("skyvern.foreign_traceback_test")
try:
diff --git a/tests/unit/test_span_exception_redaction.py b/tests/unit/test_span_exception_redaction.py
new file mode 100644
index 000000000..13ad45a05
--- /dev/null
+++ b/tests/unit/test_span_exception_redaction.py
@@ -0,0 +1,202 @@
+"""Span events are a reporting path with no logging processors in front of them.
+
+``Span.record_exception`` renders ``str(exc)`` and the traceback into ``exception.message`` /
+``exception.stacktrace`` and ships them as span attributes, so a credential inside the exception
+text reaches the exporter with nothing in its way.
+"""
+
+from __future__ import annotations
+
+import ast
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+from skyvern.forge.sdk.copilot import secret_scrub
+from skyvern.forge.sdk.copilot.secret_scrub import REDACTED_SECRET_PLACEHOLDER
+from skyvern.forge.sdk.trace import record_span_exception, traced_span
+
+_REGISTERED_CREDENTIAL = "fake-span-pa55w0rd-4b7e2d"
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+def registered_credential() -> Iterator[str]:
+ secret_scrub._SESSION_SCRUB_VALUES.clear()
+ secret_scrub._SESSION_SCRUB_VALUES["pbs_span"] = [_REGISTERED_CREDENTIAL]
+ try:
+ yield _REGISTERED_CREDENTIAL
+ finally:
+ secret_scrub._SESSION_SCRUB_VALUES.clear()
+
+
+def _drain(exporter: InMemorySpanExporter) -> tuple[str, list[str | None]]:
+ finished = exporter.get_finished_spans()
+ events_blob = "".join(str(ev.attributes) for s in finished for ev in s.events)
+ return events_blob, [s.status.description for s in finished]
+
+
+def _record_and_export(credential: str, *, set_error_status: bool = True) -> tuple[str, list[str | None]]:
+ """The helper on its own, for a span the caller handles without re-raising."""
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ with traced_span(provider.get_tracer("test"), "probe") as span:
+ try:
+ raise RuntimeError(f"(psycopg.errors) INSERT failed [parameters: ('{credential}',)]")
+ except RuntimeError as exc:
+ record_span_exception(span, exc, set_error_status=set_error_status)
+ provider.force_flush()
+ return _drain(exporter)
+
+
+def _raise_through_span(credential: str) -> tuple[str, list[str | None]]:
+ """The production shape: the exception PROPAGATES out of the span context.
+
+ This is what `@traced` and the inline `traced_span` sites do. OTel's `use_span` records the
+ exception on the way out, so a span left at the default `record_exception=True` appends a
+ second, unscrubbed event and overwrites the scrubbed status.
+ """
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ with pytest.raises(RuntimeError):
+ with traced_span(provider.get_tracer("test"), "probe"):
+ raise RuntimeError(f"(psycopg.errors) INSERT failed [parameters: ('{credential}',)]")
+ provider.force_flush()
+ return _drain(exporter)
+
+
+def test_credential_is_redacted_from_the_span_exception_event(registered_credential: str) -> None:
+ events, _ = _record_and_export(registered_credential)
+ assert registered_credential not in events
+ assert REDACTED_SECRET_PLACEHOLDER in events
+
+
+def test_credential_is_redacted_from_the_span_error_status(registered_credential: str) -> None:
+ _, descriptions = _record_and_export(registered_credential)
+ assert descriptions and all(d is not None for d in descriptions)
+ blob = "".join(d or "" for d in descriptions)
+ assert registered_credential not in blob
+ assert REDACTED_SECRET_PLACEHOLDER in blob
+
+
+def test_span_exception_stays_useful_for_debugging(registered_credential: str) -> None:
+ events, _ = _record_and_export(registered_credential)
+ assert "RuntimeError" in events
+ assert "psycopg.errors" in events
+ assert "Traceback (most recent call last)" in events
+ assert "_record_and_export" in events
+
+
+def test_status_is_left_alone_when_not_requested(registered_credential: str) -> None:
+ """The copilot turn span records the exception without claiming the span failed."""
+ _, descriptions = _record_and_export(registered_credential, set_error_status=False)
+ assert descriptions == [None]
+
+
+def test_propagating_exception_is_redacted_and_recorded_once(registered_credential: str) -> None:
+ """Regression for the use_span double-record: exactly one event, and it is scrubbed."""
+ events, descriptions = _raise_through_span(registered_credential)
+
+ assert events.count("exception.stacktrace") == 1, "use_span appended a second exception event"
+ assert registered_credential not in events
+ assert registered_credential not in "".join(d or "" for d in descriptions)
+ assert REDACTED_SECRET_PLACEHOLDER in events
+
+
+def test_propagating_exception_stays_useful_for_debugging(registered_credential: str) -> None:
+ events, descriptions = _raise_through_span(registered_credential)
+
+ assert "RuntimeError" in events
+ assert "psycopg.errors" in events
+ assert "Traceback (most recent call last)" in events
+ assert any(d and d.startswith("RuntimeError: ") for d in descriptions)
+
+
+def _bare_span_start_sites() -> list[str]:
+ """Every bare ``*.start_as_current_span(...)`` outside the helper, found via AST not grep."""
+ offenders: list[str] = []
+ helper = _REPO_ROOT / "skyvern" / "forge" / "sdk" / "trace" / "__init__.py"
+ for package in ("skyvern", "cloud"):
+ root = _REPO_ROOT / package
+ if not root.is_dir():
+ # This file is synced to the OSS repo, where `cloud/` does not exist. Skipping
+ # explicitly rather than relying on rglob() silently yielding nothing on a missing dir.
+ continue
+ for path in root.rglob("*.py"):
+ if path == helper:
+ continue
+ try:
+ tree = ast.parse(path.read_text(encoding="utf-8"))
+ except (SyntaxError, UnicodeDecodeError):
+ continue
+ for node in ast.walk(tree):
+ if (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "start_as_current_span"
+ ):
+ offenders.append(f"{path.relative_to(_REPO_ROOT)}:{node.lineno}")
+ return sorted(offenders)
+
+
+def test_no_bare_start_as_current_span_calls_remain() -> None:
+ """A bare span start re-opens the leak: use_span records the raw exception on the way out.
+
+ Use ``traced_span`` instead, which disables that and records the scrubbed exception itself.
+ """
+ assert _bare_span_start_sites() == []
+
+
+def _bare_record_exception_sites() -> list[str]:
+ """Every ``*.record_exception(...)`` call outside the helper, found via AST not grep."""
+ offenders: list[str] = []
+ helper = _REPO_ROOT / "skyvern" / "forge" / "sdk" / "trace" / "__init__.py"
+ for package in ("skyvern", "cloud"):
+ root = _REPO_ROOT / package
+ if not root.is_dir():
+ # This file is synced to the OSS repo, where `cloud/` does not exist. Skipping
+ # explicitly rather than relying on rglob() silently yielding nothing on a missing dir.
+ continue
+ for path in root.rglob("*.py"):
+ if path == helper:
+ continue
+ try:
+ tree = ast.parse(path.read_text(encoding="utf-8"))
+ except (SyntaxError, UnicodeDecodeError):
+ continue
+ for node in ast.walk(tree):
+ if (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "record_exception"
+ ):
+ offenders.append(f"{path.relative_to(_REPO_ROOT)}:{node.lineno}")
+ return sorted(offenders)
+
+
+def test_no_bare_record_exception_calls_remain() -> None:
+ """Remove the capability to err: a bare ``record_exception`` re-opens the leak silently.
+
+ Use ``record_span_exception`` instead. If a call genuinely must bypass scrubbing, widen the
+ helper rather than this guard.
+ """
+ assert _bare_record_exception_sites() == []
+
+
+def test_the_guard_can_actually_fail(tmp_path: Path) -> None:
+ """Non-vacuity: the AST walk must flag a bare call, not pass because it found nothing."""
+ offending = tmp_path / "offender.py"
+ offending.write_text("def f(span, exc):\n span.record_exception(exc)\n", encoding="utf-8")
+ tree = ast.parse(offending.read_text(encoding="utf-8"))
+ found = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "record_exception"
+ ]
+ assert len(found) == 1