Skip to content

Automate browser based workflows with AI

fix(extraction-shadow): follow-up to SKY-8871 — cycle, dead code, comment nits (#5486)

Aaron Perezff198cb6f588
diff --git a/skyvern-frontend/src/api/sse.test.ts b/skyvern-frontend/src/api/sse.test.ts
new file mode 100644
index 000000000..a85f6fda7
--- /dev/null
+++ b/skyvern-frontend/src/api/sse.test.ts
@@ -0,0 +1,36 @@
+import { describe, it, expect, vi } from "vitest";
+import type { FetchEventSourceInit } from "@microsoft/fetch-event-source";
+
+const mockFetchEventSource = vi.fn(
+ (_url: string, opts: FetchEventSourceInit) => {
+ // Send a message so the inner promise can settle
+ opts.onmessage?.({
+ id: "",
+ event: "done",
+ data: '{"done":true}',
+ retry: undefined,
+ });
+ return Promise.resolve();
+ },
+);
+
+vi.mock("@microsoft/fetch-event-source", () => ({
+ fetchEventSource: mockFetchEventSource,
+}));
+
+const { fetchStreamingSse } = await import("./sse");
+
+describe("fetchStreamingSse", () => {
+ it("passes openWhenHidden: true to fetchEventSource", async () => {
+ await fetchStreamingSse(
+ "http://localhost/test",
+ { method: "POST", headers: {}, body: "{}" },
+ () => true, // return true to resolve the promise on first message
+ );
+
+ expect(mockFetchEventSource).toHaveBeenCalledWith(
+ "http://localhost/test",
+ expect.objectContaining({ openWhenHidden: true }),
+ );
+ });
+});
diff --git a/skyvern-frontend/src/api/sse.ts b/skyvern-frontend/src/api/sse.ts
index 2081a705c..57b1e8031 100644
--- a/skyvern-frontend/src/api/sse.ts
+++ b/skyvern-frontend/src/api/sse.ts
@@ -66,6 +66,10 @@ export async function fetchStreamingSse<T>(
headers: init.headers as Record<string, string>,
body: init.body,
signal: controller.signal,
+ // Keep the SSE connection alive when the browser tab is hidden.
+ // Without this, the library closes and reconnects on tab focus,
+ // which restarts the copilot agent from scratch.
+ openWhenHidden: true,
onmessage: (event) => {
if (!event.data || !event.data.trim()) {
return;
diff --git a/skyvern-frontend/src/util/env.ts b/skyvern-frontend/src/util/env.ts
index e27d18d1a..91a80506b 100644
--- a/skyvern-frontend/src/util/env.ts
+++ b/skyvern-frontend/src/util/env.ts
@@ -43,7 +43,7 @@ try {
}
newWssBaseUrl = url.toString();
} catch (e) {
- newWssBaseUrl = wssBaseUrl.replace("/api", "");
+ newWssBaseUrl = wssBaseUrl?.replace("/api", "") ?? "";
}
// Base URL for the Runs API (strip a leading `/api` segment: /api/v1 -> /v1)
@@ -55,7 +55,7 @@ const runsApiBaseUrl = (() => {
}
return `${url.origin}${url.pathname}`;
} catch (e) {
- return apiBaseUrl.replace("/api", "");
+ return apiBaseUrl?.replace("/api", "") ?? "";
}
})();
diff --git a/skyvern/cli/mcp_tools/workflow.py b/skyvern/cli/mcp_tools/workflow.py
index 18fec0fa7..116bb159c 100644
--- a/skyvern/cli/mcp_tools/workflow.py
+++ b/skyvern/cli/mcp_tools/workflow.py
@@ -22,6 +22,7 @@ from skyvern.client.types import WorkflowCreateYamlRequest
from skyvern.forge.sdk.workflow.models.parameter import ParameterType, WorkflowParameterType
from skyvern.schemas.runs import ProxyLocation
from skyvern.schemas.workflows import WorkflowCreateYAMLRequest as WorkflowCreateYAMLRequestSchema
+from skyvern.utils.yaml_loader import safe_load_no_dates
from ._common import ErrorCode, Timer, make_error, make_result
from ._session import get_skyvern
@@ -526,7 +527,7 @@ def _load_definition_dict(definition: str, fmt: str) -> tuple[dict[str, Any] | N
if fmt == "yaml":
try:
- return _as_dict(yaml.safe_load(definition), "yaml")
+ return _as_dict(safe_load_no_dates(definition), "yaml")
except yaml.YAMLError:
return None, None
@@ -534,7 +535,7 @@ def _load_definition_dict(definition: str, fmt: str) -> tuple[dict[str, Any] | N
return _as_dict(json.loads(definition), "json")
except (json.JSONDecodeError, TypeError):
try:
- return _as_dict(yaml.safe_load(definition), "yaml")
+ return _as_dict(safe_load_no_dates(definition), "yaml")
except yaml.YAMLError:
return None, None
diff --git a/skyvern/forge/failure_classifier.py b/skyvern/forge/failure_classifier.py
index c06059d66..c4b6030fa 100644
--- a/skyvern/forge/failure_classifier.py
+++ b/skyvern/forge/failure_classifier.py
@@ -16,11 +16,12 @@ def classify_from_failure_reason(
where the absence of a classification may simply mean the termination was
user-guided / expected.
- Categories:
+ Categories (16):
ANTI_BOT_DETECTION, PROXY_ERROR, BROWSER_ERROR, NAVIGATION_FAILURE,
PAGE_LOAD_TIMEOUT, AUTH_FAILURE, LLM_ERROR, CREDENTIAL_ERROR,
DATA_EXTRACTION_FAILURE, ELEMENT_NOT_FOUND, WRONG_PAGE_STATE,
- MAX_STEPS_EXCEEDED, INFRASTRUCTURE_ERROR, UNKNOWN
+ MAX_STEPS_EXCEEDED, LLM_REASONING_ERROR, INFRASTRUCTURE_ERROR,
+ PARAMETER_BINDING_ERROR, UNKNOWN
"""
if not failure_reason and not exception:
return None
@@ -191,6 +192,21 @@ def classify_from_failure_reason(
}
)
+ # Internal configuration mismatch — not a site/selector failure.
+ _param_binding_keywords = [
+ "should have already been set through workflow run parameters",
+ "should have already been set through workflow run context init",
+ "pre-run invariant: workflow_definition and persisted parameter rows disagree",
+ ]
+ if any(kw in reason for kw in _param_binding_keywords):
+ categories.append(
+ {
+ "category": "PARAMETER_BINDING_ERROR",
+ "confidence_float": 0.95,
+ "reasoning": "Keywords matched",
+ }
+ )
+
if not categories:
if fallback_to_unknown:
return [{"category": "UNKNOWN", "confidence_float": 0.5, "reasoning": "No keyword match found"}]
diff --git a/skyvern/forge/request_logging.py b/skyvern/forge/request_logging.py
index eec1812b8..059ca5380 100644
--- a/skyvern/forge/request_logging.py
+++ b/skyvern/forge/request_logging.py
@@ -23,6 +23,9 @@ _SENSITIVE_ENDPOINTS = {
"POST /v1/credentials",
"POST /v1/credentials/onepassword/create",
"POST /v1/credentials/azure_credential/create",
+ "POST /v1/credentials/totp",
+ "POST /api/v1/totp",
+ "GET /v1/credentials/totp",
}
_MAX_BODY_LENGTH = 1000
_MAX_RESPONSE_READ_BYTES = 1024 * 1024 # 1 MB — skip logging bodies larger than this
@@ -47,6 +50,11 @@ _SENSITIVE_FIELDS: set[str] = {
"auth",
"authorization",
"secret_key",
+ "totp",
+ "otp",
+ "one_time_code",
+ "one_time_password",
+ "mfa_code",
}
diff --git a/skyvern/forge/sdk/cache/extraction_shadow.py b/skyvern/forge/sdk/cache/extraction_shadow.py
index 4c49dde2e..a3a019284 100644
--- a/skyvern/forge/sdk/cache/extraction_shadow.py
+++ b/skyvern/forge/sdk/cache/extraction_shadow.py
@@ -9,7 +9,6 @@ errors are swallowed — shadow is best-effort.
from __future__ import annotations
import asyncio
-import json
import re
import time
from collections import Counter
@@ -99,9 +98,9 @@ def _collect_unique_item_paths(
seen_refs = frozenset()
ref = schema.get("$ref")
- if isinstance(ref, str):
- if ref in seen_refs:
- return paths
+ if isinstance(ref, str) and ref not in seen_refs:
+ # Skip re-expanding a ref already in the current path (cycle guard),
+ # but fall through to scan any sibling keywords on this node.
resolved = _resolve_ref(ref, root)
if resolved is not None:
paths.update(_collect_unique_item_paths(resolved, prefix, root=root, seen_refs=seen_refs | {ref}))
@@ -128,10 +127,6 @@ def _collect_unique_item_paths(
return paths
-def _json_key(value: Any) -> str:
- return json.dumps(value, sort_keys=True, default=str)
-
-
def _canonical_form(value: Any, *, unique_item_paths: set[str], prefix: str) -> Any:
"""Hashable canonical form for set-equality inside ``uniqueItems`` comparisons.
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index bec0d8419..067e111d5 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -4377,6 +4377,8 @@ async def extract_information_for_navigation_goal(
cache_key=cache_key,
workflow_run_id=task.workflow_run_id,
cached_value=lookup_result.value,
+ # -1.0 sentinel marks "age unknown" so it's distinguishable in
+ # Datadog from "just-cached (0.0)".
cached_age_seconds=lookup_result.age_seconds if lookup_result.age_seconds is not None else -1.0,
llm_call=_shadow_llm_call,
schema=shadow_schema,
diff --git a/tests/unit/test_extraction_shadow.py b/tests/unit/test_extraction_shadow.py
index 42daeb0b0..ab986cd17 100644
--- a/tests/unit/test_extraction_shadow.py
+++ b/tests/unit/test_extraction_shadow.py
@@ -300,6 +300,35 @@ def test_compare_semantic_unique_items_behind_ref() -> None:
assert result.match is True
+def test_compare_semantic_unique_items_ref_cycle_does_not_skip_siblings() -> None:
+ """Hitting a cycle must not short-circuit the rest of the current node's keys.
+
+ Regression guard: the cycle check used to `return` early, which dropped
+ sibling traversal (properties/items/combinators) on any node that
+ contained a `$ref` already in the current expansion path.
+ """
+ # Both the outer "ids" ref and the sibling uniqueItems must be picked up.
+ schema = {
+ "$defs": {
+ "Container": {
+ "type": "object",
+ "properties": {
+ # Self-referential: Container.parent → Container
+ "parent": {"$ref": "#/$defs/Container"},
+ # Sibling that depends on cycle-guard allowing traversal.
+ "ids": {"type": "array", "uniqueItems": True, "items": {"type": "integer"}},
+ },
+ },
+ },
+ "$ref": "#/$defs/Container",
+ }
+ # Reorder-only diff at ids — must match because uniqueItems is detected.
+ cached = {"ids": [1, 2, 3], "parent": {"ids": [1, 2, 3]}}
+ fresh = {"ids": [3, 2, 1], "parent": {"ids": [1, 2, 3]}}
+ result = extraction_shadow.compare_results(cached, fresh, schema=schema)
+ assert result.match is True
+
+
def test_compare_semantic_unique_items_ref_circular_safe() -> None:
"""Circular $ref must not cause infinite recursion in the collector."""
schema = {
@@ -774,8 +803,8 @@ async def test_schedule_shadow_check_skips_when_gate_returns_false() -> None:
logger=captured,
)
await task
- # One debug log confirming the gate evaluated to False — useful for
- # verifying the sampling rate in production without running the shadow LLM.
+ # One info log confirming the gate evaluated to False — used as the
+ # sampling-rate denominator (status:skipped) alongside status:ok/error.
assert len(captured.calls) == 1
event, fields = captured.calls[0]
assert event == "extract_information.shadow_comparison"
diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.py
index 1064cdfa2..230de1278 100644
--- a/tests/unit/test_failure_classifier.py
+++ b/tests/unit/test_failure_classifier.py
@@ -310,6 +310,22 @@ class TestLLMReasoningError:
assert "LLM_REASONING_ERROR" in categories
+class TestParameterBindingError:
+ @pytest.mark.parametrize(
+ "reason",
+ [
+ "Value should have already been set through workflow run parameters",
+ "Context should have already been set through workflow run context init",
+ "Pre-run invariant: workflow_definition and persisted parameter rows disagree",
+ ],
+ )
+ def test_keywords_match(self, reason: str) -> None:
+ result = classify_from_failure_reason(reason)
+ assert result is not None
+ categories = [r["category"] for r in result]
+ assert "PARAMETER_BINDING_ERROR" in categories
+
+
class TestMultipleCategories:
def test_captcha_and_max_steps(self) -> None:
"""A max_steps failure caused by captcha should return both categories."""
diff --git a/tests/unit/test_request_logging.py b/tests/unit/test_request_logging.py
index 4385c9eb0..1febf41bb 100644
--- a/tests/unit/test_request_logging.py
+++ b/tests/unit/test_request_logging.py
@@ -12,6 +12,7 @@ from skyvern.forge.request_logging import (
_is_loggable_content_type,
_is_sensitive_key,
_redact_sensitive_fields,
+ _sanitize_body,
_sanitize_response_body,
)
@@ -44,6 +45,12 @@ class TestIsSensitiveKey:
"auth",
"authorization",
"secret_key",
+ "totp",
+ "TOTP",
+ "otp",
+ "one_time_code",
+ "one_time_password",
+ "mfa_code",
],
)
def test_sensitive_keys_are_redacted(self, key: str) -> None:
@@ -117,6 +124,18 @@ class TestRedactSensitiveFields:
for key in data:
assert result[key] == _REDACTED, f"Expected {key} to be redacted"
+ def test_redacts_totp_and_otp_fields(self) -> None:
+ data = {
+ "totp": "123456",
+ "otp": "999999",
+ "one_time_code": "abc123",
+ "one_time_password": "xyz789",
+ "mfa_code": "mfa42",
+ }
+ result = _redact_sensitive_fields(data)
+ for key in data:
+ assert result[key] == _REDACTED, f"Expected {key} to be redacted"
+
def test_preserves_non_sensitive_suffixed_keys(self) -> None:
"""Fields like credential_id and page_token must NOT be redacted."""
data = {
@@ -257,3 +276,49 @@ class TestSanitizeResponseBody:
request = _make_request("POST", "/api/v1/credentials/")
result = _sanitize_response_body(request, '{"data": "value"}', "application/json")
assert result == _REDACTED
+
+ @pytest.mark.parametrize(
+ ("method", "path"),
+ [
+ ("POST", "/v1/credentials/totp"),
+ ("POST", "/v1/credentials/totp/"),
+ ("POST", "/api/v1/totp"),
+ ("POST", "/api/v1/totp/"),
+ ("GET", "/v1/credentials/totp"),
+ ("GET", "/v1/credentials/totp/"),
+ ],
+ )
+ def test_totp_endpoints_response_redacted(self, method: str, path: str) -> None:
+ request = _make_request(method, path)
+ body = json.dumps({"code": "123456", "content": "Your code is 123456"})
+ result = _sanitize_response_body(request, body, "application/json")
+ assert result == _REDACTED
+
+
+class TestSanitizeBody:
+ def test_sensitive_endpoint_request_fully_redacted(self) -> None:
+ request = _make_request("POST", "/v1/credentials")
+ result = _sanitize_body(request, b'{"password": "hunter2"}', "application/json")
+ assert result == _REDACTED
+
+ def test_non_sensitive_endpoint_request_preserved(self) -> None:
+ request = _make_request("GET", "/v1/tasks")
+ result = _sanitize_body(request, b'{"user": "alice"}', "application/json")
+ assert result == '{"user": "alice"}'
+
+ @pytest.mark.parametrize(
+ ("method", "path"),
+ [
+ ("POST", "/v1/credentials/totp"),
+ ("POST", "/v1/credentials/totp/"),
+ ("POST", "/api/v1/totp"),
+ ("POST", "/api/v1/totp/"),
+ ("GET", "/v1/credentials/totp"),
+ ("GET", "/v1/credentials/totp/"),
+ ],
+ )
+ def test_totp_endpoints_request_redacted(self, method: str, path: str) -> None:
+ request = _make_request(method, path)
+ body = b'{"totp_identifier": "[email protected]", "content": "Your code is 123456"}'
+ result = _sanitize_body(request, body, "application/json")
+ assert result == _REDACTED