Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(extraction-shadow): follow-up to SKY-8871 — cycle, dead code, comment nits (#5486)
Aaron Perezff198cb6f588
diff --git a/skyvern-frontend/src/api/sse.test.ts b/skyvern-frontend/src/api/sse.test.tsnew file mode 100644index 000000000..a85f6fda7--- /dev/null+++ b/skyvern-frontend/src/api/sse.test.ts@@ -0,0 +1,36 @@+import { describe, it, expect, vi } from "vitest";+import type { FetchEventSourceInit } from "@microsoft/fetch-event-source";++const mockFetchEventSource = vi.fn(+ (_url: string, opts: FetchEventSourceInit) => {+ // Send a message so the inner promise can settle+ opts.onmessage?.({+ id: "",+ event: "done",+ data: '{"done":true}',+ retry: undefined,+ });+ return Promise.resolve();+ },+);++vi.mock("@microsoft/fetch-event-source", () => ({+ fetchEventSource: mockFetchEventSource,+}));++const { fetchStreamingSse } = await import("./sse");++describe("fetchStreamingSse", () => {+ it("passes openWhenHidden: true to fetchEventSource", async () => {+ await fetchStreamingSse(+ "http://localhost/test",+ { method: "POST", headers: {}, body: "{}" },+ () => true, // return true to resolve the promise on first message+ );++ expect(mockFetchEventSource).toHaveBeenCalledWith(+ "http://localhost/test",+ expect.objectContaining({ openWhenHidden: true }),+ );+ });+});diff --git a/skyvern-frontend/src/api/sse.ts b/skyvern-frontend/src/api/sse.tsindex 2081a705c..57b1e8031 100644--- a/skyvern-frontend/src/api/sse.ts+++ b/skyvern-frontend/src/api/sse.ts@@ -66,6 +66,10 @@ export async function fetchStreamingSse<T>(headers: init.headers as Record<string, string>,body: init.body,signal: controller.signal,+ // Keep the SSE connection alive when the browser tab is hidden.+ // Without this, the library closes and reconnects on tab focus,+ // which restarts the copilot agent from scratch.+ openWhenHidden: true,onmessage: (event) => {if (!event.data || !event.data.trim()) {return;diff --git a/skyvern-frontend/src/util/env.ts b/skyvern-frontend/src/util/env.tsindex e27d18d1a..91a80506b 100644--- a/skyvern-frontend/src/util/env.ts+++ b/skyvern-frontend/src/util/env.ts@@ -43,7 +43,7 @@ try {}newWssBaseUrl = url.toString();} catch (e) {- newWssBaseUrl = wssBaseUrl.replace("/api", "");+ newWssBaseUrl = wssBaseUrl?.replace("/api", "") ?? "";}// Base URL for the Runs API (strip a leading `/api` segment: /api/v1 -> /v1)@@ -55,7 +55,7 @@ const runsApiBaseUrl = (() => {}return `${url.origin}${url.pathname}`;} catch (e) {- return apiBaseUrl.replace("/api", "");+ return apiBaseUrl?.replace("/api", "") ?? "";}})();diff --git a/skyvern/cli/mcp_tools/workflow.py b/skyvern/cli/mcp_tools/workflow.pyindex 18fec0fa7..116bb159c 100644--- a/skyvern/cli/mcp_tools/workflow.py+++ b/skyvern/cli/mcp_tools/workflow.py@@ -22,6 +22,7 @@ from skyvern.client.types import WorkflowCreateYamlRequestfrom skyvern.forge.sdk.workflow.models.parameter import ParameterType, WorkflowParameterTypefrom skyvern.schemas.runs import ProxyLocationfrom skyvern.schemas.workflows import WorkflowCreateYAMLRequest as WorkflowCreateYAMLRequestSchema+from skyvern.utils.yaml_loader import safe_load_no_datesfrom ._common import ErrorCode, Timer, make_error, make_resultfrom ._session import get_skyvern@@ -526,7 +527,7 @@ def _load_definition_dict(definition: str, fmt: str) -> tuple[dict[str, Any] | Nif fmt == "yaml":try:- return _as_dict(yaml.safe_load(definition), "yaml")+ return _as_dict(safe_load_no_dates(definition), "yaml")except yaml.YAMLError:return None, None@@ -534,7 +535,7 @@ def _load_definition_dict(definition: str, fmt: str) -> tuple[dict[str, Any] | Nreturn _as_dict(json.loads(definition), "json")except (json.JSONDecodeError, TypeError):try:- return _as_dict(yaml.safe_load(definition), "yaml")+ return _as_dict(safe_load_no_dates(definition), "yaml")except yaml.YAMLError:return None, Nonediff --git a/skyvern/forge/failure_classifier.py b/skyvern/forge/failure_classifier.pyindex c06059d66..c4b6030fa 100644--- a/skyvern/forge/failure_classifier.py+++ b/skyvern/forge/failure_classifier.py@@ -16,11 +16,12 @@ def classify_from_failure_reason(where the absence of a classification may simply mean the termination wasuser-guided / expected.- Categories:+ Categories (16):ANTI_BOT_DETECTION, PROXY_ERROR, BROWSER_ERROR, NAVIGATION_FAILURE,PAGE_LOAD_TIMEOUT, AUTH_FAILURE, LLM_ERROR, CREDENTIAL_ERROR,DATA_EXTRACTION_FAILURE, ELEMENT_NOT_FOUND, WRONG_PAGE_STATE,- MAX_STEPS_EXCEEDED, INFRASTRUCTURE_ERROR, UNKNOWN+ MAX_STEPS_EXCEEDED, LLM_REASONING_ERROR, INFRASTRUCTURE_ERROR,+ PARAMETER_BINDING_ERROR, UNKNOWN"""if not failure_reason and not exception:return None@@ -191,6 +192,21 @@ def classify_from_failure_reason(})+ # Internal configuration mismatch — not a site/selector failure.+ _param_binding_keywords = [+ "should have already been set through workflow run parameters",+ "should have already been set through workflow run context init",+ "pre-run invariant: workflow_definition and persisted parameter rows disagree",+ ]+ if any(kw in reason for kw in _param_binding_keywords):+ categories.append(+ {+ "category": "PARAMETER_BINDING_ERROR",+ "confidence_float": 0.95,+ "reasoning": "Keywords matched",+ }+ )+if not categories:if fallback_to_unknown:return [{"category": "UNKNOWN", "confidence_float": 0.5, "reasoning": "No keyword match found"}]diff --git a/skyvern/forge/request_logging.py b/skyvern/forge/request_logging.pyindex eec1812b8..059ca5380 100644--- a/skyvern/forge/request_logging.py+++ b/skyvern/forge/request_logging.py@@ -23,6 +23,9 @@ _SENSITIVE_ENDPOINTS = {"POST /v1/credentials","POST /v1/credentials/onepassword/create","POST /v1/credentials/azure_credential/create",+ "POST /v1/credentials/totp",+ "POST /api/v1/totp",+ "GET /v1/credentials/totp",}_MAX_BODY_LENGTH = 1000_MAX_RESPONSE_READ_BYTES = 1024 * 1024 # 1 MB — skip logging bodies larger than this@@ -47,6 +50,11 @@ _SENSITIVE_FIELDS: set[str] = {"auth","authorization","secret_key",+ "totp",+ "otp",+ "one_time_code",+ "one_time_password",+ "mfa_code",}diff --git a/skyvern/forge/sdk/cache/extraction_shadow.py b/skyvern/forge/sdk/cache/extraction_shadow.pyindex 4c49dde2e..a3a019284 100644--- a/skyvern/forge/sdk/cache/extraction_shadow.py+++ b/skyvern/forge/sdk/cache/extraction_shadow.py@@ -9,7 +9,6 @@ errors are swallowed — shadow is best-effort.from __future__ import annotationsimport asyncio-import jsonimport reimport timefrom collections import Counter@@ -99,9 +98,9 @@ def _collect_unique_item_paths(seen_refs = frozenset()ref = schema.get("$ref")- if isinstance(ref, str):- if ref in seen_refs:- return paths+ if isinstance(ref, str) and ref not in seen_refs:+ # Skip re-expanding a ref already in the current path (cycle guard),+ # but fall through to scan any sibling keywords on this node.resolved = _resolve_ref(ref, root)if resolved is not None:paths.update(_collect_unique_item_paths(resolved, prefix, root=root, seen_refs=seen_refs | {ref}))@@ -128,10 +127,6 @@ def _collect_unique_item_paths(return paths-def _json_key(value: Any) -> str:- return json.dumps(value, sort_keys=True, default=str)--def _canonical_form(value: Any, *, unique_item_paths: set[str], prefix: str) -> Any:"""Hashable canonical form for set-equality inside ``uniqueItems`` comparisons.diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex bec0d8419..067e111d5 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -4377,6 +4377,8 @@ async def extract_information_for_navigation_goal(cache_key=cache_key,workflow_run_id=task.workflow_run_id,cached_value=lookup_result.value,+ # -1.0 sentinel marks "age unknown" so it's distinguishable in+ # Datadog from "just-cached (0.0)".cached_age_seconds=lookup_result.age_seconds if lookup_result.age_seconds is not None else -1.0,llm_call=_shadow_llm_call,schema=shadow_schema,diff --git a/tests/unit/test_extraction_shadow.py b/tests/unit/test_extraction_shadow.pyindex 42daeb0b0..ab986cd17 100644--- a/tests/unit/test_extraction_shadow.py+++ b/tests/unit/test_extraction_shadow.py@@ -300,6 +300,35 @@ def test_compare_semantic_unique_items_behind_ref() -> None:assert result.match is True+def test_compare_semantic_unique_items_ref_cycle_does_not_skip_siblings() -> None:+ """Hitting a cycle must not short-circuit the rest of the current node's keys.++ Regression guard: the cycle check used to `return` early, which dropped+ sibling traversal (properties/items/combinators) on any node that+ contained a `$ref` already in the current expansion path.+ """+ # Both the outer "ids" ref and the sibling uniqueItems must be picked up.+ schema = {+ "$defs": {+ "Container": {+ "type": "object",+ "properties": {+ # Self-referential: Container.parent → Container+ "parent": {"$ref": "#/$defs/Container"},+ # Sibling that depends on cycle-guard allowing traversal.+ "ids": {"type": "array", "uniqueItems": True, "items": {"type": "integer"}},+ },+ },+ },+ "$ref": "#/$defs/Container",+ }+ # Reorder-only diff at ids — must match because uniqueItems is detected.+ cached = {"ids": [1, 2, 3], "parent": {"ids": [1, 2, 3]}}+ fresh = {"ids": [3, 2, 1], "parent": {"ids": [1, 2, 3]}}+ result = extraction_shadow.compare_results(cached, fresh, schema=schema)+ assert result.match is True++def test_compare_semantic_unique_items_ref_circular_safe() -> None:"""Circular $ref must not cause infinite recursion in the collector."""schema = {@@ -774,8 +803,8 @@ async def test_schedule_shadow_check_skips_when_gate_returns_false() -> None:logger=captured,)await task- # One debug log confirming the gate evaluated to False — useful for- # verifying the sampling rate in production without running the shadow LLM.+ # One info log confirming the gate evaluated to False — used as the+ # sampling-rate denominator (status:skipped) alongside status:ok/error.assert len(captured.calls) == 1event, fields = captured.calls[0]assert event == "extract_information.shadow_comparison"diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.pyindex 1064cdfa2..230de1278 100644--- a/tests/unit/test_failure_classifier.py+++ b/tests/unit/test_failure_classifier.py@@ -310,6 +310,22 @@ class TestLLMReasoningError:assert "LLM_REASONING_ERROR" in categories+class TestParameterBindingError:+ @pytest.mark.parametrize(+ "reason",+ [+ "Value should have already been set through workflow run parameters",+ "Context should have already been set through workflow run context init",+ "Pre-run invariant: workflow_definition and persisted parameter rows disagree",+ ],+ )+ def test_keywords_match(self, reason: str) -> None:+ result = classify_from_failure_reason(reason)+ assert result is not None+ categories = [r["category"] for r in result]+ assert "PARAMETER_BINDING_ERROR" in categories++class TestMultipleCategories:def test_captcha_and_max_steps(self) -> None:"""A max_steps failure caused by captcha should return both categories."""diff --git a/tests/unit/test_request_logging.py b/tests/unit/test_request_logging.pyindex 4385c9eb0..1febf41bb 100644--- a/tests/unit/test_request_logging.py+++ b/tests/unit/test_request_logging.py@@ -12,6 +12,7 @@ from skyvern.forge.request_logging import (_is_loggable_content_type,_is_sensitive_key,_redact_sensitive_fields,+ _sanitize_body,_sanitize_response_body,)@@ -44,6 +45,12 @@ class TestIsSensitiveKey:"auth","authorization","secret_key",+ "totp",+ "TOTP",+ "otp",+ "one_time_code",+ "one_time_password",+ "mfa_code",],)def test_sensitive_keys_are_redacted(self, key: str) -> None:@@ -117,6 +124,18 @@ class TestRedactSensitiveFields:for key in data:assert result[key] == _REDACTED, f"Expected {key} to be redacted"+ def test_redacts_totp_and_otp_fields(self) -> None:+ data = {+ "totp": "123456",+ "otp": "999999",+ "one_time_code": "abc123",+ "one_time_password": "xyz789",+ "mfa_code": "mfa42",+ }+ result = _redact_sensitive_fields(data)+ for key in data:+ assert result[key] == _REDACTED, f"Expected {key} to be redacted"+def test_preserves_non_sensitive_suffixed_keys(self) -> None:"""Fields like credential_id and page_token must NOT be redacted."""data = {@@ -257,3 +276,49 @@ class TestSanitizeResponseBody:request = _make_request("POST", "/api/v1/credentials/")result = _sanitize_response_body(request, '{"data": "value"}', "application/json")assert result == _REDACTED++ @pytest.mark.parametrize(+ ("method", "path"),+ [+ ("POST", "/v1/credentials/totp"),+ ("POST", "/v1/credentials/totp/"),+ ("POST", "/api/v1/totp"),+ ("POST", "/api/v1/totp/"),+ ("GET", "/v1/credentials/totp"),+ ("GET", "/v1/credentials/totp/"),+ ],+ )+ def test_totp_endpoints_response_redacted(self, method: str, path: str) -> None:+ request = _make_request(method, path)+ body = json.dumps({"code": "123456", "content": "Your code is 123456"})+ result = _sanitize_response_body(request, body, "application/json")+ assert result == _REDACTED+++class TestSanitizeBody:+ def test_sensitive_endpoint_request_fully_redacted(self) -> None:+ request = _make_request("POST", "/v1/credentials")+ result = _sanitize_body(request, b'{"password": "hunter2"}', "application/json")+ assert result == _REDACTED++ def test_non_sensitive_endpoint_request_preserved(self) -> None:+ request = _make_request("GET", "/v1/tasks")+ result = _sanitize_body(request, b'{"user": "alice"}', "application/json")+ assert result == '{"user": "alice"}'++ @pytest.mark.parametrize(+ ("method", "path"),+ [+ ("POST", "/v1/credentials/totp"),+ ("POST", "/v1/credentials/totp/"),+ ("POST", "/api/v1/totp"),+ ("POST", "/api/v1/totp/"),+ ("GET", "/v1/credentials/totp"),+ ("GET", "/v1/credentials/totp/"),+ ],+ )+ def test_totp_endpoints_request_redacted(self, method: str, path: str) -> None:+ request = _make_request(method, path)+ body = b'{"totp_identifier": "[email protected]", "content": "Your code is 123456"}'+ result = _sanitize_body(request, body, "application/json")+ assert result == _REDACTED