Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-8879) copilot-stack/03: copilot utilities + openai-agents dep (#5512)
Andrew Neilsonbf9a3675040e
diff --git a/pyproject.toml b/pyproject.tomlindex 19cf9de73..670c724fc 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -81,6 +81,16 @@ dependencies = ["aiosqlite>=0.21.0,<0.23",]+# Optional extras gate capabilities that are dormant in this stack. The+# copilot extra pulls openai-agents, required once the copilot runtime is+# wired up in the activation PR. Until then, any environment that imports+# skyvern.forge.sdk.copilot (tests, dev activation) should install+# `skyvern[copilot]` instead of plain `skyvern`.+[project.optional-dependencies]+copilot = [+ "openai-agents>=0.13.4,<0.14",+]+[dependency-groups]cloud = ["stripe>=9.7.0,<10",diff --git a/skyvern/cli/core/session_manager.py b/skyvern/cli/core/session_manager.pyindex 2b8cc316a..226f59082 100644--- a/skyvern/cli/core/session_manager.py+++ b/skyvern/cli/core/session_manager.py@@ -69,6 +69,26 @@ _current_session: ContextVar[SessionState | None] = ContextVar("mcp_session", de_global_session: SessionState | None = None_stateless_http_mode = False+# Process-wide registry for copilot browser sessions. Keyed by browser_session_id.+# This bypasses ContextVar propagation issues when FastMCP runs tool handlers+# in a separate task whose context snapshot predates scoped_session().+_copilot_sessions: dict[str, SessionState] = {}+++def register_copilot_session(session_id: str, state: SessionState) -> None:+ """Register a pre-configured browser session for cross-task lookup.++ The registry is process-local and in-memory: entries do not survive a+ process restart and are not shared across uvicorn workers. Callers that+ need cross-process continuity must reconnect via the cloud session API.+ """+ _copilot_sessions[session_id] = state+++def unregister_copilot_session(session_id: str) -> None:+ """Remove a copilot browser session from the process-local registry."""+ _copilot_sessions.pop(session_id, None)+def get_current_session() -> SessionState:global _global_session@@ -98,6 +118,20 @@ def set_current_session(state: SessionState) -> None:_current_session.set(state)+@asynccontextmanager+async def scoped_session(state: SessionState) -> AsyncIterator[None]:+ """Temporarily push a SessionState into ContextVar scope.++ Restores the previous value on exit. Does NOT touch _global_session,+ so it is safe for concurrent API-server requests.+ """+ token = _current_session.set(state)+ try:+ yield+ finally:+ _current_session.reset(token)++def set_stateless_http_mode(enabled: bool) -> None:global _stateless_http_mode_stateless_http_mode = enabled@@ -160,6 +194,19 @@ async def resolve_browser(return current.browser, current.contextactive_api_key_hash = _api_key_hash(get_active_api_key())++ # Check copilot session registry (cross-task fallback when ContextVar+ # does not propagate through FastMCP in-process transport).+ registered = _copilot_sessions.get(session_id) if session_id else None+ if (+ registered is not None+ and registered.browser is not None+ and registered.context is not None+ and registered.api_key_hash == active_api_key_hash+ ):+ _current_session.set(registered)+ return registered.browser, registered.context+browser: SkyvernBrowser | None = Nonetry:if session_id:@@ -228,6 +275,8 @@ async def close_current_session() -> None:if current.browser is not None:await current.browser.close()finally:+ if current.context and current.context.session_id:+ unregister_copilot_session(current.context.session_id)set_current_session(SessionState())diff --git a/skyvern/forge/request_logging.py b/skyvern/forge/request_logging.pyindex 059ca5380..d58a5d3d7 100644--- a/skyvern/forge/request_logging.py+++ b/skyvern/forge/request_logging.py@@ -87,7 +87,7 @@ def _is_sensitive_key(key: str) -> bool:return key.lower() in _SENSITIVE_FIELDS-def _redact_sensitive_fields(obj: typing.Any, _depth: int = 0) -> typing.Any:+def redact_sensitive_fields(obj: typing.Any, _depth: int = 0) -> typing.Any:"""Redact dict values whose *key name* exactly matches a known sensitive field.Uses exact-match (case-insensitive) rather than substring/regex to avoid@@ -101,10 +101,10 @@ def _redact_sensitive_fields(obj: typing.Any, _depth: int = 0) -> typing.Any:return objif isinstance(obj, dict):return {- k: _REDACTED if _is_sensitive_key(k) else _redact_sensitive_fields(v, _depth + 1) for k, v in obj.items()+ k: _REDACTED if _is_sensitive_key(k) else redact_sensitive_fields(v, _depth + 1) for k, v in obj.items()}if isinstance(obj, list):- return [_redact_sensitive_fields(item, _depth + 1) for item in obj]+ return [redact_sensitive_fields(item, _depth + 1) for item in obj]return obj@@ -125,7 +125,7 @@ def _sanitize_response_body(request: Request, body_str: str | None, content_typereturn _BINARY_PLACEHOLDERtry:parsed = json.loads(body_str)- redacted = _redact_sensitive_fields(parsed)+ redacted = redact_sensitive_fields(parsed)text = json.dumps(redacted)except (json.JSONDecodeError, TypeError):text = body_strdiff --git a/skyvern/forge/sdk/copilot/__init__.py b/skyvern/forge/sdk/copilot/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pynew file mode 100644index 000000000..24f7abc65--- /dev/null+++ b/skyvern/forge/sdk/copilot/context.py@@ -0,0 +1,95 @@+"""Structured context for copilot cross-turn memory."""++from __future__ import annotations++import re++from pydantic import BaseModel, Field+++class UrlVisit(BaseModel):+ url: str+ summary: str = ""+++class FieldFilled(BaseModel):+ selector: str = ""+ label: str = ""+ value: str = ""+++class CredentialCheck(BaseModel):+ credential_name: str = ""+ credential_id: str | None = None+ found: bool = False+++class StructuredContext(BaseModel):+ user_goal: str = ""+ urls_visited: list[UrlVisit] = Field(default_factory=list)+ fields_filled: list[FieldFilled] = Field(default_factory=list)+ credentials_checked: list[CredentialCheck] = Field(default_factory=list)+ decisions_made: list[str] = Field(default_factory=list)+ workflow_state: str = ""++ def to_json_str(self) -> str:+ return self.model_dump_json(indent=2)++ @classmethod+ def from_json_str(cls, raw: str | None) -> StructuredContext:+ if not raw:+ return cls()+ raw = raw.strip()+ if raw.startswith("{"):+ try:+ return cls.model_validate_json(raw)+ except Exception:+ return cls(user_goal=raw)+ return cls(user_goal=raw)++ def merge_turn_summary(self, tool_activity: list[dict]) -> None:+ for entry in tool_activity:+ tool = entry.get("tool", "")+ summary = entry.get("summary", "")++ if tool == "navigate_browser":+ url = summary.removeprefix("Navigated to ").strip()+ if url and not any(v.url == url for v in self.urls_visited):+ self.urls_visited.append(UrlVisit(url=url, summary=""))++ elif tool == "list_credentials":+ match = re.search(r"Found (\d+)", summary)+ found = int(match.group(1)) > 0 if match else False+ self.credentials_checked.append(CredentialCheck(credential_name=summary, found=found))++ elif tool == "type_text":+ parts = summary.split("into ")+ selector = parts[-1].strip("'\"") if len(parts) > 1 else ""+ # Intentionally omit value: typed text may contain PII / credentials.+ self.fields_filled.append(FieldFilled(selector=selector, label=selector))++ elif tool == "update_workflow":+ self.workflow_state = summary++ elif tool in ("click", "evaluate", "run_blocks_and_collect_debug", "get_run_results"):+ self.decisions_made.append(f"{tool}: {summary}")++ elif tool == "get_browser_screenshot":+ if "(" in summary and ")" in summary:+ url = summary.split("(", 1)[1].rsplit(")", 1)[0]+ if url and not any(v.url == url for v in self.urls_visited):+ self.urls_visited.append(UrlVisit(url=url, summary="screenshot"))++ output = entry.get("output_preview")+ if output and tool in ("run_blocks_and_collect_debug", "get_run_results"):+ preview = output[:300] if len(output) > 300 else output+ self.decisions_made.append(f" output: {preview}")++ if len(self.decisions_made) > 20:+ self.decisions_made = self.decisions_made[-15:]+ if len(self.urls_visited) > 50:+ self.urls_visited = self.urls_visited[-40:]+ if len(self.fields_filled) > 50:+ self.fields_filled = self.fields_filled[-40:]+ if len(self.credentials_checked) > 50:+ self.credentials_checked = self.credentials_checked[-40:]diff --git a/skyvern/forge/sdk/copilot/exceptions.py b/skyvern/forge/sdk/copilot/exceptions.pynew file mode 100644index 000000000..470ddcea7--- /dev/null+++ b/skyvern/forge/sdk/copilot/exceptions.py@@ -0,0 +1,7 @@+from __future__ import annotations++from skyvern.exceptions import SkyvernException+++class CopilotClientDisconnectedError(SkyvernException):+ """Raised when the SSE client disconnects during a streaming copilot run."""diff --git a/skyvern/forge/sdk/copilot/failure_tracking.py b/skyvern/forge/sdk/copilot/failure_tracking.pynew file mode 100644index 000000000..9d007810e--- /dev/null+++ b/skyvern/forge/sdk/copilot/failure_tracking.py@@ -0,0 +1,234 @@+"""Repeated-failure tracking for the copilot agent loop.++Computes two normalized keys per run:++- **failure signature**: "is this the same failure as last time?" —+ frontier_start_label + normalized failure reason + top failure category ++ suspicious-success flag.+- **frontier fingerprint**: SHA256 of the executed blocks' canonical config.+ Changes whenever the agent edits any block in the executed suffix.++A streak counter increments only when BOTH keys repeat. It resets on:+- a meaningful-data success+- a different frontier fingerprint+- a different failure signature++Enforcement uses the streak count to escalate nudges (see ``enforcement.py``).+This module does not itself decide when to stop the loop — it only maintains+the state enforcement reads.+"""++from __future__ import annotations++import hashlib+import json+from typing import TYPE_CHECKING, Any++if TYPE_CHECKING:+ from skyvern.forge.sdk.copilot.agent import CopilotContext+ from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition++_FAILURE_REASON_MAX_CHARS = 200+++def _normalize_failure_reason(raw: str | None) -> str:+ if not raw:+ return ""+ collapsed = " ".join(raw.split())+ if len(collapsed) > _FAILURE_REASON_MAX_CHARS:+ collapsed = collapsed[:_FAILURE_REASON_MAX_CHARS]+ return collapsed.lower()+++def _top_failure_category(failure_categories: list[dict] | None) -> str:+ if not failure_categories:+ return ""+ first = failure_categories[0]+ if isinstance(first, dict):+ return str(first.get("category") or "")+ return ""+++def compute_failure_signature(+ frontier_start_label: str | None,+ failure_reason: str | None,+ failure_categories: list[dict] | None,+ suspicious_success: bool,+) -> str | None:+ """Return a normalized signature for the current failure, or ``None`` on success.++ ``None`` means "no signature — this was a real success". A suspicious-success+ run (status=completed but data-producing blocks produced no output) still+ generates a signature so repeated no-data runs can be counted as repeats.+ """+ normalized = _normalize_failure_reason(failure_reason)+ has_signal = bool(normalized) or suspicious_success+ if not has_signal:+ return None+ safe_label = frontier_start_label if isinstance(frontier_start_label, str) else ""+ top_category = _top_failure_category(failure_categories)+ # PARAMETER_BINDING_ERROR failure_reason embeds the offending key name, so+ # collapse it to a stable constant or repeats on different keys won't hash+ # to the same signature.+ if top_category == "PARAMETER_BINDING_ERROR":+ normalized = "parameter_binding_error"+ parts = [+ safe_label,+ normalized,+ top_category,+ "suspicious" if suspicious_success else "failed",+ ]+ return "|".join(parts)+++def _canonical_block_config(block: Any) -> dict[str, Any]:+ dump = getattr(block, "model_dump", None)+ if callable(dump):+ try:+ return dump(mode="json", exclude_none=True)+ except TypeError:+ return dump()+ if isinstance(block, dict):+ return block+ return {"repr": repr(block)}+++def compute_frontier_fingerprint(+ executed_labels: list[str],+ workflow_definition: WorkflowDefinition | None,+) -> str:+ """SHA256 of the executed blocks' canonical config, joined by label order.++ The fingerprint changes whenever a block's material config changes or the+ executed suffix itself changes. Returns an empty string when the workflow+ definition is missing — the caller treats "" as "can't fingerprint" and+ does not increment the streak on that run.+ """+ if not executed_labels or workflow_definition is None:+ return ""+ by_label: dict[str, Any] = {}+ blocks = getattr(workflow_definition, "blocks", None) or []+ for block in blocks:+ label = getattr(block, "label", None)+ if isinstance(label, str):+ by_label[label] = block+ payload: list[dict[str, Any]] = []+ for label in executed_labels:+ block = by_label.get(label)+ if block is None:+ payload.append({"label": label, "missing": True})+ continue+ config = _canonical_block_config(block)+ config.pop("output_parameter", None)+ payload.append({"label": label, "config": config})+ try:+ serialized = json.dumps(payload, sort_keys=True, default=str, separators=(",", ":"))+ except (TypeError, ValueError):+ serialized = repr(payload)+ return hashlib.sha256(serialized.encode("utf-8")).hexdigest()+++def _has_meaningful_success(result: dict[str, Any], suspicious_success: bool) -> bool:+ """Successful run AND the suffix produced meaningful data (not suspicious)."""+ return bool(result.get("ok")) and not suspicious_success+++def update_repeated_failure_state(+ ctx: CopilotContext,+ result: dict[str, Any],+) -> None:+ """Update ``repeated_failure_streak_count`` and related fields on ``ctx``.++ Called after ``_record_run_blocks_result`` has populated the fail-mode+ fields (``last_test_suspicious_success``, ``last_test_anti_bot``,+ ``last_test_failure_reason``) and ``_run_blocks_and_collect_debug`` has+ set ``last_executed_block_labels`` and ``last_frontier_start_label``.+ """+ data = result.get("data") if isinstance(result, dict) else None+ failure_categories = None+ if isinstance(data, dict):+ raw_cats = data.get("failure_categories")+ if isinstance(raw_cats, list):+ failure_categories = raw_cats++ suspicious_success_raw = getattr(ctx, "last_test_suspicious_success", False)+ suspicious_success = bool(suspicious_success_raw) if isinstance(suspicious_success_raw, (bool, int)) else False+ failure_reason_raw = getattr(ctx, "last_test_failure_reason", None)+ failure_reason = failure_reason_raw if isinstance(failure_reason_raw, str) else None+ frontier_start_raw = getattr(ctx, "last_frontier_start_label", None)+ frontier_start_label = frontier_start_raw if isinstance(frontier_start_raw, str) else None+ executed_labels_raw = getattr(ctx, "last_executed_block_labels", None)+ executed_labels = (+ [label for label in executed_labels_raw if isinstance(label, str)]+ if isinstance(executed_labels_raw, list)+ else []+ )+ workflow_definition = None+ last_workflow = getattr(ctx, "last_workflow", None)+ if last_workflow is not None:+ candidate = getattr(last_workflow, "workflow_definition", None)+ if candidate is not None and hasattr(candidate, "blocks"):+ workflow_definition = candidate++ new_action_fingerprint_raw = getattr(ctx, "pending_action_sequence_fingerprint", None)+ new_action_fingerprint = new_action_fingerprint_raw if isinstance(new_action_fingerprint_raw, str) else None+ prior_action_fingerprint_raw = getattr(ctx, "last_action_sequence_fingerprint", None)+ prior_action_fingerprint = prior_action_fingerprint_raw if isinstance(prior_action_fingerprint_raw, str) else None++ if _has_meaningful_success(result, suspicious_success):+ ctx.last_failure_signature = None+ ctx.last_frontier_fingerprint = compute_frontier_fingerprint(executed_labels, workflow_definition)+ ctx.repeated_failure_streak_count = 0+ ctx.repeated_failure_nudge_emitted_at_streak = 0+ # Success resets the action-sequence streak. Promote the pending+ # fingerprint so the next failure run can compare against it.+ ctx.last_action_sequence_fingerprint = new_action_fingerprint+ ctx.pending_action_sequence_fingerprint = None+ ctx.repeated_action_fingerprint_streak_count = 0+ return++ signature = compute_failure_signature(+ frontier_start_label=frontier_start_label,+ failure_reason=failure_reason,+ failure_categories=failure_categories,+ suspicious_success=suspicious_success,+ )+ fingerprint = compute_frontier_fingerprint(executed_labels, workflow_definition)++ # Action-sequence streak runs independently of the frontier streak: a+ # repeated action sequence can fire even when the failure-reason text+ # changes turn to turn (e.g. different validation messages).+ if new_action_fingerprint is not None and new_action_fingerprint == prior_action_fingerprint:+ prior_action_streak_raw = getattr(ctx, "repeated_action_fingerprint_streak_count", 0)+ prior_action_streak = prior_action_streak_raw if isinstance(prior_action_streak_raw, int) else 0+ ctx.repeated_action_fingerprint_streak_count = prior_action_streak + 1+ elif new_action_fingerprint is not None:+ ctx.repeated_action_fingerprint_streak_count = 1+ else:+ # No action trace on this run (e.g. all blocks succeeded or no failed+ # blocks had a task_id). Don't reset — a transient empty trace between+ # two repeats shouldn't erase an in-progress streak.+ pass+ ctx.last_action_sequence_fingerprint = new_action_fingerprint+ ctx.pending_action_sequence_fingerprint = None++ if not signature or not fingerprint:+ ctx.last_failure_signature = signature+ ctx.last_frontier_fingerprint = fingerprint+ return++ prior_signature_raw = getattr(ctx, "last_failure_signature", None)+ prior_signature = prior_signature_raw if isinstance(prior_signature_raw, str) else None+ prior_fingerprint_raw = getattr(ctx, "last_frontier_fingerprint", None)+ prior_fingerprint = prior_fingerprint_raw if isinstance(prior_fingerprint_raw, str) else None+ if signature == prior_signature and fingerprint == prior_fingerprint:+ prior_streak_raw = getattr(ctx, "repeated_failure_streak_count", 0)+ prior_streak = prior_streak_raw if isinstance(prior_streak_raw, int) else 0+ ctx.repeated_failure_streak_count = prior_streak + 1+ else:+ ctx.repeated_failure_streak_count = 1+ # New frontier/signature restarts the nudge escalation cycle.+ ctx.repeated_failure_nudge_emitted_at_streak = 0++ ctx.last_failure_signature = signature+ ctx.last_frontier_fingerprint = fingerprintdiff --git a/skyvern/forge/sdk/copilot/loop_detection.py b/skyvern/forge/sdk/copilot/loop_detection.pynew file mode 100644index 000000000..57b770c4d--- /dev/null+++ b/skyvern/forge/sdk/copilot/loop_detection.py@@ -0,0 +1,34 @@+"""Shared loop detection utilities for copilot tool dispatch.++Detects consecutive same-tool streaks (e.g., A-A-A). Does not detect+oscillating patterns (e.g., A-B-A-B) — those are left for higher-layer+enforcement to catch.+"""++from __future__ import annotations++MAX_CONSECUTIVE_SAME_TOOL = 3+++def detect_tool_loop(+ tracker: list[str],+ tool_name: str,+ threshold: int = MAX_CONSECUTIVE_SAME_TOOL,+) -> str | None:+ """Track tool invocation order and return a loop error message when threshold is hit."""+ tracker.append(tool_name)++ if len(tracker) >= threshold and len(set(tracker[-threshold:])) == 1:+ tracker.clear()+ return (+ f"LOOP DETECTED: '{tool_name}' has been called "+ f"{threshold} times consecutively. "+ "This tool will not run again. Use a DIFFERENT tool "+ "to continue, or produce your final JSON response."+ )++ if len(tracker) >= 2 and tracker[-1] != tracker[-2]:+ tracker.clear()+ tracker.append(tool_name)++ return Nonediff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pynew file mode 100644index 000000000..333a64e55--- /dev/null+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -0,0 +1,216 @@+"""Shared output formatting helpers for copilot."""++from __future__ import annotations++import base64+import binascii+import json+from typing import TYPE_CHECKING, Any++if TYPE_CHECKING:+ from agents.result import RunResultStreaming++_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"+_JPEG_PREFIX = b"\xff\xd8\xff"+++def extract_final_text(result: RunResultStreaming) -> str:+ """Pull the model's final textual output from a streamed run result."""+ if result.final_output is not None:+ if isinstance(result.final_output, str):+ return result.final_output+ if hasattr(result.final_output, "model_dump"):+ return json.dumps(result.final_output.model_dump())+ return json.dumps(result.final_output)++ for item in reversed(result.new_items):+ if hasattr(item, "output") and isinstance(item.output, list):+ for part in item.output:+ part_type = part.get("type") if isinstance(part, dict) else getattr(part, "type", None)+ if part_type == "text":+ text = part.get("text", "") if isinstance(part, dict) else getattr(part, "text", "")+ if text:+ return text+ if hasattr(item, "text") and item.text:+ return item.text+ return ""+++def parse_final_response(text: str) -> dict[str, Any]:+ """Parse the agent's final JSON envelope, stripping markdown code fences."""+ cleaned = text.strip()+ for prefix in ("```json", "```"):+ if cleaned.startswith(prefix):+ cleaned = cleaned[len(prefix) :]+ break+ if cleaned.endswith("```"):+ cleaned = cleaned[:-3]+ cleaned = cleaned.strip()++ try:+ parsed = json.loads(cleaned)+ if isinstance(parsed, dict):+ return parsed+ except json.JSONDecodeError:+ pass++ return {"type": "REPLY", "user_response": text}+++def extract_screenshot_b64(result: dict[str, Any]) -> str | None:+ """Extract screenshot_base64 from a copilot result dict, if present."""+ data = result.get("data")+ if isinstance(data, dict):+ return data.get("screenshot_base64")+ return None+++def is_valid_image_base64(value: str | None) -> bool:+ """Return True if value looks like valid base64-encoded PNG or JPEG data."""+ if not value or not isinstance(value, str) or len(value) < 100:+ return False+ try:+ header = base64.b64decode(value[:24], validate=True)+ return header[:8] == _PNG_SIGNATURE or header[:3] == _JPEG_PREFIX+ except (binascii.Error, ValueError):+ return False+++def _summarize_extracted_data(extracted: Any) -> str:+ """Summarize extracted data to prevent the LLM from echoing raw values."""+ if isinstance(extracted, list):+ if not extracted:+ return "Extracted empty list."+ if isinstance(extracted[0], dict):+ keys = sorted(extracted[0].keys())+ return f"Extracted {len(extracted)} items. Keys: {', '.join(keys)}"+ return f"Extracted list with {len(extracted)} items."+ if isinstance(extracted, dict):+ keys = sorted(extracted.keys())+ return f"Extracted object with keys: {', '.join(keys)}"+ if isinstance(extracted, str):+ return f"Extracted text ({len(extracted)} chars)."+ return "Extracted data present."+++def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict[str, Any]:+ """Strip large/binary fields from tool results before sending to the LLM."""+ sanitized = dict(result)+ for key in ("action", "browser_context", "artifacts", "timing_ms"):+ sanitized.pop(key, None)++ data = sanitized.get("data")+ if isinstance(data, dict):+ data = dict(data)+ if "screenshot_base64" in data:+ data["screenshot_base64"] = "[base64 image omitted — screenshot was taken successfully]"+ if "schema" in data and isinstance(data["schema"], dict):+ schema_str = json.dumps(data["schema"])+ # 2000 chars ~= 500 LLM tokens — enough for the model to see the+ # overall shape without consuming a meaningful slice of the prompt+ # budget. Over this, point the model at get_block_schema instead.+ if len(schema_str) > 2000:+ data["schema"] = {+ "_truncated": True,+ "message": (+ f"Schema too large ({len(schema_str)} chars). Use get_block_schema for the specific block type."+ ),+ }+ data.pop("sdk_equivalent", None)+ if tool_name == "run_blocks_and_collect_debug":+ blocks = data.get("blocks")+ if isinstance(blocks, list):+ data["blocks"] = [+ {**block, "extracted_data": _summarize_extracted_data(block["extracted_data"])}+ if isinstance(block, dict) and "extracted_data" in block+ else block+ for block in blocks+ ]+ if tool_name == "get_run_results":+ # _attach_failed_block_screenshots puts base64 bytes on each failed+ # block. They would otherwise flow straight into the LLM context as+ # raw image data — strip them while preserving the existence signal.+ blocks = data.get("blocks")+ if isinstance(blocks, list):+ data["blocks"] = [+ {**block, "screenshot_b64": "[base64 image omitted — screenshot was taken successfully]"}+ if isinstance(block, dict) and "screenshot_b64" in block+ else block+ for block in blocks+ ]+ sanitized["data"] = data+ sanitized.pop("_workflow", None)+ return sanitized+++def summarize_tool_result(tool_name: str, result: dict[str, Any]) -> str:+ """Create a brief human-readable summary of a tool result."""+ if not result.get("ok", False):+ return f"Failed: {result.get('error', 'Unknown error')[:200]}"++ data = result.get("data") or {}++ if tool_name == "update_workflow":+ return f"Workflow updated ({data.get('block_count', '?')} blocks)"+ if tool_name == "list_credentials":+ return f"Found {data.get('count', 0)} credential(s)"+ if tool_name == "get_block_schema":+ if "block_types" in data:+ return f"Listed {data.get('count', '?')} block types"+ return f"Schema for {data.get('block_type', '?')}"+ if tool_name == "validate_block":+ if data.get("valid"):+ return f"Block '{data.get('label', '?')}' is valid"+ return "Block validation failed"+ if tool_name == "run_blocks_and_collect_debug":+ if not isinstance(data, dict):+ return "Run debug completed"+ executed = data.get("executed_block_labels") or [b.get("label", "?") for b in data.get("blocks", [])]+ status = data.get("overall_status", "?")+ requested = data.get("requested_block_labels") or []+ if requested and executed and list(executed) != list(requested):+ skipped = [label for label in requested if label not in set(executed)]+ suffix = f" (skipped prefix from cache: {', '.join(skipped)})" if skipped else ""+ return f"Run {', '.join(executed)}: {status}{suffix}"+ return f"Run {', '.join(executed)}: {status}"+ if tool_name == "get_browser_screenshot":+ return f"Screenshot taken ({data.get('url', '?')[:80]})"+ if tool_name == "navigate_browser":+ url = result.get("url") or data.get("url", "?")+ return f"Navigated to {url[:80]}"+ if tool_name == "evaluate":+ result_val = data.get("result")+ preview = str(result_val)[:100] if result_val is not None else "undefined"+ return f"JS result: {preview}"+ if tool_name == "click":+ return f"Clicked '{data.get('selector', '?')}'"+ if tool_name == "type_text":+ length = data.get("typed_length") or data.get("text_length", "?")+ return f"Typed {length} chars into '{data.get('selector', '?')}'"+ if tool_name == "scroll":+ return f"Scrolled {data.get('direction', '?')}"+ if tool_name == "console_messages":+ count = data.get("count", 0)+ return f"Read {count} console message(s)"+ if tool_name == "select_option":+ return f"Selected '{data.get('value', '?')}'"+ if tool_name == "press_key":+ return f"Pressed '{data.get('key', '?')}'"+ return "OK"+++def truncate_output(output: Any, max_chars: int = 2000) -> str | None:+ if output is None:+ return None++ if isinstance(output, str):+ text = output+ else:+ try:+ text = json.dumps(output, default=str)+ except (TypeError, ValueError):+ text = str(output)++ if len(text) > max_chars:+ return text[:max_chars] + "\n... [truncated]"+ return textdiff --git a/skyvern/forge/sdk/copilot/screenshot_utils.py b/skyvern/forge/sdk/copilot/screenshot_utils.pynew file mode 100644index 000000000..fe30a699d--- /dev/null+++ b/skyvern/forge/sdk/copilot/screenshot_utils.py@@ -0,0 +1,73 @@+"""Screenshot resize/compress utilities for the copilot agent."""++from __future__ import annotations++import base64+import io+from dataclasses import dataclass+from typing import Any++import structlog+from PIL import Image++COPILOT_SCREENSHOT_MAX_WIDTH = 1024+COPILOT_SCREENSHOT_MAX_HEIGHT = 768+COPILOT_JPEG_QUALITY = 60++LOG = structlog.get_logger()+++@dataclass+class ScreenshotEntry:+ b64: str+ mime: str # "image/jpeg" or "image/png"+++def resize_screenshot_b64(b64_png: str) -> ScreenshotEntry:+ """Resize a base64 PNG to 1024x768 max and compress to JPEG.++ Uses thumbnail() to preserve aspect ratio — only shrinks, never enlarges.+ Converts RGBA/P modes to RGB for JPEG compatibility.+ """+ raw = base64.b64decode(b64_png)+ img = Image.open(io.BytesIO(raw))+ img.thumbnail(+ (COPILOT_SCREENSHOT_MAX_WIDTH, COPILOT_SCREENSHOT_MAX_HEIGHT),+ Image.Resampling.LANCZOS,+ )+ if img.mode in ("RGBA", "P"):+ img = img.convert("RGB")+ buf = io.BytesIO()+ img.save(buf, format="JPEG", quality=COPILOT_JPEG_QUALITY)+ return ScreenshotEntry(+ b64=base64.b64encode(buf.getvalue()).decode("ascii"),+ mime="image/jpeg",+ )+++def enqueue_screenshot(ctx: Any, b64_png: str) -> None:+ """Resize, compress, and store a screenshot for later LLM injection.++ Replaces any previously pending screenshot (cap to 1).+ No-op if the context does not support vision.+ """+ if not getattr(ctx, "supports_vision", False):+ return+ try:+ entry = resize_screenshot_b64(b64_png)+ except Exception:+ LOG.warning("Failed to resize copilot screenshot", exc_info=True)+ return+ pending = getattr(ctx, "pending_screenshots", None)+ if isinstance(pending, list):+ pending.clear()+ pending.append(entry)+++def enqueue_screenshot_from_result(ctx: Any, result: dict[str, Any]) -> None:+ """Extract, validate, and enqueue a screenshot from a copilot result dict."""+ from skyvern.forge.sdk.copilot.output_utils import extract_screenshot_b64, is_valid_image_base64++ screenshot_b64 = extract_screenshot_b64(result)+ if screenshot_b64 and is_valid_image_base64(screenshot_b64):+ enqueue_screenshot(ctx, screenshot_b64)diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.pynew file mode 100644index 000000000..8fea21a2b--- /dev/null+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py@@ -0,0 +1,242 @@+"""Map OpenAI Agents SDK stream events to Skyvern SSE payloads."""++from __future__ import annotations++import asyncio+import json+from typing import TYPE_CHECKING, Any++import structlog++# Reuse the HTTP-logging redactor so SSE tool inputs and request-body logs+# share one exact-match sensitive-key policy.+from skyvern.forge.request_logging import redact_sensitive_fields+from skyvern.forge.sdk.copilot.exceptions import CopilotClientDisconnectedError+from skyvern.forge.sdk.copilot.output_utils import summarize_tool_result+from skyvern.forge.sdk.schemas.workflow_copilot import (+ WorkflowCopilotStreamMessageType,+ WorkflowCopilotToolCallUpdate,+ WorkflowCopilotToolResultUpdate,+)++if TYPE_CHECKING:+ from agents.result import RunResultStreaming++ from skyvern.forge.sdk.routes.event_source_stream import EventSourceStream++LOG = structlog.get_logger()++_OBSERVATION_TOOLS = {+ "evaluate",+ "get_browser_screenshot",+ "click",+ "type_text",+ "run_blocks_and_collect_debug",+ "scroll",+ "console_messages",+ "select_option",+ "press_key",+}+++async def stream_to_sse(+ result: RunResultStreaming,+ stream: EventSourceStream,+ ctx: Any,+) -> None:+ """Consume SDK stream events and emit SSE payloads to the client.++ *ctx* is a CopilotContext object with enforcement state attributes such as+ ``update_workflow_called``, ``test_after_update_done``,+ ``post_update_nudge_count``, ``navigate_called``, and+ ``observation_after_navigate``.++ A true client disconnect -- detected by ``stream.is_disconnected()`` or a+ failed ``stream.send()`` -- cancels the agent run and raises+ ``CopilotClientDisconnectedError``. A plain ``asyncio.CancelledError``+ from some other source (task-group cancel, upstream timeout, parent+ abort) is allowed to propagate unchanged so asyncio's cancellation+ machinery runs normally; callers that want the two to share a UX path+ should catch both at the call site.+ """+ from agents.stream_events import RunItemStreamEvent++ call_id_to_name: dict[str, str] = {}+ # Counts completed tool round-trips (tool_called + tool_output pair), not+ # raw stream events. Both TOOL_CALL and TOOL_RESULT for the same round+ # carry the same iteration value; it advances after the matching result.+ iteration = 0++ try:+ async for event in result.stream_events():+ if await stream.is_disconnected():+ result.cancel()+ raise CopilotClientDisconnectedError()+ if not isinstance(event, RunItemStreamEvent):+ continue++ if event.name == "tool_called":+ raw = event.item.raw_item+ call_id = _get_raw_field(raw, "call_id") or _get_raw_field(raw, "id") or ""+ tool_name = _get_raw_field(raw, "name") or "unknown"+ call_id_to_name[call_id] = tool_name++ raw_args = _get_raw_field(raw, "arguments")+ tool_input: dict[str, Any] = {}+ if isinstance(raw_args, str):+ try:+ tool_input = json.loads(raw_args)+ except (json.JSONDecodeError, TypeError):+ tool_input = {"raw": raw_args}+ elif isinstance(raw_args, dict):+ tool_input = raw_args++ if not await stream.send(+ WorkflowCopilotToolCallUpdate(+ type=WorkflowCopilotStreamMessageType.TOOL_CALL,+ tool_name=tool_name,+ tool_input=_sanitize_input(tool_input),+ iteration=iteration,+ tool_call_id=call_id,+ )+ ):+ result.cancel()+ raise CopilotClientDisconnectedError()++ elif event.name == "tool_output":+ raw = event.item.raw_item+ call_id = _get_raw_field(raw, "call_id") or _get_raw_field(raw, "id") or ""+ tool_name = call_id_to_name.get(call_id, "unknown")++ output = getattr(event.item, "output", None)+ parsed = parse_tool_output(output)+ summary = summarize_tool_result(tool_name, parsed)+ success = parsed.get("ok", True)++ if not await stream.send(+ WorkflowCopilotToolResultUpdate(+ type=WorkflowCopilotStreamMessageType.TOOL_RESULT,+ tool_name=tool_name,+ success=success,+ summary=summary,+ iteration=iteration,+ tool_call_id=call_id,+ )+ ):+ result.cancel()+ raise CopilotClientDisconnectedError()++ _update_enforcement_from_tool(ctx, tool_name, parsed)+ iteration += 1+ except asyncio.CancelledError:+ # Don't relabel generic cancellation as a client disconnect -- the+ # inline is_disconnected / send-failure branches above already raise+ # CopilotClientDisconnectedError when there is real evidence of a+ # dropped client. Preserve cancellation semantics by re-raising so+ # asyncio's task machinery sees the cancel and any upstream+ # except Exception does NOT swallow it.+ result.cancel()+ raise+++def _get_raw_field(raw: Any, key: str) -> Any:+ if isinstance(raw, dict):+ return raw.get(key)+ return getattr(raw, key, None)+++def _extract_text_content(item: Any) -> str | None:+ """Extract text from a dict or object with type='text', returning None otherwise."""+ if isinstance(item, dict):+ if item.get("type") == "text":+ return item.get("text")+ return None+ if getattr(item, "type", None) == "text":+ return getattr(item, "text", None)+ return None+++def parse_tool_output(output: Any) -> dict[str, Any]:+ if output is None:+ return {"ok": True}++ if isinstance(output, list):+ for item in output:+ text = _extract_text_content(item)+ if text is not None:+ try:+ return _to_result_dict(json.loads(text))+ except (json.JSONDecodeError, TypeError):+ pass+ return {"ok": True, "data": str(output)}++ if isinstance(output, dict):+ if output.get("type") == "text":+ try:+ return _to_result_dict(json.loads(output["text"]))+ except (json.JSONDecodeError, TypeError, KeyError):+ return {"ok": True, "data": str(output)}+ if "ok" in output:+ return output+ return {"ok": True, "data": output}++ text = _extract_text_content(output)+ if text is not None:+ try:+ return _to_result_dict(json.loads(text))+ except (json.JSONDecodeError, TypeError):+ pass++ if isinstance(output, str):+ try:+ return _to_result_dict(json.loads(output))+ except (json.JSONDecodeError, TypeError):+ pass++ return {"ok": True, "data": str(output)}+++def _to_result_dict(value: Any) -> dict[str, Any]:+ if isinstance(value, dict):+ return value+ return {"ok": True, "data": value}+++def _update_enforcement_from_tool(+ ctx: Any,+ tool_name: str,+ output: dict[str, Any],+) -> None:+ data = output.get("data")+ has_blocks = isinstance(data, dict) and data.get("block_count", 0) > 0++ if tool_name in ("update_workflow", "update_and_run_blocks") and output.get("ok") and has_blocks:+ ctx.update_workflow_called = True+ ctx.test_after_update_done = False+ ctx.post_update_nudge_count = 0++ if tool_name in ("run_blocks_and_collect_debug", "update_and_run_blocks"):+ ctx.test_after_update_done = True++ if tool_name == "navigate_browser" and output.get("ok"):+ ctx.navigate_called = True+ ctx.observation_after_navigate = False++ if tool_name in _OBSERVATION_TOOLS:+ ctx.observation_after_navigate = True+++def _sanitize_input(raw_args: dict[str, Any]) -> dict[str, Any]:+ # Redacts tool-call args before they hit the SSE payload sent to the UI.+ # Distinct from output_utils.sanitize_tool_result_for_llm, which shapes+ # tool *results* for LLM context consumption.+ # Drop the large workflow YAML blob (it's displayed elsewhere in the UI),+ # then run the remaining args through the shared exact-match redactor to+ # strip values under sensitive key names like `password`, `api_key`,+ # `totp`, etc. Benign identifiers (`credential_id`, `page_token`,+ # `username`) pass through unchanged.+ trimmed = {k: v for k, v in raw_args.items() if k != "workflow_yaml"}+ redacted = redact_sensitive_fields(trimmed)+ if isinstance(redacted, dict):+ return redacted+ return trimmeddiff --git a/tests/unit/test_request_logging.py b/tests/unit/test_request_logging.pyindex 1febf41bb..89d977ffe 100644--- a/tests/unit/test_request_logging.py+++ b/tests/unit/test_request_logging.py@@ -11,9 +11,9 @@ from skyvern.forge.request_logging import (_REDACTED,_is_loggable_content_type,_is_sensitive_key,- _redact_sensitive_fields,_sanitize_body,_sanitize_response_body,+ redact_sensitive_fields,)# ---------------------------------------------------------------------------@@ -86,26 +86,26 @@ class TestIsSensitiveKey:# ----------------------------------------------------------------------------# _redact_sensitive_fields+# redact_sensitive_fields# ---------------------------------------------------------------------------class TestRedactSensitiveFields:def test_redacts_password(self) -> None:data = {"username": "alice", "password": "secret123"}- result = _redact_sensitive_fields(data)+ result = redact_sensitive_fields(data)assert result["username"] == "alice"assert result["password"] == _REDACTEDdef test_redacts_nested_keys(self) -> None:data = {"user": {"api_key": "key123", "name": "bob"}}- result = _redact_sensitive_fields(data)+ result = redact_sensitive_fields(data)assert result["user"]["api_key"] == _REDACTEDassert result["user"]["name"] == "bob"def test_redacts_in_lists(self) -> None:data = [{"token": "abc"}, {"name": "ok"}]- result = _redact_sensitive_fields(data)+ result = redact_sensitive_fields(data)assert result[0]["token"] == _REDACTEDassert result[1]["name"] == "ok"@@ -120,7 +120,7 @@ class TestRedactSensitiveFields:"api_key": "g","Authorization": "h",}- result = _redact_sensitive_fields(data)+ result = redact_sensitive_fields(data)for key in data:assert result[key] == _REDACTED, f"Expected {key} to be redacted"@@ -132,7 +132,7 @@ class TestRedactSensitiveFields:"one_time_password": "xyz789","mfa_code": "mfa42",}- result = _redact_sensitive_fields(data)+ result = redact_sensitive_fields(data)for key in data:assert result[key] == _REDACTED, f"Expected {key} to be redacted"@@ -145,7 +145,7 @@ class TestRedactSensitiveFields:"author": "alice","token_count": 42,}- result = _redact_sensitive_fields(data)+ result = redact_sensitive_fields(data)assert result == datadef test_depth_limit_prevents_crash(self) -> None:@@ -156,7 +156,7 @@ class TestRedactSensitiveFields:current = current["nested"]current["password"] = "should_not_crash"- result = _redact_sensitive_fields(deep)+ result = redact_sensitive_fields(deep)assert result is not None # should not raise RecursionErrordef test_depth_limit_still_redacts_keys_at_boundary(self) -> None:@@ -170,7 +170,7 @@ class TestRedactSensitiveFields:current["password"] = "leak_me"current["safe"] = "visible"- result = _redact_sensitive_fields(deep)+ result = redact_sensitive_fields(deep)node = result["level"]for _ in range(19):node = node["next"]@@ -179,13 +179,13 @@ class TestRedactSensitiveFields:def test_preserves_non_sensitive_values(self) -> None:data = {"status": "ok", "count": 42, "items": [1, 2, 3]}- result = _redact_sensitive_fields(data)+ result = redact_sensitive_fields(data)assert result == datadef test_handles_non_dict_non_list(self) -> None:- assert _redact_sensitive_fields("hello") == "hello"- assert _redact_sensitive_fields(42) == 42- assert _redact_sensitive_fields(None) is None+ assert redact_sensitive_fields("hello") == "hello"+ assert redact_sensitive_fields(42) == 42+ assert redact_sensitive_fields(None) is None# ---------------------------------------------------------------------------diff --git a/uv.lock b/uv.lockindex c14a6f5bc..1abb9e80d 100644--- a/uv.lock+++ b/uv.lock@@ -14,7 +14,7 @@ resolution-markers = [][options]-exclude-newer = "2026-04-07T19:07:49.173655Z"+exclude-newer = "2026-04-08T21:26:23.576693Z"exclude-newer-span = "P7D"[manifest]@@ -1664,6 +1664,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/62/6b/a89f8456dcb06becff288f563618e9f20deed8dd29beea14f9a168aef64b/greenlet-3.3.2-cp313-cp313-win_arm64.whl", hash = "sha256:394ead29063ee3515b4e775216cb756b2e3b4a7e55ae8fd884f17fa579e6b327", size = 230221, upload-time = "2026-02-20T20:17:37.152Z" },]+[[package]]+name = "griffelib"+version = "2.0.2"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/9d/82/74f4a3310cdabfbb10da554c3a672847f1ed33c6f61dd472681ce7f1fe67/griffelib-2.0.2.tar.gz", hash = "sha256:3cf20b3bc470e83763ffbf236e0076b1211bac1bc67de13daf494640f2de707e", size = 166461, upload-time = "2026-03-27T11:34:51.091Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/11/8c/c9138d881c79aa0ea9ed83cbd58d5ca75624378b38cee225dcf5c42cc91f/griffelib-2.0.2-py3-none-any.whl", hash = "sha256:925c857658fb1ba40c0772c37acbc2ab650bd794d9c1b9726922e36ea4117ea1", size = 142357, upload-time = "2026-03-27T11:34:46.275Z" },+]+[[package]]name = "grpc-google-iam-v1"version = "0.14.3"@@ -3380,6 +3389,24 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/2a/9e/5bfa2270f902d5b92ab7d41ce0475b8630572e71e349b2a4996d14bdda93/openai-2.30.0-py3-none-any.whl", hash = "sha256:9a5ae616888eb2748ec5e0c5b955a51592e0b201a11f4262db920f2a78c5231d", size = 1146656, upload-time = "2026-03-25T22:08:58.2Z" },]+[[package]]+name = "openai-agents"+version = "0.13.5"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "griffelib" },+ { name = "mcp" },+ { name = "openai" },+ { name = "pydantic" },+ { name = "requests" },+ { name = "types-requests" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/1a/27/e3e60cc77abf588fd95e27b15b1a6806fd7107bb9fc4b2f36acfaaeff64c/openai_agents-0.13.5.tar.gz", hash = "sha256:ebe5bfb3d7d702d133ff9fb335718d61b741300de25b409ab4a6c8212ee945c0", size = 2699666, upload-time = "2026-04-06T04:11:47.575Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/df/54/5f85e18235f5bf7947c84323e3214757c6a22531529914f44077eb5a1198/openai_agents-0.13.5-py3-none-any.whl", hash = "sha256:672c76830d25b7eb3d85580539ba7caca975288df4395fe80c84cb4edfe4665f", size = 470837, upload-time = "2026-04-06T04:11:45.552Z" },+]+[[package]]name = "openapi-pydantic"version = "0.5.1"@@ -5628,6 +5655,11 @@ dependencies = [{ name = "zstandard" },]+[package.optional-dependencies]+copilot = [+ { name = "openai-agents" },+]+[package.dev-dependencies]cloud = [{ name = "kr8s" },@@ -5710,6 +5742,7 @@ requires-dist = [{ name = "litellm", specifier = ">=1.83.0" },{ name = "onepassword-sdk", specifier = "==0.4.0" },{ name = "openai", specifier = ">=1.68.2" },+ { name = "openai-agents", marker = "extra == 'copilot'", specifier = ">=0.13.4,<0.14" },{ name = "opentelemetry-api", specifier = ">=1.39.0,<2" },{ name = "orjson", specifier = ">=3.9.10,<4" },{ name = "pandas", specifier = ">=2.3.1,<4" },@@ -5751,6 +5784,7 @@ requires-dist = [{ name = "websockets", specifier = ">=12.0,<15.1" },{ name = "zstandard", specifier = ">=0.25.0" },]+provides-extras = ["copilot"][package.metadata.requires-dev]cloud = [