Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Report session identity and dispatch facts when a copilot browser call dies mid-protocol (SKY-15786) (#8513)
Andrew Neilson68e6f1ad2f9f
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.pyindex 2786642b9..9b2d9ded2 100644--- a/skyvern/forge/sdk/copilot/mcp_adapter.py+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py@@ -360,8 +360,33 @@ def _project_browser_call_outcome() -> dict[str, Any]:if outcome.protocol_error_detail is not None:detail = outcome.protocol_error_detail- error = f"{display_tool_name} failed: {detail}" if detail else f"{display_tool_name} failed"- result: dict[str, Any] = {"ok": False, "error": error}+ failure = f"{display_tool_name} failed: {detail}" if detail else f"{display_tool_name} failed"+ if outcome.dispatched:+ guidance = (+ "No result was returned, so whether the call ran is unknown. Observe the page again "+ "before relying on state that may have changed."+ )+ effect = "unknown"+ else:+ guidance = "The call was never sent to the browser, so it had no effect."+ effect = "none"+ result: dict[str, Any] = {+ "ok": False,+ "error": f"{failure}. {guidance}",+ "data": {+ "browser_call_continuity": {+ "source": "direct_mcp",+ "failed_tool": display_tool_name,+ "browser_session_id": outcome.source_browser_session_id,+ "browser_session_generation": outcome.source_browser_session_generation,+ "browser_session_id_after": outcome.completion_browser_session_id,+ "browser_session_generation_after": outcome.completion_browser_session_generation,+ "dispatched": outcome.dispatched,+ "result_delivered": False,+ "prior_action_effect": effect,+ }+ },+ }else:raw_result = outcome.raw_result()result = mcp_to_copilot(raw_result) if raw_result else {}@@ -2019,10 +2044,15 @@ class SkyvernOverlayMCPServer(MCPServer):dispatched=dispatch_started,exception=exc,)+ # The call was bound to one browser; a run-targeted call must not report the+ # chat's browser as where it ended up. The generation counter belongs to the chat's+ # browser only, so a fixed target has none to report.outcome = replace(outcome,- completion_browser_session_id=copilot_ctx.browser_session_id,- completion_browser_session_generation=copilot_ctx.browser_session_continuity_generation,+ completion_browser_session_id=binding.session_id_for(copilot_ctx),+ completion_browser_session_generation=(+ None if binding.session_id_override else copilot_ctx.browser_session_continuity_generation+ ),)_record_browser_call_outcome(copilot_ctx, outcome, call_path="model")err = _project_browser_call_outcome(@@ -2330,8 +2360,12 @@ class SkyvernOverlayMCPServer(MCPServer):)outcome = replace(_scrub_browser_call_outcome(ctx, outcome),- completion_browser_session_id=ctx.browser_session_id,- completion_browser_session_generation=ctx.browser_session_continuity_generation,+ completion_browser_session_id=call_browser_session_id,+ completion_browser_session_generation=(+ None+ if (requested_session_id or call_session_override)+ else ctx.browser_session_continuity_generation+ ),)_record_browser_call_outcome(ctx, outcome, call_path="internal")return _InternalToolCallResult(diff --git a/tests/unit/test_copilot_mcp_adapter.py b/tests/unit/test_copilot_mcp_adapter.pyindex 68eb80796..d02b25d57 100644--- a/tests/unit/test_copilot_mcp_adapter.py+++ b/tests/unit/test_copilot_mcp_adapter.py@@ -1099,6 +1099,116 @@ class TestSharedBrowserCallOutcome:assert len(records) == 2assert all(record["dispatched"] is False for record in records)+ def test_a_protocol_failure_reports_the_model_surface_tool_and_both_session_identities(self) -> None:+ """A bare 'evaluate failed' leaves the model no way to tell where it now is, so the failed+ call has to name a tool the model can actually call and both ends of the session identity."""+ outcome = replace(+ mcp_adapter._browser_protocol_exception_outcome(+ raw_tool_name="skyvern_evaluate",+ source_browser_session_id="pbs_source",+ source_browser_session_generation=2,+ dispatched=True,+ exception=RuntimeError("transport closed"),+ ),+ completion_browser_session_id="pbs_after",+ completion_browser_session_generation=3,+ )++ projected = mcp_adapter._project_browser_call_outcome(outcome, display_tool_name="evaluate")+ continuity = projected["data"]["browser_call_continuity"]++ assert continuity["failed_tool"] == "evaluate"+ assert continuity["browser_session_id"] == "pbs_source"+ assert continuity["browser_session_generation"] == 2+ assert continuity["browser_session_id_after"] == "pbs_after"+ assert continuity["browser_session_generation_after"] == 3++ def test_a_protocol_failure_states_no_liveness_or_read_fact_the_outcome_cannot_prove(self) -> None:+ """Dispatch is recorded before the await, so a dispatched call cannot tell pre- from+ post-execution failure — the projection may report delivery and effect, never liveness."""+ outcome = mcp_adapter._browser_protocol_exception_outcome(+ raw_tool_name="skyvern_evaluate",+ source_browser_session_id="pbs_source",+ source_browser_session_generation=0,+ dispatched=True,+ exception=RuntimeError("transport closed"),+ )++ projected = mcp_adapter._project_browser_call_outcome(outcome, display_tool_name="evaluate")+ continuity = projected["data"]["browser_call_continuity"]++ assert continuity["dispatched"] is True+ assert continuity["result_delivered"] is False+ assert continuity["prior_action_effect"] == "unknown"+ assert "session_state" not in continuity+ assert "read_completed" not in continuity+ assert "Observe the page again" in projected["error"]++ @pytest.mark.asyncio+ async def test_a_failure_before_dispatch_reports_no_effect_on_the_targeted_browser(self) -> None:+ """A pre-hook that dies never sends the call, so the block must say so instead of inviting a+ re-observation, and a run-targeted call must name the run browser as where it ended, not the+ chat's debug browser."""++ async def _explode(_args: dict[str, Any], _ctx: AgentContext) -> dict[str, Any]:+ raise RuntimeError("hook transport closed")++ ctx = make_copilot_ctx(browser_session_id="pbs_debug")+ ctx.last_run_blocks_browser_session_id = "pbs_run"+ ctx.last_run_blocks_workflow_run_id = "wr_1"+ ctx.browser_session_continuity_generation = 4+ server = _make_server(+ ctx,+ {"ok": True},+ SchemaOverlay(requires_browser=True, pre_hook=_explode),+ alias_map={"evaluate": "skyvern_evaluate"},+ )++ with capture_logs() as captured:+ projected = await server.call_tool("evaluate", {"target": "last_run"})++ payload = json.loads(projected.content[0].text)+ continuity = payload["data"]["browser_call_continuity"]+ assert continuity["dispatched"] is False+ assert continuity["prior_action_effect"] == "none"+ assert continuity["browser_session_id"] == "pbs_run"+ assert continuity["browser_session_id_after"] == "pbs_run"+ assert continuity["browser_session_generation_after"] is None+ assert "had no effect" in payload["error"]+ assert "Observe the page again" not in payload["error"]+ records = _browser_outcome_records(captured)+ assert records[-1]["completion_browser_session_id"] == "pbs_run"++ def test_a_protocol_failure_does_not_borrow_the_session_loss_continuity_block(self) -> None:+ """The session is not known to be lost here; reusing the loss block would tell the model a+ replacement browser is ready when nothing established that."""+ protocol = mcp_adapter._project_browser_call_outcome(+ mcp_adapter._browser_protocol_exception_outcome(+ raw_tool_name="skyvern_evaluate",+ source_browser_session_id="pbs_source",+ source_browser_session_generation=0,+ dispatched=True,+ exception=RuntimeError("transport closed"),+ ),+ display_tool_name="evaluate",+ )+ loss = mcp_adapter._project_browser_call_outcome(+ mcp_adapter._not_dispatched_browser_call_outcome(+ raw_tool_name="skyvern_evaluate",+ source_browser_session_id="pbs_lost",+ source_browser_session_generation=0,+ raw_result={},+ ctx=make_copilot_ctx(browser_session_id="pbs_replacement"),+ session_loss_disposition="reestablished",+ ),+ display_tool_name="evaluate",+ )++ assert "browser_session_continuity" not in protocol["data"]+ assert "error_code" not in protocol+ assert loss["data"]["browser_session_continuity"]["disposition"] == "reestablished"+ assert "browser_call_continuity" not in loss["data"]+@pytest.mark.usefixtures("_stub_browser_session")class TestMCPToolTiming: