Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Send $ref-bearing tool results to Gemini as plain text so schemas stay literal (SKY-16493) (#8685)
Andrew Neilson08754e089d59
diff --git a/skyvern/forge/sdk/api/llm/copilot_model_usage.py b/skyvern/forge/sdk/api/llm/copilot_model_usage.pyindex 75d945f2e..f1dd18a82 100644--- a/skyvern/forge/sdk/api/llm/copilot_model_usage.py+++ b/skyvern/forge/sdk/api/llm/copilot_model_usage.py@@ -76,6 +76,7 @@ class CopilotModelUsageEvent:cache_mode: CacheMode | None = Nonecache_breakpoint_count: int | None = Nonecache_stable_prefix_chars: int | None = None+ ref_tool_outputs_escaped: int | None = Nonedef log_fields(self) -> dict[str, UsageScalar]:provider_name = None@@ -111,6 +112,7 @@ class CopilotModelUsageEvent:("copilot.cache.mode", self.cache_mode),("copilot.cache.breakpoint_count", self.cache_breakpoint_count),("copilot.cache.stable_prefix_chars", self.cache_stable_prefix_chars),+ ("copilot.ref_tool_outputs_escaped", self.ref_tool_outputs_escaped),)fields.update((key, value) for key, value in optional_fields if value is not None)return fieldsdiff --git a/skyvern/forge/sdk/copilot/model_telemetry.py b/skyvern/forge/sdk/copilot/model_telemetry.pyindex a52db1cc6..3cfbfb7af 100644--- a/skyvern/forge/sdk/copilot/model_telemetry.py+++ b/skyvern/forge/sdk/copilot/model_telemetry.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport contextlibimport contextvars+import jsonimport reimport timefrom collections.abc import AsyncIterator, Callable, Iterator@@ -33,6 +34,9 @@ from openai import AsyncStream, omitfrom openai.types.chat import ChatCompletionChunkfrom openai.types.completion_usage import CompletionUsagefrom openai.types.responses import Response+from openai.types.responses.response_function_call_output_item_list_param import (+ ResponseFunctionCallOutputItemListParam,+)from skyvern.forge.sdk.api.llm.copilot_model_usage import (CopilotModelUsageEvent,@@ -105,6 +109,7 @@ class CopilotModelCallTelemetry:cache_mode: Literal["implicit", "explicit"] = "implicit"cache_breakpoint_count: int = 0cache_stable_prefix_chars: int | None = None+ ref_tool_outputs_escaped: int = 0input_tokens: int | None = Noneoutput_tokens: int | None = Nonecache_read_tokens: int | None = None@@ -235,6 +240,7 @@ def _log_model_call_usage(cache_mode=telemetry.cache_mode,cache_breakpoint_count=telemetry.cache_breakpoint_count,cache_stable_prefix_chars=telemetry.cache_stable_prefix_chars,+ ref_tool_outputs_escaped=telemetry.ref_tool_outputs_escaped or None,),logger=LOG,)@@ -377,6 +383,87 @@ def _capture_usage(LOG.warning("Failed to capture Copilot model usage", error=repr(exc))+_GEMINI_PROVIDERS = frozenset({"vertex_ai", "vertex_ai_beta", "gemini"})+++def _is_gemini_model(model: str) -> bool:+ try:+ return litellm.get_llm_provider(model)[1] in _GEMINI_PROVIDERS+ except litellm.exceptions.BadRequestError:+ return False+++def _gemini_on_model_chain(model: str, model_settings: ModelSettings) -> bool:+ fallbacks: list[str] = (model_settings.extra_args or {}).get("fallbacks") or []+ return any(_is_gemini_model(candidate) for candidate in [model, *fallbacks])+++def _has_ref_key(value: object) -> bool:+ if isinstance(value, dict):+ return "$ref" in value or any(_has_ref_key(child) for child in value.values())+ if isinstance(value, list):+ return any(_has_ref_key(child) for child in value)+ return False+++def _escape_ref_text(text: str) -> str | None:+ # LiteLLM's Gemini converter sends a tool result that parses to a JSON object as the literal+ # function_response.response, where Vertex reads a $ref key as a media reference; its plain-text shape is safe.+ if not text.strip().startswith("{"):+ return None+ try:+ parsed = json.loads(text)+ has_ref = isinstance(parsed, dict) and _has_ref_key(parsed)+ except ValueError:+ return None+ except RecursionError:+ # Too deep to walk; wrapping is lossless, and the converter then parses only the shallow envelope.+ has_ref = True+ return json.dumps({"content": text}, ensure_ascii=False) if has_ref else None+++def _escape_ref_output(+ output: str | ResponseFunctionCallOutputItemListParam,+) -> str | ResponseFunctionCallOutputItemListParam | None:+ if isinstance(output, str):+ return _escape_ref_text(output)+ texts = [part["text"] for part in output if part["type"] == "input_text"]+ if len(texts) != len(output):+ return None+ escaped = _escape_ref_text("".join(texts))+ return None if escaped is None else [{"type": "input_text", "text": escaped}]+++def _escape_ref_tool_outputs_for_gemini(+ input: str | list[TResponseInputItem],+ model: str,+ model_settings: ModelSettings,+) -> str | list[TResponseInputItem]:+ if isinstance(input, str) or not _gemini_on_model_chain(model, model_settings):+ return input+ rewritten: list[TResponseInputItem] | None = None+ escaped_count = 0+ for index, item in enumerate(input):+ tool_output = Converter.maybe_function_tool_call_output(item)+ if tool_output is None:+ continue+ escaped = _escape_ref_output(tool_output["output"])+ if escaped is None:+ continue+ if rewritten is None:+ rewritten = list(input)+ escaped_item = tool_output.copy()+ escaped_item["output"] = escaped+ rewritten[index] = escaped_item+ escaped_count += 1+ if rewritten is None:+ return input+ telemetry = current_model_call_telemetry()+ if telemetry is not None:+ telemetry.ref_tool_outputs_escaped = escaped_count+ return rewritten++class CopilotLitellmModel(LitellmModel):def __init__(self,@@ -497,6 +584,7 @@ class CopilotLitellmModel(LitellmModel):stream: bool = False,prompt: Any | None = None,) -> LiteLLMModelResponse | tuple[Response, AsyncStream[ChatCompletionChunk]]:+ input = _escape_ref_tool_outputs_for_gemini(input, self.model, model_settings)explicit_cache_envelope = build_explicit_cache_envelope(model=self.model,base_url=self.base_url,diff --git a/tests/unit/test_copilot_model_telemetry.py b/tests/unit/test_copilot_model_telemetry.pyindex 2240751b4..0834a484a 100644--- a/tests/unit/test_copilot_model_telemetry.py+++ b/tests/unit/test_copilot_model_telemetry.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport contextlib+import copyimport jsonfrom collections.abc import AsyncIterator, Iteratorfrom types import SimpleNamespace@@ -9,22 +10,29 @@ from typing import Any, Self, castfrom unittest.mock import AsyncMock, MagicMockimport pytest-from agents import ModelSettings, function_tool+from agents import ItemHelpers, ModelSettings, RunContextWrapper, function_toolfrom agents.extensions.models.litellm_model import LitellmModel+from agents.items import TResponseInputItem+from agents.mcp import MCPServer, MCPUtilfrom agents.models.interface import ModelTracing+from litellm.llms.vertex_ai.gemini.transformation import _gemini_convert_messages_with_historyfrom litellm.types.llms.openai import ResponsesAPIResponsefrom litellm.types.utils import Deltafrom litellm.types.utils import ModelResponse as LiteLLMModelResponsefrom litellm.types.utils import ModelResponseStream, StreamingChoices, Usage+from mcp.types import Tool as MCPToolfrom openai import AsyncStreamfrom openai.types.chat import ChatCompletionChunk+from openai.types.responses import ResponseFunctionToolCallfrom structlog.testing import capture_logs+from skyvern.cli.mcp_tools.blocks import skyvern_block_schemafrom skyvern.forge.sdk.copilot import agent as copilot_agent_modulefrom skyvern.forge.sdk.copilot import model_telemetry as model_telemetry_modulefrom skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalModefrom skyvern.forge.sdk.copilot.cache_envelope import CacheableSystemInstructionsfrom skyvern.forge.sdk.copilot.config import CopilotConfig+from skyvern.forge.sdk.copilot.mcp_adapter import _copilot_to_call_tool_resultfrom skyvern.forge.sdk.copilot.model_telemetry import (CopilotLitellmModel,current_model_attempt_telemetry,@@ -1301,3 +1309,206 @@ async def test_browser_ablation_timeout_reports_active_model_and_browser_work(assert result.browser_ablation_metadata is not Noneassert result.browser_ablation_metadata["eval_mode"] == "browser_ablation"assert result.browser_ablation_metadata["tool_activity"] == [{"tool_name": "navigate_browser", "success": True}]+++_GEMINI = "vertex_ai/gemini-2.5-flash"+_SCHEMA_CALL = ResponseFunctionToolCall(+ type="function_call",+ call_id="call_schema",+ name="get_block_schema",+ arguments='{"block_type": "code"}',+)+_REF_RESULT = json.dumps({"ok": True, "data": {"schema": {"$ref": "#/$defs/Step", "$defs": {"Step": {}}}}})+_ESCAPED_REF_KEY_RESULT = '{"ok": true, "data": {"schema": {"\\u0024ref": "#/$defs/Step", "$defs": {"Step": {}}}}}'+++def _tool_turn(output: object) -> list[TResponseInputItem]:+ return [+ {"role": "user", "content": "Add a block"},+ cast(TResponseInputItem, _SCHEMA_CALL.model_dump()),+ ItemHelpers.tool_call_output_item(_SCHEMA_CALL, output),+ ]+++async def _block_schema_tool_output(block_type: str) -> tuple[object, str]:+ call_result = _copilot_to_call_tool_result(await skyvern_block_schema(block_type), "get_block_schema")+ server = SimpleNamespace(+ name="skyvern", use_structured_content=False, call_tool=AsyncMock(return_value=call_result)+ )+ output = await MCPUtil.invoke_mcp_tool(+ cast(MCPServer, server),+ MCPTool(name="get_block_schema", inputSchema={"type": "object"}),+ RunContextWrapper(None),+ _SCHEMA_CALL.arguments,+ )+ return output, cast(Any, call_result.content[0]).text+++async def _sent_messages(+ monkeypatch: pytest.MonkeyPatch,+ model: LitellmModel,+ turn: list[TResponseInputItem],+ extra_args: dict[str, Any] | None,+ stream: bool = False,+) -> list[dict[str, Any]]:+ requests: list[dict[str, Any]] = []++ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse | AsyncStream[ChatCompletionChunk]:+ requests.append(kwargs)+ return cast(AsyncStream[ChatCompletionChunk], _ChunkStream(_stream_chunks())) if stream else _completion()++ monkeypatch.setattr("litellm.acompletion", fake_acompletion)+ call = {+ "system_instructions": "You are concise.",+ "input": turn,+ "model_settings": ModelSettings(extra_args=extra_args),+ "tools": [],+ "output_schema": None,+ "handoffs": [],+ "tracing": ModelTracing.DISABLED,+ }+ if stream:+ async for _ in model.stream_response(**call):+ pass+ else:+ await model.get_response(**call)+ return requests[0]["messages"]+++def _tool_message(messages: list[dict[str, Any]]) -> dict[str, Any]:+ return next(message for message in messages if message["role"] == "tool")+++def _tool_text(messages: list[dict[str, Any]]) -> str:+ content = _tool_message(messages)["content"]+ return content if isinstance(content, str) else "".join(part["text"] for part in content)+++def _gemini_function_responses(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:+ contents = _gemini_convert_messages_with_history(+ messages=cast(Any, [message for message in messages if message["role"] != "system"]),+ model="gemini-2.5-flash",+ custom_llm_provider="vertex_ai",+ )+ return [+ part["function_response"] for content in contents for part in content["parts"] if "function_response" in part+ ]+++def _json_keys(value: object) -> set[str]:+ if isinstance(value, dict):+ return set(value).union(*(_json_keys(child) for child in value.values()))+ if isinstance(value, list):+ return set().union(*(_json_keys(child) for child in value))+ return set()+++ ("payload", "model_name", "extra_args", "stream"),+ [+ ("code", _GEMINI, None, False),+ ("code", "openai/gpt-5.6", {"fallbacks": [_GEMINI]}, True),+ ("for_loop", _GEMINI, None, True),+ ("for_loop", "azure/gpt-5.6-sol", {"fallbacks": [_GEMINI]}, False),+ ("escaped_ref_key", _GEMINI, None, False),+ ],+)+async def test_gemini_on_the_model_chain_receives_ref_tool_results_as_literal_text(+ monkeypatch: pytest.MonkeyPatch,+ payload: str,+ model_name: str,+ extra_args: dict[str, Any] | None,+ stream: bool,+) -> None:+ if payload == "escaped_ref_key":+ output, text = _ESCAPED_REF_KEY_RESULT, _ESCAPED_REF_KEY_RESULT+ else:+ output, text = await _block_schema_tool_output(payload)+ turn = _tool_turn(output)+ original_turn = copy.deepcopy(turn)++ raw = await _sent_messages(monkeypatch, LitellmModel(model=model_name), turn, extra_args)+ usage_events: list[dict[str, Any]] = []+ monkeypatch.setattr(model_telemetry_module.LOG, "info", lambda event, **fields: usage_events.append(fields))+ sent = await _sent_messages(+ monkeypatch, CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1), turn, extra_args, stream+ )++ assert [event["copilot.ref_tool_outputs_escaped"] for event in usage_events] == [1]+ assert _tool_text(raw) == text+ assert "$ref" in _json_keys([response["response"] for response in _gemini_function_responses(raw)])+ [function_response] = _gemini_function_responses(sent)+ assert function_response["name"] == "get_block_schema"+ assert function_response["response"] == {"content": text}+ assert json.loads(_tool_text(sent)) == {"content": text}+ assert _tool_message(sent)["tool_call_id"] == "call_schema"+ assert turn == original_turn+++async def test_a_tool_result_too_deep_to_walk_is_still_sent_to_a_gemini_chain(monkeypatch: pytest.MonkeyPatch) -> None:+ deep = "{" + '"a": {' * 600 + "}" * 600 + "}"++ sent = await _sent_messages(+ monkeypatch,+ CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 1),+ _tool_turn(deep),+ {"fallbacks": [_GEMINI]},+ )++ assert json.loads(_tool_text(sent)) == {"content": deep}+++ ("model_name", "output"),+ [+ ("azure/gpt-5.6-sol", [{"type": "text", "text": _REF_RESULT}]),+ ("openai/gpt-5.6", _REF_RESULT),+ ("anthropic/claude-sonnet-4-5", [{"type": "text", "text": _REF_RESULT}]),+ ("foo/bar", _REF_RESULT),+ (_GEMINI, '{"ok": true, "data": {"count": 3}}'),+ (_GEMINI, [{"type": "text", "text": '{"ok": false, "error": "Timed out waiting for the page"}'}]),+ (_GEMINI, "An error occurred while running the tool."),+ (_GEMINI, json.dumps([{"$ref": "#/$defs/Step"}])),+ ],+)+async def test_tool_results_off_the_gemini_escape_reach_the_provider_unchanged(+ monkeypatch: pytest.MonkeyPatch,+ model_name: str,+ output: object,+) -> None:+ turn = _tool_turn(output)+ original_turn = copy.deepcopy(turn)++ raw = await _sent_messages(monkeypatch, LitellmModel(model=model_name), turn, None)+ sent = await _sent_messages(+ monkeypatch, CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1), turn, None+ )++ assert sent == raw+ assert turn == original_turn+++async def test_gemini_route_leaves_an_unresolved_media_reference_for_the_provider(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ media_reference = {"screenshot": {"$ref": "page.png"}}+ turn = _tool_turn(+ [+ {"type": "text", "text": json.dumps(media_reference)},+ {"type": "image", "image_url": "data:image/png;base64,iVBORw0KGgo="},+ ]+ )++ raw = await _sent_messages(monkeypatch, LitellmModel(model=_GEMINI), turn, None)+ sent = await _sent_messages(+ monkeypatch, CopilotLitellmModel(model=_GEMINI, next_model_call_index=lambda: 1), turn, None+ )++ assert sent == raw+ [function_response] = _gemini_function_responses(sent)+ assert function_response["response"] == media_reference+ assert [set(part) for part in function_response["parts"]] == [{"inline_data"}]diff --git a/tests/unit/test_copilot_model_usage.py b/tests/unit/test_copilot_model_usage.pyindex 33814290d..8cb4b036a 100644--- a/tests/unit/test_copilot_model_usage.py+++ b/tests/unit/test_copilot_model_usage.py@@ -119,6 +119,7 @@ def test_direct_event_is_a_closed_content_free_scalar_schema() -> None:"cache_mode","cache_breakpoint_count","cache_stable_prefix_chars",+ "ref_tool_outputs_escaped",}assert normalize_gen_ai_provider(event.provider_name, event.response_model) == "gcp.vertex_ai"