Skip to content

Automate browser based workflows with AI

Send $ref-bearing tool results to Gemini as plain text so schemas stay literal (SKY-16493) (#8685)

Andrew Neilson08754e089d59
diff --git a/skyvern/forge/sdk/api/llm/copilot_model_usage.py b/skyvern/forge/sdk/api/llm/copilot_model_usage.py
index 75d945f2e..f1dd18a82 100644
--- a/skyvern/forge/sdk/api/llm/copilot_model_usage.py
+++ b/skyvern/forge/sdk/api/llm/copilot_model_usage.py
@@ -76,6 +76,7 @@ class CopilotModelUsageEvent:
cache_mode: CacheMode | None = None
cache_breakpoint_count: int | None = None
cache_stable_prefix_chars: int | None = None
+ ref_tool_outputs_escaped: int | None = None
def log_fields(self) -> dict[str, UsageScalar]:
provider_name = None
@@ -111,6 +112,7 @@ class CopilotModelUsageEvent:
("copilot.cache.mode", self.cache_mode),
("copilot.cache.breakpoint_count", self.cache_breakpoint_count),
("copilot.cache.stable_prefix_chars", self.cache_stable_prefix_chars),
+ ("copilot.ref_tool_outputs_escaped", self.ref_tool_outputs_escaped),
)
fields.update((key, value) for key, value in optional_fields if value is not None)
return fields
diff --git a/skyvern/forge/sdk/copilot/model_telemetry.py b/skyvern/forge/sdk/copilot/model_telemetry.py
index a52db1cc6..3cfbfb7af 100644
--- a/skyvern/forge/sdk/copilot/model_telemetry.py
+++ b/skyvern/forge/sdk/copilot/model_telemetry.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import contextlib
import contextvars
+import json
import re
import time
from collections.abc import AsyncIterator, Callable, Iterator
@@ -33,6 +34,9 @@ from openai import AsyncStream, omit
from openai.types.chat import ChatCompletionChunk
from openai.types.completion_usage import CompletionUsage
from openai.types.responses import Response
+from openai.types.responses.response_function_call_output_item_list_param import (
+ ResponseFunctionCallOutputItemListParam,
+)
from skyvern.forge.sdk.api.llm.copilot_model_usage import (
CopilotModelUsageEvent,
@@ -105,6 +109,7 @@ class CopilotModelCallTelemetry:
cache_mode: Literal["implicit", "explicit"] = "implicit"
cache_breakpoint_count: int = 0
cache_stable_prefix_chars: int | None = None
+ ref_tool_outputs_escaped: int = 0
input_tokens: int | None = None
output_tokens: int | None = None
cache_read_tokens: int | None = None
@@ -235,6 +240,7 @@ def _log_model_call_usage(
cache_mode=telemetry.cache_mode,
cache_breakpoint_count=telemetry.cache_breakpoint_count,
cache_stable_prefix_chars=telemetry.cache_stable_prefix_chars,
+ ref_tool_outputs_escaped=telemetry.ref_tool_outputs_escaped or None,
),
logger=LOG,
)
@@ -377,6 +383,87 @@ def _capture_usage(
LOG.warning("Failed to capture Copilot model usage", error=repr(exc))
+_GEMINI_PROVIDERS = frozenset({"vertex_ai", "vertex_ai_beta", "gemini"})
+
+
+def _is_gemini_model(model: str) -> bool:
+ try:
+ return litellm.get_llm_provider(model)[1] in _GEMINI_PROVIDERS
+ except litellm.exceptions.BadRequestError:
+ return False
+
+
+def _gemini_on_model_chain(model: str, model_settings: ModelSettings) -> bool:
+ fallbacks: list[str] = (model_settings.extra_args or {}).get("fallbacks") or []
+ return any(_is_gemini_model(candidate) for candidate in [model, *fallbacks])
+
+
+def _has_ref_key(value: object) -> bool:
+ if isinstance(value, dict):
+ return "$ref" in value or any(_has_ref_key(child) for child in value.values())
+ if isinstance(value, list):
+ return any(_has_ref_key(child) for child in value)
+ return False
+
+
+def _escape_ref_text(text: str) -> str | None:
+ # LiteLLM's Gemini converter sends a tool result that parses to a JSON object as the literal
+ # function_response.response, where Vertex reads a $ref key as a media reference; its plain-text shape is safe.
+ if not text.strip().startswith("{"):
+ return None
+ try:
+ parsed = json.loads(text)
+ has_ref = isinstance(parsed, dict) and _has_ref_key(parsed)
+ except ValueError:
+ return None
+ except RecursionError:
+ # Too deep to walk; wrapping is lossless, and the converter then parses only the shallow envelope.
+ has_ref = True
+ return json.dumps({"content": text}, ensure_ascii=False) if has_ref else None
+
+
+def _escape_ref_output(
+ output: str | ResponseFunctionCallOutputItemListParam,
+) -> str | ResponseFunctionCallOutputItemListParam | None:
+ if isinstance(output, str):
+ return _escape_ref_text(output)
+ texts = [part["text"] for part in output if part["type"] == "input_text"]
+ if len(texts) != len(output):
+ return None
+ escaped = _escape_ref_text("".join(texts))
+ return None if escaped is None else [{"type": "input_text", "text": escaped}]
+
+
+def _escape_ref_tool_outputs_for_gemini(
+ input: str | list[TResponseInputItem],
+ model: str,
+ model_settings: ModelSettings,
+) -> str | list[TResponseInputItem]:
+ if isinstance(input, str) or not _gemini_on_model_chain(model, model_settings):
+ return input
+ rewritten: list[TResponseInputItem] | None = None
+ escaped_count = 0
+ for index, item in enumerate(input):
+ tool_output = Converter.maybe_function_tool_call_output(item)
+ if tool_output is None:
+ continue
+ escaped = _escape_ref_output(tool_output["output"])
+ if escaped is None:
+ continue
+ if rewritten is None:
+ rewritten = list(input)
+ escaped_item = tool_output.copy()
+ escaped_item["output"] = escaped
+ rewritten[index] = escaped_item
+ escaped_count += 1
+ if rewritten is None:
+ return input
+ telemetry = current_model_call_telemetry()
+ if telemetry is not None:
+ telemetry.ref_tool_outputs_escaped = escaped_count
+ return rewritten
+
+
class CopilotLitellmModel(LitellmModel):
def __init__(
self,
@@ -497,6 +584,7 @@ class CopilotLitellmModel(LitellmModel):
stream: bool = False,
prompt: Any | None = None,
) -> LiteLLMModelResponse | tuple[Response, AsyncStream[ChatCompletionChunk]]:
+ input = _escape_ref_tool_outputs_for_gemini(input, self.model, model_settings)
explicit_cache_envelope = build_explicit_cache_envelope(
model=self.model,
base_url=self.base_url,
diff --git a/tests/unit/test_copilot_model_telemetry.py b/tests/unit/test_copilot_model_telemetry.py
index 2240751b4..0834a484a 100644
--- a/tests/unit/test_copilot_model_telemetry.py
+++ b/tests/unit/test_copilot_model_telemetry.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import contextlib
+import copy
import json
from collections.abc import AsyncIterator, Iterator
from types import SimpleNamespace
@@ -9,22 +10,29 @@ from typing import Any, Self, cast
from unittest.mock import AsyncMock, MagicMock
import pytest
-from agents import ModelSettings, function_tool
+from agents import ItemHelpers, ModelSettings, RunContextWrapper, function_tool
from agents.extensions.models.litellm_model import LitellmModel
+from agents.items import TResponseInputItem
+from agents.mcp import MCPServer, MCPUtil
from agents.models.interface import ModelTracing
+from litellm.llms.vertex_ai.gemini.transformation import _gemini_convert_messages_with_history
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import Delta
from litellm.types.utils import ModelResponse as LiteLLMModelResponse
from litellm.types.utils import ModelResponseStream, StreamingChoices, Usage
+from mcp.types import Tool as MCPTool
from openai import AsyncStream
from openai.types.chat import ChatCompletionChunk
+from openai.types.responses import ResponseFunctionToolCall
from structlog.testing import capture_logs
+from skyvern.cli.mcp_tools.blocks import skyvern_block_schema
from skyvern.forge.sdk.copilot import agent as copilot_agent_module
from skyvern.forge.sdk.copilot import model_telemetry as model_telemetry_module
from skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalMode
from skyvern.forge.sdk.copilot.cache_envelope import CacheableSystemInstructions
from skyvern.forge.sdk.copilot.config import CopilotConfig
+from skyvern.forge.sdk.copilot.mcp_adapter import _copilot_to_call_tool_result
from skyvern.forge.sdk.copilot.model_telemetry import (
CopilotLitellmModel,
current_model_attempt_telemetry,
@@ -1301,3 +1309,206 @@ async def test_browser_ablation_timeout_reports_active_model_and_browser_work(
assert result.browser_ablation_metadata is not None
assert result.browser_ablation_metadata["eval_mode"] == "browser_ablation"
assert result.browser_ablation_metadata["tool_activity"] == [{"tool_name": "navigate_browser", "success": True}]
+
+
+_GEMINI = "vertex_ai/gemini-2.5-flash"
+_SCHEMA_CALL = ResponseFunctionToolCall(
+ type="function_call",
+ call_id="call_schema",
+ name="get_block_schema",
+ arguments='{"block_type": "code"}',
+)
+_REF_RESULT = json.dumps({"ok": True, "data": {"schema": {"$ref": "#/$defs/Step", "$defs": {"Step": {}}}}})
+_ESCAPED_REF_KEY_RESULT = '{"ok": true, "data": {"schema": {"\\u0024ref": "#/$defs/Step", "$defs": {"Step": {}}}}}'
+
+
+def _tool_turn(output: object) -> list[TResponseInputItem]:
+ return [
+ {"role": "user", "content": "Add a block"},
+ cast(TResponseInputItem, _SCHEMA_CALL.model_dump()),
+ ItemHelpers.tool_call_output_item(_SCHEMA_CALL, output),
+ ]
+
+
+async def _block_schema_tool_output(block_type: str) -> tuple[object, str]:
+ call_result = _copilot_to_call_tool_result(await skyvern_block_schema(block_type), "get_block_schema")
+ server = SimpleNamespace(
+ name="skyvern", use_structured_content=False, call_tool=AsyncMock(return_value=call_result)
+ )
+ output = await MCPUtil.invoke_mcp_tool(
+ cast(MCPServer, server),
+ MCPTool(name="get_block_schema", inputSchema={"type": "object"}),
+ RunContextWrapper(None),
+ _SCHEMA_CALL.arguments,
+ )
+ return output, cast(Any, call_result.content[0]).text
+
+
+async def _sent_messages(
+ monkeypatch: pytest.MonkeyPatch,
+ model: LitellmModel,
+ turn: list[TResponseInputItem],
+ extra_args: dict[str, Any] | None,
+ stream: bool = False,
+) -> list[dict[str, Any]]:
+ requests: list[dict[str, Any]] = []
+
+ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse | AsyncStream[ChatCompletionChunk]:
+ requests.append(kwargs)
+ return cast(AsyncStream[ChatCompletionChunk], _ChunkStream(_stream_chunks())) if stream else _completion()
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+ call = {
+ "system_instructions": "You are concise.",
+ "input": turn,
+ "model_settings": ModelSettings(extra_args=extra_args),
+ "tools": [],
+ "output_schema": None,
+ "handoffs": [],
+ "tracing": ModelTracing.DISABLED,
+ }
+ if stream:
+ async for _ in model.stream_response(**call):
+ pass
+ else:
+ await model.get_response(**call)
+ return requests[0]["messages"]
+
+
+def _tool_message(messages: list[dict[str, Any]]) -> dict[str, Any]:
+ return next(message for message in messages if message["role"] == "tool")
+
+
+def _tool_text(messages: list[dict[str, Any]]) -> str:
+ content = _tool_message(messages)["content"]
+ return content if isinstance(content, str) else "".join(part["text"] for part in content)
+
+
+def _gemini_function_responses(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ contents = _gemini_convert_messages_with_history(
+ messages=cast(Any, [message for message in messages if message["role"] != "system"]),
+ model="gemini-2.5-flash",
+ custom_llm_provider="vertex_ai",
+ )
+ return [
+ part["function_response"] for content in contents for part in content["parts"] if "function_response" in part
+ ]
+
+
+def _json_keys(value: object) -> set[str]:
+ if isinstance(value, dict):
+ return set(value).union(*(_json_keys(child) for child in value.values()))
+ if isinstance(value, list):
+ return set().union(*(_json_keys(child) for child in value))
+ return set()
+
+
+ ("payload", "model_name", "extra_args", "stream"),
+ [
+ ("code", _GEMINI, None, False),
+ ("code", "openai/gpt-5.6", {"fallbacks": [_GEMINI]}, True),
+ ("for_loop", _GEMINI, None, True),
+ ("for_loop", "azure/gpt-5.6-sol", {"fallbacks": [_GEMINI]}, False),
+ ("escaped_ref_key", _GEMINI, None, False),
+ ],
+)
+async def test_gemini_on_the_model_chain_receives_ref_tool_results_as_literal_text(
+ monkeypatch: pytest.MonkeyPatch,
+ payload: str,
+ model_name: str,
+ extra_args: dict[str, Any] | None,
+ stream: bool,
+) -> None:
+ if payload == "escaped_ref_key":
+ output, text = _ESCAPED_REF_KEY_RESULT, _ESCAPED_REF_KEY_RESULT
+ else:
+ output, text = await _block_schema_tool_output(payload)
+ turn = _tool_turn(output)
+ original_turn = copy.deepcopy(turn)
+
+ raw = await _sent_messages(monkeypatch, LitellmModel(model=model_name), turn, extra_args)
+ usage_events: list[dict[str, Any]] = []
+ monkeypatch.setattr(model_telemetry_module.LOG, "info", lambda event, **fields: usage_events.append(fields))
+ sent = await _sent_messages(
+ monkeypatch, CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1), turn, extra_args, stream
+ )
+
+ assert [event["copilot.ref_tool_outputs_escaped"] for event in usage_events] == [1]
+ assert _tool_text(raw) == text
+ assert "$ref" in _json_keys([response["response"] for response in _gemini_function_responses(raw)])
+ [function_response] = _gemini_function_responses(sent)
+ assert function_response["name"] == "get_block_schema"
+ assert function_response["response"] == {"content": text}
+ assert json.loads(_tool_text(sent)) == {"content": text}
+ assert _tool_message(sent)["tool_call_id"] == "call_schema"
+ assert turn == original_turn
+
+
+async def test_a_tool_result_too_deep_to_walk_is_still_sent_to_a_gemini_chain(monkeypatch: pytest.MonkeyPatch) -> None:
+ deep = "{" + '"a": {' * 600 + "}" * 600 + "}"
+
+ sent = await _sent_messages(
+ monkeypatch,
+ CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 1),
+ _tool_turn(deep),
+ {"fallbacks": [_GEMINI]},
+ )
+
+ assert json.loads(_tool_text(sent)) == {"content": deep}
+
+
+ ("model_name", "output"),
+ [
+ ("azure/gpt-5.6-sol", [{"type": "text", "text": _REF_RESULT}]),
+ ("openai/gpt-5.6", _REF_RESULT),
+ ("anthropic/claude-sonnet-4-5", [{"type": "text", "text": _REF_RESULT}]),
+ ("foo/bar", _REF_RESULT),
+ (_GEMINI, '{"ok": true, "data": {"count": 3}}'),
+ (_GEMINI, [{"type": "text", "text": '{"ok": false, "error": "Timed out waiting for the page"}'}]),
+ (_GEMINI, "An error occurred while running the tool."),
+ (_GEMINI, json.dumps([{"$ref": "#/$defs/Step"}])),
+ ],
+)
+async def test_tool_results_off_the_gemini_escape_reach_the_provider_unchanged(
+ monkeypatch: pytest.MonkeyPatch,
+ model_name: str,
+ output: object,
+) -> None:
+ turn = _tool_turn(output)
+ original_turn = copy.deepcopy(turn)
+
+ raw = await _sent_messages(monkeypatch, LitellmModel(model=model_name), turn, None)
+ sent = await _sent_messages(
+ monkeypatch, CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1), turn, None
+ )
+
+ assert sent == raw
+ assert turn == original_turn
+
+
+async def test_gemini_route_leaves_an_unresolved_media_reference_for_the_provider(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ media_reference = {"screenshot": {"$ref": "page.png"}}
+ turn = _tool_turn(
+ [
+ {"type": "text", "text": json.dumps(media_reference)},
+ {"type": "image", "image_url": "data:image/png;base64,iVBORw0KGgo="},
+ ]
+ )
+
+ raw = await _sent_messages(monkeypatch, LitellmModel(model=_GEMINI), turn, None)
+ sent = await _sent_messages(
+ monkeypatch, CopilotLitellmModel(model=_GEMINI, next_model_call_index=lambda: 1), turn, None
+ )
+
+ assert sent == raw
+ [function_response] = _gemini_function_responses(sent)
+ assert function_response["response"] == media_reference
+ assert [set(part) for part in function_response["parts"]] == [{"inline_data"}]
diff --git a/tests/unit/test_copilot_model_usage.py b/tests/unit/test_copilot_model_usage.py
index 33814290d..8cb4b036a 100644
--- a/tests/unit/test_copilot_model_usage.py
+++ b/tests/unit/test_copilot_model_usage.py
@@ -119,6 +119,7 @@ def test_direct_event_is_a_closed_content_free_scalar_schema() -> None:
"cache_mode",
"cache_breakpoint_count",
"cache_stable_prefix_chars",
+ "ref_tool_outputs_escaped",
}
assert normalize_gen_ai_provider(event.provider_name, event.response_model) == "gcp.vertex_ai"