Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-9841 extract shared agent context helpers (#5988)
Andrew Neilsona8d7a358a2ba
diff --git a/skyvern/forge/sdk/agents/__init__.py b/skyvern/forge/sdk/agents/__init__.pynew file mode 100644index 000000000..22a896e0f--- /dev/null+++ b/skyvern/forge/sdk/agents/__init__.py@@ -0,0 +1 @@+"""Shared agent-adjacent SDK utilities."""diff --git a/skyvern/forge/sdk/agents/context.py b/skyvern/forge/sdk/agents/context.pynew file mode 100644index 000000000..9c17cf643--- /dev/null+++ b/skyvern/forge/sdk/agents/context.py@@ -0,0 +1,274 @@+"""Generic helpers for shaping model-facing agent context.++These helpers are intentionally mechanical. Caller-owned product policy,+including reviewer-specific persist validation and Copilot enforcement rules,+stays with the caller.+"""++from __future__ import annotations++import copy+import json+from collections.abc import Callable, Collection, Mapping+from typing import Any++import structlog++DEFAULT_TRUNCATION_SUFFIX = "\n... [truncated]"+ReplacementValue = object++LOG = structlog.get_logger()+++def get_agent_message_field(item: Any, name: str) -> Any:+ if isinstance(item, dict):+ return item.get(name)+ return getattr(item, name, None)+++def replace_agent_message_field(item: Any, name: str, value: Any) -> Any:+ if isinstance(item, dict):+ return {**item, name: value}+ try:+ updated = copy.copy(item)+ setattr(updated, name, value)+ return updated+ except (AttributeError, TypeError) as exc:+ LOG.warning(+ "Failed to rewrite agent message field; leaving item as-is",+ field=name,+ item_type=type(item).__name__,+ error=str(exc),+ )+ return item+++def compact_agent_messages_for_llm(+ messages: list[Any],+ *,+ keep_recent_tool_outputs: int,+ max_recent_tool_output_chars: int,+ summarize_tool_output: Callable[[str], str] | None = None,+ summarize_tool_arguments: Callable[[str], str] | None = None,+ tool_output_truncation_suffix: str = DEFAULT_TRUNCATION_SUFFIX,+ token_budget: int | None = None,+ estimate_tokens: Callable[[list[Any]], int] | None = None,+ is_synthetic_message: Callable[[Any], bool] | None = None,+ synthetic_message_placeholder: Any | Callable[[Any], Any] | None = None,+ emergency_tool_output_char_cap: int | None = None,+ aggressive_prune: Callable[[list[Any]], list[Any]] | None = None,+) -> list[Any]:+ """Compact model-facing messages without depending on one agent runtime."""+ _validate_non_negative("keep_recent_tool_outputs", keep_recent_tool_outputs)+ _validate_non_negative("max_recent_tool_output_chars", max_recent_tool_output_chars)+ if token_budget is not None:+ _validate_non_negative("token_budget", token_budget)+ if estimate_tokens is None:+ raise ValueError("estimate_tokens is required when token_budget is set")+ if emergency_tool_output_char_cap is not None:+ _validate_non_negative("emergency_tool_output_char_cap", emergency_tool_output_char_cap)++ items = _compact_tool_payloads(+ list(messages),+ keep_recent_tool_outputs=keep_recent_tool_outputs,+ max_recent_tool_output_chars=max_recent_tool_output_chars,+ summarize_tool_output=summarize_tool_output,+ summarize_tool_arguments=summarize_tool_arguments,+ suffix=tool_output_truncation_suffix,+ )+ if token_budget is None:+ return items++ assert estimate_tokens is not None+ if estimate_tokens(items) <= token_budget:+ return items++ if is_synthetic_message is not None and synthetic_message_placeholder is not None:+ synthetic_indices = [i for i, item in enumerate(items) if is_synthetic_message(item)]+ drop_indices = set(synthetic_indices[:-1])+ items = [+ _replacement(synthetic_message_placeholder, item) if i in drop_indices else item+ for i, item in enumerate(items)+ ]+ if estimate_tokens(items) <= token_budget:+ return items++ if emergency_tool_output_char_cap is not None:+ items = [+ _truncate_tool_output_item(item, emergency_tool_output_char_cap, tool_output_truncation_suffix)+ for item in items+ ]+ if estimate_tokens(items) <= token_budget:+ return items++ return aggressive_prune(items) if aggressive_prune is not None else items+++def sanitize_agent_tool_result_for_llm(+ tool_name: str,+ result: Mapping[str, Any],+ *,+ drop_top_level_keys: Collection[str] | None = None,+ drop_data_keys: Collection[str] | None = None,+ replacement_fields: Mapping[str, ReplacementValue] | None = None,+ large_fields: Collection[str] | None = None,+ max_chars: int = 4000,+ truncation_suffix: str = DEFAULT_TRUNCATION_SUFFIX,+) -> dict[str, Any]:+ """Return a model-facing copy of a tool result shaped by caller config."""+ _validate_non_negative("max_chars", max_chars)+ top_drops = set(drop_top_level_keys or ())+ data_drops = set(drop_data_keys or ())+ replacements = replacement_fields or {}+ large_field_names = set(large_fields or ())++ sanitized: dict[str, Any] = {}+ for key, value in result.items():+ if key in top_drops:+ continue+ if key == "data" and isinstance(value, Mapping):+ value = {data_key: data_value for data_key, data_value in value.items() if data_key not in data_drops}+ sanitized[key] = _sanitize_tool_value(+ value,+ field_name=key,+ replacements=replacements,+ large_fields=large_field_names,+ max_chars=max_chars,+ suffix=truncation_suffix,+ )+ return sanitized+++def _compact_tool_payloads(+ items: list[Any],+ *,+ keep_recent_tool_outputs: int,+ max_recent_tool_output_chars: int,+ summarize_tool_output: Callable[[str], str] | None,+ summarize_tool_arguments: Callable[[str], str] | None,+ suffix: str,+) -> list[Any]:+ output_indices = [i for i, item in enumerate(items) if _tool_output_field(item) is not None]+ call_indices = [i for i, item in enumerate(items) if get_agent_message_field(item, "type") == "function_call"]+ recent_outputs = set(output_indices[-keep_recent_tool_outputs:]) if keep_recent_tool_outputs > 0 else set()+ recent_calls = set(call_indices[-keep_recent_tool_outputs:]) if keep_recent_tool_outputs > 0 else set()++ compacted: list[Any] = []+ for i, item in enumerate(items):+ output_field = _tool_output_field(item)+ if output_field is not None:+ output = get_agent_message_field(item, output_field)+ if isinstance(output, str):+ new_output = (+ _truncate_text(output, max_recent_tool_output_chars, suffix)+ if i in recent_outputs+ else _summarize_or_truncate(output, summarize_tool_output, max_recent_tool_output_chars, suffix)+ )+ item = replace_agent_message_field(item, output_field, new_output) if new_output != output else item+ elif get_agent_message_field(item, "type") == "function_call" and i not in recent_calls:+ arguments = get_agent_message_field(item, "arguments")+ if isinstance(arguments, str):+ new_arguments = _summarize_or_truncate(+ arguments,+ summarize_tool_arguments,+ max_recent_tool_output_chars,+ suffix,+ )+ item = (+ replace_agent_message_field(item, "arguments", new_arguments)+ if new_arguments != arguments+ else item+ )+ compacted.append(item)+ return compacted+++def _tool_output_field(item: Any) -> str | None:+ if get_agent_message_field(item, "type") == "function_call_output":+ return "output"+ if get_agent_message_field(item, "role") == "tool":+ return "content"+ return None+++def _truncate_tool_output_item(item: Any, max_chars: int, suffix: str) -> Any:+ output_field = _tool_output_field(item)+ if output_field is None:+ return item+ output = get_agent_message_field(item, output_field)+ if not isinstance(output, str):+ return item+ new_output = _truncate_text(output, max_chars, suffix)+ return replace_agent_message_field(item, output_field, new_output) if new_output != output else item+++def _sanitize_tool_value(+ value: Any,+ *,+ field_name: str | None,+ replacements: Mapping[str, ReplacementValue],+ large_fields: set[str],+ max_chars: int,+ suffix: str,+) -> Any:+ if field_name is not None and field_name in replacements:+ return _replacement(replacements[field_name], value)+ if field_name is not None and field_name in large_fields:+ return _truncate_large_value(value, max_chars, suffix)+ if isinstance(value, Mapping):+ return {+ key: _sanitize_tool_value(+ nested_value,+ field_name=str(key),+ replacements=replacements,+ large_fields=large_fields,+ max_chars=max_chars,+ suffix=suffix,+ )+ for key, nested_value in value.items()+ }+ if isinstance(value, list):+ return [+ _sanitize_tool_value(+ item,+ field_name=None,+ replacements=replacements,+ large_fields=large_fields,+ max_chars=max_chars,+ suffix=suffix,+ )+ for item in value+ ]+ return value+++def _truncate_large_value(value: Any, max_chars: int, suffix: str) -> Any:+ if isinstance(value, str):+ return _truncate_text(value, max_chars, suffix)+ try:+ serialized = json.dumps(value, separators=(",", ":"), default=str)+ except (TypeError, ValueError):+ return _truncate_text(str(value), max_chars, suffix)+ return copy.deepcopy(value) if len(serialized) <= max_chars else _truncate_text(serialized, max_chars, suffix)+++def _summarize_or_truncate(+ value: str,+ summarizer: Callable[[str], str] | None,+ max_chars: int,+ suffix: str,+) -> str:+ return summarizer(value) if summarizer is not None else _truncate_text(value, max_chars, suffix)+++def _truncate_text(value: str, max_chars: int, suffix: str) -> str:+ return value[:max_chars] + suffix if len(value) > max_chars else value+++def _replacement(replacement: ReplacementValue, original: Any) -> Any:+ return replacement(original) if callable(replacement) else copy.deepcopy(replacement)+++def _validate_non_negative(name: str, value: int) -> None:+ if value < 0:+ raise ValueError(f"{name} must be >= 0")diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex 6025e0784..2eb68b933 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -9,6 +9,7 @@ import refrom collections.abc import Iteratorfrom typing import TYPE_CHECKING, Any+from skyvern.forge.sdk.agents.context import sanitize_agent_tool_result_for_llm as sanitize_generic_tool_result_for_llmfrom skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPESfrom skyvern.forge.sdk.copilot.loop_detection import LOOP_DETECTED_MARKER@@ -16,6 +17,7 @@ if TYPE_CHECKING:from agents.result import RunResultStreaming_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY = "_copilot_internal_run_cancelled_by_watchdog"+_BASE64_IMAGE_OMITTED_MESSAGE = "[base64 image omitted — screenshot was taken successfully]"_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"_JPEG_PREFIX = b"\xff\xd8\xff"@@ -181,15 +183,24 @@ def _summarize_extracted_data(extracted: Any) -> str:def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict[str, Any]:"""Strip large/binary fields from tool results before sending to the LLM."""- sanitized = dict(result)- for key in ("action", "browser_context", "artifacts", "timing_ms"):- sanitized.pop(key, None)+ sanitized = sanitize_generic_tool_result_for_llm(+ tool_name,+ result,+ drop_top_level_keys=(+ "action",+ "browser_context",+ "artifacts",+ "timing_ms",+ "_workflow",+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,+ ),+ drop_data_keys=("sdk_equivalent",),+ replacement_fields={"screenshot_base64": _BASE64_IMAGE_OMITTED_MESSAGE},+ )data = sanitized.get("data")if isinstance(data, dict):data = dict(data)- if "screenshot_base64" in data:- data["screenshot_base64"] = "[base64 image omitted — screenshot was taken successfully]"if "schema" in data and isinstance(data["schema"], dict):schema_str = json.dumps(data["schema"])# 2000 chars ~= 500 LLM tokens — enough for the model to see the@@ -219,14 +230,12 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dictblocks = data.get("blocks")if isinstance(blocks, list):data["blocks"] = [- {**block, "screenshot_b64": "[base64 image omitted — screenshot was taken successfully]"}+ {**block, "screenshot_b64": _BASE64_IMAGE_OMITTED_MESSAGE}if isinstance(block, dict) and "screenshot_b64" in blockelse blockfor block in blocks]sanitized["data"] = data- sanitized.pop("_workflow", None)- sanitized.pop(_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY, None)return sanitizeddiff --git a/skyvern/forge/sdk/copilot/session_factory.py b/skyvern/forge/sdk/copilot/session_factory.pyindex 9b471ccec..ba7408261 100644--- a/skyvern/forge/sdk/copilot/session_factory.py+++ b/skyvern/forge/sdk/copilot/session_factory.py@@ -9,6 +9,11 @@ import structlogfrom agents.memory.sqlite_session import SQLiteSessionfrom agents.run_config import CallModelData, ModelInputData+from skyvern.forge.sdk.agents.context import (+ compact_agent_messages_for_llm,+ get_agent_message_field,+ replace_agent_message_field,+)from skyvern.forge.sdk.copilot.enforcement import (_RECENT_TOOL_OUTPUT_CHAR_CAP,_TOOL_OUTPUT_HEAD_TRUNCATION_SUFFIX,@@ -34,71 +39,15 @@ def create_copilot_session(chat_id: str) -> SQLiteSession:return SQLiteSession(session_id=chat_id, db_path=":memory:")-def _item_field(item: Any, name: str) -> Any:- """Read *name* from an item that is either a dict or attr-style object."""- if isinstance(item, dict):- return item.get(name)- return getattr(item, name, None)---def _replace_item_field(item: Any, name: str, value: Any) -> Any:- """Return *item* with *name* set to *value*. Dict items get a shallow copy;- attr-style items are mutated in place. When setattr fails (frozen dataclass- / __slots__ object), the item is returned unmodified and we log a warning- so a budget overrun from a silently-unpatched item shows up in the logs- instead of disappearing."""- if isinstance(item, dict):- return {**item, name: value}- try:- setattr(item, name, value)- except (AttributeError, TypeError) as exc:- LOG.warning(- "Failed to rewrite session item field; leaving it as-is",- field=name,- item_type=type(item).__name__,- error=str(exc),- )- return item--def _compact_tool_items(items: list[Any]) -> list[Any]:- """Compact older function_call_output and function_call items using the- same KEEP_RECENT_TOOL_OUTPUTS rule as ``enforcement._prune_input_list``.-- The last ``KEEP_RECENT_TOOL_OUTPUTS`` items in each category stay full- (head-truncated only if very large). Older items get the JSON synopsis- compression. This is the session-path mirror of the non-session- ``_prune_input_list`` behavior, so first-turn transcripts with a long- tool chain get compacted just like the non-session path."""- fco_indices = [i for i, it in enumerate(items) if _item_field(it, "type") == "function_call_output"]- fc_indices = [i for i, it in enumerate(items) if _item_field(it, "type") == "function_call"]- recent_fco_set = set(fco_indices[-KEEP_RECENT_TOOL_OUTPUTS:]) if fco_indices else set()- recent_fc_set = set(fc_indices[-KEEP_RECENT_TOOL_OUTPUTS:]) if fc_indices else set()-- result: list[Any] = []- for i, item in enumerate(items):- item_type = _item_field(item, "type")- if item_type == "function_call_output":- output = _item_field(item, "output")- if isinstance(output, str):- if i in recent_fco_set:- new_output = (- output[:_RECENT_TOOL_OUTPUT_CHAR_CAP] + _TOOL_OUTPUT_HEAD_TRUNCATION_SUFFIX- if len(output) > _RECENT_TOOL_OUTPUT_CHAR_CAP- else output- )- else:- new_output = _summarize_tool_output(output)- if new_output != output:- item = _replace_item_field(item, "output", new_output)- elif item_type == "function_call" and i not in recent_fc_set:- args = _item_field(item, "arguments")- if isinstance(args, str):- new_args = _summarize_tool_arguments(args)- if new_args != args:- item = _replace_item_field(item, "arguments", new_args)- result.append(item)- return result+ return compact_agent_messages_for_llm(+ items,+ keep_recent_tool_outputs=KEEP_RECENT_TOOL_OUTPUTS,+ max_recent_tool_output_chars=_RECENT_TOOL_OUTPUT_CHAR_CAP,+ summarize_tool_output=_summarize_tool_output,+ summarize_tool_arguments=_summarize_tool_arguments,+ tool_output_truncation_suffix=_TOOL_OUTPUT_HEAD_TRUNCATION_SUFFIX,+ )def copilot_session_input_callback(@@ -129,7 +78,7 @@ def copilot_session_input_callback(# which double-emitted every non-goal item — fixed here.# * boundary == 0 — first-turn shape (fewer real users than recent_turns).# Treat everything after the goal as "middle" so the KEEP_RECENT_TOOL_OUTPUTS- # compaction inside _compact_tool_items can still fire on the long tool+ # compaction inside compact_agent_messages_for_llm can still fire on the long tool# chain. Recent is empty.if boundary >= 1:middle = history_items[1:boundary]@@ -221,17 +170,12 @@ def _copilot_call_model_input_filter(data: CallModelData[Any], *, token_budget:def _truncate_tool_output(item: Any, max_chars: int) -> Any:- """Truncate a function_call_output item's output if it exceeds max_chars.-- Handles both dict and attr-style items via the shared ``_item_field`` /- ``_replace_item_field`` helpers so Layer 3 emergency truncation stays- consistent with the KEEP_RECENT_TOOL_OUTPUTS path in ``_compact_tool_items``.- """- if _item_field(item, "type") != "function_call_output":+ """Truncate a function_call_output item's output if it exceeds max_chars."""+ if get_agent_message_field(item, "type") != "function_call_output":return item- output = _item_field(item, "output")+ output = get_agent_message_field(item, "output")if isinstance(output, str) and len(output) > max_chars:- return _replace_item_field(item, "output", output[:max_chars] + _TOOL_OUTPUT_HEAD_TRUNCATION_SUFFIX)+ return replace_agent_message_field(item, "output", output[:max_chars] + _TOOL_OUTPUT_HEAD_TRUNCATION_SUFFIX)return item@@ -244,7 +188,7 @@ def _find_real_user_boundary(items: list[Any], recent_turns: int = 2) -> int:real_count = 0for i in range(len(items) - 1, -1, -1):item = items[i]- if _item_field(item, "role") == "user" and not is_synthetic_user_message(item):+ if get_agent_message_field(item, "role") == "user" and not is_synthetic_user_message(item):real_count += 1if real_count >= recent_turns:return idiff --git a/tests/unit/test_agent_context.py b/tests/unit/test_agent_context.pynew file mode 100644index 000000000..287b0ccf9--- /dev/null+++ b/tests/unit/test_agent_context.py@@ -0,0 +1,108 @@+"""Tests for generic agent context message and tool-result helpers."""++from __future__ import annotations++import json+from typing import Any++from skyvern.forge.sdk.agents.context import (+ compact_agent_messages_for_llm,+ sanitize_agent_tool_result_for_llm,+)+++def test_compact_agent_messages_summarizes_old_tool_items_and_caps_recent_outputs() -> None:+ def summarize_output(output: str) -> str:+ return json.dumps({"_summarized": True, "length": len(output)})++ def summarize_arguments(arguments: str) -> str:+ return json.dumps({"_summarized": True, "length": len(arguments)})++ older_output = "x" * 100+ recent_output = "y" * 100+ older_args = json.dumps({"workflow_yaml": "z" * 100})+ recent_args = json.dumps({"workflow_yaml": "a" * 100})+ messages: list[dict[str, Any]] = [+ {"role": "user", "content": "build a workflow"},+ {"type": "function_call_output", "call_id": "out-old-1", "output": older_output},+ {"role": "tool", "tool_call_id": "out-old-2", "content": older_output},+ {"type": "function_call_output", "call_id": "out-new-1", "output": recent_output},+ {"type": "function_call_output", "call_id": "out-new-2", "output": recent_output},+ {"type": "function_call", "call_id": "call-old", "arguments": older_args},+ {"type": "function_call", "call_id": "call-mid", "arguments": recent_args},+ {"type": "function_call", "call_id": "call-new", "arguments": recent_args},+ ]++ compacted = compact_agent_messages_for_llm(+ messages,+ keep_recent_tool_outputs=2,+ max_recent_tool_output_chars=12,+ summarize_tool_output=summarize_output,+ summarize_tool_arguments=summarize_arguments,+ )++ outputs = [item for item in compacted if item.get("type") == "function_call_output" or item.get("role") == "tool"]+ assert json.loads(outputs[0]["output"]) == {"_summarized": True, "length": 100}+ assert json.loads(outputs[1]["content"]) == {"_summarized": True, "length": 100}+ assert outputs[2]["output"] == "y" * 12 + "\n... [truncated]"+ assert outputs[3]["output"] == "y" * 12 + "\n... [truncated]"+ calls = [item for item in compacted if item.get("type") == "function_call"]+ assert json.loads(calls[0]["arguments"]) == {"_summarized": True, "length": len(older_args)}+ assert calls[1]["arguments"] == recent_args+ assert calls[2]["arguments"] == recent_args+ assert messages[1]["output"] == older_output+++def test_compact_agent_messages_replaces_old_synthetic_messages_when_over_budget() -> None:+ messages: list[dict[str, Any]] = [+ {"role": "user", "content": "goal"},+ {"role": "user", "content": "[screenshot] old"},+ {"role": "assistant", "content": "middle"},+ {"role": "user", "content": "[screenshot] recent"},+ ]++ compacted = compact_agent_messages_for_llm(+ messages,+ keep_recent_tool_outputs=3,+ max_recent_tool_output_chars=2000,+ token_budget=1,+ estimate_tokens=lambda items: sum(len(str(item)) for item in items),+ is_synthetic_message=lambda item: item.get("content", "").startswith("[screenshot]"),+ synthetic_message_placeholder={"role": "user", "content": "[screenshot omitted]"},+ )++ assert compacted[1] == {"role": "user", "content": "[screenshot omitted]"}+ assert compacted[3] == messages[3]+ assert messages[1]["content"] == "[screenshot] old"+++def test_sanitize_agent_tool_result_for_llm_shapes_configured_fields_without_mutating_original() -> None:+ raw = {+ "ok": True,+ "action": "inspect_page",+ "data": {+ "content": "a" * 30,+ "sdk_equivalent": "await page.content()",+ "screenshot_base64": "iVBORw0KGgo" + "A" * 200,+ "nested": [{"html": "b" * 30}],+ },+ }++ sanitized = sanitize_agent_tool_result_for_llm(+ tool_name="inspect_page",+ result=raw,+ drop_top_level_keys={"action"},+ drop_data_keys={"sdk_equivalent"},+ replacement_fields={"screenshot_base64": "[image omitted]"},+ large_fields={"content", "html"},+ max_chars=10,+ )++ assert "action" not in sanitized+ assert "sdk_equivalent" not in sanitized["data"]+ assert sanitized["data"]["screenshot_base64"] == "[image omitted]"+ assert sanitized["data"]["content"] == "a" * 10 + "\n... [truncated]"+ assert sanitized["data"]["nested"][0]["html"] == "b" * 10 + "\n... [truncated]"+ assert raw["action"] == "inspect_page"+ assert raw["data"]["content"] == "a" * 30+ assert raw["data"]["nested"][0]["html"] == "b" * 30