Skip to content

Automate browser based workflows with AI

fix(SKY-10626): cap raw HTML + screenshots in the max-retries failure summary (#6361)

pedrohsdb13376f12d23b
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index bcc54ca87..7fc183b4d 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -130,7 +130,7 @@ from skyvern.utils.prompt_engine import (
enforce_prompt_ceiling_tracked,
load_prompt_with_elements,
)
-from skyvern.utils.prompt_truncation import truncate_extraction_schema
+from skyvern.utils.prompt_truncation import truncate_extraction_schema, truncate_page_html_for_summary
from skyvern.utils.token_counter import count_tokens
from skyvern.utils.url_validators import strip_query_params
from skyvern.webeye.actions.action_types import ActionType
@@ -5408,8 +5408,10 @@ class ForgeAgent:
if page is not None:
skyvern_frame = await SkyvernFrame.create_instance(frame=page)
- html = await skyvern_frame.get_content()
- screenshots = await SkyvernFrame.take_split_screenshots(page=page, url=page.url)
+ html = truncate_page_html_for_summary(await skyvern_frame.get_content())
+ # scroll=False: one current-viewport shot (the failing region) is enough for a
+ # failure summary; avoids 10 full-page strips. SKY-10626.
+ screenshots = await SkyvernFrame.take_split_screenshots(page=page, url=page.url, scroll=False)
prompt = prompt_engine.load_prompt(
"summarize-max-retries-reason",
diff --git a/skyvern/utils/prompt_truncation.py b/skyvern/utils/prompt_truncation.py
index 6873bef51..35fc5a111 100644
--- a/skyvern/utils/prompt_truncation.py
+++ b/skyvern/utils/prompt_truncation.py
@@ -1,8 +1,10 @@
"""Helpers for capping prompt inputs before templating.
Separate from prompt_engine.py so per-field caps can be applied at the call
-boundary without reaching into the element-tree truncation logic. Reuses
-skyvern.utils.token_counter.count_tokens for consistent measurement.
+boundary without reaching into the element-tree truncation logic. Token-based
+caps reuse skyvern.utils.token_counter.count_tokens for consistent measurement;
+hot-path caps (see truncate_page_html_for_summary) slice by characters instead
+to stay off tiktoken.
"""
from __future__ import annotations
@@ -20,6 +22,13 @@ PREVIOUS_EXTRACTED_INFO_MAX_TOKENS = 20_000
EXTRACTION_SCHEMA_MAX_TOKENS = 10_000
EXTRACTION_GOAL_MAX_TOKENS = 150_000
+# ~150k chars ≈ ~37k tokens; char-based for the hot path (see truncate_page_html_for_summary).
+# A small head keeps page identity; the rest of the budget goes to the body tail, where the
+# failing element usually lives (the raw <head> is low-signal).
+SUMMARY_PAGE_HTML_MAX_CHARS = 150_000
+SUMMARY_HTML_HEAD_CHARS = 10_000
+_SUMMARY_HTML_TRUNCATION_MARKER = "\n<!-- [truncated by Skyvern: page HTML exceeded the failure-summary budget] -->"
+
def _crop_string(value: str, max_tokens: int) -> str:
if max_tokens <= 0:
@@ -98,6 +107,26 @@ def truncate_previous_extracted_information(
return result
+def truncate_page_html_for_summary(
+ html: str,
+ max_chars: int = SUMMARY_PAGE_HTML_MAX_CHARS,
+) -> str:
+ # Char-based, not token-based: this fires on every max-retry step (high volume in
+ # retry storms) on 1M+ token HTML, so running tiktoken on the full string is too costly here.
+ if not html or len(html) <= max_chars:
+ return html
+ head_chars = min(SUMMARY_HTML_HEAD_CHARS, max_chars)
+ tail_chars = max_chars - head_chars
+ LOG.debug(
+ "Truncated page_html for failure summary",
+ before_chars=len(html),
+ max_chars=max_chars,
+ )
+ if tail_chars <= 0:
+ return html[:head_chars] + _SUMMARY_HTML_TRUNCATION_MARKER
+ return html[:head_chars] + _SUMMARY_HTML_TRUNCATION_MARKER + html[-tail_chars:]
+
+
def truncate_extraction_schema(
schema: Any,
max_tokens: int = EXTRACTION_SCHEMA_MAX_TOKENS,
diff --git a/tests/unit/test_prompt_truncation.py b/tests/unit/test_prompt_truncation.py
index 51fd84b76..01a08fc83 100644
--- a/tests/unit/test_prompt_truncation.py
+++ b/tests/unit/test_prompt_truncation.py
@@ -134,3 +134,52 @@ def test_truncate_extraction_schema_preserves_array_top_level() -> None:
result = truncate_extraction_schema(schema, max_tokens=2_000)
assert count_tokens(json.dumps(result)) <= 2_200
assert result["type"] == "array"
+
+
+def test_truncate_page_html_short_returns_unchanged() -> None:
+ from skyvern.utils.prompt_truncation import truncate_page_html_for_summary
+
+ html = "<html><body>small page</body></html>"
+ assert truncate_page_html_for_summary(html, max_chars=10_000) == html
+
+
+def test_truncate_page_html_empty_returns_unchanged() -> None:
+ from skyvern.utils.prompt_truncation import truncate_page_html_for_summary
+
+ assert truncate_page_html_for_summary("", max_chars=10_000) == ""
+
+
+def test_truncate_page_html_long_is_bounded_and_marked() -> None:
+ from skyvern.utils.prompt_truncation import (
+ _SUMMARY_HTML_TRUNCATION_MARKER,
+ truncate_page_html_for_summary,
+ )
+
+ html = "<div>" + ("x" * 1_000_000) + "</div>"
+ result = truncate_page_html_for_summary(html, max_chars=120_000)
+ assert len(result) == 120_000 + len(_SUMMARY_HTML_TRUNCATION_MARKER)
+ assert result.startswith("<div>")
+ assert _SUMMARY_HTML_TRUNCATION_MARKER in result
+
+
+def test_truncate_page_html_keeps_head_and_tail() -> None:
+ from skyvern.utils.prompt_truncation import truncate_page_html_for_summary
+
+ html = "HEAD_MARKER" + ("x" * 500_000) + "TAIL_MARKER"
+ result = truncate_page_html_for_summary(html, max_chars=50_000)
+ assert "HEAD_MARKER" in result # small head kept for page identity
+ assert "TAIL_MARKER" in result # tail kept — head-only slicing would have dropped this
+ assert len(result) < len(html) # middle dropped
+
+
+def test_truncate_page_html_tiny_budget_keeps_head_only() -> None:
+ from skyvern.utils.prompt_truncation import (
+ _SUMMARY_HTML_TRUNCATION_MARKER,
+ SUMMARY_HTML_HEAD_CHARS,
+ truncate_page_html_for_summary,
+ )
+
+ budget = SUMMARY_HTML_HEAD_CHARS // 2 # tail_chars == 0, so the head-only branch runs
+ html = "h" * (budget * 4)
+ result = truncate_page_html_for_summary(html, max_chars=budget)
+ assert result == html[:budget] + _SUMMARY_HTML_TRUNCATION_MARKER