Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-10626): cap raw HTML + screenshots in the max-retries failure summary (#6361)
pedrohsdb13376f12d23b
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex bcc54ca87..7fc183b4d 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -130,7 +130,7 @@ from skyvern.utils.prompt_engine import (enforce_prompt_ceiling_tracked,load_prompt_with_elements,)-from skyvern.utils.prompt_truncation import truncate_extraction_schema+from skyvern.utils.prompt_truncation import truncate_extraction_schema, truncate_page_html_for_summaryfrom skyvern.utils.token_counter import count_tokensfrom skyvern.utils.url_validators import strip_query_paramsfrom skyvern.webeye.actions.action_types import ActionType@@ -5408,8 +5408,10 @@ class ForgeAgent:if page is not None:skyvern_frame = await SkyvernFrame.create_instance(frame=page)- html = await skyvern_frame.get_content()- screenshots = await SkyvernFrame.take_split_screenshots(page=page, url=page.url)+ html = truncate_page_html_for_summary(await skyvern_frame.get_content())+ # scroll=False: one current-viewport shot (the failing region) is enough for a+ # failure summary; avoids 10 full-page strips. SKY-10626.+ screenshots = await SkyvernFrame.take_split_screenshots(page=page, url=page.url, scroll=False)prompt = prompt_engine.load_prompt("summarize-max-retries-reason",diff --git a/skyvern/utils/prompt_truncation.py b/skyvern/utils/prompt_truncation.pyindex 6873bef51..35fc5a111 100644--- a/skyvern/utils/prompt_truncation.py+++ b/skyvern/utils/prompt_truncation.py@@ -1,8 +1,10 @@"""Helpers for capping prompt inputs before templating.Separate from prompt_engine.py so per-field caps can be applied at the call-boundary without reaching into the element-tree truncation logic. Reuses-skyvern.utils.token_counter.count_tokens for consistent measurement.+boundary without reaching into the element-tree truncation logic. Token-based+caps reuse skyvern.utils.token_counter.count_tokens for consistent measurement;+hot-path caps (see truncate_page_html_for_summary) slice by characters instead+to stay off tiktoken."""from __future__ import annotations@@ -20,6 +22,13 @@ PREVIOUS_EXTRACTED_INFO_MAX_TOKENS = 20_000EXTRACTION_SCHEMA_MAX_TOKENS = 10_000EXTRACTION_GOAL_MAX_TOKENS = 150_000+# ~150k chars ≈ ~37k tokens; char-based for the hot path (see truncate_page_html_for_summary).+# A small head keeps page identity; the rest of the budget goes to the body tail, where the+# failing element usually lives (the raw <head> is low-signal).+SUMMARY_PAGE_HTML_MAX_CHARS = 150_000+SUMMARY_HTML_HEAD_CHARS = 10_000+_SUMMARY_HTML_TRUNCATION_MARKER = "\n<!-- [truncated by Skyvern: page HTML exceeded the failure-summary budget] -->"+def _crop_string(value: str, max_tokens: int) -> str:if max_tokens <= 0:@@ -98,6 +107,26 @@ def truncate_previous_extracted_information(return result+def truncate_page_html_for_summary(+ html: str,+ max_chars: int = SUMMARY_PAGE_HTML_MAX_CHARS,+) -> str:+ # Char-based, not token-based: this fires on every max-retry step (high volume in+ # retry storms) on 1M+ token HTML, so running tiktoken on the full string is too costly here.+ if not html or len(html) <= max_chars:+ return html+ head_chars = min(SUMMARY_HTML_HEAD_CHARS, max_chars)+ tail_chars = max_chars - head_chars+ LOG.debug(+ "Truncated page_html for failure summary",+ before_chars=len(html),+ max_chars=max_chars,+ )+ if tail_chars <= 0:+ return html[:head_chars] + _SUMMARY_HTML_TRUNCATION_MARKER+ return html[:head_chars] + _SUMMARY_HTML_TRUNCATION_MARKER + html[-tail_chars:]++def truncate_extraction_schema(schema: Any,max_tokens: int = EXTRACTION_SCHEMA_MAX_TOKENS,diff --git a/tests/unit/test_prompt_truncation.py b/tests/unit/test_prompt_truncation.pyindex 51fd84b76..01a08fc83 100644--- a/tests/unit/test_prompt_truncation.py+++ b/tests/unit/test_prompt_truncation.py@@ -134,3 +134,52 @@ def test_truncate_extraction_schema_preserves_array_top_level() -> None:result = truncate_extraction_schema(schema, max_tokens=2_000)assert count_tokens(json.dumps(result)) <= 2_200assert result["type"] == "array"+++def test_truncate_page_html_short_returns_unchanged() -> None:+ from skyvern.utils.prompt_truncation import truncate_page_html_for_summary++ html = "<html><body>small page</body></html>"+ assert truncate_page_html_for_summary(html, max_chars=10_000) == html+++def test_truncate_page_html_empty_returns_unchanged() -> None:+ from skyvern.utils.prompt_truncation import truncate_page_html_for_summary++ assert truncate_page_html_for_summary("", max_chars=10_000) == ""+++def test_truncate_page_html_long_is_bounded_and_marked() -> None:+ from skyvern.utils.prompt_truncation import (+ _SUMMARY_HTML_TRUNCATION_MARKER,+ truncate_page_html_for_summary,+ )++ html = "<div>" + ("x" * 1_000_000) + "</div>"+ result = truncate_page_html_for_summary(html, max_chars=120_000)+ assert len(result) == 120_000 + len(_SUMMARY_HTML_TRUNCATION_MARKER)+ assert result.startswith("<div>")+ assert _SUMMARY_HTML_TRUNCATION_MARKER in result+++def test_truncate_page_html_keeps_head_and_tail() -> None:+ from skyvern.utils.prompt_truncation import truncate_page_html_for_summary++ html = "HEAD_MARKER" + ("x" * 500_000) + "TAIL_MARKER"+ result = truncate_page_html_for_summary(html, max_chars=50_000)+ assert "HEAD_MARKER" in result # small head kept for page identity+ assert "TAIL_MARKER" in result # tail kept — head-only slicing would have dropped this+ assert len(result) < len(html) # middle dropped+++def test_truncate_page_html_tiny_budget_keeps_head_only() -> None:+ from skyvern.utils.prompt_truncation import (+ _SUMMARY_HTML_TRUNCATION_MARKER,+ SUMMARY_HTML_HEAD_CHARS,+ truncate_page_html_for_summary,+ )++ budget = SUMMARY_HTML_HEAD_CHARS // 2 # tail_chars == 0, so the head-only branch runs+ html = "h" * (budget * 4)+ result = truncate_page_html_for_summary(html, max_chars=budget)+ assert result == html[:budget] + _SUMMARY_HTML_TRUNCATION_MARKER