AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/copilot): preserve main content in web_fetch and warn on JS-only shells (#14875)
Johnny Wilson Doughertya98edc442b2a
diff --git a/autogpt_platform/backend/backend/copilot/tools/web_fetch.py b/autogpt_platform/backend/backend/copilot/tools/web_fetch.pyindex 3fc5e53781..a2bb33e7e5 100644--- a/autogpt_platform/backend/backend/copilot/tools/web_fetch.py+++ b/autogpt_platform/backend/backend/copilot/tools/web_fetch.py@@ -1,8 +1,10 @@"""Web fetch tool — safely retrieve public web page content."""+import asyncioimport loggingimport re-from html import unescape+from html import escape, unescape+from html.parser import HTMLParserfrom typing import Anyimport aiohttp@@ -17,7 +19,10 @@ from .models import ErrorResponse, ToolResponseBase, WebFetchResponselogger = logging.getLogger(__name__)# Limits-_MAX_CONTENT_BYTES = 102_400 # 100 KB download cap+_MAX_DOWNLOAD_BYTES = (+ 2_097_152 # 2 MB of the response body is decoded and converted to text+)+_MAX_TEXT_CHARS = 100_000 # 100K characters text budget for the model_REQUEST_TIMEOUT = aiohttp.ClientTimeout(total=15)# Content types we'll read as text@@ -38,6 +43,73 @@ _TEXT_CONTENT_TYPES = {"application/ld+json",}+# SPA container and fallback signals+_SPA_SHELL_PATTERNS = re.compile(+ r"""(?<![-a-zA-Z0-9_])id\s*=\s*['"]?(?:root|__next|app)['"]?(?=[\s>/]|$)""",+ re.IGNORECASE,+)+_SPA_FALLBACK_TEXT = "you need to enable javascript to run this app"+++class _HTMLCleaner(HTMLParser):+ """Filter out non-content elements (script, style, noscript, svg) before text extraction.++ While html2text ignores script and style contents natively, this cleaner additionally+ drops noscript and svg elements and strips non-content tags before markdown conversion.+ """++ _DROP_TAGS = frozenset({"script", "style", "noscript", "svg"})++ def __init__(self) -> None:+ super().__init__(convert_charrefs=False)+ self._skip_depth = 0+ self._pieces: list[str] = []++ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:+ if tag.lower() in self._DROP_TAGS:+ self._skip_depth += 1+ elif self._skip_depth == 0:+ attr_str = "".join(+ f' {k}="{escape(v, quote=True)}"' if v is not None else f" {k}"+ for k, v in attrs+ )+ self._pieces.append(f"<{tag}{attr_str}>")++ def handle_endtag(self, tag: str) -> None:+ if tag.lower() in self._DROP_TAGS:+ if self._skip_depth > 0:+ self._skip_depth -= 1+ if self._skip_depth == 0:+ self._pieces.append(" ")+ elif self._skip_depth == 0:+ self._pieces.append(f"</{tag}>")++ def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:+ if tag.lower() in self._DROP_TAGS:+ if self._skip_depth == 0:+ self._pieces.append(" ")+ elif self._skip_depth == 0:+ attr_str = "".join(+ f' {k}="{escape(v, quote=True)}"' if v is not None else f" {k}"+ for k, v in attrs+ )+ self._pieces.append(f"<{tag}{attr_str} />")++ def handle_data(self, data: str) -> None:+ if self._skip_depth == 0:+ self._pieces.append(data)++ def handle_entityref(self, name: str) -> None:+ if self._skip_depth == 0:+ self._pieces.append(f"&{name};")++ def handle_charref(self, name: str) -> None:+ if self._skip_depth == 0:+ self._pieces.append(f"&#{name};")++ def get_cleaned_html(self) -> str:+ return "".join(self._pieces)+def _is_text_content(content_type: str) -> bool:base = content_type.split(";")[0].strip().lower()@@ -45,11 +117,20 @@ def _is_text_content(content_type: str) -> bool:def _html_to_text(html: str) -> str:+ cleaner = _HTMLCleaner()+ try:+ cleaner.feed(html)+ cleaned_html = cleaner.get_cleaned_html()+ except Exception as err:+ logger.debug(+ "HTML cleaner encountered an error, falling back to raw html: %s", err+ )+ cleaned_html = htmlh = html2text.HTML2Text()h.ignore_links = Falseh.ignore_images = Trueh.body_width = 0- return h.handle(html)+ return h.handle(cleaned_html).strip()_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)@@ -63,6 +144,15 @@ def _extract_title(html: str) -> str | None:return title or None+def _is_client_rendered_shell(raw_html: str, extracted_text: str) -> bool:+ """Near-empty readable text on a page that carries an SPA mount point or noscript fallback."""+ if len(extracted_text.strip()) >= 200:+ return False+ return bool(_SPA_SHELL_PATTERNS.search(raw_html)) or (+ _SPA_FALLBACK_TEXT in raw_html.lower()+ )++class WebFetchTool(BaseTool):"""Safely fetch content from a public URL using SSRF-protected HTTP."""@@ -140,23 +230,62 @@ class WebFetchTool(BaseTool):session_id=session_id,)- raw = response.content[:_MAX_CONTENT_BYTES]- text = raw.decode("utf-8", errors="replace")+ raw_bytes = response.content[:_MAX_DOWNLOAD_BYTES]+ raw_text = raw_bytes.decode("utf-8", errors="replace")title = None- if "html" in content_type.lower():- title = _extract_title(text)+ is_html = "html" in content_type.lower()+ raw_truncated = len(response.content) > _MAX_DOWNLOAD_BYTES+ text_truncated = False++ if is_html:+ title = _extract_title(raw_text)if extract_text:- text = _html_to_text(text)+ text = await asyncio.to_thread(_html_to_text, raw_text)+ else:+ text = raw_text+ else:+ text = raw_text++ # Enforce character budget on the extracted text+ if len(text) > _MAX_TEXT_CHARS:+ text = text[:_MAX_TEXT_CHARS]+ text_truncated = True++ truncated = raw_truncated or text_truncated++ message = f"Fetched {url}"++ # Detect JavaScript-rendered SPA shells and surface an actionable hint+ if is_html and extract_text and _is_client_rendered_shell(raw_text, text):+ hint = (+ "[Notice: Content not rendered. This page appears to require JavaScript "+ "to render its content. Use the 'tool:browser_navigate' tool instead.]"+ )+ text = f"{hint}\n\n{text}".strip()+ message = f"Fetched {url} — warning: content not rendered (use tool:browser_navigate)"++ if text_truncated and raw_truncated:+ message += f" (download capped at {_MAX_DOWNLOAD_BYTES:,} bytes, text truncated to {_MAX_TEXT_CHARS:,} chars)"+ text += (+ f"\n\n[Content truncated — response exceeded {_MAX_DOWNLOAD_BYTES:,} bytes "+ f"and text was capped at {_MAX_TEXT_CHARS:,} characters]"+ )+ elif text_truncated:+ message += f" (truncated to {_MAX_TEXT_CHARS:,} chars)"+ text += f"\n\n[Content truncated — limit of {_MAX_TEXT_CHARS:,} characters reached]"+ elif raw_truncated:+ message += f" (raw content truncated at {_MAX_DOWNLOAD_BYTES:,} bytes)"+ text += f"\n\n[Content truncated — response body exceeded network cap of {_MAX_DOWNLOAD_BYTES:,} bytes]"return WebFetchResponse(- message=f"Fetched {url}",+ message=message,url=response.url,status_code=response.status,content_type=content_type.split(";")[0].strip(),content=text,title=title,content_length=len(response.content),- truncated=len(response.content) > _MAX_CONTENT_BYTES,+ truncated=truncated,session_id=session_id,)diff --git a/autogpt_platform/backend/backend/copilot/tools/web_fetch_test.py b/autogpt_platform/backend/backend/copilot/tools/web_fetch_test.pyindex 2d77e9255f..f71bc711c8 100644--- a/autogpt_platform/backend/backend/copilot/tools/web_fetch_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/web_fetch_test.py@@ -4,7 +4,14 @@ import pytestfrom ._test_data import make_sessionfrom .models import WebFetchResponse-from .web_fetch import WebFetchTool, _extract_title+from .web_fetch import (+ _MAX_DOWNLOAD_BYTES,+ _MAX_TEXT_CHARS,+ WebFetchTool,+ _extract_title,+ _html_to_text,+ _is_client_rendered_shell,+)def test_extract_title_decodes_entities_and_normalizes_whitespace():@@ -18,12 +25,76 @@ def test_extract_title_returns_none_when_missing():assert _extract_title("<html><body>No title</body></html>") is None+def test_is_client_rendered_shell_detects_root_div():+ html = '<html><head><script src="/app.js"></script></head><body><div id="root"></div></body></html>'+ assert _is_client_rendered_shell(html, "Home Menu Contact") is True+++def test_is_client_rendered_shell_detects_single_quoted_root():+ html = "<html><head><script src='/app.js'></script></head><body><div id='root'></div></body></html>"+ assert _is_client_rendered_shell(html, "Home Menu Contact") is True+++def test_is_client_rendered_shell_ignores_unrelated_attributes_with_root():+ html = '<html><body><div data-testid="root">Content</div><div id="root-heading">Title</div></body></html>'+ assert _is_client_rendered_shell(html, "Content Title") is False+++def test_html_cleaner_filters_scripts_and_styles_safely():+ dirty_html = (+ "<html><head><script type='text/javascript'>const a = '<p>code</p>';</script>"+ "<style>body { color: red; }</style></head>"+ "<body><h1>Hello</h1><svg><path d='M0 0' /></svg>"+ "<noscript><p>Please enable JS</p></noscript>"+ "<p>World & Universe</p></body></html>"+ )+ cleaned = _html_to_text(dirty_html)+ assert "Hello" in cleaned+ assert "World & Universe" in cleaned+ assert "color: red" not in cleaned+ assert "const a" not in cleaned+ assert "Please enable JS" not in cleaned+++def test_is_client_rendered_shell_false_for_normal_content():+ html = (+ "<html><body>"+ + ("<p>Detailed documentation section.</p>" * 30)+ + "</body></html>"+ )+ text = "Detailed documentation section. " * 30+ assert _is_client_rendered_shell(html, text) is False+++def test_is_client_rendered_shell_false_for_short_page_with_script():+ html = (+ "<html><head><script>/* 3.5kb analytics payload */\n"+ + ("ga('send', 'pageview');\n" * 150)+ + "</script></head>"+ "<body><h1>Opening Hours</h1><p>Mon-Fri: 9am - 5pm. Sat: 10am - 2pm. Closed Sundays.</p></body></html>"+ )+ text = "Opening Hours\n\nMon-Fri: 9am - 5pm. Sat: 10am - 2pm. Closed Sundays."+ assert _is_client_rendered_shell(html, text) is False+++def test_is_client_rendered_shell_false_for_content_exceeding_threshold_in_app_div():+ html = (+ '<html><body><div id="app"><h1>Changelog</h1><p>'+ + ("Version update details and notes. " * 15)+ + "</p></div></body></html>"+ )+ text = "Changelog\n\n" + ("Version update details and notes. " * 15)+ assert len(text.strip()) > 200+ assert _is_client_rendered_shell(html, text) is False++@pytest.mark.asyncio(loop_scope="session")async def test_execute_returns_page_metadata_and_truncation():response = MagicMock()response.headers = {"content-type": "text/html; charset=utf-8"}response.content = (- b"<html><title>Example</title><body>Hello</body></html>" + b" " * 102_400+ b"<html><title>Example</title><body>Hello</body></html>"+ + b" " * (_MAX_DOWNLOAD_BYTES + 100))response.url = "https://example.com/"response.status = 200@@ -41,6 +112,73 @@ async def test_execute_returns_page_metadata_and_truncation():assert result.title == "Example"assert result.content_length == len(response.content)assert result.truncated is True+ assert "truncated" in result.message++[email protected](loop_scope="session")+async def test_execute_extracts_content_below_fold_past_100kb_head():+ # Construct an HTML document with 120 KB of style/script head noise,+ # followed by the critical body content+ large_head = "<style>" + ("/* css rule */\n" * 7000) + "</style>"+ body_content = "### Enterprise Pricing Table\nTier 1: $100/mo\nTier 2: $500/mo"+ html = f"<html><head><title>Docs</title>{large_head}</head><body><h1>Docs</h1><p>{body_content}</p></body></html>"++ response = MagicMock()+ response.headers = {"content-type": "text/html; charset=utf-8"}+ response.content = html.encode("utf-8")+ response.url = "https://docs.example.com/pricing"+ response.status = 200++ client = MagicMock()+ client.get = AsyncMock(return_value=response)++ with patch("backend.copilot.tools.web_fetch.Requests", return_value=client):+ result = await WebFetchTool()._execute(+ user_id="test-user",+ session=make_session(user_id="test-user"),+ url="https://docs.example.com/pricing",+ extract_text=True,+ )++ assert isinstance(result, WebFetchResponse)+ assert result.title == "Docs"+ # Verify content below the 100 KB fold survived extraction+ assert "Enterprise Pricing Table" in result.content+ assert "Tier 1: $100/mo" in result.content+ assert "/* css rule */" not in result.content # Head bloat was cleanly stripped++[email protected](loop_scope="session")+async def test_execute_detects_js_rendered_shell_and_emits_hint():+ spa_html = (+ "<!DOCTYPE html><html><head><title>SPA App</title>"+ '<script src="/bundle.js"></script></head>'+ '<body><div id="root"></div>'+ '<nav><a href="/home">Home</a><a href="/login">Login</a></nav>'+ "</body></html>"+ )++ response = MagicMock()+ response.headers = {"content-type": "text/html; charset=utf-8"}+ response.content = spa_html.encode("utf-8")+ response.url = "https://app.example.com"+ response.status = 200++ client = MagicMock()+ client.get = AsyncMock(return_value=response)++ with patch("backend.copilot.tools.web_fetch.Requests", return_value=client):+ result = await WebFetchTool()._execute(+ user_id="test-user",+ session=make_session(user_id="test-user"),+ url="https://app.example.com",+ extract_text=True,+ )++ assert isinstance(result, WebFetchResponse)+ assert "tool:browser_navigate" in result.message+ assert "tool:browser_navigate" in result.content+ assert "Content not rendered" in result.content@pytest.mark.asyncio(loop_scope="session")@@ -66,3 +204,64 @@ async def test_execute_reports_small_original_body_size_after_text_extraction():assert "<body>" not in result.contentassert result.content_length == len(response.content)assert result.truncated is False++[email protected](loop_scope="session")+async def test_execute_handles_raw_truncation_only():+ large_html = (+ "<html><head><script>"+ + ("var x = 1;\n" * 250_000)+ + "</script></head><body><p>Short content</p></body></html>"+ )+ content_bytes = large_html.encode("utf-8")+ assert len(content_bytes) > _MAX_DOWNLOAD_BYTES++ response = MagicMock()+ response.headers = {"content-type": "text/html; charset=utf-8"}+ response.content = content_bytes+ response.url = "https://example.com/bloat"+ response.status = 200+ client = MagicMock()+ client.get = AsyncMock(return_value=response)++ with patch("backend.copilot.tools.web_fetch.Requests", return_value=client):+ result = await WebFetchTool()._execute(+ user_id="test-user",+ session=make_session(user_id="test-user"),+ url="https://example.com/bloat",+ extract_text=True,+ )++ assert isinstance(result, WebFetchResponse)+ assert result.truncated is True+ assert "raw content truncated at" in result.message+ assert "exceeded network cap" in result.content++[email protected](loop_scope="session")+async def test_execute_handles_text_truncation_only():+ content_str = "Word " * 25_000+ content_bytes = content_str.encode("utf-8")+ assert len(content_bytes) < _MAX_DOWNLOAD_BYTES+ assert len(content_str) > _MAX_TEXT_CHARS++ response = MagicMock()+ response.headers = {"content-type": "text/plain; charset=utf-8"}+ response.content = content_bytes+ response.url = "https://example.com/long.txt"+ response.status = 200+ client = MagicMock()+ client.get = AsyncMock(return_value=response)++ with patch("backend.copilot.tools.web_fetch.Requests", return_value=client):+ result = await WebFetchTool()._execute(+ user_id="test-user",+ session=make_session(user_id="test-user"),+ url="https://example.com/long.txt",+ )++ assert isinstance(result, WebFetchResponse)+ assert result.truncated is True+ assert f"truncated to {_MAX_TEXT_CHARS:,} chars" in result.message+ assert f"limit of {_MAX_TEXT_CHARS:,} characters reached" in result.content+ assert len(result.content) < _MAX_TEXT_CHARS + 200