Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Filter out json and metadata from markdown
Magnus Müller84395f1641b0
diff --git a/browser_use/dom/serializer/html_serializer.py b/browser_use/dom/serializer/html_serializer.pyindex afddf3604..5050cfa17 100644--- a/browser_use/dom/serializer/html_serializer.py+++ b/browser_use/dom/serializer/html_serializer.py@@ -71,6 +71,23 @@ class HTMLSerializer:if tag_name in {'style', 'script', 'head', 'meta', 'link', 'title'}:return ''+ # Skip code tags with display:none - these often contain JSON state for SPAs+ if tag_name == 'code' and node.attributes:+ style = node.attributes.get('style', '')+ # Check if element is hidden (display:none) - likely JSON data+ if 'display:none' in style.replace(' ', '') or 'display: none' in style:+ return ''+ # Also check for bpr-guid IDs (LinkedIn's JSON data pattern)+ element_id = node.attributes.get('id', '')+ if 'bpr-guid' in element_id or 'data' in element_id or 'state' in element_id:+ return ''++ # Skip base64 inline images - these are usually placeholders or tracking pixels+ if tag_name == 'img' and node.attributes:+ src = node.attributes.get('src', '')+ if src.startswith('data:image/'):+ return ''+# Opening tagparts.append(f'<{tag_name}')@@ -158,6 +175,11 @@ class HTMLSerializer:if not self.extract_links and key == 'href':continue+ # Skip data-* attributes as they often contain JSON payloads+ # These are used by modern SPAs (React, Vue, Angular) for state management+ if key.startswith('data-'):+ continue+# Handle boolean attributesif value == '' or value is None:parts.append(key)diff --git a/browser_use/tools/service.py b/browser_use/tools/service.pyindex 79c0b6739..ea8ae6847 100644--- a/browser_use/tools/service.py+++ b/browser_use/tools/service.py@@ -1080,6 +1080,12 @@ You will be given a query and the markdown of a webpage that has been filtered tstrip=['script', 'style'], # Remove these tagsbullets='-', # Use - for unordered listscode_language='', # Don't add language to code blocks+ escape_asterisks=False, # Don't escape asterisks (cleaner output)+ escape_underscores=False, # Don't escape underscores (cleaner output)+ escape_misc=False, # Don't escape other characters (cleaner output)+ autolinks=False, # Don't convert URLs to <> format+ default_title=False, # Don't add default title attributes+ keep_inline_images_in=[], # Don't keep inline images in any tags (we already filter base64 in HTML))initial_markdown_length = len(content)@@ -1109,10 +1115,10 @@ You will be given a query and the markdown of a webpage that has been filtered tdef _preprocess_markdown_content(self, content: str, max_newlines: int = 3) -> tuple[str, int]:"""- Light preprocessing of html2text output - minimal cleanup since html2text is already clean.+ Light preprocessing of markdown output - minimal cleanup with JSON blob removal.Args:- content: Markdown content from html2text to lightly filter+ content: Markdown content to lightly filtermax_newlines: Maximum consecutive newlines to allowReturns:@@ -1122,6 +1128,14 @@ You will be given a query and the markdown of a webpage that has been filtered toriginal_length = len(content)+ # Remove JSON blobs (common in SPAs like LinkedIn, Facebook, etc.)+ # These are often embedded as `{"key":"value",...}` and can be massive+ # Match JSON objects/arrays that are at least 100 chars long+ # This catches SPA state/config data without removing small inline JSON+ content = re.sub(r'`\{["\w].*?\}`', '', content, flags=re.DOTALL) # Remove JSON in code blocks+ content = re.sub(r'\{"\$type":[^}]{100,}\}', '', content) # Remove JSON with $type fields (common pattern)+ content = re.sub(r'\{"[^"]{5,}":\{[^}]{100,}\}', '', content) # Remove nested JSON objects+# Compress consecutive newlines (4+ newlines become max_newlines)content = re.sub(r'\n{4,}', '\n' * max_newlines, content)@@ -1130,8 +1144,11 @@ You will be given a query and the markdown of a webpage that has been filtered tfiltered_lines = []for line in lines:stripped = line.strip()- # Keep lines with substantial content (html2text output is already clean)+ # Keep lines with substantial contentif len(stripped) > 2:+ # Skip lines that look like JSON (start with { or [ and are very long)+ if (stripped.startswith('{') or stripped.startswith('[')) and len(stripped) > 100:+ continuefiltered_lines.append(line)content = '\n'.join(filtered_lines)