Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Filter out json and metadata from markdown

Magnus Müller84395f1641b0
diff --git a/browser_use/dom/serializer/html_serializer.py b/browser_use/dom/serializer/html_serializer.py
index afddf3604..5050cfa17 100644
--- a/browser_use/dom/serializer/html_serializer.py
+++ b/browser_use/dom/serializer/html_serializer.py
@@ -71,6 +71,23 @@ class HTMLSerializer:
if tag_name in {'style', 'script', 'head', 'meta', 'link', 'title'}:
return ''
+ # Skip code tags with display:none - these often contain JSON state for SPAs
+ if tag_name == 'code' and node.attributes:
+ style = node.attributes.get('style', '')
+ # Check if element is hidden (display:none) - likely JSON data
+ if 'display:none' in style.replace(' ', '') or 'display: none' in style:
+ return ''
+ # Also check for bpr-guid IDs (LinkedIn's JSON data pattern)
+ element_id = node.attributes.get('id', '')
+ if 'bpr-guid' in element_id or 'data' in element_id or 'state' in element_id:
+ return ''
+
+ # Skip base64 inline images - these are usually placeholders or tracking pixels
+ if tag_name == 'img' and node.attributes:
+ src = node.attributes.get('src', '')
+ if src.startswith('data:image/'):
+ return ''
+
# Opening tag
parts.append(f'<{tag_name}')
@@ -158,6 +175,11 @@ class HTMLSerializer:
if not self.extract_links and key == 'href':
continue
+ # Skip data-* attributes as they often contain JSON payloads
+ # These are used by modern SPAs (React, Vue, Angular) for state management
+ if key.startswith('data-'):
+ continue
+
# Handle boolean attributes
if value == '' or value is None:
parts.append(key)
diff --git a/browser_use/tools/service.py b/browser_use/tools/service.py
index 79c0b6739..ea8ae6847 100644
--- a/browser_use/tools/service.py
+++ b/browser_use/tools/service.py
@@ -1080,6 +1080,12 @@ You will be given a query and the markdown of a webpage that has been filtered t
strip=['script', 'style'], # Remove these tags
bullets='-', # Use - for unordered lists
code_language='', # Don't add language to code blocks
+ escape_asterisks=False, # Don't escape asterisks (cleaner output)
+ escape_underscores=False, # Don't escape underscores (cleaner output)
+ escape_misc=False, # Don't escape other characters (cleaner output)
+ autolinks=False, # Don't convert URLs to <> format
+ default_title=False, # Don't add default title attributes
+ keep_inline_images_in=[], # Don't keep inline images in any tags (we already filter base64 in HTML)
)
initial_markdown_length = len(content)
@@ -1109,10 +1115,10 @@ You will be given a query and the markdown of a webpage that has been filtered t
def _preprocess_markdown_content(self, content: str, max_newlines: int = 3) -> tuple[str, int]:
"""
- Light preprocessing of html2text output - minimal cleanup since html2text is already clean.
+ Light preprocessing of markdown output - minimal cleanup with JSON blob removal.
Args:
- content: Markdown content from html2text to lightly filter
+ content: Markdown content to lightly filter
max_newlines: Maximum consecutive newlines to allow
Returns:
@@ -1122,6 +1128,14 @@ You will be given a query and the markdown of a webpage that has been filtered t
original_length = len(content)
+ # Remove JSON blobs (common in SPAs like LinkedIn, Facebook, etc.)
+ # These are often embedded as `{"key":"value",...}` and can be massive
+ # Match JSON objects/arrays that are at least 100 chars long
+ # This catches SPA state/config data without removing small inline JSON
+ content = re.sub(r'`\{["\w].*?\}`', '', content, flags=re.DOTALL) # Remove JSON in code blocks
+ content = re.sub(r'\{"\$type":[^}]{100,}\}', '', content) # Remove JSON with $type fields (common pattern)
+ content = re.sub(r'\{"[^"]{5,}":\{[^}]{100,}\}', '', content) # Remove nested JSON objects
+
# Compress consecutive newlines (4+ newlines become max_newlines)
content = re.sub(r'\n{4,}', '\n' * max_newlines, content)
@@ -1130,8 +1144,11 @@ You will be given a query and the markdown of a webpage that has been filtered t
filtered_lines = []
for line in lines:
stripped = line.strip()
- # Keep lines with substantial content (html2text output is already clean)
+ # Keep lines with substantial content
if len(stripped) > 2:
+ # Skip lines that look like JSON (start with { or [ and are very long)
+ if (stripped.startswith('{') or stripped.startswith('[')) and len(stripped) > 100:
+ continue
filtered_lines.append(line)
content = '\n'.join(filtered_lines)