Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Fix markdown extraction destroying URLs and dropping long link lines

Saurav Pandaab08dea62ce1
diff --git a/browser_use/dom/markdown_extractor.py b/browser_use/dom/markdown_extractor.py
index aff9736d4..668ba7d99 100644
--- a/browser_use/dom/markdown_extractor.py
+++ b/browser_use/dom/markdown_extractor.py
@@ -5,6 +5,7 @@ This module provides a unified interface for extracting clean markdown from brow
used by both the tools service and page actor.
"""
+import json
import re
from dataclasses import dataclass
from enum import Enum, auto
@@ -67,33 +68,7 @@ async def extract_clean_markdown(
original_html_length = len(page_html)
- # Use markdownify for clean markdown conversion
- from markdownify import markdownify as md
-
- # 'td', 'th', and headings are the only elements where markdownify sets the _inline context,
- # which causes img elements to be stripped to just alt text when keep_inline_images_in=[]
- _keep_inline_images_in = ['td', 'th', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'] if extract_images else []
- content = md(
- page_html,
- heading_style='ATX', # Use # style headings
- strip=['script', 'style'], # Remove these tags
- bullets='-', # Use - for unordered lists
- code_language='', # Don't add language to code blocks
- escape_asterisks=False, # Don't escape asterisks (cleaner output)
- escape_underscores=False, # Don't escape underscores (cleaner output)
- escape_misc=False, # Don't escape other characters (cleaner output)
- autolinks=False, # Don't convert URLs to <> format
- default_title=False, # Don't add default title attributes
- keep_inline_images_in=_keep_inline_images_in, # Include image src URLs when extract_images=True
- )
-
- initial_markdown_length = len(content)
-
- # Minimal cleanup - markdownify already does most of the work
- content = re.sub(r'%[0-9A-Fa-f]{2}', '', content) # Remove any remaining URL encoding
-
- # Apply light preprocessing to clean up excessive whitespace
- content, chars_filtered = _preprocess_markdown_content(content)
+ content, initial_markdown_length, chars_filtered = convert_html_to_markdown(page_html, extract_images=extract_images)
final_filtered_length = len(content)
@@ -135,6 +110,39 @@ async def _get_enhanced_dom_tree_from_browser_session(browser_session: 'BrowserS
# Legacy aliases removed - all code now uses the unified extract_clean_markdown function
+def convert_html_to_markdown(page_html: str, extract_images: bool = False) -> tuple[str, int, int]:
+ """Convert serialized page HTML to filtered markdown.
+
+ Returns:
+ tuple: (filtered_markdown, initial_markdown_chars, chars_filtered)
+ """
+ from markdownify import markdownify as md
+
+ # 'td', 'th', and headings are the only elements where markdownify sets the _inline context,
+ # which causes img elements to be stripped to just alt text when keep_inline_images_in=[]
+ _keep_inline_images_in = ['td', 'th', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'] if extract_images else []
+ content = md(
+ page_html,
+ heading_style='ATX', # Use # style headings
+ strip=['script', 'style'], # Remove these tags
+ bullets='-', # Use - for unordered lists
+ code_language='', # Don't add language to code blocks
+ escape_asterisks=False, # Don't escape asterisks (cleaner output)
+ escape_underscores=False, # Don't escape underscores (cleaner output)
+ escape_misc=False, # Don't escape other characters (cleaner output)
+ autolinks=False, # Don't convert URLs to <> format
+ default_title=False, # Don't add default title attributes
+ keep_inline_images_in=_keep_inline_images_in, # Include image src URLs when extract_images=True
+ )
+
+ initial_markdown_length = len(content)
+
+ # Apply light preprocessing to clean up excessive whitespace
+ content, chars_filtered = _preprocess_markdown_content(content)
+
+ return content, initial_markdown_length, chars_filtered
+
+
def _preprocess_markdown_content(content: str, max_newlines: int = 3) -> tuple[str, int]:
"""
Light preprocessing of markdown output - minimal cleanup with JSON blob removal.
@@ -166,9 +174,14 @@ def _preprocess_markdown_content(content: str, max_newlines: int = 3) -> tuple[s
stripped = line.strip()
# Keep all non-empty lines
if stripped:
- # Skip lines that look like JSON (start with { or [ and are very long)
- if (stripped.startswith('{') or stripped.startswith('[')) and len(stripped) > 100:
- continue
+ # Skip long lines that actually parse as JSON (SPA state blobs). A prefix
+ # check alone is not enough: markdown links/images also start with '['.
+ if len(stripped) > 100 and stripped[0] in '{[':
+ try:
+ json.loads(stripped)
+ continue
+ except ValueError:
+ pass
filtered_lines.append(line)
content = '\n'.join(filtered_lines)
diff --git a/tests/ci/test_markdown_extractor.py b/tests/ci/test_markdown_extractor.py
index 14f21b53b..d163b03a1 100644
--- a/tests/ci/test_markdown_extractor.py
+++ b/tests/ci/test_markdown_extractor.py
@@ -101,3 +101,46 @@ class TestPreprocessMarkdownContent:
assert not filtered.startswith('\n')
assert not filtered.endswith(' ')
assert not filtered.endswith('\n')
+
+
+class TestPreservesLinksAndEncoding:
+ """Regression tests: markdown links and percent-encoded URLs must survive filtering."""
+
+ def test_preserves_long_markdown_link_lines(self):
+ """A long markdown link line (>100 chars) must not be dropped by the JSON heuristic."""
+ link = '[Read the full quarterly earnings report for fiscal year 2025](https://example.com/investor-relations/reports/q4-2025-earnings-full.pdf)'
+ assert len(link) > 100
+ content = f'Header\n{link}\nFooter'
+ filtered, _ = _preprocess_markdown_content(content)
+
+ assert link in filtered
+
+ def test_preserves_long_image_link_lines(self):
+ """A long clickable-image line (starts with [![) must not be dropped."""
+ line = '[![Product photo of the deluxe widget](https://cdn.example.com/images/products/deluxe-widget-hero.jpg)](https://example.com/products/deluxe-widget)'
+ assert len(line) > 100
+ content = f'Intro\n{line}\nOutro'
+ filtered, _ = _preprocess_markdown_content(content)
+
+ assert line in filtered
+
+ def test_removes_long_json_array_lines(self):
+ """A valid JSON array blob >100 chars should still be dropped."""
+ json_array = '[' + ', '.join(f'{{"id": {i}, "name": "item-{i}"}}' for i in range(10)) + ']'
+ assert len(json_array) > 100
+ content = f'Header\n{json_array}\nFooter'
+ filtered, _ = _preprocess_markdown_content(content)
+
+ assert json_array not in filtered
+ assert 'Header' in filtered
+
+ def test_preserves_percent_encoded_urls(self):
+ """Percent-encodings in URLs must survive HTML -> markdown conversion."""
+ from browser_use.dom.markdown_extractor import convert_html_to_markdown
+
+ html = '<p>See <a href="https://example.com/my%20file%2Fv2?q=a%26b">the doc</a> here</p>'
+ content, _, _ = convert_html_to_markdown(html)
+
+ assert '%20' in content
+ assert '%2F' in content
+ assert '%26' in content