Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Refactor markdown preprocessing and add tests

Cursor Agent24e276d5f460
diff --git a/browser_use/dom/markdown_extractor.py b/browser_use/dom/markdown_extractor.py
index d9392664d..ba4a7bde8 100644
--- a/browser_use/dom/markdown_extractor.py
+++ b/browser_use/dom/markdown_extractor.py
@@ -151,13 +151,13 @@ def _preprocess_markdown_content(content: str, max_newlines: int = 3) -> tuple[s
# Compress consecutive newlines (4+ newlines become max_newlines)
content = re.sub(r'\n{4,}', '\n' * max_newlines, content)
- # Remove lines that are only whitespace or very short (likely artifacts)
+ # Remove lines that are only whitespace
lines = content.split('\n')
filtered_lines = []
for line in lines:
stripped = line.strip()
- # Keep lines with substantial content
- if len(stripped) > 2:
+ # Keep all non-empty lines
+ if stripped:
# Skip lines that look like JSON (start with { or [ and are very long)
if (stripped.startswith('{') or stripped.startswith('[')) and len(stripped) > 100:
continue
diff --git a/tests/ci/test_markdown_extractor.py b/tests/ci/test_markdown_extractor.py
new file mode 100644
index 000000000..14f21b53b
--- /dev/null
+++ b/tests/ci/test_markdown_extractor.py
@@ -0,0 +1,103 @@
+"""Tests for markdown extractor preprocessing."""
+
+from browser_use.dom.markdown_extractor import _preprocess_markdown_content
+
+
+class TestPreprocessMarkdownContent:
+ """Tests for _preprocess_markdown_content function."""
+
+ def test_preserves_short_lines(self):
+ """Short lines (1-2 chars) should be preserved, not removed."""
+ content = '# Items\na\nb\nc\nOK\nNo'
+ filtered, _ = _preprocess_markdown_content(content)
+
+ assert 'a' in filtered.split('\n')
+ assert 'b' in filtered.split('\n')
+ assert 'c' in filtered.split('\n')
+ assert 'OK' in filtered.split('\n')
+ assert 'No' in filtered.split('\n')
+
+ def test_preserves_single_digit_numbers(self):
+ """Single digit page numbers should be preserved."""
+ content = 'Page navigation:\n1\n2\n3\n10'
+ filtered, _ = _preprocess_markdown_content(content)
+
+ lines = filtered.split('\n')
+ assert '1' in lines
+ assert '2' in lines
+ assert '3' in lines
+ assert '10' in lines
+
+ def test_preserves_markdown_list_items(self):
+ """Markdown list items with short content should be preserved."""
+ content = 'Shopping list:\n- a\n- b\n- OK\n- No'
+ filtered, _ = _preprocess_markdown_content(content)
+
+ assert '- a' in filtered
+ assert '- b' in filtered
+ assert '- OK' in filtered
+ assert '- No' in filtered
+
+ def test_preserves_state_codes(self):
+ """Two-letter state codes should be preserved."""
+ content = 'States:\nCA\nNY\nTX'
+ filtered, _ = _preprocess_markdown_content(content)
+
+ lines = filtered.split('\n')
+ assert 'CA' in lines
+ assert 'NY' in lines
+ assert 'TX' in lines
+
+ def test_removes_empty_lines(self):
+ """Empty and whitespace-only lines should be removed."""
+ content = 'Header\n\n \n\nContent'
+ filtered, _ = _preprocess_markdown_content(content)
+
+ # Should not have empty lines
+ for line in filtered.split('\n'):
+ assert line.strip(), f'Found empty line in output: {repr(line)}'
+
+ def test_removes_large_json_blobs(self):
+ """Large JSON-like lines (>100 chars) should be removed."""
+ # Create a JSON blob > 100 chars
+ json_blob = '{"key": "' + 'x' * 100 + '"}'
+ content = f'Header\n{json_blob}\nFooter'
+ filtered, _ = _preprocess_markdown_content(content)
+
+ assert json_blob not in filtered
+ assert 'Header' in filtered
+ assert 'Footer' in filtered
+
+ def test_preserves_small_json(self):
+ """Small JSON objects (<100 chars) should be preserved."""
+ small_json = '{"key": "value"}'
+ content = f'Header\n{small_json}\nFooter'
+ filtered, _ = _preprocess_markdown_content(content)
+
+ assert small_json in filtered
+
+ def test_compresses_multiple_newlines(self):
+ """4+ consecutive newlines should be compressed to max_newlines."""
+ content = 'Header\n\n\n\n\nFooter'
+ filtered, _ = _preprocess_markdown_content(content, max_newlines=2)
+
+ # After filtering empty lines, we should have just Header and Footer
+ lines = [line for line in filtered.split('\n') if line.strip()]
+ assert lines == ['Header', 'Footer']
+
+ def test_returns_chars_filtered_count(self):
+ """Should return count of characters removed."""
+ content = 'Header\n\n\n\n\nFooter'
+ _, chars_filtered = _preprocess_markdown_content(content)
+
+ assert chars_filtered > 0
+
+ def test_strips_result(self):
+ """Result should be stripped of leading/trailing whitespace."""
+ content = ' \n\nContent\n\n '
+ filtered, _ = _preprocess_markdown_content(content)
+
+ assert not filtered.startswith(' ')
+ assert not filtered.startswith('\n')
+ assert not filtered.endswith(' ')
+ assert not filtered.endswith('\n')