Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

New library for extraction

Magnus Müller9fbc78b0b66b
diff --git a/browser_use/tools/service.py b/browser_use/tools/service.py
index d6acffad8..3f1f82c9f 100644
--- a/browser_use/tools/service.py
+++ b/browser_use/tools/service.py
@@ -3,7 +3,7 @@ import enum
import json
import logging
import os
-from typing import Generic, TypeVar
+from typing import Any, Generic, TypeVar
try:
from lmnr import Laminar # type: ignore
@@ -558,105 +558,15 @@ Use start_from_char to start extraction from a specific character position (use
):
# Constants
MAX_CHAR_LIMIT = 30000
- MAX_NEWLINES = 3
-
- cdp_session = await browser_session.get_or_create_cdp_session()
-
- # Wait for the page to be ready (same pattern used in DOM service)
- try:
- ready_state = await cdp_session.cdp_client.send.Runtime.evaluate(
- params={'expression': 'document.readyState'}, session_id=cdp_session.session_id
- )
- except Exception:
- pass # Page might not be ready yet
+ # Extract clean markdown using the new method
try:
- # Get the HTML content
- body_id = await cdp_session.cdp_client.send.DOM.getDocument(session_id=cdp_session.session_id)
- page_html_result = await cdp_session.cdp_client.send.DOM.getOuterHTML(
- params={'backendNodeId': body_id['root']['backendNodeId']}, session_id=cdp_session.session_id
- )
+ content, content_stats = await self.extract_clean_markdown(browser_session, extract_links)
except Exception as e:
- raise RuntimeError(f"Couldn't extract page content: {e}")
-
- page_html = page_html_result['outerHTML']
-
- # Enhanced markdown conversion and content filtering
- try:
- import re
-
- import markdownify
-
- original_html_length = len(page_html)
-
- if extract_links:
- content = markdownify.markdownify(page_html, heading_style='ATX', bullets='-')
- else:
- content = markdownify.markdownify(page_html, heading_style='ATX', bullets='-', strip=['a'])
- # Remove all markdown links and images, keep only the text
- content = re.sub(r'!\[.*?\]\([^)]*\)', '', content, flags=re.MULTILINE | re.DOTALL) # Remove images
- content = re.sub(
- r'\[([^\]]*)\]\([^)]*\)', r'\1', content, flags=re.MULTILINE | re.DOTALL
- ) # Convert [text](url) -> text
-
- initial_markdown_length = len(content)
-
- # Enhanced content filtering - remove low-quality patterns
- patterns_to_remove = [
- # Positioning artifacts and debug info
- r'❓\s*\[\d+\]\s*\w+.*?Position:.*?Size:.*?\n?',
- r'Primary: UNKNOWN\n\nNo specific evidence found',
- r'UNKNOWN CONFIDENCE',
- r'!\[\]\(\)',
- # Navigation breadcrumbs that are too verbose
- r'^>.*?>.*?>.*?>.*?$',
- # Empty or near-empty table cells
- r'^\|\s*\|\s*\|\s*\|*\s*$',
- # Excessive whitespace between table elements
- r'\|\s{10,}\|',
- # Long strings of special characters (likely formatting artifacts)
- r'[^\w\s]{15,}',
- # Extremely long words (likely encoded data)
- r'\b\w{50,}\b',
- # CSS/style artifacts that leak into content
- r'(?:style|class|id)=["\'][^"\']*["\']',
- # Empty list items
- r'^-\s*$',
- ]
-
- chars_filtered = 0
- for pattern in patterns_to_remove:
- before_len = len(content)
- content = re.sub(pattern, '', content, flags=re.MULTILINE | re.DOTALL | re.IGNORECASE)
- chars_filtered += before_len - len(content)
-
- # Compress consecutive newlines (4+ newlines become 3 newlines)
- content = re.sub(r'\n{4,}', '\n' * MAX_NEWLINES, content)
-
- # Remove lines that are mostly punctuation or very short
- lines = content.split('\n')
- filtered_lines = []
- for line in lines:
- stripped = line.strip()
- # Keep line if it has substantial content
- if len(stripped) > 3 and len(re.sub(r'[^\w\s]', '', stripped)) > len(stripped) * 0.3:
- filtered_lines.append(line)
-
- content = '\n'.join(filtered_lines)
- content = content.strip()
+ raise RuntimeError(f'Could not extract clean markdown: {type(e).__name__}')
- final_filtered_length = len(content)
-
- except Exception as e:
- raise RuntimeError(f'Could not convert html to markdown: {type(e).__name__}')
-
- # Apply start_from_char offset if specified
- content_stats = {
- 'original_html_chars': original_html_length,
- 'initial_markdown_chars': initial_markdown_length,
- 'filtered_chars_removed': chars_filtered,
- 'final_filtered_chars': final_filtered_length,
- }
+ # Original content length for processing
+ final_filtered_length = content_stats['final_filtered_chars']
if start_from_char > 0:
if start_from_char >= len(content):
@@ -689,6 +599,10 @@ Use start_from_char to start extraction from a specific character position (use
content_stats['next_start_char'] = next_start
# Add content statistics to the result
+ original_html_length = content_stats['original_html_chars']
+ initial_markdown_length = content_stats['initial_markdown_chars']
+ chars_filtered = content_stats['filtered_chars_removed']
+
stats_summary = f"""Content processed: {original_html_length:,} HTML chars → {initial_markdown_length:,} initial markdown → {final_filtered_length:,} filtered markdown"""
if start_from_char > 0:
stats_summary += f' (started from char {start_from_char:,})'
@@ -976,6 +890,149 @@ You will be given a query and the markdown of a webpage that has been filtered t
)
# Custom done action for structured output
+ async def extract_clean_markdown(self, browser_session: BrowserSession, extract_links: bool = False) -> tuple[str, dict[str, Any]]:
+ """Extract clean markdown from the current page.
+
+ Args:
+ browser_session: Browser session to extract content from
+ extract_links: Whether to preserve links in markdown
+
+ Returns:
+ tuple: (clean_markdown_content, content_statistics)
+ """
+ import re
+
+ import markdownify
+
+ # Get HTML content from current page
+ cdp_session = await browser_session.get_or_create_cdp_session()
+ try:
+ body_id = await cdp_session.cdp_client.send.DOM.getDocument(session_id=cdp_session.session_id)
+ page_html_result = await cdp_session.cdp_client.send.DOM.getOuterHTML(
+ params={'backendNodeId': body_id['root']['backendNodeId']},
+ session_id=cdp_session.session_id
+ )
+ page_html = page_html_result['outerHTML']
+ current_url = await browser_session.get_current_page_url()
+ except Exception as e:
+ raise RuntimeError(f"Couldn't extract page content: {e}")
+
+ original_html_length = len(page_html)
+
+ # Use html2text for much better markdown conversion
+ try:
+ import html2text
+ h = html2text.HTML2Text()
+ h.ignore_links = not extract_links
+ h.ignore_images = True
+ h.ignore_emphasis = False
+ h.body_width = 0 # Don't wrap lines
+ h.unicode_snob = True
+ h.skip_internal_links = True
+ content = h.handle(page_html)
+ except ImportError:
+ # Fallback to markdownify with better settings
+ if extract_links:
+ content = markdownify.markdownify(
+ page_html,
+ heading_style='ATX',
+ bullets='-',
+ strip=['script', 'style']
+ )
+ else:
+ content = markdownify.markdownify(
+ page_html,
+ heading_style='ATX',
+ bullets='-',
+ strip=['a', 'script', 'style']
+ )
+ # Remove all markdown links and images, keep only the text
+ content = re.sub(r'!\[.*?\]\([^)]*\)', '', content, flags=re.MULTILINE | re.DOTALL)
+ content = re.sub(r'\[([^\]]*)\]\([^)]*\)', r'\1', content, flags=re.MULTILINE | re.DOTALL)
+
+ initial_markdown_length = len(content)
+
+ # Light cleanup - only remove obvious artifacts
+ content = re.sub(r'%[0-9A-Fa-f]{2}', '', content) # Remove URL encoding
+ content = re.sub(r'\bfill=[\'"][^\'\"]*[\'"]', '', content) # Remove SVG fill attributes
+ content = re.sub(r'\bstroke=[\'"][^\'\"]*[\'"]', '', content) # Remove SVG stroke attributes
+
+ # Apply comprehensive preprocessing to remove noise patterns
+ content, chars_filtered = self._preprocess_markdown_content(content)
+
+ final_filtered_length = len(content)
+
+ # Content statistics
+ stats = {
+ 'url': current_url,
+ 'original_html_chars': original_html_length,
+ 'initial_markdown_chars': initial_markdown_length,
+ 'filtered_chars_removed': chars_filtered,
+ 'final_filtered_chars': final_filtered_length,
+ }
+
+ return content, stats
+
+ def _preprocess_markdown_content(self, content: str, max_newlines: int = 3) -> tuple[str, int]:
+ """
+ Preprocess markdown content by removing noise patterns and low-quality lines.
+
+ Args:
+ content: Raw markdown content to filter
+ max_newlines: Maximum consecutive newlines to allow
+
+ Returns:
+ tuple: (filtered_content, chars_filtered)
+ """
+ import re
+
+ original_length = len(content)
+
+ # Enhanced content filtering - remove low-quality patterns
+ patterns_to_remove = [
+ # Positioning artifacts and debug info
+ r'❓\s*\[\d+\]\s*\w+.*?Position:.*?Size:.*?\n?',
+ r'Primary: UNKNOWN\n\nNo specific evidence found',
+ r'UNKNOWN CONFIDENCE',
+ r'!\[\]\(\)',
+ # Empty or near-empty table cells
+ r'^\|\s*\|\s*\|\s*\|*\s*$',
+ # Excessive whitespace between table elements
+ r'\|\s{10,}\|',
+ # Long strings of special characters (likely formatting artifacts)
+ r'[^\w\s]{15,}',
+ # Extremely long words (likely encoded data)
+ r'\b\w{50,}\b',
+ # CSS/style artifacts that leak into content
+ r'(?:style|class|id)=["\'][^"\']*["\']',
+ # Empty list items
+ r'^-\s*$',
+ ]
+
+ chars_filtered = 0
+ for pattern in patterns_to_remove:
+ before_len = len(content)
+ content = re.sub(pattern, '', content, flags=re.MULTILINE | re.DOTALL | re.IGNORECASE)
+ chars_filtered += before_len - len(content)
+
+ # Compress consecutive newlines (4+ newlines become max_newlines)
+ content = re.sub(r'\n{4,}', '\n' * max_newlines, content)
+
+ # Remove lines that are mostly punctuation or very short
+ lines = content.split('\n')
+ filtered_lines = []
+ for line in lines:
+ stripped = line.strip()
+ # Keep line if it has substantial content
+ if len(stripped) > 3 and len(re.sub(r'[^\w\s]', '', stripped)) > len(stripped) * 0.3:
+ filtered_lines.append(line)
+
+ content = '\n'.join(filtered_lines)
+ content = content.strip()
+
+ chars_filtered += original_length - len(content)
+ return content, chars_filtered
+
def _register_done_action(self, output_model: type[T] | None, display_files_in_done_text: bool = True):
if output_model is not None:
self.display_files_in_done_text = display_files_in_done_text
diff --git a/pyproject.toml b/pyproject.toml
index 93a240e01..716b5f78a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,6 +41,7 @@ dependencies = [
"pypdf>=5.7.0",
"cdp-use>=1.4.0",
"markdown-pdf==1.5",
+ "html2text>=2025.4.15",
]
# google-api-core: only used for Google LLM APIs
# pyperclip: only used for examples that use copy/paste