Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Improve Markdown Chunker
Saurav Pandae66eac8a3704
diff --git a/browser_use/dom/markdown_extractor.py b/browser_use/dom/markdown_extractor.pyindex ba4a7bde8..a2d244721 100644--- a/browser_use/dom/markdown_extractor.py+++ b/browser_use/dom/markdown_extractor.py@@ -6,10 +6,13 @@ used by both the tools service and page actor."""import re+from dataclasses import dataclass+from enum import Enum, autofrom typing import TYPE_CHECKING, Anyfrom browser_use.dom.serializer.html_serializer import HTMLSerializerfrom browser_use.dom.service import DomService+from browser_use.dom.views import MarkdownChunkif TYPE_CHECKING:from browser_use.browser.session import BrowserSession@@ -168,3 +171,333 @@ def _preprocess_markdown_content(content: str, max_newlines: int = 3) -> tuple[schars_filtered = original_length - len(content)return content, chars_filtered+++# ---------------------------------------------------------------------------+# Structure-aware markdown chunking+# ---------------------------------------------------------------------------+++class _BlockType(Enum):+ HEADER = auto()+ CODE_FENCE = auto()+ TABLE = auto()+ LIST_ITEM = auto()+ PARAGRAPH = auto()+ BLANK = auto()+++@dataclass(slots=True)+class _AtomicBlock:+ block_type: _BlockType+ lines: list[str]+ char_start: int # offset in original content+ char_end: int # offset in original content (exclusive)+++_TABLE_ROW_RE = re.compile(r'^\s*\|.*\|\s*$')+_LIST_ITEM_RE = re.compile(r'^(\s*)([-*+]|\d+[.)]) ')+_LIST_CONTINUATION_RE = re.compile(r'^(\s{2,}|\t)')+++def _parse_atomic_blocks(content: str) -> list[_AtomicBlock]:+ """Phase 1: Walk lines, group into unsplittable blocks."""+ lines = content.split('\n')+ blocks: list[_AtomicBlock] = []+ i = 0+ offset = 0 # char offset tracking++ while i < len(lines):+ line = lines[i]+ line_len = len(line) + 1 # +1 for the newline we split on++ # BLANK+ if not line.strip():+ blocks.append(_AtomicBlock(+ block_type=_BlockType.BLANK,+ lines=[line],+ char_start=offset,+ char_end=offset + line_len,+ ))+ offset += line_len+ i += 1+ continue++ # CODE FENCE+ if line.strip().startswith('```'):+ fence_lines = [line]+ fence_end = offset + line_len+ i += 1+ # Consume until closing fence or EOF+ while i < len(lines):+ fence_line = lines[i]+ fence_line_len = len(fence_line) + 1+ fence_lines.append(fence_line)+ fence_end += fence_line_len+ i += 1+ if fence_line.strip().startswith('```') and len(fence_lines) > 1:+ break+ blocks.append(_AtomicBlock(+ block_type=_BlockType.CODE_FENCE,+ lines=fence_lines,+ char_start=offset,+ char_end=fence_end,+ ))+ offset = fence_end+ continue++ # HEADER+ if line.lstrip().startswith('#'):+ blocks.append(_AtomicBlock(+ block_type=_BlockType.HEADER,+ lines=[line],+ char_start=offset,+ char_end=offset + line_len,+ ))+ offset += line_len+ i += 1+ continue++ # TABLE (consecutive |...| lines)+ # Header + separator row stay together; each data row is its own block+ if _TABLE_ROW_RE.match(line):+ # Collect header line+ header_lines = [line]+ header_end = offset + line_len+ i += 1+ # Check if next line is separator (contains ---)+ if i < len(lines) and _TABLE_ROW_RE.match(lines[i]) and '---' in lines[i]:+ sep = lines[i]+ sep_len = len(sep) + 1+ header_lines.append(sep)+ header_end += sep_len+ i += 1+ # Emit header+separator as one atomic block+ blocks.append(_AtomicBlock(+ block_type=_BlockType.TABLE,+ lines=header_lines,+ char_start=offset,+ char_end=header_end,+ ))+ offset = header_end+ # Each subsequent table row is its own TABLE block (splittable between rows)+ while i < len(lines) and _TABLE_ROW_RE.match(lines[i]):+ row = lines[i]+ row_len = len(row) + 1+ blocks.append(_AtomicBlock(+ block_type=_BlockType.TABLE,+ lines=[row],+ char_start=offset,+ char_end=offset + row_len,+ ))+ offset += row_len+ i += 1+ continue++ # LIST ITEM (with indented continuations)+ if _LIST_ITEM_RE.match(line):+ list_lines = [line]+ list_end = offset + line_len+ i += 1+ # Consume continuation lines (indented or blank between items)+ while i < len(lines):+ next_line = lines[i]+ next_len = len(next_line) + 1+ # Another list item at same or deeper indent → still part of this block+ if _LIST_ITEM_RE.match(next_line):+ list_lines.append(next_line)+ list_end += next_len+ i += 1+ continue+ # Indented continuation+ if next_line.strip() and _LIST_CONTINUATION_RE.match(next_line):+ list_lines.append(next_line)+ list_end += next_len+ i += 1+ continue+ break+ blocks.append(_AtomicBlock(+ block_type=_BlockType.LIST_ITEM,+ lines=list_lines,+ char_start=offset,+ char_end=list_end,+ ))+ offset = list_end+ continue++ # PARAGRAPH (everything else, up to next blank line)+ para_lines = [line]+ para_end = offset + line_len+ i += 1+ while i < len(lines) and lines[i].strip():+ # Stop if next line starts a different block type+ nl = lines[i]+ if (+ nl.lstrip().startswith('#')+ or nl.strip().startswith('```')+ or _TABLE_ROW_RE.match(nl)+ or _LIST_ITEM_RE.match(nl)+ ):+ break+ nl_len = len(nl) + 1+ para_lines.append(nl)+ para_end += nl_len+ i += 1+ blocks.append(_AtomicBlock(+ block_type=_BlockType.PARAGRAPH,+ lines=para_lines,+ char_start=offset,+ char_end=para_end,+ ))+ offset = para_end++ # Fix last block char_end: content may not end with \n+ if blocks and content and not content.endswith('\n'):+ blocks[-1] = _AtomicBlock(+ block_type=blocks[-1].block_type,+ lines=blocks[-1].lines,+ char_start=blocks[-1].char_start,+ char_end=len(content),+ )++ return blocks+++def _block_text(block: _AtomicBlock) -> str:+ return '\n'.join(block.lines)+++def _get_table_header(block: _AtomicBlock) -> str | None:+ """Extract table header + separator rows from a TABLE block."""+ assert block.block_type == _BlockType.TABLE+ if len(block.lines) < 2:+ return None+ # Header is first line, separator is second line (must contain ---)+ sep_line = block.lines[1]+ if '---' in sep_line or '- -' in sep_line:+ return block.lines[0] + '\n' + block.lines[1]+ return None+++def chunk_markdown_by_structure(+ content: str,+ max_chunk_chars: int = 100_000,+ overlap_lines: int = 5,+ start_from_char: int = 0,+) -> list[MarkdownChunk]:+ """Split markdown into structure-aware chunks.++ Algorithm:+ Phase 1 — Parse atomic blocks (headers, code fences, tables, list items, paragraphs).+ Phase 2 — Greedy chunk assembly: accumulate blocks until exceeding max_chunk_chars.+ A single block exceeding the limit is allowed (soft limit).+ Phase 3 — Build overlap prefixes for context carry between chunks.++ Args:+ content: Full markdown string.+ max_chunk_chars: Target maximum chars per chunk (soft limit for single blocks).+ overlap_lines: Number of trailing lines from previous chunk to prepend.+ start_from_char: Return chunks starting from the chunk that contains this offset.++ Returns:+ List of MarkdownChunk. Empty if start_from_char is past end of content.+ """+ if not content:+ return [MarkdownChunk(+ content='',+ chunk_index=0,+ total_chunks=1,+ char_offset_start=0,+ char_offset_end=0,+ overlap_prefix='',+ has_more=False,+ )]++ if start_from_char >= len(content):+ return []++ # Phase 1: parse atomic blocks+ blocks = _parse_atomic_blocks(content)+ if not blocks:+ return []++ # Phase 2: greedy chunk assembly+ raw_chunks: list[list[_AtomicBlock]] = []+ current_chunk: list[_AtomicBlock] = []+ current_size = 0++ for block in blocks:+ block_size = block.char_end - block.char_start+ # If adding this block would exceed limit AND we already have content, emit chunk+ if current_size + block_size > max_chunk_chars and current_chunk:+ raw_chunks.append(current_chunk)+ current_chunk = []+ current_size = 0+ current_chunk.append(block)+ current_size += block_size++ if current_chunk:+ raw_chunks.append(current_chunk)++ total_chunks = len(raw_chunks)++ # Phase 3: build MarkdownChunk objects with overlap prefixes+ chunks: list[MarkdownChunk] = []+ # Track table header from previous chunk for table continuations+ prev_chunk_last_table_header: str | None = None++ for idx, chunk_blocks in enumerate(raw_chunks):+ chunk_text = '\n'.join(_block_text(b) for b in chunk_blocks)+ char_start = chunk_blocks[0].char_start+ char_end = chunk_blocks[-1].char_end++ # Build overlap prefix+ overlap = ''+ if idx > 0:+ prev_blocks = raw_chunks[idx - 1]+ prev_text = '\n'.join(_block_text(b) for b in prev_blocks)+ prev_lines = prev_text.split('\n')++ # Check if current chunk starts with a table continuation+ first_block = chunk_blocks[0]+ if first_block.block_type == _BlockType.TABLE and prev_chunk_last_table_header:+ # Always prepend table header for continuation+ trailing = prev_lines[-(overlap_lines):] if overlap_lines > 0 else []+ header_lines = prev_chunk_last_table_header.split('\n')+ # Deduplicate: don't repeat header lines if they're already in trailing+ combined = list(header_lines)+ for tl in trailing:+ if tl not in combined:+ combined.append(tl)+ overlap = '\n'.join(combined)+ elif overlap_lines > 0:+ overlap = '\n'.join(prev_lines[-(overlap_lines):])++ # Track table header from this chunk for next iteration.+ # Scan all blocks: the header+separator block could be anywhere in the chunk.+ prev_chunk_last_table_header = None+ for b in chunk_blocks:+ if b.block_type == _BlockType.TABLE:+ hdr = _get_table_header(b)+ if hdr is not None:+ prev_chunk_last_table_header = hdr++ has_more = idx < total_chunks - 1+ chunks.append(MarkdownChunk(+ content=chunk_text,+ chunk_index=idx,+ total_chunks=total_chunks,+ char_offset_start=char_start,+ char_offset_end=char_end,+ overlap_prefix=overlap,+ has_more=has_more,+ ))++ # Apply start_from_char filter: return chunks from the one containing that offset+ if start_from_char > 0:+ for i, chunk in enumerate(chunks):+ if chunk.char_offset_end > start_from_char:+ return chunks[i:]+ return [] # offset past all chunks++ return chunksdiff --git a/browser_use/dom/serializer/html_serializer.py b/browser_use/dom/serializer/html_serializer.pyindex 8aaf31df7..dd3d1c6b6 100644--- a/browser_use/dom/serializer/html_serializer.py+++ b/browser_use/dom/serializer/html_serializer.py@@ -119,8 +119,12 @@ class HTMLSerializer:parts.append('>')+ # Handle table normalization (ensure thead/tbody for markdownify)+ if tag_name == 'table':+ table_html = self._serialize_table_children(node, depth)+ parts.append(table_html)# Handle iframe content document- if tag_name in {'iframe', 'frame'} and node.content_document:+ elif tag_name in {'iframe', 'frame'} and node.content_document:# Serialize iframe contentfor child in node.content_document.children_nodes or []:child_html = self.serialize(child, depth + 1)@@ -159,6 +163,85 @@ class HTMLSerializer:# Unknown node type - skipreturn ''+ def _serialize_table_children(self, table_node: EnhancedDOMTreeNode, depth: int) -> str:+ """Normalize table structure to ensure thead/tbody for markdownify.++ When a <table> has no <thead> but the first <tr> contains <th> cells,+ wrap that row in <thead> and remaining rows in <tbody>.+ """+ children = table_node.children+ if not children:+ return ''++ # Check if table already has thead+ child_tags = [c.tag_name for c in children if c.node_type == NodeType.ELEMENT_NODE]+ has_thead = 'thead' in child_tags+ has_tbody = 'tbody' in child_tags++ if has_thead or not child_tags:+ # Already normalized or empty — serialize normally+ parts = []+ for child in children:+ child_html = self.serialize(child, depth + 1)+ if child_html:+ parts.append(child_html)+ return ''.join(parts)++ # Find the first <tr> with <th> cells+ first_tr = None+ first_tr_idx = -1+ for i, child in enumerate(children):+ if child.node_type == NodeType.ELEMENT_NODE and child.tag_name == 'tr':+ # Check if this row contains <th> cells+ has_th = any(+ c.node_type == NodeType.ELEMENT_NODE and c.tag_name == 'th'+ for c in child.children+ )+ if has_th:+ first_tr = child+ first_tr_idx = i+ break # Only check the first <tr>++ if first_tr is None:+ # No header row detected — serialize normally+ parts = []+ for child in children:+ child_html = self.serialize(child, depth + 1)+ if child_html:+ parts.append(child_html)+ return ''.join(parts)++ # Wrap first_tr in <thead>, remaining <tr> in <tbody>+ parts = []++ # Emit any children before the header row (e.g. colgroup, caption)+ for child in children[:first_tr_idx]:+ child_html = self.serialize(child, depth + 1)+ if child_html:+ parts.append(child_html)++ # Emit <thead>+ parts.append('<thead>')+ parts.append(self.serialize(first_tr, depth + 2))+ parts.append('</thead>')++ # Collect remaining rows+ remaining = children[first_tr_idx + 1:]+ if remaining and not has_tbody:+ parts.append('<tbody>')+ for child in remaining:+ child_html = self.serialize(child, depth + 2)+ if child_html:+ parts.append(child_html)+ parts.append('</tbody>')+ else:+ for child in remaining:+ child_html = self.serialize(child, depth + 1)+ if child_html:+ parts.append(child_html)++ return ''.join(parts)+def _serialize_attributes(self, attributes: dict[str, str]) -> str:"""Serialize element attributes to HTML attribute string.diff --git a/browser_use/dom/views.py b/browser_use/dom/views.pyindex 42aafb68b..7243f2df3 100644--- a/browser_use/dom/views.py+++ b/browser_use/dom/views.py@@ -901,6 +901,19 @@ class EnhancedDOMTreeNode:DOMSelectorMap = dict[int, EnhancedDOMTreeNode]+@dataclass(slots=True)+class MarkdownChunk:+ """A structure-aware chunk of markdown content."""++ content: str+ chunk_index: int+ total_chunks: int+ char_offset_start: int # in original content+ char_offset_end: int # in original content+ overlap_prefix: str # context from prev chunk (e.g. table headers)+ has_more: bool++@dataclassclass SerializedDOMState:_root: SimplifiedNode | Nonediff --git a/browser_use/tools/service.py b/browser_use/tools/service.pyindex 1ef6b1650..18bfb5c8c 100644--- a/browser_use/tools/service.py+++ b/browser_use/tools/service.py@@ -951,35 +951,24 @@ class Tools(Generic[Context]):# Original content length for processingfinal_filtered_length = content_stats['final_filtered_chars']+ # Structure-aware chunking replaces naive char-based truncation+ from browser_use.dom.markdown_extractor import chunk_markdown_by_structure++ chunks = chunk_markdown_by_structure(content, max_chunk_chars=MAX_CHAR_LIMIT, start_from_char=start_from_char)+ if not chunks:+ return ActionResult(+ error=f'start_from_char ({start_from_char}) exceeds content length {final_filtered_length} characters.'+ )+ chunk = chunks[0]+ content = chunk.content+ truncated = chunk.has_moreif start_from_char > 0:- if start_from_char >= len(content):- return ActionResult(- error=f'start_from_char ({start_from_char}) exceeds content length {final_filtered_length} characters.'- )- content = content[start_from_char:]content_stats['started_from_char'] = start_from_char-- # Smart truncation with context preservation- truncated = False- if len(content) > MAX_CHAR_LIMIT:- # Try to truncate at a natural break point (paragraph, sentence)- truncate_at = MAX_CHAR_LIMIT-- # Look for paragraph break within last 500 chars of limit- paragraph_break = content.rfind('\n\n', MAX_CHAR_LIMIT - 500, MAX_CHAR_LIMIT)- if paragraph_break > 0:- truncate_at = paragraph_break- else:- # Look for sentence break within last 200 chars of limit- sentence_break = content.rfind('.', MAX_CHAR_LIMIT - 200, MAX_CHAR_LIMIT)- if sentence_break > 0:- truncate_at = sentence_break + 1-- content = content[:truncate_at]- truncated = True- next_start = (start_from_char or 0) + truncate_at- content_stats['truncated_at_char'] = truncate_at- content_stats['next_start_char'] = next_start+ if truncated:+ content_stats['truncated_at_char'] = chunk.char_offset_end+ content_stats['next_start_char'] = chunk.char_offset_end+ content_stats['chunk_index'] = chunk.chunk_index+ content_stats['total_chunks'] = chunk.total_chunks# Add content statistics to the resultoriginal_html_length = content_stats['original_html_chars']diff --git a/tests/ci/test_markdown_chunking.py b/tests/ci/test_markdown_chunking.pynew file mode 100644index 000000000..d47184419--- /dev/null+++ b/tests/ci/test_markdown_chunking.py@@ -0,0 +1,312 @@+"""Tests for structure-aware markdown chunking."""++import pytest+from markdownify import markdownify as md+from pytest_httpserver import HTTPServer++from browser_use.dom.markdown_extractor import chunk_markdown_by_structure+from browser_use.dom.serializer.html_serializer import HTMLSerializer+from browser_use.dom.views import MarkdownChunk+++# ---------------------------------------------------------------------------+# Unit tests — synchronous, no browser needed+# ---------------------------------------------------------------------------+++class TestChunkMarkdownBasic:+ """Basic chunking behaviour."""++ def test_short_content_single_chunk(self):+ content = '# Hello\n\nSome short content.'+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=100_000)+ assert len(chunks) == 1+ assert chunks[0].content == content+ assert chunks[0].chunk_index == 0+ assert chunks[0].total_chunks == 1+ assert chunks[0].has_more is False++ def test_empty_content(self):+ chunks = chunk_markdown_by_structure('', max_chunk_chars=100)+ assert len(chunks) == 1+ assert chunks[0].content == ''+ assert chunks[0].has_more is False++ def test_chunk_offsets_cover_full_content(self):+ """Chunk offsets should cover the entire original content without gaps."""+ content = '# Header\n\nParagraph one.\n\n# Header 2\n\nParagraph two.'+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=20)+ # Verify no gaps between consecutive chunks+ for i in range(1, len(chunks)):+ assert chunks[i].char_offset_start == chunks[i - 1].char_offset_end, (+ f'Gap between chunk {i - 1} end ({chunks[i - 1].char_offset_end}) '+ f'and chunk {i} start ({chunks[i].char_offset_start})'+ )+ # First chunk starts at 0+ assert chunks[0].char_offset_start == 0+ # Last chunk ends at content length+ assert chunks[-1].char_offset_end == len(content)+++class TestChunkMarkdownHeaders:+ """Header boundary splitting."""++ def test_splits_at_header_boundary(self):+ """Chunks should prefer splitting at header boundaries."""+ section_a = '# Section A\n\n' + 'x' * 50+ section_b = '\n\n# Section B\n\n' + 'y' * 50+ content = section_a + section_b+ # Set limit so section_a fits but section_a + section_b doesn't+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=len(section_a) + 5)+ assert len(chunks) >= 2+ # First chunk should contain Section A header+ assert '# Section A' in chunks[0].content+ # Second chunk should start with or contain Section B header+ assert '# Section B' in chunks[1].content+++class TestChunkMarkdownCodeFence:+ """Code fence blocks never split."""++ def test_code_fence_not_split(self):+ code_block = '```python\n' + 'x = 1\n' * 100 + '```'+ content = '# Title\n\n' + code_block + '\n\n# Footer\n\nDone.'+ # Limit smaller than the code block — it should still stay in one chunk (soft limit)+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=50)+ # Find the chunk containing the code block+ code_chunks = [c for c in chunks if '```python' in c.content and '```' in c.content.split('```python')[1]]+ assert len(code_chunks) >= 1, 'Code fence should appear intact in at least one chunk'++ def test_unclosed_code_fence(self):+ """Unclosed code fence should still be kept as one block."""+ content = '# Title\n\n```python\nx = 1\ny = 2'+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=100_000)+ assert len(chunks) == 1+ assert '```python' in chunks[0].content+ assert 'y = 2' in chunks[0].content+++class TestChunkMarkdownTable:+ """Table rows never split mid-row."""++ def test_table_not_split_mid_row(self):+ header = '| Name | Value |'+ separator = '| --- | --- |'+ rows = [f'| item{i} | val{i} |' for i in range(50)]+ table = '\n'.join([header, separator] + rows)+ content = '# Data\n\n' + table++ # Use a limit that would fall in the middle of the table+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=200)++ for chunk in chunks:+ lines = chunk.content.split('\n')+ for line in lines:+ stripped = line.strip()+ if stripped.startswith('|') and stripped.endswith('|'):+ # Each table row line should be complete (start and end with |)+ assert stripped.count('|') >= 3, f'Incomplete table row: {stripped}'++ def test_table_header_in_overlap_for_continuation(self):+ """When a table spans multiple chunks, the header should be in the overlap prefix."""+ header = '| Col1 | Col2 |'+ separator = '| --- | --- |'+ rows = [f'| r{i} | d{i} |' for i in range(100)]+ table = '\n'.join([header, separator] + rows)+ content = table++ # Force split within the table+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=300)++ if len(chunks) > 1:+ # Second chunk should have table header in overlap+ assert '| Col1 | Col2 |' in chunks[1].overlap_prefix+ assert '| --- | --- |' in chunks[1].overlap_prefix+++class TestChunkMarkdownListItems:+ """List item continuations stay together."""++ def test_list_items_not_split(self):+ items = '\n'.join([f'- Item {i} with some description text' for i in range(50)])+ content = '# List\n\n' + items++ chunks = chunk_markdown_by_structure(content, max_chunk_chars=200)+ for chunk in chunks:+ lines = chunk.content.split('\n')+ for line in lines:+ stripped = line.strip()+ if stripped.startswith('- '):+ # Each list item should be a complete item+ assert 'Item' in stripped+++class TestChunkMarkdownStartFromChar:+ """start_from_char parameter returns correct chunk."""++ def test_start_from_char_returns_correct_chunk(self):+ section_a = '# A\n\nContent A here.'+ section_b = '\n\n# B\n\nContent B here.'+ content = section_a + section_b+ # Chunk at header boundaries+ all_chunks = chunk_markdown_by_structure(content, max_chunk_chars=len(section_a) + 5)++ if len(all_chunks) > 1:+ # Request from char offset within second chunk+ mid = all_chunks[1].char_offset_start + 1+ filtered = chunk_markdown_by_structure(content, max_chunk_chars=len(section_a) + 5, start_from_char=mid)+ assert len(filtered) >= 1+ assert filtered[0].chunk_index == all_chunks[1].chunk_index++ def test_start_from_char_past_end_returns_empty(self):+ content = '# Hello\n\nWorld.'+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=100_000, start_from_char=99999)+ assert chunks == []++ def test_start_from_char_zero_returns_all(self):+ content = '# Hello\n\nWorld.'+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=100_000, start_from_char=0)+ assert len(chunks) == 1+++class TestChunkMarkdownOverlap:+ """Overlap lines carry context."""++ def test_overlap_lines_carry_context(self):+ lines_content = '\n'.join([f'Line {i}' for i in range(100)])+ content = lines_content++ chunks = chunk_markdown_by_structure(content, max_chunk_chars=200, overlap_lines=3)+ if len(chunks) > 1:+ # Second chunk should have overlap from first chunk+ assert chunks[1].overlap_prefix != ''+ # Overlap should contain lines from the end of the previous chunk+ overlap_lines = chunks[1].overlap_prefix.split('\n')+ assert len(overlap_lines) <= 3 + 2 # some flexibility for table headers etc.++ def test_no_overlap_on_first_chunk(self):+ content = '# A\n\nSome content.\n\n# B\n\nMore content.'+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=25)+ assert chunks[0].overlap_prefix == ''+++class TestChunkMarkdownMixed:+ """Mixed content scenarios."""++ def test_paragraph_splitting(self):+ """Paragraphs separated by blank lines are separate blocks."""+ p1 = 'First paragraph with text.'+ p2 = 'Second paragraph with more text.'+ content = f'{p1}\n\n{p2}'+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=30)+ # Should produce multiple chunks+ assert len(chunks) >= 2++ def test_single_oversized_block_allowed(self):+ """A single block bigger than max_chunk_chars is allowed (soft limit)."""+ big_para = 'x' * 200+ content = big_para+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=50)+ assert len(chunks) == 1+ assert chunks[0].content == big_para+++# ---------------------------------------------------------------------------+# HTML → markdown → chunk pipeline tests+# ---------------------------------------------------------------------------+++class TestHTMLToMarkdownChunking:+ """End-to-end: HTML table → markdown → chunks."""++ def test_large_table_produces_valid_chunks(self):+ """200-row HTML table → markdown → chunks should produce valid table rows in every chunk."""+ rows = ''.join(f'<tr><td>Row {i}</td><td>Val {i}</td></tr>' for i in range(200))+ html = f'<table><thead><tr><th>Name</th><th>Value</th></tr></thead><tbody>{rows}</tbody></table>'+ markdown = md(html, heading_style='ATX')++ chunks = chunk_markdown_by_structure(markdown, max_chunk_chars=500)+ assert len(chunks) > 1, 'Should produce multiple chunks for 200 rows'++ for chunk in chunks:+ lines = chunk.content.strip().split('\n')+ for line in lines:+ s = line.strip()+ if s.startswith('|') and s.endswith('|'):+ # Every table line should have consistent column count+ assert s.count('|') >= 3++ def test_table_without_thead_normalization(self):+ """Table with <th> in first <tr> but no <thead> should still produce proper markdown."""+ html = '<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr><tr><td>3</td><td>4</td></tr></table>'+ markdown = md(html, heading_style='ATX')+ # Verify markdownify produced a proper table (with separator row)+ assert '---' in markdown or '| A |' in markdown+++# ---------------------------------------------------------------------------+# Integration tests — require browser + httpserver+# ---------------------------------------------------------------------------+++class TestTableNormalizationIntegration:+ """Integration tests using browser session and httpserver."""++ async def test_table_without_thead_normalized_via_serializer(self, browser_session, httpserver: HTTPServer):+ """Tables without <thead> should get normalized by HTMLSerializer during extraction."""+ html = """+ <html><body>+ <table>+ <tr><th>Header1</th><th>Header2</th></tr>+ <tr><td>data1</td><td>data2</td></tr>+ <tr><td>data3</td><td>data4</td></tr>+ </table>+ </body></html>+ """+ httpserver.expect_request('/table-test').respond_with_data(html, content_type='text/html')+ url = httpserver.url_for('/table-test')++ await browser_session.navigate_to(url)++ from browser_use.dom.markdown_extractor import extract_clean_markdown++ content, _ = await extract_clean_markdown(browser_session=browser_session)++ # Should have proper markdown table with separator+ assert '|' in content+ # The header should be present+ assert 'Header1' in content+ assert 'Header2' in content++ async def test_large_table_extraction_preserves_structure(self, browser_session, httpserver: HTTPServer):+ """Large table extraction should produce structure-aware chunks."""+ rows = ''.join(f'<tr><td>Name{i}</td><td>Value{i}</td></tr>' for i in range(300))+ html = f"""+ <html><body>+ <table>+ <tr><th>Name</th><th>Value</th></tr>+ {rows}+ </table>+ </body></html>+ """+ httpserver.expect_request('/big-table').respond_with_data(html, content_type='text/html')+ url = httpserver.url_for('/big-table')++ await browser_session.navigate_to(url)++ from browser_use.dom.markdown_extractor import extract_clean_markdown++ content, _ = await extract_clean_markdown(browser_session=browser_session)++ # Chunk with a small limit to force multiple chunks+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=2000)++ # Should produce multiple chunks+ assert len(chunks) > 1++ # Each chunk should have complete table rows+ for chunk in chunks:+ for line in chunk.content.split('\n'):+ s = line.strip()+ if s.startswith('|') and s.endswith('|'):+ assert s.count('|') >= 3, f'Incomplete table row: {s}'