Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Improve Markdown Chunker

Saurav Pandae66eac8a3704
diff --git a/browser_use/dom/markdown_extractor.py b/browser_use/dom/markdown_extractor.py
index ba4a7bde8..a2d244721 100644
--- a/browser_use/dom/markdown_extractor.py
+++ b/browser_use/dom/markdown_extractor.py
@@ -6,10 +6,13 @@ used by both the tools service and page actor.
"""
import re
+from dataclasses import dataclass
+from enum import Enum, auto
from typing import TYPE_CHECKING, Any
from browser_use.dom.serializer.html_serializer import HTMLSerializer
from browser_use.dom.service import DomService
+from browser_use.dom.views import MarkdownChunk
if TYPE_CHECKING:
from browser_use.browser.session import BrowserSession
@@ -168,3 +171,333 @@ def _preprocess_markdown_content(content: str, max_newlines: int = 3) -> tuple[s
chars_filtered = original_length - len(content)
return content, chars_filtered
+
+
+# ---------------------------------------------------------------------------
+# Structure-aware markdown chunking
+# ---------------------------------------------------------------------------
+
+
+class _BlockType(Enum):
+ HEADER = auto()
+ CODE_FENCE = auto()
+ TABLE = auto()
+ LIST_ITEM = auto()
+ PARAGRAPH = auto()
+ BLANK = auto()
+
+
+@dataclass(slots=True)
+class _AtomicBlock:
+ block_type: _BlockType
+ lines: list[str]
+ char_start: int # offset in original content
+ char_end: int # offset in original content (exclusive)
+
+
+_TABLE_ROW_RE = re.compile(r'^\s*\|.*\|\s*$')
+_LIST_ITEM_RE = re.compile(r'^(\s*)([-*+]|\d+[.)]) ')
+_LIST_CONTINUATION_RE = re.compile(r'^(\s{2,}|\t)')
+
+
+def _parse_atomic_blocks(content: str) -> list[_AtomicBlock]:
+ """Phase 1: Walk lines, group into unsplittable blocks."""
+ lines = content.split('\n')
+ blocks: list[_AtomicBlock] = []
+ i = 0
+ offset = 0 # char offset tracking
+
+ while i < len(lines):
+ line = lines[i]
+ line_len = len(line) + 1 # +1 for the newline we split on
+
+ # BLANK
+ if not line.strip():
+ blocks.append(_AtomicBlock(
+ block_type=_BlockType.BLANK,
+ lines=[line],
+ char_start=offset,
+ char_end=offset + line_len,
+ ))
+ offset += line_len
+ i += 1
+ continue
+
+ # CODE FENCE
+ if line.strip().startswith('```'):
+ fence_lines = [line]
+ fence_end = offset + line_len
+ i += 1
+ # Consume until closing fence or EOF
+ while i < len(lines):
+ fence_line = lines[i]
+ fence_line_len = len(fence_line) + 1
+ fence_lines.append(fence_line)
+ fence_end += fence_line_len
+ i += 1
+ if fence_line.strip().startswith('```') and len(fence_lines) > 1:
+ break
+ blocks.append(_AtomicBlock(
+ block_type=_BlockType.CODE_FENCE,
+ lines=fence_lines,
+ char_start=offset,
+ char_end=fence_end,
+ ))
+ offset = fence_end
+ continue
+
+ # HEADER
+ if line.lstrip().startswith('#'):
+ blocks.append(_AtomicBlock(
+ block_type=_BlockType.HEADER,
+ lines=[line],
+ char_start=offset,
+ char_end=offset + line_len,
+ ))
+ offset += line_len
+ i += 1
+ continue
+
+ # TABLE (consecutive |...| lines)
+ # Header + separator row stay together; each data row is its own block
+ if _TABLE_ROW_RE.match(line):
+ # Collect header line
+ header_lines = [line]
+ header_end = offset + line_len
+ i += 1
+ # Check if next line is separator (contains ---)
+ if i < len(lines) and _TABLE_ROW_RE.match(lines[i]) and '---' in lines[i]:
+ sep = lines[i]
+ sep_len = len(sep) + 1
+ header_lines.append(sep)
+ header_end += sep_len
+ i += 1
+ # Emit header+separator as one atomic block
+ blocks.append(_AtomicBlock(
+ block_type=_BlockType.TABLE,
+ lines=header_lines,
+ char_start=offset,
+ char_end=header_end,
+ ))
+ offset = header_end
+ # Each subsequent table row is its own TABLE block (splittable between rows)
+ while i < len(lines) and _TABLE_ROW_RE.match(lines[i]):
+ row = lines[i]
+ row_len = len(row) + 1
+ blocks.append(_AtomicBlock(
+ block_type=_BlockType.TABLE,
+ lines=[row],
+ char_start=offset,
+ char_end=offset + row_len,
+ ))
+ offset += row_len
+ i += 1
+ continue
+
+ # LIST ITEM (with indented continuations)
+ if _LIST_ITEM_RE.match(line):
+ list_lines = [line]
+ list_end = offset + line_len
+ i += 1
+ # Consume continuation lines (indented or blank between items)
+ while i < len(lines):
+ next_line = lines[i]
+ next_len = len(next_line) + 1
+ # Another list item at same or deeper indent → still part of this block
+ if _LIST_ITEM_RE.match(next_line):
+ list_lines.append(next_line)
+ list_end += next_len
+ i += 1
+ continue
+ # Indented continuation
+ if next_line.strip() and _LIST_CONTINUATION_RE.match(next_line):
+ list_lines.append(next_line)
+ list_end += next_len
+ i += 1
+ continue
+ break
+ blocks.append(_AtomicBlock(
+ block_type=_BlockType.LIST_ITEM,
+ lines=list_lines,
+ char_start=offset,
+ char_end=list_end,
+ ))
+ offset = list_end
+ continue
+
+ # PARAGRAPH (everything else, up to next blank line)
+ para_lines = [line]
+ para_end = offset + line_len
+ i += 1
+ while i < len(lines) and lines[i].strip():
+ # Stop if next line starts a different block type
+ nl = lines[i]
+ if (
+ nl.lstrip().startswith('#')
+ or nl.strip().startswith('```')
+ or _TABLE_ROW_RE.match(nl)
+ or _LIST_ITEM_RE.match(nl)
+ ):
+ break
+ nl_len = len(nl) + 1
+ para_lines.append(nl)
+ para_end += nl_len
+ i += 1
+ blocks.append(_AtomicBlock(
+ block_type=_BlockType.PARAGRAPH,
+ lines=para_lines,
+ char_start=offset,
+ char_end=para_end,
+ ))
+ offset = para_end
+
+ # Fix last block char_end: content may not end with \n
+ if blocks and content and not content.endswith('\n'):
+ blocks[-1] = _AtomicBlock(
+ block_type=blocks[-1].block_type,
+ lines=blocks[-1].lines,
+ char_start=blocks[-1].char_start,
+ char_end=len(content),
+ )
+
+ return blocks
+
+
+def _block_text(block: _AtomicBlock) -> str:
+ return '\n'.join(block.lines)
+
+
+def _get_table_header(block: _AtomicBlock) -> str | None:
+ """Extract table header + separator rows from a TABLE block."""
+ assert block.block_type == _BlockType.TABLE
+ if len(block.lines) < 2:
+ return None
+ # Header is first line, separator is second line (must contain ---)
+ sep_line = block.lines[1]
+ if '---' in sep_line or '- -' in sep_line:
+ return block.lines[0] + '\n' + block.lines[1]
+ return None
+
+
+def chunk_markdown_by_structure(
+ content: str,
+ max_chunk_chars: int = 100_000,
+ overlap_lines: int = 5,
+ start_from_char: int = 0,
+) -> list[MarkdownChunk]:
+ """Split markdown into structure-aware chunks.
+
+ Algorithm:
+ Phase 1 — Parse atomic blocks (headers, code fences, tables, list items, paragraphs).
+ Phase 2 — Greedy chunk assembly: accumulate blocks until exceeding max_chunk_chars.
+ A single block exceeding the limit is allowed (soft limit).
+ Phase 3 — Build overlap prefixes for context carry between chunks.
+
+ Args:
+ content: Full markdown string.
+ max_chunk_chars: Target maximum chars per chunk (soft limit for single blocks).
+ overlap_lines: Number of trailing lines from previous chunk to prepend.
+ start_from_char: Return chunks starting from the chunk that contains this offset.
+
+ Returns:
+ List of MarkdownChunk. Empty if start_from_char is past end of content.
+ """
+ if not content:
+ return [MarkdownChunk(
+ content='',
+ chunk_index=0,
+ total_chunks=1,
+ char_offset_start=0,
+ char_offset_end=0,
+ overlap_prefix='',
+ has_more=False,
+ )]
+
+ if start_from_char >= len(content):
+ return []
+
+ # Phase 1: parse atomic blocks
+ blocks = _parse_atomic_blocks(content)
+ if not blocks:
+ return []
+
+ # Phase 2: greedy chunk assembly
+ raw_chunks: list[list[_AtomicBlock]] = []
+ current_chunk: list[_AtomicBlock] = []
+ current_size = 0
+
+ for block in blocks:
+ block_size = block.char_end - block.char_start
+ # If adding this block would exceed limit AND we already have content, emit chunk
+ if current_size + block_size > max_chunk_chars and current_chunk:
+ raw_chunks.append(current_chunk)
+ current_chunk = []
+ current_size = 0
+ current_chunk.append(block)
+ current_size += block_size
+
+ if current_chunk:
+ raw_chunks.append(current_chunk)
+
+ total_chunks = len(raw_chunks)
+
+ # Phase 3: build MarkdownChunk objects with overlap prefixes
+ chunks: list[MarkdownChunk] = []
+ # Track table header from previous chunk for table continuations
+ prev_chunk_last_table_header: str | None = None
+
+ for idx, chunk_blocks in enumerate(raw_chunks):
+ chunk_text = '\n'.join(_block_text(b) for b in chunk_blocks)
+ char_start = chunk_blocks[0].char_start
+ char_end = chunk_blocks[-1].char_end
+
+ # Build overlap prefix
+ overlap = ''
+ if idx > 0:
+ prev_blocks = raw_chunks[idx - 1]
+ prev_text = '\n'.join(_block_text(b) for b in prev_blocks)
+ prev_lines = prev_text.split('\n')
+
+ # Check if current chunk starts with a table continuation
+ first_block = chunk_blocks[0]
+ if first_block.block_type == _BlockType.TABLE and prev_chunk_last_table_header:
+ # Always prepend table header for continuation
+ trailing = prev_lines[-(overlap_lines):] if overlap_lines > 0 else []
+ header_lines = prev_chunk_last_table_header.split('\n')
+ # Deduplicate: don't repeat header lines if they're already in trailing
+ combined = list(header_lines)
+ for tl in trailing:
+ if tl not in combined:
+ combined.append(tl)
+ overlap = '\n'.join(combined)
+ elif overlap_lines > 0:
+ overlap = '\n'.join(prev_lines[-(overlap_lines):])
+
+ # Track table header from this chunk for next iteration.
+ # Scan all blocks: the header+separator block could be anywhere in the chunk.
+ prev_chunk_last_table_header = None
+ for b in chunk_blocks:
+ if b.block_type == _BlockType.TABLE:
+ hdr = _get_table_header(b)
+ if hdr is not None:
+ prev_chunk_last_table_header = hdr
+
+ has_more = idx < total_chunks - 1
+ chunks.append(MarkdownChunk(
+ content=chunk_text,
+ chunk_index=idx,
+ total_chunks=total_chunks,
+ char_offset_start=char_start,
+ char_offset_end=char_end,
+ overlap_prefix=overlap,
+ has_more=has_more,
+ ))
+
+ # Apply start_from_char filter: return chunks from the one containing that offset
+ if start_from_char > 0:
+ for i, chunk in enumerate(chunks):
+ if chunk.char_offset_end > start_from_char:
+ return chunks[i:]
+ return [] # offset past all chunks
+
+ return chunks
diff --git a/browser_use/dom/serializer/html_serializer.py b/browser_use/dom/serializer/html_serializer.py
index 8aaf31df7..dd3d1c6b6 100644
--- a/browser_use/dom/serializer/html_serializer.py
+++ b/browser_use/dom/serializer/html_serializer.py
@@ -119,8 +119,12 @@ class HTMLSerializer:
parts.append('>')
+ # Handle table normalization (ensure thead/tbody for markdownify)
+ if tag_name == 'table':
+ table_html = self._serialize_table_children(node, depth)
+ parts.append(table_html)
# Handle iframe content document
- if tag_name in {'iframe', 'frame'} and node.content_document:
+ elif tag_name in {'iframe', 'frame'} and node.content_document:
# Serialize iframe content
for child in node.content_document.children_nodes or []:
child_html = self.serialize(child, depth + 1)
@@ -159,6 +163,85 @@ class HTMLSerializer:
# Unknown node type - skip
return ''
+ def _serialize_table_children(self, table_node: EnhancedDOMTreeNode, depth: int) -> str:
+ """Normalize table structure to ensure thead/tbody for markdownify.
+
+ When a <table> has no <thead> but the first <tr> contains <th> cells,
+ wrap that row in <thead> and remaining rows in <tbody>.
+ """
+ children = table_node.children
+ if not children:
+ return ''
+
+ # Check if table already has thead
+ child_tags = [c.tag_name for c in children if c.node_type == NodeType.ELEMENT_NODE]
+ has_thead = 'thead' in child_tags
+ has_tbody = 'tbody' in child_tags
+
+ if has_thead or not child_tags:
+ # Already normalized or empty — serialize normally
+ parts = []
+ for child in children:
+ child_html = self.serialize(child, depth + 1)
+ if child_html:
+ parts.append(child_html)
+ return ''.join(parts)
+
+ # Find the first <tr> with <th> cells
+ first_tr = None
+ first_tr_idx = -1
+ for i, child in enumerate(children):
+ if child.node_type == NodeType.ELEMENT_NODE and child.tag_name == 'tr':
+ # Check if this row contains <th> cells
+ has_th = any(
+ c.node_type == NodeType.ELEMENT_NODE and c.tag_name == 'th'
+ for c in child.children
+ )
+ if has_th:
+ first_tr = child
+ first_tr_idx = i
+ break # Only check the first <tr>
+
+ if first_tr is None:
+ # No header row detected — serialize normally
+ parts = []
+ for child in children:
+ child_html = self.serialize(child, depth + 1)
+ if child_html:
+ parts.append(child_html)
+ return ''.join(parts)
+
+ # Wrap first_tr in <thead>, remaining <tr> in <tbody>
+ parts = []
+
+ # Emit any children before the header row (e.g. colgroup, caption)
+ for child in children[:first_tr_idx]:
+ child_html = self.serialize(child, depth + 1)
+ if child_html:
+ parts.append(child_html)
+
+ # Emit <thead>
+ parts.append('<thead>')
+ parts.append(self.serialize(first_tr, depth + 2))
+ parts.append('</thead>')
+
+ # Collect remaining rows
+ remaining = children[first_tr_idx + 1:]
+ if remaining and not has_tbody:
+ parts.append('<tbody>')
+ for child in remaining:
+ child_html = self.serialize(child, depth + 2)
+ if child_html:
+ parts.append(child_html)
+ parts.append('</tbody>')
+ else:
+ for child in remaining:
+ child_html = self.serialize(child, depth + 1)
+ if child_html:
+ parts.append(child_html)
+
+ return ''.join(parts)
+
def _serialize_attributes(self, attributes: dict[str, str]) -> str:
"""Serialize element attributes to HTML attribute string.
diff --git a/browser_use/dom/views.py b/browser_use/dom/views.py
index 42aafb68b..7243f2df3 100644
--- a/browser_use/dom/views.py
+++ b/browser_use/dom/views.py
@@ -901,6 +901,19 @@ class EnhancedDOMTreeNode:
DOMSelectorMap = dict[int, EnhancedDOMTreeNode]
+@dataclass(slots=True)
+class MarkdownChunk:
+ """A structure-aware chunk of markdown content."""
+
+ content: str
+ chunk_index: int
+ total_chunks: int
+ char_offset_start: int # in original content
+ char_offset_end: int # in original content
+ overlap_prefix: str # context from prev chunk (e.g. table headers)
+ has_more: bool
+
+
@dataclass
class SerializedDOMState:
_root: SimplifiedNode | None
diff --git a/browser_use/tools/service.py b/browser_use/tools/service.py
index 1ef6b1650..18bfb5c8c 100644
--- a/browser_use/tools/service.py
+++ b/browser_use/tools/service.py
@@ -951,35 +951,24 @@ class Tools(Generic[Context]):
# Original content length for processing
final_filtered_length = content_stats['final_filtered_chars']
+ # Structure-aware chunking replaces naive char-based truncation
+ from browser_use.dom.markdown_extractor import chunk_markdown_by_structure
+
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=MAX_CHAR_LIMIT, start_from_char=start_from_char)
+ if not chunks:
+ return ActionResult(
+ error=f'start_from_char ({start_from_char}) exceeds content length {final_filtered_length} characters.'
+ )
+ chunk = chunks[0]
+ content = chunk.content
+ truncated = chunk.has_more
if start_from_char > 0:
- if start_from_char >= len(content):
- return ActionResult(
- error=f'start_from_char ({start_from_char}) exceeds content length {final_filtered_length} characters.'
- )
- content = content[start_from_char:]
content_stats['started_from_char'] = start_from_char
-
- # Smart truncation with context preservation
- truncated = False
- if len(content) > MAX_CHAR_LIMIT:
- # Try to truncate at a natural break point (paragraph, sentence)
- truncate_at = MAX_CHAR_LIMIT
-
- # Look for paragraph break within last 500 chars of limit
- paragraph_break = content.rfind('\n\n', MAX_CHAR_LIMIT - 500, MAX_CHAR_LIMIT)
- if paragraph_break > 0:
- truncate_at = paragraph_break
- else:
- # Look for sentence break within last 200 chars of limit
- sentence_break = content.rfind('.', MAX_CHAR_LIMIT - 200, MAX_CHAR_LIMIT)
- if sentence_break > 0:
- truncate_at = sentence_break + 1
-
- content = content[:truncate_at]
- truncated = True
- next_start = (start_from_char or 0) + truncate_at
- content_stats['truncated_at_char'] = truncate_at
- content_stats['next_start_char'] = next_start
+ if truncated:
+ content_stats['truncated_at_char'] = chunk.char_offset_end
+ content_stats['next_start_char'] = chunk.char_offset_end
+ content_stats['chunk_index'] = chunk.chunk_index
+ content_stats['total_chunks'] = chunk.total_chunks
# Add content statistics to the result
original_html_length = content_stats['original_html_chars']
diff --git a/tests/ci/test_markdown_chunking.py b/tests/ci/test_markdown_chunking.py
new file mode 100644
index 000000000..d47184419
--- /dev/null
+++ b/tests/ci/test_markdown_chunking.py
@@ -0,0 +1,312 @@
+"""Tests for structure-aware markdown chunking."""
+
+import pytest
+from markdownify import markdownify as md
+from pytest_httpserver import HTTPServer
+
+from browser_use.dom.markdown_extractor import chunk_markdown_by_structure
+from browser_use.dom.serializer.html_serializer import HTMLSerializer
+from browser_use.dom.views import MarkdownChunk
+
+
+# ---------------------------------------------------------------------------
+# Unit tests — synchronous, no browser needed
+# ---------------------------------------------------------------------------
+
+
+class TestChunkMarkdownBasic:
+ """Basic chunking behaviour."""
+
+ def test_short_content_single_chunk(self):
+ content = '# Hello\n\nSome short content.'
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=100_000)
+ assert len(chunks) == 1
+ assert chunks[0].content == content
+ assert chunks[0].chunk_index == 0
+ assert chunks[0].total_chunks == 1
+ assert chunks[0].has_more is False
+
+ def test_empty_content(self):
+ chunks = chunk_markdown_by_structure('', max_chunk_chars=100)
+ assert len(chunks) == 1
+ assert chunks[0].content == ''
+ assert chunks[0].has_more is False
+
+ def test_chunk_offsets_cover_full_content(self):
+ """Chunk offsets should cover the entire original content without gaps."""
+ content = '# Header\n\nParagraph one.\n\n# Header 2\n\nParagraph two.'
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=20)
+ # Verify no gaps between consecutive chunks
+ for i in range(1, len(chunks)):
+ assert chunks[i].char_offset_start == chunks[i - 1].char_offset_end, (
+ f'Gap between chunk {i - 1} end ({chunks[i - 1].char_offset_end}) '
+ f'and chunk {i} start ({chunks[i].char_offset_start})'
+ )
+ # First chunk starts at 0
+ assert chunks[0].char_offset_start == 0
+ # Last chunk ends at content length
+ assert chunks[-1].char_offset_end == len(content)
+
+
+class TestChunkMarkdownHeaders:
+ """Header boundary splitting."""
+
+ def test_splits_at_header_boundary(self):
+ """Chunks should prefer splitting at header boundaries."""
+ section_a = '# Section A\n\n' + 'x' * 50
+ section_b = '\n\n# Section B\n\n' + 'y' * 50
+ content = section_a + section_b
+ # Set limit so section_a fits but section_a + section_b doesn't
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=len(section_a) + 5)
+ assert len(chunks) >= 2
+ # First chunk should contain Section A header
+ assert '# Section A' in chunks[0].content
+ # Second chunk should start with or contain Section B header
+ assert '# Section B' in chunks[1].content
+
+
+class TestChunkMarkdownCodeFence:
+ """Code fence blocks never split."""
+
+ def test_code_fence_not_split(self):
+ code_block = '```python\n' + 'x = 1\n' * 100 + '```'
+ content = '# Title\n\n' + code_block + '\n\n# Footer\n\nDone.'
+ # Limit smaller than the code block — it should still stay in one chunk (soft limit)
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=50)
+ # Find the chunk containing the code block
+ code_chunks = [c for c in chunks if '```python' in c.content and '```' in c.content.split('```python')[1]]
+ assert len(code_chunks) >= 1, 'Code fence should appear intact in at least one chunk'
+
+ def test_unclosed_code_fence(self):
+ """Unclosed code fence should still be kept as one block."""
+ content = '# Title\n\n```python\nx = 1\ny = 2'
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=100_000)
+ assert len(chunks) == 1
+ assert '```python' in chunks[0].content
+ assert 'y = 2' in chunks[0].content
+
+
+class TestChunkMarkdownTable:
+ """Table rows never split mid-row."""
+
+ def test_table_not_split_mid_row(self):
+ header = '| Name | Value |'
+ separator = '| --- | --- |'
+ rows = [f'| item{i} | val{i} |' for i in range(50)]
+ table = '\n'.join([header, separator] + rows)
+ content = '# Data\n\n' + table
+
+ # Use a limit that would fall in the middle of the table
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=200)
+
+ for chunk in chunks:
+ lines = chunk.content.split('\n')
+ for line in lines:
+ stripped = line.strip()
+ if stripped.startswith('|') and stripped.endswith('|'):
+ # Each table row line should be complete (start and end with |)
+ assert stripped.count('|') >= 3, f'Incomplete table row: {stripped}'
+
+ def test_table_header_in_overlap_for_continuation(self):
+ """When a table spans multiple chunks, the header should be in the overlap prefix."""
+ header = '| Col1 | Col2 |'
+ separator = '| --- | --- |'
+ rows = [f'| r{i} | d{i} |' for i in range(100)]
+ table = '\n'.join([header, separator] + rows)
+ content = table
+
+ # Force split within the table
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=300)
+
+ if len(chunks) > 1:
+ # Second chunk should have table header in overlap
+ assert '| Col1 | Col2 |' in chunks[1].overlap_prefix
+ assert '| --- | --- |' in chunks[1].overlap_prefix
+
+
+class TestChunkMarkdownListItems:
+ """List item continuations stay together."""
+
+ def test_list_items_not_split(self):
+ items = '\n'.join([f'- Item {i} with some description text' for i in range(50)])
+ content = '# List\n\n' + items
+
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=200)
+ for chunk in chunks:
+ lines = chunk.content.split('\n')
+ for line in lines:
+ stripped = line.strip()
+ if stripped.startswith('- '):
+ # Each list item should be a complete item
+ assert 'Item' in stripped
+
+
+class TestChunkMarkdownStartFromChar:
+ """start_from_char parameter returns correct chunk."""
+
+ def test_start_from_char_returns_correct_chunk(self):
+ section_a = '# A\n\nContent A here.'
+ section_b = '\n\n# B\n\nContent B here.'
+ content = section_a + section_b
+ # Chunk at header boundaries
+ all_chunks = chunk_markdown_by_structure(content, max_chunk_chars=len(section_a) + 5)
+
+ if len(all_chunks) > 1:
+ # Request from char offset within second chunk
+ mid = all_chunks[1].char_offset_start + 1
+ filtered = chunk_markdown_by_structure(content, max_chunk_chars=len(section_a) + 5, start_from_char=mid)
+ assert len(filtered) >= 1
+ assert filtered[0].chunk_index == all_chunks[1].chunk_index
+
+ def test_start_from_char_past_end_returns_empty(self):
+ content = '# Hello\n\nWorld.'
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=100_000, start_from_char=99999)
+ assert chunks == []
+
+ def test_start_from_char_zero_returns_all(self):
+ content = '# Hello\n\nWorld.'
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=100_000, start_from_char=0)
+ assert len(chunks) == 1
+
+
+class TestChunkMarkdownOverlap:
+ """Overlap lines carry context."""
+
+ def test_overlap_lines_carry_context(self):
+ lines_content = '\n'.join([f'Line {i}' for i in range(100)])
+ content = lines_content
+
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=200, overlap_lines=3)
+ if len(chunks) > 1:
+ # Second chunk should have overlap from first chunk
+ assert chunks[1].overlap_prefix != ''
+ # Overlap should contain lines from the end of the previous chunk
+ overlap_lines = chunks[1].overlap_prefix.split('\n')
+ assert len(overlap_lines) <= 3 + 2 # some flexibility for table headers etc.
+
+ def test_no_overlap_on_first_chunk(self):
+ content = '# A\n\nSome content.\n\n# B\n\nMore content.'
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=25)
+ assert chunks[0].overlap_prefix == ''
+
+
+class TestChunkMarkdownMixed:
+ """Mixed content scenarios."""
+
+ def test_paragraph_splitting(self):
+ """Paragraphs separated by blank lines are separate blocks."""
+ p1 = 'First paragraph with text.'
+ p2 = 'Second paragraph with more text.'
+ content = f'{p1}\n\n{p2}'
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=30)
+ # Should produce multiple chunks
+ assert len(chunks) >= 2
+
+ def test_single_oversized_block_allowed(self):
+ """A single block bigger than max_chunk_chars is allowed (soft limit)."""
+ big_para = 'x' * 200
+ content = big_para
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=50)
+ assert len(chunks) == 1
+ assert chunks[0].content == big_para
+
+
+# ---------------------------------------------------------------------------
+# HTML → markdown → chunk pipeline tests
+# ---------------------------------------------------------------------------
+
+
+class TestHTMLToMarkdownChunking:
+ """End-to-end: HTML table → markdown → chunks."""
+
+ def test_large_table_produces_valid_chunks(self):
+ """200-row HTML table → markdown → chunks should produce valid table rows in every chunk."""
+ rows = ''.join(f'<tr><td>Row {i}</td><td>Val {i}</td></tr>' for i in range(200))
+ html = f'<table><thead><tr><th>Name</th><th>Value</th></tr></thead><tbody>{rows}</tbody></table>'
+ markdown = md(html, heading_style='ATX')
+
+ chunks = chunk_markdown_by_structure(markdown, max_chunk_chars=500)
+ assert len(chunks) > 1, 'Should produce multiple chunks for 200 rows'
+
+ for chunk in chunks:
+ lines = chunk.content.strip().split('\n')
+ for line in lines:
+ s = line.strip()
+ if s.startswith('|') and s.endswith('|'):
+ # Every table line should have consistent column count
+ assert s.count('|') >= 3
+
+ def test_table_without_thead_normalization(self):
+ """Table with <th> in first <tr> but no <thead> should still produce proper markdown."""
+ html = '<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr><tr><td>3</td><td>4</td></tr></table>'
+ markdown = md(html, heading_style='ATX')
+ # Verify markdownify produced a proper table (with separator row)
+ assert '---' in markdown or '| A |' in markdown
+
+
+# ---------------------------------------------------------------------------
+# Integration tests — require browser + httpserver
+# ---------------------------------------------------------------------------
+
+
+class TestTableNormalizationIntegration:
+ """Integration tests using browser session and httpserver."""
+
+ async def test_table_without_thead_normalized_via_serializer(self, browser_session, httpserver: HTTPServer):
+ """Tables without <thead> should get normalized by HTMLSerializer during extraction."""
+ html = """
+ <html><body>
+ <table>
+ <tr><th>Header1</th><th>Header2</th></tr>
+ <tr><td>data1</td><td>data2</td></tr>
+ <tr><td>data3</td><td>data4</td></tr>
+ </table>
+ </body></html>
+ """
+ httpserver.expect_request('/table-test').respond_with_data(html, content_type='text/html')
+ url = httpserver.url_for('/table-test')
+
+ await browser_session.navigate_to(url)
+
+ from browser_use.dom.markdown_extractor import extract_clean_markdown
+
+ content, _ = await extract_clean_markdown(browser_session=browser_session)
+
+ # Should have proper markdown table with separator
+ assert '|' in content
+ # The header should be present
+ assert 'Header1' in content
+ assert 'Header2' in content
+
+ async def test_large_table_extraction_preserves_structure(self, browser_session, httpserver: HTTPServer):
+ """Large table extraction should produce structure-aware chunks."""
+ rows = ''.join(f'<tr><td>Name{i}</td><td>Value{i}</td></tr>' for i in range(300))
+ html = f"""
+ <html><body>
+ <table>
+ <tr><th>Name</th><th>Value</th></tr>
+ {rows}
+ </table>
+ </body></html>
+ """
+ httpserver.expect_request('/big-table').respond_with_data(html, content_type='text/html')
+ url = httpserver.url_for('/big-table')
+
+ await browser_session.navigate_to(url)
+
+ from browser_use.dom.markdown_extractor import extract_clean_markdown
+
+ content, _ = await extract_clean_markdown(browser_session=browser_session)
+
+ # Chunk with a small limit to force multiple chunks
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=2000)
+
+ # Should produce multiple chunks
+ assert len(chunks) > 1
+
+ # Each chunk should have complete table rows
+ for chunk in chunks:
+ for line in chunk.content.split('\n'):
+ s = line.strip()
+ if s.startswith('|') and s.endswith('|'):
+ assert s.count('|') >= 3, f'Incomplete table row: {s}'