Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

improve markdown splitting based on header

Saurav Panda7c05e73bbfa6
diff --git a/browser_use/dom/markdown_extractor.py b/browser_use/dom/markdown_extractor.py
index cd86ccaec..0c940a7f3 100644
--- a/browser_use/dom/markdown_extractor.py
+++ b/browser_use/dom/markdown_extractor.py
@@ -421,7 +421,7 @@ def chunk_markdown_by_structure(
if not blocks:
return []
- # Phase 2: greedy chunk assembly
+ # Phase 2: greedy chunk assembly with header-preferred splitting
raw_chunks: list[list[_AtomicBlock]] = []
current_chunk: list[_AtomicBlock] = []
current_size = 0
@@ -430,9 +430,21 @@ def chunk_markdown_by_structure(
block_size = block.char_end - block.char_start
# If adding this block would exceed limit AND we already have content, emit chunk
if current_size + block_size > max_chunk_chars and current_chunk:
- raw_chunks.append(current_chunk)
- current_chunk = []
- current_size = 0
+ # Prefer splitting at a header boundary within the current chunk.
+ # Scan backwards for the last HEADER block; if found and it wouldn't
+ # create a tiny chunk (< 50% of limit), split right before it so the
+ # header starts the next chunk for better semantic coherence.
+ best_split = len(current_chunk)
+ for j in range(len(current_chunk) - 1, 0, -1):
+ if current_chunk[j].block_type == _BlockType.HEADER:
+ prefix_size = sum(b.char_end - b.char_start for b in current_chunk[:j])
+ if prefix_size >= max_chunk_chars * 0.5:
+ best_split = j
+ break
+ raw_chunks.append(current_chunk[:best_split])
+ # Carry remaining blocks (from the header onward) into the next chunk
+ current_chunk = current_chunk[best_split:]
+ current_size = sum(b.char_end - b.char_start for b in current_chunk)
current_chunk.append(block)
current_size += block_size
diff --git a/browser_use/tools/service.py b/browser_use/tools/service.py
index 18bfb5c8c..c0c112098 100644
--- a/browser_use/tools/service.py
+++ b/browser_use/tools/service.py
@@ -962,6 +962,11 @@ class Tools(Generic[Context]):
chunk = chunks[0]
content = chunk.content
truncated = chunk.has_more
+
+ # Prepend overlap context for continuation chunks (e.g. table headers)
+ if chunk.overlap_prefix:
+ content = chunk.overlap_prefix + '\n' + content
+
if start_from_char > 0:
content_stats['started_from_char'] = start_from_char
if truncated:
@@ -979,7 +984,8 @@ class Tools(Generic[Context]):
if start_from_char > 0:
stats_summary += f' (started from char {start_from_char:,})'
if truncated:
- stats_summary += f' → {len(content):,} final chars (truncated, use start_from_char={content_stats["next_start_char"]} to continue)'
+ chunk_info = f'chunk {chunk.chunk_index + 1} of {chunk.total_chunks}, '
+ stats_summary += f' → {len(content):,} final chars ({chunk_info}use start_from_char={content_stats["next_start_char"]} to continue)'
elif chars_filtered > 0:
stats_summary += f' (filtered {chars_filtered:,} chars of noise)'
diff --git a/tests/ci/test_markdown_chunking.py b/tests/ci/test_markdown_chunking.py
index 68b178087..f208d2f37 100644
--- a/tests/ci/test_markdown_chunking.py
+++ b/tests/ci/test_markdown_chunking.py
@@ -65,6 +65,38 @@ class TestChunkMarkdownHeaders:
assert '# Section B' in chunks[1].content
+class TestChunkMarkdownHeaderPreferred:
+ """Header-preferred splitting ensures chunks start at semantic boundaries."""
+
+ def test_header_preferred_split_moves_header_to_next_chunk(self):
+ """When a header sits in the middle of an overflowing chunk, split before it."""
+ # Build content: para_a (big) + header_b + para_b (small)
+ para_a = 'A' * 600
+ header_b = '# Section B'
+ para_b = 'B' * 100
+ content = f'{para_a}\n\n{header_b}\n\n{para_b}'
+ # Limit forces a split; header is near end of first chunk
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=700)
+ assert len(chunks) >= 2
+ # The header should be the START of the second chunk, not the end of the first
+ assert chunks[1].content.lstrip().startswith('# Section B')
+ # First chunk should NOT contain the header
+ assert '# Section B' not in chunks[0].content
+
+ def test_header_preferred_split_doesnt_create_tiny_chunks(self):
+ """Don't split at a header that would make the prefix chunk < 50% of limit."""
+ header_a = '# Section A'
+ para_a = 'A' * 30 # very small before header
+ header_b = '# Section B'
+ para_b = 'B' * 600
+ content = f'{header_a}\n\n{para_a}\n\n{header_b}\n\n{para_b}'
+ # With a limit of 700, header_b is near the start — splitting there would
+ # leave a tiny prefix chunk. The algo should NOT split there.
+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=700)
+ # First chunk should contain both headers (no tiny split)
+ assert '# Section A' in chunks[0].content
+
+
class TestChunkMarkdownCodeFence:
"""Code fence blocks never split."""