Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
improve markdown splitting based on header
Saurav Panda7c05e73bbfa6
diff --git a/browser_use/dom/markdown_extractor.py b/browser_use/dom/markdown_extractor.pyindex cd86ccaec..0c940a7f3 100644--- a/browser_use/dom/markdown_extractor.py+++ b/browser_use/dom/markdown_extractor.py@@ -421,7 +421,7 @@ def chunk_markdown_by_structure(if not blocks:return []- # Phase 2: greedy chunk assembly+ # Phase 2: greedy chunk assembly with header-preferred splittingraw_chunks: list[list[_AtomicBlock]] = []current_chunk: list[_AtomicBlock] = []current_size = 0@@ -430,9 +430,21 @@ def chunk_markdown_by_structure(block_size = block.char_end - block.char_start# If adding this block would exceed limit AND we already have content, emit chunkif current_size + block_size > max_chunk_chars and current_chunk:- raw_chunks.append(current_chunk)- current_chunk = []- current_size = 0+ # Prefer splitting at a header boundary within the current chunk.+ # Scan backwards for the last HEADER block; if found and it wouldn't+ # create a tiny chunk (< 50% of limit), split right before it so the+ # header starts the next chunk for better semantic coherence.+ best_split = len(current_chunk)+ for j in range(len(current_chunk) - 1, 0, -1):+ if current_chunk[j].block_type == _BlockType.HEADER:+ prefix_size = sum(b.char_end - b.char_start for b in current_chunk[:j])+ if prefix_size >= max_chunk_chars * 0.5:+ best_split = j+ break+ raw_chunks.append(current_chunk[:best_split])+ # Carry remaining blocks (from the header onward) into the next chunk+ current_chunk = current_chunk[best_split:]+ current_size = sum(b.char_end - b.char_start for b in current_chunk)current_chunk.append(block)current_size += block_sizediff --git a/browser_use/tools/service.py b/browser_use/tools/service.pyindex 18bfb5c8c..c0c112098 100644--- a/browser_use/tools/service.py+++ b/browser_use/tools/service.py@@ -962,6 +962,11 @@ class Tools(Generic[Context]):chunk = chunks[0]content = chunk.contenttruncated = chunk.has_more++ # Prepend overlap context for continuation chunks (e.g. table headers)+ if chunk.overlap_prefix:+ content = chunk.overlap_prefix + '\n' + content+if start_from_char > 0:content_stats['started_from_char'] = start_from_charif truncated:@@ -979,7 +984,8 @@ class Tools(Generic[Context]):if start_from_char > 0:stats_summary += f' (started from char {start_from_char:,})'if truncated:- stats_summary += f' → {len(content):,} final chars (truncated, use start_from_char={content_stats["next_start_char"]} to continue)'+ chunk_info = f'chunk {chunk.chunk_index + 1} of {chunk.total_chunks}, '+ stats_summary += f' → {len(content):,} final chars ({chunk_info}use start_from_char={content_stats["next_start_char"]} to continue)'elif chars_filtered > 0:stats_summary += f' (filtered {chars_filtered:,} chars of noise)'diff --git a/tests/ci/test_markdown_chunking.py b/tests/ci/test_markdown_chunking.pyindex 68b178087..f208d2f37 100644--- a/tests/ci/test_markdown_chunking.py+++ b/tests/ci/test_markdown_chunking.py@@ -65,6 +65,38 @@ class TestChunkMarkdownHeaders:assert '# Section B' in chunks[1].content+class TestChunkMarkdownHeaderPreferred:+ """Header-preferred splitting ensures chunks start at semantic boundaries."""++ def test_header_preferred_split_moves_header_to_next_chunk(self):+ """When a header sits in the middle of an overflowing chunk, split before it."""+ # Build content: para_a (big) + header_b + para_b (small)+ para_a = 'A' * 600+ header_b = '# Section B'+ para_b = 'B' * 100+ content = f'{para_a}\n\n{header_b}\n\n{para_b}'+ # Limit forces a split; header is near end of first chunk+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=700)+ assert len(chunks) >= 2+ # The header should be the START of the second chunk, not the end of the first+ assert chunks[1].content.lstrip().startswith('# Section B')+ # First chunk should NOT contain the header+ assert '# Section B' not in chunks[0].content++ def test_header_preferred_split_doesnt_create_tiny_chunks(self):+ """Don't split at a header that would make the prefix chunk < 50% of limit."""+ header_a = '# Section A'+ para_a = 'A' * 30 # very small before header+ header_b = '# Section B'+ para_b = 'B' * 600+ content = f'{header_a}\n\n{para_a}\n\n{header_b}\n\n{para_b}'+ # With a limit of 700, header_b is near the start — splitting there would+ # leave a tiny prefix chunk. The algo should NOT split there.+ chunks = chunk_markdown_by_structure(content, max_chunk_chars=700)+ # First chunk should contain both headers (no tiny split)+ assert '# Section A' in chunks[0].content++class TestChunkMarkdownCodeFence:"""Code fence blocks never split."""