The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
text-splitters: fix stale header metadata in ExperimentalMarkdownSyntaxTextSplitter (#31622)
Raghu Kapur2c9859956a00
diff --git a/libs/text-splitters/langchain_text_splitters/markdown.py b/libs/text-splitters/langchain_text_splitters/markdown.pyindex 1db585bef4..ae885bbb0a 100644--- a/libs/text-splitters/langchain_text_splitters/markdown.py+++ b/libs/text-splitters/langchain_text_splitters/markdown.py@@ -376,10 +376,10 @@ class ExperimentalMarkdownSyntaxTextSplitter:def _resolve_header_stack(self, header_depth: int, header_text: str) -> None:for i, (depth, _) in enumerate(self.current_header_stack):- if depth == header_depth:- self.current_header_stack[i] = (header_depth, header_text)- self.current_header_stack = self.current_header_stack[: i + 1]- return+ if depth >= header_depth:+ # Truncate everything from this level onward+ self.current_header_stack = self.current_header_stack[:i]+ breakself.current_header_stack.append((header_depth, header_text))def _resolve_code_chunk(self, current_line: str, raw_lines: List[str]) -> str:diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.pyindex 85c6caace1..9aeb4b1520 100644--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py@@ -1464,6 +1464,10 @@ EXPERIMENTAL_MARKDOWN_DOCUMENT = ("Content for header 1\n""## Header 2\n""Content for header 2\n"+ "### Header 3\n"+ "Content for header 3\n"+ "## Header 2 Again\n"+ "This should be tagged with Header 1 and Header 2 Again\n""```python\n""def func_definition():\n"" print('Keep the whitespace consistent')\n"@@ -1491,6 +1495,18 @@ def test_experimental_markdown_syntax_text_splitter() -> None:page_content="Content for header 2\n",metadata={"Header 1": "My Header 1", "Header 2": "Header 2"},),+ Document(+ page_content="Content for header 3\n",+ metadata={+ "Header 1": "My Header 1",+ "Header 2": "Header 2",+ "Header 3": "Header 3",+ },+ ),+ Document(+ page_content="This should be tagged with Header 1 and Header 2 Again\n",+ metadata={"Header 1": "My Header 1", "Header 2": "Header 2 Again"},+ ),Document(page_content=("```python\ndef func_definition():\n "@@ -1499,7 +1515,7 @@ def test_experimental_markdown_syntax_text_splitter() -> None:metadata={"Code": "python","Header 1": "My Header 1",- "Header 2": "Header 2",+ "Header 2": "Header 2 Again",},),Document(@@ -1530,7 +1546,15 @@ def test_experimental_markdown_syntax_text_splitter_header_configuration() -> Noexpected_output = [Document(- page_content="Content for header 1\n## Header 2\nContent for header 2\n",+ page_content=(+ "Content for header 1\n"+ "## Header 2\n"+ "Content for header 2\n"+ "### Header 3\n"+ "Content for header 3\n"+ "## Header 2 Again\n"+ "This should be tagged with Header 1 and Header 2 Again\n"+ ),metadata={"Encabezamiento 1": "My Header 1"},),Document(@@ -1571,6 +1595,21 @@ def test_experimental_markdown_syntax_text_splitter_with_headers() -> None:page_content="## Header 2\nContent for header 2\n",metadata={"Header 1": "My Header 1", "Header 2": "Header 2"},),+ Document(+ page_content="### Header 3\nContent for header 3\n",+ metadata={+ "Header 1": "My Header 1",+ "Header 2": "Header 2",+ "Header 3": "Header 3",+ },+ ),+ Document(+ page_content=(+ "## Header 2 Again\n"+ "This should be tagged with Header 1 and Header 2 Again\n"+ ),+ metadata={"Header 1": "My Header 1", "Header 2": "Header 2 Again"},+ ),Document(page_content=("```python\ndef func_definition():\n "@@ -1579,7 +1618,7 @@ def test_experimental_markdown_syntax_text_splitter_with_headers() -> None:metadata={"Code": "python","Header 1": "My Header 1",- "Header 2": "Header 2",+ "Header 2": "Header 2 Again",},),Document(@@ -1614,12 +1653,24 @@ def test_experimental_markdown_syntax_text_splitter_split_lines() -> None:page_content="Content for header 2",metadata={"Header 1": "My Header 1", "Header 2": "Header 2"},),+ Document(+ page_content="Content for header 3",+ metadata={+ "Header 1": "My Header 1",+ "Header 2": "Header 2",+ "Header 3": "Header 3",+ },+ ),+ Document(+ page_content="This should be tagged with Header 1 and Header 2 Again",+ metadata={"Header 1": "My Header 1", "Header 2": "Header 2 Again"},+ ),Document(page_content="```python",metadata={"Code": "python","Header 1": "My Header 1",- "Header 2": "Header 2",+ "Header 2": "Header 2 Again",},),Document(@@ -1627,7 +1678,7 @@ def test_experimental_markdown_syntax_text_splitter_split_lines() -> None:metadata={"Code": "python","Header 1": "My Header 1",- "Header 2": "Header 2",+ "Header 2": "Header 2 Again",},),Document(@@ -1635,7 +1686,7 @@ def test_experimental_markdown_syntax_text_splitter_split_lines() -> None:metadata={"Code": "python","Header 1": "My Header 1",- "Header 2": "Header 2",+ "Header 2": "Header 2 Again",},),Document(@@ -1643,7 +1694,7 @@ def test_experimental_markdown_syntax_text_splitter_split_lines() -> None:metadata={"Code": "python","Header 1": "My Header 1",- "Header 2": "Header 2",+ "Header 2": "Header 2 Again",},),Document(