The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
text-splitters: fix state persistence issue in ExperimentalMarkdownSyntaxTextSplitter (#28373)
Yuxin Chen3256b5d6ae4f
diff --git a/libs/text-splitters/langchain_text_splitters/markdown.py b/libs/text-splitters/langchain_text_splitters/markdown.pyindex 34c7d2197d..f8d3807dc5 100644--- a/libs/text-splitters/langchain_text_splitters/markdown.py+++ b/libs/text-splitters/langchain_text_splitters/markdown.py@@ -324,6 +324,11 @@ class ExperimentalMarkdownSyntaxTextSplitter:chunks of the input text. If `return_each_line` is enabled, each lineis returned as a separate `Document`."""+ # Reset the state for each new file processed+ self.chunks.clear()+ self.current_chunk = Document(page_content="")+ self.current_header_stack.clear()+raw_lines = text.splitlines(keepends=True)while raw_lines:diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.pyindex 95f170d52b..ee86ae6a60 100644--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py@@ -1527,6 +1527,407 @@ def test_experimental_markdown_syntax_text_splitter_split_lines() -> None:assert output == expected_output+EXPERIMENTAL_MARKDOWN_DOCUMENTS = [+ (+ "# My Header 1 From Document 1\n"+ "Content for header 1 from Document 1\n"+ "## Header 2 From Document 1\n"+ "Content for header 2 from Document 1\n"+ "```python\n"+ "def func_definition():\n"+ " print('Keep the whitespace consistent')\n"+ "```\n"+ "# Header 1 again From Document 1\n"+ "We should also split on the horizontal line\n"+ "----\n"+ "This will be a new doc but with the same header metadata\n\n"+ "And it includes a new paragraph"+ ),+ (+ "# My Header 1 From Document 2\n"+ "Content for header 1 from Document 2\n"+ "## Header 2 From Document 2\n"+ "Content for header 2 from Document 2\n"+ "```python\n"+ "def func_definition():\n"+ " print('Keep the whitespace consistent')\n"+ "```\n"+ "# Header 1 again From Document 2\n"+ "We should also split on the horizontal line\n"+ "----\n"+ "This will be a new doc but with the same header metadata\n\n"+ "And it includes a new paragraph"+ ),+]+++def test_experimental_markdown_syntax_text_splitter_on_multi_files() -> None:+ """Test experimental markdown syntax splitter split+ on default called consecutively on two files."""+ markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter()+ output = []+ for experimental_markdown_document in EXPERIMENTAL_MARKDOWN_DOCUMENTS:+ output += markdown_splitter.split_text(experimental_markdown_document)++ expected_output = [+ Document(+ page_content="Content for header 1 from Document 1\n",+ metadata={"Header 1": "My Header 1 From Document 1"},+ ),+ Document(+ page_content="Content for header 2 from Document 1\n",+ metadata={+ "Header 1": "My Header 1 From Document 1",+ "Header 2": "Header 2 From Document 1",+ },+ ),+ Document(+ page_content=(+ "```python\ndef func_definition():\n "+ "print('Keep the whitespace consistent')\n```\n"+ ),+ metadata={+ "Code": "python",+ "Header 1": "My Header 1 From Document 1",+ "Header 2": "Header 2 From Document 1",+ },+ ),+ Document(+ page_content="We should also split on the horizontal line\n",+ metadata={"Header 1": "Header 1 again From Document 1"},+ ),+ Document(+ page_content=(+ "This will be a new doc but with the same header metadata\n\n"+ "And it includes a new paragraph"+ ),+ metadata={"Header 1": "Header 1 again From Document 1"},+ ),+ Document(+ page_content="Content for header 1 from Document 2\n",+ metadata={"Header 1": "My Header 1 From Document 2"},+ ),+ Document(+ page_content="Content for header 2 from Document 2\n",+ metadata={+ "Header 1": "My Header 1 From Document 2",+ "Header 2": "Header 2 From Document 2",+ },+ ),+ Document(+ page_content=(+ "```python\ndef func_definition():\n "+ "print('Keep the whitespace consistent')\n```\n"+ ),+ metadata={+ "Code": "python",+ "Header 1": "My Header 1 From Document 2",+ "Header 2": "Header 2 From Document 2",+ },+ ),+ Document(+ page_content="We should also split on the horizontal line\n",+ metadata={"Header 1": "Header 1 again From Document 2"},+ ),+ Document(+ page_content=(+ "This will be a new doc but with the same header metadata\n\n"+ "And it includes a new paragraph"+ ),+ metadata={"Header 1": "Header 1 again From Document 2"},+ ),+ ]++ assert output == expected_output+++def test_experimental_markdown_syntax_text_splitter_split_lines_on_multi_files() -> (+ None+):+ """Test experimental markdown syntax splitter split+ on each line called consecutively on two files."""+ markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(return_each_line=True)+ output = []+ for experimental_markdown_document in EXPERIMENTAL_MARKDOWN_DOCUMENTS:+ output += markdown_splitter.split_text(experimental_markdown_document)+ expected_output = [+ Document(+ page_content="Content for header 1 from Document 1",+ metadata={"Header 1": "My Header 1 From Document 1"},+ ),+ Document(+ page_content="Content for header 2 from Document 1",+ metadata={+ "Header 1": "My Header 1 From Document 1",+ "Header 2": "Header 2 From Document 1",+ },+ ),+ Document(+ page_content="```python",+ metadata={+ "Code": "python",+ "Header 1": "My Header 1 From Document 1",+ "Header 2": "Header 2 From Document 1",+ },+ ),+ Document(+ page_content="def func_definition():",+ metadata={+ "Code": "python",+ "Header 1": "My Header 1 From Document 1",+ "Header 2": "Header 2 From Document 1",+ },+ ),+ Document(+ page_content=" print('Keep the whitespace consistent')",+ metadata={+ "Code": "python",+ "Header 1": "My Header 1 From Document 1",+ "Header 2": "Header 2 From Document 1",+ },+ ),+ Document(+ page_content="```",+ metadata={+ "Code": "python",+ "Header 1": "My Header 1 From Document 1",+ "Header 2": "Header 2 From Document 1",+ },+ ),+ Document(+ page_content="We should also split on the horizontal line",+ metadata={"Header 1": "Header 1 again From Document 1"},+ ),+ Document(+ page_content="This will be a new doc but with the same header metadata",+ metadata={"Header 1": "Header 1 again From Document 1"},+ ),+ Document(+ page_content="And it includes a new paragraph",+ metadata={"Header 1": "Header 1 again From Document 1"},+ ),+ Document(+ page_content="Content for header 1 from Document 2",+ metadata={"Header 1": "My Header 1 From Document 2"},+ ),+ Document(+ page_content="Content for header 2 from Document 2",+ metadata={+ "Header 1": "My Header 1 From Document 2",+ "Header 2": "Header 2 From Document 2",+ },+ ),+ Document(+ page_content="```python",+ metadata={+ "Code": "python",+ "Header 1": "My Header 1 From Document 2",+ "Header 2": "Header 2 From Document 2",+ },+ ),+ Document(+ page_content="def func_definition():",+ metadata={+ "Code": "python",+ "Header 1": "My Header 1 From Document 2",+ "Header 2": "Header 2 From Document 2",+ },+ ),+ Document(+ page_content=" print('Keep the whitespace consistent')",+ metadata={+ "Code": "python",+ "Header 1": "My Header 1 From Document 2",+ "Header 2": "Header 2 From Document 2",+ },+ ),+ Document(+ page_content="```",+ metadata={+ "Code": "python",+ "Header 1": "My Header 1 From Document 2",+ "Header 2": "Header 2 From Document 2",+ },+ ),+ Document(+ page_content="We should also split on the horizontal line",+ metadata={"Header 1": "Header 1 again From Document 2"},+ ),+ Document(+ page_content="This will be a new doc but with the same header metadata",+ metadata={"Header 1": "Header 1 again From Document 2"},+ ),+ Document(+ page_content="And it includes a new paragraph",+ metadata={"Header 1": "Header 1 again From Document 2"},+ ),+ ]++ assert output == expected_output+++def test_experimental_markdown_syntax_text_splitter_with_header_on_multi_files() -> (+ None+):+ """Test experimental markdown splitter+ by header called consecutively on two files"""++ markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(strip_headers=False)+ output = []+ for experimental_markdown_document in EXPERIMENTAL_MARKDOWN_DOCUMENTS:+ output += markdown_splitter.split_text(experimental_markdown_document)++ expected_output = [+ Document(+ page_content="# My Header 1 From Document 1\n"+ "Content for header 1 from Document 1\n",+ metadata={"Header 1": "My Header 1 From Document 1"},+ ),+ Document(+ page_content="## Header 2 From Document 1\n"+ "Content for header 2 from Document 1\n",+ metadata={+ "Header 1": "My Header 1 From Document 1",+ "Header 2": "Header 2 From Document 1",+ },+ ),+ Document(+ page_content=(+ "```python\ndef func_definition():\n "+ "print('Keep the whitespace consistent')\n```\n"+ ),+ metadata={+ "Code": "python",+ "Header 1": "My Header 1 From Document 1",+ "Header 2": "Header 2 From Document 1",+ },+ ),+ Document(+ page_content="# Header 1 again From Document 1\n"+ "We should also split on the horizontal line\n",+ metadata={"Header 1": "Header 1 again From Document 1"},+ ),+ Document(+ page_content=(+ "This will be a new doc but with the same header metadata\n\n"+ "And it includes a new paragraph"+ ),+ metadata={"Header 1": "Header 1 again From Document 1"},+ ),+ Document(+ page_content="# My Header 1 From Document 2\n"+ "Content for header 1 from Document 2\n",+ metadata={"Header 1": "My Header 1 From Document 2"},+ ),+ Document(+ page_content="## Header 2 From Document 2\n"+ "Content for header 2 from Document 2\n",+ metadata={+ "Header 1": "My Header 1 From Document 2",+ "Header 2": "Header 2 From Document 2",+ },+ ),+ Document(+ page_content=(+ "```python\ndef func_definition():\n "+ "print('Keep the whitespace consistent')\n```\n"+ ),+ metadata={+ "Code": "python",+ "Header 1": "My Header 1 From Document 2",+ "Header 2": "Header 2 From Document 2",+ },+ ),+ Document(+ page_content="# Header 1 again From Document 2\n"+ "We should also split on the horizontal line\n",+ metadata={"Header 1": "Header 1 again From Document 2"},+ ),+ Document(+ page_content=(+ "This will be a new doc but with the same header metadata\n\n"+ "And it includes a new paragraph"+ ),+ metadata={"Header 1": "Header 1 again From Document 2"},+ ),+ ]+ assert output == expected_output+++def test_experimental_markdown_syntax_text_splitter_header_config_on_multi_files() -> (+ None+):+ """Test experimental markdown splitter+ by header configuration called consecutively on two files"""++ headers_to_split_on = [("#", "Encabezamiento 1")]+ markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(+ headers_to_split_on=headers_to_split_on+ )+ output = []+ for experimental_markdown_document in EXPERIMENTAL_MARKDOWN_DOCUMENTS:+ output += markdown_splitter.split_text(experimental_markdown_document)++ expected_output = [+ Document(+ page_content="Content for header 1 from Document 1\n"+ "## Header 2 From Document 1\n"+ "Content for header 2 from Document 1\n",+ metadata={"Encabezamiento 1": "My Header 1 From Document 1"},+ ),+ Document(+ page_content=(+ "```python\ndef func_definition():\n "+ "print('Keep the whitespace consistent')\n```\n"+ ),+ metadata={+ "Code": "python",+ "Encabezamiento 1": "My Header 1 From Document 1",+ },+ ),+ Document(+ page_content="We should also split on the horizontal line\n",+ metadata={"Encabezamiento 1": "Header 1 again From Document 1"},+ ),+ Document(+ page_content=(+ "This will be a new doc but with the same header metadata\n\n"+ "And it includes a new paragraph"+ ),+ metadata={"Encabezamiento 1": "Header 1 again From Document 1"},+ ),+ Document(+ page_content="Content for header 1 from Document 2\n"+ "## Header 2 From Document 2\n"+ "Content for header 2 from Document 2\n",+ metadata={"Encabezamiento 1": "My Header 1 From Document 2"},+ ),+ Document(+ page_content=(+ "```python\ndef func_definition():\n "+ "print('Keep the whitespace consistent')\n```\n"+ ),+ metadata={+ "Code": "python",+ "Encabezamiento 1": "My Header 1 From Document 2",+ },+ ),+ Document(+ page_content="We should also split on the horizontal line\n",+ metadata={"Encabezamiento 1": "Header 1 again From Document 2"},+ ),+ Document(+ page_content=(+ "This will be a new doc but with the same header metadata\n\n"+ "And it includes a new paragraph"+ ),+ metadata={"Encabezamiento 1": "Header 1 again From Document 2"},+ ),+ ]++ assert output == expected_output++def test_solidity_code_splitter() -> None:splitter = RecursiveCharacterTextSplitter.from_language(Language.SOL, chunk_size=CHUNK_SIZE, chunk_overlap=0