Skip to content

The agent engineering platform.

fix(text-splitters): remove invalid and duplicate separators in Kotlin, Rust, and Haskell (#37039)

Deepak Bhagatcd80a805b2af
diff --git a/libs/text-splitters/langchain_text_splitters/character.py b/libs/text-splitters/langchain_text_splitters/character.py
index dd2378a787..0e2e1ab2a9 100644
--- a/libs/text-splitters/langchain_text_splitters/character.py
+++ b/libs/text-splitters/langchain_text_splitters/character.py
@@ -266,7 +266,6 @@ class RecursiveCharacterTextSplitter(TextSplitter):
"\nfor ",
"\nwhile ",
"\nwhen ",
- "\ncase ",
"\nelse ",
# Split by the normal type of lines
"\n\n",
@@ -463,7 +462,6 @@ class RecursiveCharacterTextSplitter(TextSplitter):
"\nfor ",
"\nloop ",
"\nmatch ",
- "\nconst ",
# Split by the normal type of lines
"\n\n",
"\n",
@@ -718,7 +716,6 @@ class RecursiveCharacterTextSplitter(TextSplitter):
"\ndata ",
"\nnewtype ",
"\ntype ",
- "\n:: ",
# Split along module declarations
"\nmodule ",
# Split along import statements
@@ -733,7 +730,6 @@ class RecursiveCharacterTextSplitter(TextSplitter):
# Split along guards in function definitions
"\n| ",
# Split along record field declarations
- "\ndata ",
"\n= {",
"\n, ",
# Split by the normal type of lines