The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
feat(text-splitters): add R programming language support (#34241)
rari4040f940d74b217
diff --git a/libs/text-splitters/langchain_text_splitters/base.py b/libs/text-splitters/langchain_text_splitters/base.pyindex ed06b78a49..d3bc65adb6 100644--- a/libs/text-splitters/langchain_text_splitters/base.py+++ b/libs/text-splitters/langchain_text_splitters/base.py@@ -312,6 +312,7 @@ class Language(str, Enum):PHP = "php"PROTO = "proto"PYTHON = "python"+ R = "r"RST = "rst"RUBY = "ruby"RUST = "rust"diff --git a/libs/text-splitters/langchain_text_splitters/character.py b/libs/text-splitters/langchain_text_splitters/character.pyindex 22736f1df5..5bcc62b99f 100644--- a/libs/text-splitters/langchain_text_splitters/character.py+++ b/libs/text-splitters/langchain_text_splitters/character.py@@ -362,6 +362,29 @@ class RecursiveCharacterTextSplitter(TextSplitter):" ","",]+ if language == Language.R:+ return [+ # Split along function definitions+ "\nfunction ",+ # Split along S4 class and method definitions+ "\nsetClass\\(",+ "\nsetMethod\\(",+ "\nsetGeneric\\(",+ # Split along control flow statements+ "\nif ",+ "\nelse ",+ "\nfor ",+ "\nwhile ",+ "\nrepeat ",+ # Split along package loading+ "\nlibrary\\(",+ "\nrequire\\(",+ # Split by the normal type of lines+ "\n\n",+ "\n",+ " ",+ "",+ ]if language == Language.RST:return [# Split along section titlesdiff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.pyindex 6825dce276..8919c2db27 100644--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py@@ -1071,6 +1071,35 @@ fn main() {assert chunks == ["fn main() {", 'println!("Hello', ",", 'World!");', "}"]+def test_r_code_splitter() -> None:+ splitter = RecursiveCharacterTextSplitter.from_language(+ Language.R, chunk_size=CHUNK_SIZE, chunk_overlap=0+ )+ code = """+library(dplyr)++my_func <- function(x) {+ return(x + 1)+}++if (TRUE) {+ print("Hello")+}+ """+ chunks = splitter.split_text(code)+ assert chunks == [+ "library(dplyr)",+ "my_func <-",+ "function(x) {",+ "return(x +",+ "1)",+ "}",+ "if (TRUE) {",+ 'print("Hello")',+ "}",+ ]++def test_markdown_code_splitter() -> None:splitter = RecursiveCharacterTextSplitter.from_language(Language.MARKDOWN, chunk_size=CHUNK_SIZE, chunk_overlap=0