Skip to content

The agent engineering platform.

feat(text-splitters): add R programming language support (#34241)

rari4040f940d74b217
diff --git a/libs/text-splitters/langchain_text_splitters/base.py b/libs/text-splitters/langchain_text_splitters/base.py
index ed06b78a49..d3bc65adb6 100644
--- a/libs/text-splitters/langchain_text_splitters/base.py
+++ b/libs/text-splitters/langchain_text_splitters/base.py
@@ -312,6 +312,7 @@ class Language(str, Enum):
PHP = "php"
PROTO = "proto"
PYTHON = "python"
+ R = "r"
RST = "rst"
RUBY = "ruby"
RUST = "rust"
diff --git a/libs/text-splitters/langchain_text_splitters/character.py b/libs/text-splitters/langchain_text_splitters/character.py
index 22736f1df5..5bcc62b99f 100644
--- a/libs/text-splitters/langchain_text_splitters/character.py
+++ b/libs/text-splitters/langchain_text_splitters/character.py
@@ -362,6 +362,29 @@ class RecursiveCharacterTextSplitter(TextSplitter):
" ",
"",
]
+ if language == Language.R:
+ return [
+ # Split along function definitions
+ "\nfunction ",
+ # Split along S4 class and method definitions
+ "\nsetClass\\(",
+ "\nsetMethod\\(",
+ "\nsetGeneric\\(",
+ # Split along control flow statements
+ "\nif ",
+ "\nelse ",
+ "\nfor ",
+ "\nwhile ",
+ "\nrepeat ",
+ # Split along package loading
+ "\nlibrary\\(",
+ "\nrequire\\(",
+ # Split by the normal type of lines
+ "\n\n",
+ "\n",
+ " ",
+ "",
+ ]
if language == Language.RST:
return [
# Split along section titles
diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
index 6825dce276..8919c2db27 100644
--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py
+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
@@ -1071,6 +1071,35 @@ fn main() {
assert chunks == ["fn main() {", 'println!("Hello', ",", 'World!");', "}"]
+def test_r_code_splitter() -> None:
+ splitter = RecursiveCharacterTextSplitter.from_language(
+ Language.R, chunk_size=CHUNK_SIZE, chunk_overlap=0
+ )
+ code = """
+library(dplyr)
+
+my_func <- function(x) {
+ return(x + 1)
+}
+
+if (TRUE) {
+ print("Hello")
+}
+ """
+ chunks = splitter.split_text(code)
+ assert chunks == [
+ "library(dplyr)",
+ "my_func <-",
+ "function(x) {",
+ "return(x +",
+ "1)",
+ "}",
+ "if (TRUE) {",
+ 'print("Hello")',
+ "}",
+ ]
+
+
def test_markdown_code_splitter() -> None:
splitter = RecursiveCharacterTextSplitter.from_language(
Language.MARKDOWN, chunk_size=CHUNK_SIZE, chunk_overlap=0