The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
feat(text-splitters): add Visual Basic 6 support (#31173)
Fabio Fontanafd168e1c112b
diff --git a/docs/docs/how_to/code_splitter.ipynb b/docs/docs/how_to/code_splitter.ipynbindex 3da6622478..efa4fde9a7 100644--- a/docs/docs/how_to/code_splitter.ipynb+++ b/docs/docs/how_to/code_splitter.ipynb@@ -63,7 +63,7 @@},{"cell_type": "code",- "execution_count": 4,+ "execution_count": 1,"id": "a9e37aa1","metadata": {},"outputs": [],@@ -84,7 +84,7 @@},{"cell_type": "code",- "execution_count": 3,+ "execution_count": 2,"id": "e21a2434","metadata": {},"outputs": [@@ -114,10 +114,13 @@" 'c',\n"," 'lua',\n"," 'perl',\n",- " 'haskell']"+ " 'haskell',\n",+ " 'elixir',\n",+ " 'powershell',\n",+ " 'visualbasic6']"]},- "execution_count": 3,+ "execution_count": 2,"metadata": {},"output_type": "execute_result"}@@ -136,7 +139,7 @@},{"cell_type": "code",- "execution_count": 3,+ "execution_count": 6,"id": "c92fb913","metadata": {},"outputs": [@@ -146,7 +149,7 @@"['\\nclass ', '\\ndef ', '\\n\\tdef ', '\\n\\n', '\\n', ' ', '']"]},- "execution_count": 3,+ "execution_count": 6,"metadata": {},"output_type": "execute_result"}@@ -168,18 +171,18 @@},{"cell_type": "code",- "execution_count": 5,+ "execution_count": 7,"id": "a58512b9","metadata": {},"outputs": [{"data": {"text/plain": [- "[Document(page_content='def hello_world():\\n print(\"Hello, World!\")'),\n",- " Document(page_content='# Call the function\\nhello_world()')]"+ "[Document(metadata={}, page_content='def hello_world():\\n print(\"Hello, World!\")'),\n",+ " Document(metadata={}, page_content='# Call the function\\nhello_world()')]"]},- "execution_count": 5,+ "execution_count": 7,"metadata": {},"output_type": "execute_result"}@@ -210,18 +213,18 @@},{"cell_type": "code",- "execution_count": 6,+ "execution_count": 8,"id": "7db0d486","metadata": {},"outputs": [{"data": {"text/plain": [- "[Document(page_content='function helloWorld() {\\n console.log(\"Hello, World!\");\\n}'),\n",- " Document(page_content='// Call the function\\nhelloWorld();')]"+ "[Document(metadata={}, page_content='function helloWorld() {\\n console.log(\"Hello, World!\");\\n}'),\n",+ " Document(metadata={}, page_content='// Call the function\\nhelloWorld();')]"]},- "execution_count": 6,+ "execution_count": 8,"metadata": {},"output_type": "execute_result"}@@ -254,19 +257,19 @@},{"cell_type": "code",- "execution_count": 7,+ "execution_count": 9,"id": "aee738a4","metadata": {},"outputs": [{"data": {"text/plain": [- "[Document(page_content='function helloWorld(): void {'),\n",- " Document(page_content='console.log(\"Hello, World!\");\\n}'),\n",- " Document(page_content='// Call the function\\nhelloWorld();')]"+ "[Document(metadata={}, page_content='function helloWorld(): void {'),\n",+ " Document(metadata={}, page_content='console.log(\"Hello, World!\");\\n}'),\n",+ " Document(metadata={}, page_content='// Call the function\\nhelloWorld();')]"]},- "execution_count": 7,+ "execution_count": 9,"metadata": {},"output_type": "execute_result"}@@ -300,7 +303,7 @@},{"cell_type": "code",- "execution_count": 2,+ "execution_count": 10,"id": "ac9295d3","metadata": {},"outputs": [],@@ -321,7 +324,7 @@},{"cell_type": "code",- "execution_count": 3,+ "execution_count": 11,"id": "bfa1771b-d4b0-48f8-a949-5537cd1df0dd","metadata": {},"outputs": [@@ -337,7 +340,7 @@" Document(metadata={}, page_content='are extremely open to contributions.')]"]},- "execution_count": 3,+ "execution_count": 11,"metadata": {},"output_type": "execute_result"}@@ -362,7 +365,7 @@},{"cell_type": "code",- "execution_count": 10,+ "execution_count": 12,"id": "77d1049d","metadata": {},"outputs": [],@@ -389,38 +392,38 @@},{"cell_type": "code",- "execution_count": 11,+ "execution_count": 13,"id": "4dbc47e1","metadata": {},"outputs": [{"data": {"text/plain": [- "[Document(page_content='\\\\documentclass{article}\\n\\n\\x08egin{document}\\n\\n\\\\maketitle'),\n",- " Document(page_content='\\\\section{Introduction}'),\n",- " Document(page_content='Large language models (LLMs) are a type of machine learning'),\n",- " Document(page_content='model that can be trained on vast amounts of text data to'),\n",- " Document(page_content='generate human-like language. In recent years, LLMs have'),\n",- " Document(page_content='made significant advances in a variety of natural language'),\n",- " Document(page_content='processing tasks, including language translation, text'),\n",- " Document(page_content='generation, and sentiment analysis.'),\n",- " Document(page_content='\\\\subsection{History of LLMs}'),\n",- " Document(page_content='The earliest LLMs were developed in the 1980s and 1990s,'),\n",- " Document(page_content='but they were limited by the amount of data that could be'),\n",- " Document(page_content='processed and the computational power available at the'),\n",- " Document(page_content='time. In the past decade, however, advances in hardware and'),\n",- " Document(page_content='software have made it possible to train LLMs on massive'),\n",- " Document(page_content='datasets, leading to significant improvements in'),\n",- " Document(page_content='performance.'),\n",- " Document(page_content='\\\\subsection{Applications of LLMs}'),\n",- " Document(page_content='LLMs have many applications in industry, including'),\n",- " Document(page_content='chatbots, content creation, and virtual assistants. They'),\n",- " Document(page_content='can also be used in academia for research in linguistics,'),\n",- " Document(page_content='psychology, and computational linguistics.'),\n",- " Document(page_content='\\\\end{document}')]"+ "[Document(metadata={}, page_content='\\\\documentclass{article}\\n\\n\\x08egin{document}\\n\\n\\\\maketitle'),\n",+ " Document(metadata={}, page_content='\\\\section{Introduction}'),\n",+ " Document(metadata={}, page_content='Large language models (LLMs) are a type of machine learning'),\n",+ " Document(metadata={}, page_content='model that can be trained on vast amounts of text data to'),\n",+ " Document(metadata={}, page_content='generate human-like language. In recent years, LLMs have'),\n",+ " Document(metadata={}, page_content='made significant advances in a variety of natural language'),\n",+ " Document(metadata={}, page_content='processing tasks, including language translation, text'),\n",+ " Document(metadata={}, page_content='generation, and sentiment analysis.'),\n",+ " Document(metadata={}, page_content='\\\\subsection{History of LLMs}'),\n",+ " Document(metadata={}, page_content='The earliest LLMs were developed in the 1980s and 1990s,'),\n",+ " Document(metadata={}, page_content='but they were limited by the amount of data that could be'),\n",+ " Document(metadata={}, page_content='processed and the computational power available at the'),\n",+ " Document(metadata={}, page_content='time. In the past decade, however, advances in hardware and'),\n",+ " Document(metadata={}, page_content='software have made it possible to train LLMs on massive'),\n",+ " Document(metadata={}, page_content='datasets, leading to significant improvements in'),\n",+ " Document(metadata={}, page_content='performance.'),\n",+ " Document(metadata={}, page_content='\\\\subsection{Applications of LLMs}'),\n",+ " Document(metadata={}, page_content='LLMs have many applications in industry, including'),\n",+ " Document(metadata={}, page_content='chatbots, content creation, and virtual assistants. They'),\n",+ " Document(metadata={}, page_content='can also be used in academia for research in linguistics,'),\n",+ " Document(metadata={}, page_content='psychology, and computational linguistics.'),\n",+ " Document(metadata={}, page_content='\\\\end{document}')]"]},- "execution_count": 11,+ "execution_count": 13,"metadata": {},"output_type": "execute_result"}@@ -445,7 +448,7 @@},{"cell_type": "code",- "execution_count": 12,+ "execution_count": 14,"id": "0fc78794","metadata": {},"outputs": [],@@ -479,29 +482,29 @@},{"cell_type": "code",- "execution_count": 13,+ "execution_count": 15,"id": "e3e3fca1","metadata": {},"outputs": [{"data": {"text/plain": [- "[Document(page_content='<!DOCTYPE html>\\n<html>'),\n",- " Document(page_content='<head>\\n <title>π¦οΈπ LangChain</title>'),\n",- " Document(page_content='<style>\\n body {\\n font-family: Aria'),\n",- " Document(page_content='l, sans-serif;\\n }\\n h1 {'),\n",- " Document(page_content='color: darkblue;\\n }\\n </style>\\n </head'),\n",- " Document(page_content='>'),\n",- " Document(page_content='<body>'),\n",- " Document(page_content='<div>\\n <h1>π¦οΈπ LangChain</h1>'),\n",- " Document(page_content='<p>β‘ Building applications with LLMs through composability β‘'),\n",- " Document(page_content='</p>\\n </div>'),\n",- " Document(page_content='<div>\\n As an open-source project in a rapidly dev'),\n",- " Document(page_content='eloping field, we are extremely open to contributions.'),\n",- " Document(page_content='</div>\\n </body>\\n</html>')]"+ "[Document(metadata={}, page_content='<!DOCTYPE html>\\n<html>'),\n",+ " Document(metadata={}, page_content='<head>\\n <title>π¦οΈπ LangChain</title>'),\n",+ " Document(metadata={}, page_content='<style>\\n body {\\n font-family: Aria'),\n",+ " Document(metadata={}, page_content='l, sans-serif;\\n }\\n h1 {'),\n",+ " Document(metadata={}, page_content='color: darkblue;\\n }\\n </style>\\n </head'),\n",+ " Document(metadata={}, page_content='>'),\n",+ " Document(metadata={}, page_content='<body>'),\n",+ " Document(metadata={}, page_content='<div>\\n <h1>π¦οΈπ LangChain</h1>'),\n",+ " Document(metadata={}, page_content='<p>β‘ Building applications with LLMs through composability β‘'),\n",+ " Document(metadata={}, page_content='</p>\\n </div>'),\n",+ " Document(metadata={}, page_content='<div>\\n As an open-source project in a rapidly dev'),\n",+ " Document(metadata={}, page_content='eloping field, we are extremely open to contributions.'),\n",+ " Document(metadata={}, page_content='</div>\\n </body>\\n</html>')]"]},- "execution_count": 13,+ "execution_count": 15,"metadata": {},"output_type": "execute_result"}@@ -525,18 +528,18 @@},{"cell_type": "code",- "execution_count": 14,+ "execution_count": 16,"id": "49a1df11","metadata": {},"outputs": [{"data": {"text/plain": [- "[Document(page_content='pragma solidity ^0.8.20;'),\n",- " Document(page_content='contract HelloWorld {\\n function add(uint a, uint b) pure public returns(uint) {\\n return a + b;\\n }\\n}')]"+ "[Document(metadata={}, page_content='pragma solidity ^0.8.20;'),\n",+ " Document(metadata={}, page_content='contract HelloWorld {\\n function add(uint a, uint b) pure public returns(uint) {\\n return a + b;\\n }\\n}')]"]},- "execution_count": 14,+ "execution_count": 16,"metadata": {},"output_type": "execute_result"}@@ -569,21 +572,21 @@},{"cell_type": "code",- "execution_count": 15,+ "execution_count": 17,"id": "1524ae0f","metadata": {},"outputs": [{"data": {"text/plain": [- "[Document(page_content='using System;'),\n",- " Document(page_content='class Program\\n{\\n static void Main()\\n {\\n int age = 30; // Change the age value as needed'),\n",- " Document(page_content='// Categorize the age without any console output\\n if (age < 18)\\n {\\n // Age is under 18'),\n",- " Document(page_content='}\\n else if (age >= 18 && age < 65)\\n {\\n // Age is an adult\\n }\\n else\\n {'),\n",- " Document(page_content='// Age is a senior citizen\\n }\\n }\\n}')]"+ "[Document(metadata={}, page_content='using System;'),\n",+ " Document(metadata={}, page_content='class Program\\n{\\n static void Main()\\n {\\n int age = 30; // Change the age value as needed'),\n",+ " Document(metadata={}, page_content='// Categorize the age without any console output\\n if (age < 18)\\n {\\n // Age is under 18'),\n",+ " Document(metadata={}, page_content='}\\n else if (age >= 18 && age < 65)\\n {\\n // Age is an adult\\n }\\n else\\n {'),\n",+ " Document(metadata={}, page_content='// Age is a senior citizen\\n }\\n }\\n}')]"]},- "execution_count": 15,+ "execution_count": 17,"metadata": {},"output_type": "execute_result"}@@ -631,20 +634,20 @@},{"cell_type": "code",- "execution_count": 3,+ "execution_count": 18,"id": "688185b5","metadata": {},"outputs": [{"data": {"text/plain": [- "[Document(page_content='main :: IO ()'),\n",- " Document(page_content='main = do\\n putStrLn \"Hello, World!\"\\n-- Some'),\n",- " Document(page_content='sample functions\\nadd :: Int -> Int -> Int\\nadd x y'),\n",- " Document(page_content='= x + y')]"+ "[Document(metadata={}, page_content='main :: IO ()'),\n",+ " Document(metadata={}, page_content='main = do\\n putStrLn \"Hello, World!\"\\n-- Some'),\n",+ " Document(metadata={}, page_content='sample functions\\nadd :: Int -> Int -> Int\\nadd x y'),\n",+ " Document(metadata={}, page_content='= x + y')]"]},- "execution_count": 3,+ "execution_count": 18,"metadata": {},"output_type": "execute_result"}@@ -676,23 +679,23 @@},{"cell_type": "code",- "execution_count": 2,+ "execution_count": 19,"id": "90c66e7e-87a5-4a81-bece-7949aabf2369","metadata": {},"outputs": [{"data": {"text/plain": [- "[Document(page_content='<?php\\nnamespace foo;'),\n",- " Document(page_content='class Hello {'),\n",- " Document(page_content='public function __construct() { }\\n}'),\n",- " Document(page_content='function hello() {\\n echo \"Hello World!\";\\n}'),\n",- " Document(page_content='interface Human {\\n public function breath();\\n}'),\n",- " Document(page_content='trait Foo { }\\nenum Color\\n{\\n case Red;'),\n",- " Document(page_content='case Blue;\\n}')]"+ "[Document(metadata={}, page_content='<?php\\nnamespace foo;'),\n",+ " Document(metadata={}, page_content='class Hello {'),\n",+ " Document(metadata={}, page_content='public function __construct() { }\\n}'),\n",+ " Document(metadata={}, page_content='function hello() {\\n echo \"Hello World!\";\\n}'),\n",+ " Document(metadata={}, page_content='interface Human {\\n public function breath();\\n}'),\n",+ " Document(metadata={}, page_content='trait Foo { }\\nenum Color\\n{\\n case Red;'),\n",+ " Document(metadata={}, page_content='case Blue;\\n}')]"]},- "execution_count": 2,+ "execution_count": 19,"metadata": {},"output_type": "execute_result"}@@ -733,10 +736,25 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 20,"id": "7e6893ad","metadata": {},- "outputs": [],+ "outputs": [+ {+ "data": {+ "text/plain": [+ "[Document(metadata={}, page_content='$directoryPath = Get-Location\\n\\n$items = Get-ChildItem -Path $directoryPath'),\n",+ " Document(metadata={}, page_content='$files = $items | Where-Object { -not $_.PSIsContainer }'),\n",+ " Document(metadata={}, page_content='$sortedFiles = $files | Sort-Object LastWriteTime'),\n",+ " Document(metadata={}, page_content='foreach ($file in $sortedFiles) {'),\n",+ " Document(metadata={}, page_content='Write-Output (\"Name: \" + $file.Name + \" | Last Write Time: \" + $file.LastWriteTime)\\n}')]"+ ]+ },+ "execution_count": 20,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],"source": ["POWERSHELL_CODE = \"\"\"\n","$directoryPath = Get-Location\n",@@ -757,11 +775,58 @@"powershell_docs = powershell_splitter.create_documents([POWERSHELL_CODE])\n","powershell_docs"]+ },+ {+ "cell_type": "markdown",+ "id": "3ef77730",+ "metadata": {},+ "source": [+ "## Visual Basic 6"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 21,+ "id": "1dc3c740",+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/plain": [+ "[Document(metadata={}, page_content='Option Explicit'),\n",+ " Document(metadata={}, page_content='Public Sub HelloWorld()\\n MsgBox \"Hello, World!\"\\nEnd Sub'),\n",+ " Document(metadata={}, page_content='Private Function Add(a As Integer, b As Integer) As Integer\\n Add = a + b\\nEnd Function')]"+ ]+ },+ "execution_count": 21,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "VISUALBASIC6_CODE = \"\"\"Option Explicit\n",+ "\n",+ "Public Sub HelloWorld()\n",+ " MsgBox \"Hello, World!\"\n",+ "End Sub\n",+ "\n",+ "Private Function Add(a As Integer, b As Integer) As Integer\n",+ " Add = a + b\n",+ "End Function\n",+ "\"\"\"\n",+ "visualbasic6_splitter = RecursiveCharacterTextSplitter.from_language(\n",+ " Language.VISUALBASIC6,\n",+ " chunk_size=128,\n",+ " chunk_overlap=0,\n",+ ")\n",+ "visualbasic6_docs = visualbasic6_splitter.create_documents([VISUALBASIC6_CODE])\n",+ "visualbasic6_docs"+ ]}],"metadata": {"kernelspec": {- "display_name": "Python 3 (ipykernel)",+ "display_name": "langchain","language": "python","name": "python3"},@@ -775,7 +840,7 @@"name": "python","nbconvert_exporter": "python","pygments_lexer": "ipython3",- "version": "3.10.4"+ "version": "3.10.16"}},"nbformat": 4,diff --git a/libs/text-splitters/langchain_text_splitters/base.py b/libs/text-splitters/langchain_text_splitters/base.pyindex 22e3288a19..891f988d44 100644--- a/libs/text-splitters/langchain_text_splitters/base.py+++ b/libs/text-splitters/langchain_text_splitters/base.py@@ -316,6 +316,7 @@ class Language(str, Enum):HASKELL = "haskell"ELIXIR = "elixir"POWERSHELL = "powershell"+ VISUALBASIC6 = "visualbasic6"@dataclass(frozen=True)diff --git a/libs/text-splitters/langchain_text_splitters/character.py b/libs/text-splitters/langchain_text_splitters/character.pyindex bfe4653f06..76c4d2fe85 100644--- a/libs/text-splitters/langchain_text_splitters/character.py+++ b/libs/text-splitters/langchain_text_splitters/character.py@@ -734,6 +734,32 @@ class RecursiveCharacterTextSplitter(TextSplitter):" ","",]+ if language == Language.VISUALBASIC6:+ vis = r"(?:Public|Private|Friend|Global|Static)\s+"+ return [+ # Split along definitions+ rf"\n(?!End\s){vis}?Sub\s+",+ rf"\n(?!End\s){vis}?Function\s+",+ rf"\n(?!End\s){vis}?Property\s+(?:Get|Let|Set)\s+",+ rf"\n(?!End\s){vis}?Type\s+",+ rf"\n(?!End\s){vis}?Enum\s+",+ # Split along control flow statements+ r"\n(?!End\s)If\s+",+ r"\nElseIf\s+",+ r"\nElse\s+",+ r"\nSelect\s+Case\s+",+ r"\nCase\s+",+ r"\nFor\s+",+ r"\nDo\s+",+ r"\nWhile\s+",+ r"\nWith\s+",+ # Split by the normal type of lines+ r"\n\n",+ r"\n",+ " ",+ "",+ ]+if language in Language._value2member_map_:msg = f"Language {language} is not implemented yet!"raise ValueError(msg)diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.pyindex f155dd10ab..0d72e80630 100644--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py@@ -3043,6 +3043,82 @@ $csvContent | ForEach-Object {]+FAKE_VISUALBASIC6_TEXT = """+Option Explicit++Public Function SumTwoIntegers(ByVal a As Integer, ByVal b As Integer) As Integer+ SumTwoIntegers = a + b+End Function++Public Sub Main()+ Dim i As Integer+ Dim limit As Integer++ i = 0+ limit = 50++ While i < limit+ i = SumTwoIntegers(i, 1)++ If i = limit \\ 2 Then+ MsgBox "Halfway there! i = " & i+ End If+ Wend++ MsgBox "Done! Final value of i: " & i+End Sub+"""+++def test_visualbasic6_code_splitter() -> None:+ splitter = RecursiveCharacterTextSplitter.from_language(+ Language.VISUALBASIC6,+ chunk_size=CHUNK_SIZE,+ chunk_overlap=0,+ )+ chunks = splitter.split_text(FAKE_VISUALBASIC6_TEXT)++ assert chunks == [+ "Option Explicit",+ "Public Function",+ "SumTwoIntegers(",+ "ByVal",+ "a As Integer,",+ "ByVal b As",+ "Integer) As",+ "Integer",+ "SumTwoIntegers",+ "= a + b",+ "End Function",+ "Public Sub",+ "Main()",+ "Dim i As",+ "Integer",+ "Dim limit",+ "As Integer",+ "i = 0",+ "limit = 50",+ "While i <",+ "limit",+ "i =",+ "SumTwoIntegers(",+ "i,",+ "1)",+ "If i =",+ "limit \\ 2 Then",+ 'MsgBox "Halfway',+ 'there! i = " &',+ "i",+ "End If",+ "Wend",+ "MsgBox",+ '"Done! Final',+ 'value of i: " &',+ "i",+ "End Sub",+ ]++def custom_iframe_extractor(iframe_tag: Any) -> str:iframe_src = iframe_tag.get("src", "")return f"[iframe:{iframe_src}]({iframe_src})"