💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add semantic chunking, closes #812
davidmezzettibf32498a79bb
diff --git a/docs/pipeline/data/segmentation.md b/docs/pipeline/data/segmentation.mdindex 27a855ca..9e16b5c4 100644--- a/docs/pipeline/data/segmentation.md+++ b/docs/pipeline/data/segmentation.md@@ -15,8 +15,16 @@ from txtai.pipeline import Segmentation# Create and run pipelinesegment = Segmentation(sentences=True)segment("This is a test. And another test.")++# Segment text with Chonkie chunker (word, sentence, semantic, late etc)+segment = Segmentation(chunker="semantic")+segment("This is a test. And another test.")```+The Segmentation pipeline supports segmenting `sentences`, `lines`, `paragraphs` and `sections` using a rules-based approach. Each of these modes can be set when creating the pipeline.++More advanced functionality is supported via a [Chonkie chunker](https://docs.chonkie.ai/chunkers/overview). The `chunker` keyword dynamically creates a Chonkie chunker. For example, `chunker='token'` creates a [TokenChunker](https://docs.chonkie.ai/chunkers/token-chunker), `chunker='semantic'` creates a [SemanticChunker](https://docs.chonkie.ai/chunkers/semantic-chunker) and so forth. Additional keyword arguments are passed to the chunker.+## Configuration-driven examplePipelines are run with Python or configuration. Pipelines can be instantiated in [configuration](../../../api/configuration/#pipeline) using the lower case name of the pipeline. Configuration-driven pipelines are run with [workflows](../../../workflow/#configuration-driven-example) or the [API](../../../api#local-instance).diff --git a/setup.py b/setup.pyindex 65cdf4cf..f3625356 100644--- a/setup.py+++ b/setup.py@@ -63,7 +63,7 @@ extras["pipeline-audio"] = ["webrtcvad-wheels>=2.0.14",]-extras["pipeline-data"] = ["beautifulsoup4>=4.9.3", "docling>=2.8.2", "nltk>=3.5", "pandas>=1.1.0", "tika>=1.24"]+extras["pipeline-data"] = ["beautifulsoup4>=4.9.3", "chonkie>=0.4.1", "docling>=2.8.2", "nltk>=3.5", "pandas>=1.1.0", "tika>=1.24"]extras["pipeline-image"] = ["imagehash>=4.2.1", "pillow>=7.1.2", "timm>=0.4.12"]diff --git a/src/python/txtai/pipeline/data/segmentation.py b/src/python/txtai/pipeline/data/segmentation.pyindex ec0e1a08..df8a3e57 100644--- a/src/python/txtai/pipeline/data/segmentation.py+++ b/src/python/txtai/pipeline/data/segmentation.py@@ -12,6 +12,14 @@ try:except ImportError:NLTK = False+# Conditional import+try:+ import chonkie++ CHONKIE = True+except ImportError:+ CHONKIE = False+from ..base import Pipeline@@ -20,7 +28,9 @@ class Segmentation(Pipeline):Segments text into logical units."""- def __init__(self, sentences=False, lines=False, paragraphs=False, minlength=None, join=False, sections=False, cleantext=True):+ def __init__(+ self, sentences=False, lines=False, paragraphs=False, minlength=None, join=False, sections=False, cleantext=True, chunker=None, **kwargs+ ):"""Creates a new Segmentation pipeline.@@ -32,10 +42,15 @@ class Segmentation(Pipeline):join: joins tokenized sections back together if True, defaults to Falsesections: tokenizes text into sections if True, defaults to False. Splits using section or page breaks, depending on what's availablecleantext: apply text cleaning rules, defaults to True+ chunker: creates a chonkie chunker to tokenize text if set, defaults to None+ kwargs: additional keyword arguments"""- if not NLTK:- raise ImportError('Segmentation pipeline is not available - install "pipeline" extra to enable')+ if not NLTK and sentences:+ raise ImportError('NLTK is not available - install "pipeline" extra to enable')++ if not CHONKIE and chunker:+ raise ImportError('Chonkie is not available - install "pipeline" extra to enable')self.sentences = sentencesself.lines = lines@@ -45,6 +60,9 @@ class Segmentation(Pipeline):self.join = joinself.cleantext = cleantext+ # Create a chonkie chunker, if applicable+ self.chunker = self.createchunker(chunker, **kwargs) if chunker else None+def __call__(self, text):"""Segments text into semantic units.@@ -97,7 +115,10 @@ class Segmentation(Pipeline):content = None- if self.sentences:+ if self.chunker:+ # pylint: disable=E1102+ content = [self.clean(x.text) for x in self.chunker(text)]+ elif self.sentences:content = [self.clean(x) for x in sent_tokenize(text)]elif self.lines:content = [self.clean(x) for x in re.split(r"\n{1,}", text)]@@ -139,3 +160,19 @@ class Segmentation(Pipeline):# If minlength enabled, require at least minlength charsreturn text if not self.minlength or len(text) >= self.minlength else None++ def createchunker(self, chunker=None, **kwargs):+ """+ Creates a new Chonkie chunker++ Args:+ chunker: name of chonkie chunker to create+ kwargs: additional keyword arguments++ Returns:+ new chunker+ """++ # Resolve and create Chonkie chunker+ chunker = f"{chunker[0].upper() + chunker[1:]}Chunker"+ return getattr(chonkie, chunker)(**kwargs)diff --git a/src/python/txtai/pipeline/data/textractor.py b/src/python/txtai/pipeline/data/textractor.pyindex 8b9eb1da..fa7835f6 100644--- a/src/python/txtai/pipeline/data/textractor.py+++ b/src/python/txtai/pipeline/data/textractor.py@@ -19,6 +19,7 @@ class Textractor(Segmentation):Extracts text from files."""+ # pylint: disable=R0913def __init__(self,sentences=False,@@ -28,11 +29,12 @@ class Textractor(Segmentation):join=False,sections=False,cleantext=True,+ chunker=None,headers=None,backend="available",**kwargs):- super().__init__(sentences, lines, paragraphs, minlength, join, sections, cleantext)+ super().__init__(sentences, lines, paragraphs, minlength, join, sections, cleantext, chunker, **kwargs)# Get backend parameter - handle legacy tika flagbackend = "tika" if "tika" in kwargs and kwargs["tika"] else None if "tika" in kwargs else backenddiff --git a/test/python/testoptional.py b/test/python/testoptional.pyindex d54980e3..01037ce7 100644--- a/test/python/testoptional.py+++ b/test/python/testoptional.py@@ -23,6 +23,7 @@ class TestOptional(unittest.TestCase):modules = ["annoy","bs4",+ "chonkie","croniter","docling.document_converter","duckdb",@@ -252,7 +253,10 @@ class TestOptional(unittest.TestCase):Objects()with self.assertRaises(ImportError):- Segmentation()+ Segmentation(sentences=True)++ with self.assertRaises(ImportError):+ Segmentation(chunker="token")with self.assertRaises(ImportError):Tabular()diff --git a/test/python/testpipeline/testdata/testtextractor.py b/test/python/testpipeline/testdata/testtextractor.pyindex f58040f2..0cacf6c8 100644--- a/test/python/testpipeline/testdata/testtextractor.py+++ b/test/python/testpipeline/testdata/testtextractor.py@@ -33,6 +33,19 @@ class TestTextractor(unittest.TestCase):textractor = Textractor(cleantext=False, minlength=10)self.assertEqual(textractor(" a b c "), " a b c ")+ def testChonkie(self):+ """+ Test a chonkie chunker+ """++ # Test chonkie chunking+ textractor = Textractor(chunker="sentence", chunk_size=5, chunk_overlap=0)+ self.assertEqual(textractor("This is a test. And another test."), ["This is a test.", "And another test."])++ # Test bad chunker throws an exception+ with self.assertRaises(AttributeError):+ textractor = Textractor(chunker="badchunker")+def testDefault(self):"""Test default text extraction