Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add semantic chunking, closes #812

davidmezzettibf32498a79bb
diff --git a/docs/pipeline/data/segmentation.md b/docs/pipeline/data/segmentation.md
index 27a855ca..9e16b5c4 100644
--- a/docs/pipeline/data/segmentation.md
+++ b/docs/pipeline/data/segmentation.md
@@ -15,8 +15,16 @@ from txtai.pipeline import Segmentation
# Create and run pipeline
segment = Segmentation(sentences=True)
segment("This is a test. And another test.")
+
+# Segment text with Chonkie chunker (word, sentence, semantic, late etc)
+segment = Segmentation(chunker="semantic")
+segment("This is a test. And another test.")
```
+The Segmentation pipeline supports segmenting `sentences`, `lines`, `paragraphs` and `sections` using a rules-based approach. Each of these modes can be set when creating the pipeline.
+
+More advanced functionality is supported via a [Chonkie chunker](https://docs.chonkie.ai/chunkers/overview). The `chunker` keyword dynamically creates a Chonkie chunker. For example, `chunker='token'` creates a [TokenChunker](https://docs.chonkie.ai/chunkers/token-chunker), `chunker='semantic'` creates a [SemanticChunker](https://docs.chonkie.ai/chunkers/semantic-chunker) and so forth. Additional keyword arguments are passed to the chunker.
+
## Configuration-driven example
Pipelines are run with Python or configuration. Pipelines can be instantiated in [configuration](../../../api/configuration/#pipeline) using the lower case name of the pipeline. Configuration-driven pipelines are run with [workflows](../../../workflow/#configuration-driven-example) or the [API](../../../api#local-instance).
diff --git a/setup.py b/setup.py
index 65cdf4cf..f3625356 100644
--- a/setup.py
+++ b/setup.py
@@ -63,7 +63,7 @@ extras["pipeline-audio"] = [
"webrtcvad-wheels>=2.0.14",
]
-extras["pipeline-data"] = ["beautifulsoup4>=4.9.3", "docling>=2.8.2", "nltk>=3.5", "pandas>=1.1.0", "tika>=1.24"]
+extras["pipeline-data"] = ["beautifulsoup4>=4.9.3", "chonkie>=0.4.1", "docling>=2.8.2", "nltk>=3.5", "pandas>=1.1.0", "tika>=1.24"]
extras["pipeline-image"] = ["imagehash>=4.2.1", "pillow>=7.1.2", "timm>=0.4.12"]
diff --git a/src/python/txtai/pipeline/data/segmentation.py b/src/python/txtai/pipeline/data/segmentation.py
index ec0e1a08..df8a3e57 100644
--- a/src/python/txtai/pipeline/data/segmentation.py
+++ b/src/python/txtai/pipeline/data/segmentation.py
@@ -12,6 +12,14 @@ try:
except ImportError:
NLTK = False
+# Conditional import
+try:
+ import chonkie
+
+ CHONKIE = True
+except ImportError:
+ CHONKIE = False
+
from ..base import Pipeline
@@ -20,7 +28,9 @@ class Segmentation(Pipeline):
Segments text into logical units.
"""
- def __init__(self, sentences=False, lines=False, paragraphs=False, minlength=None, join=False, sections=False, cleantext=True):
+ def __init__(
+ self, sentences=False, lines=False, paragraphs=False, minlength=None, join=False, sections=False, cleantext=True, chunker=None, **kwargs
+ ):
"""
Creates a new Segmentation pipeline.
@@ -32,10 +42,15 @@ class Segmentation(Pipeline):
join: joins tokenized sections back together if True, defaults to False
sections: tokenizes text into sections if True, defaults to False. Splits using section or page breaks, depending on what's available
cleantext: apply text cleaning rules, defaults to True
+ chunker: creates a chonkie chunker to tokenize text if set, defaults to None
+ kwargs: additional keyword arguments
"""
- if not NLTK:
- raise ImportError('Segmentation pipeline is not available - install "pipeline" extra to enable')
+ if not NLTK and sentences:
+ raise ImportError('NLTK is not available - install "pipeline" extra to enable')
+
+ if not CHONKIE and chunker:
+ raise ImportError('Chonkie is not available - install "pipeline" extra to enable')
self.sentences = sentences
self.lines = lines
@@ -45,6 +60,9 @@ class Segmentation(Pipeline):
self.join = join
self.cleantext = cleantext
+ # Create a chonkie chunker, if applicable
+ self.chunker = self.createchunker(chunker, **kwargs) if chunker else None
+
def __call__(self, text):
"""
Segments text into semantic units.
@@ -97,7 +115,10 @@ class Segmentation(Pipeline):
content = None
- if self.sentences:
+ if self.chunker:
+ # pylint: disable=E1102
+ content = [self.clean(x.text) for x in self.chunker(text)]
+ elif self.sentences:
content = [self.clean(x) for x in sent_tokenize(text)]
elif self.lines:
content = [self.clean(x) for x in re.split(r"\n{1,}", text)]
@@ -139,3 +160,19 @@ class Segmentation(Pipeline):
# If minlength enabled, require at least minlength chars
return text if not self.minlength or len(text) >= self.minlength else None
+
+ def createchunker(self, chunker=None, **kwargs):
+ """
+ Creates a new Chonkie chunker
+
+ Args:
+ chunker: name of chonkie chunker to create
+ kwargs: additional keyword arguments
+
+ Returns:
+ new chunker
+ """
+
+ # Resolve and create Chonkie chunker
+ chunker = f"{chunker[0].upper() + chunker[1:]}Chunker"
+ return getattr(chonkie, chunker)(**kwargs)
diff --git a/src/python/txtai/pipeline/data/textractor.py b/src/python/txtai/pipeline/data/textractor.py
index 8b9eb1da..fa7835f6 100644
--- a/src/python/txtai/pipeline/data/textractor.py
+++ b/src/python/txtai/pipeline/data/textractor.py
@@ -19,6 +19,7 @@ class Textractor(Segmentation):
Extracts text from files.
"""
+ # pylint: disable=R0913
def __init__(
self,
sentences=False,
@@ -28,11 +29,12 @@ class Textractor(Segmentation):
join=False,
sections=False,
cleantext=True,
+ chunker=None,
headers=None,
backend="available",
**kwargs
):
- super().__init__(sentences, lines, paragraphs, minlength, join, sections, cleantext)
+ super().__init__(sentences, lines, paragraphs, minlength, join, sections, cleantext, chunker, **kwargs)
# Get backend parameter - handle legacy tika flag
backend = "tika" if "tika" in kwargs and kwargs["tika"] else None if "tika" in kwargs else backend
diff --git a/test/python/testoptional.py b/test/python/testoptional.py
index d54980e3..01037ce7 100644
--- a/test/python/testoptional.py
+++ b/test/python/testoptional.py
@@ -23,6 +23,7 @@ class TestOptional(unittest.TestCase):
modules = [
"annoy",
"bs4",
+ "chonkie",
"croniter",
"docling.document_converter",
"duckdb",
@@ -252,7 +253,10 @@ class TestOptional(unittest.TestCase):
Objects()
with self.assertRaises(ImportError):
- Segmentation()
+ Segmentation(sentences=True)
+
+ with self.assertRaises(ImportError):
+ Segmentation(chunker="token")
with self.assertRaises(ImportError):
Tabular()
diff --git a/test/python/testpipeline/testdata/testtextractor.py b/test/python/testpipeline/testdata/testtextractor.py
index f58040f2..0cacf6c8 100644
--- a/test/python/testpipeline/testdata/testtextractor.py
+++ b/test/python/testpipeline/testdata/testtextractor.py
@@ -33,6 +33,19 @@ class TestTextractor(unittest.TestCase):
textractor = Textractor(cleantext=False, minlength=10)
self.assertEqual(textractor(" a b c "), " a b c ")
+ def testChonkie(self):
+ """
+ Test a chonkie chunker
+ """
+
+ # Test chonkie chunking
+ textractor = Textractor(chunker="sentence", chunk_size=5, chunk_overlap=0)
+ self.assertEqual(textractor("This is a test. And another test."), ["This is a test.", "And another test."])
+
+ # Test bad chunker throws an exception
+ with self.assertRaises(AttributeError):
+ textractor = Textractor(chunker="badchunker")
+
def testDefault(self):
"""
Test default text extraction