💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Update docs
davidmezzetti1f65a64dc3e3
diff --git a/src/python/txtai/pipeline/data/segmentation.py b/src/python/txtai/pipeline/data/segmentation.pyindex 51d89615..fa8caef3 100644--- a/src/python/txtai/pipeline/data/segmentation.py+++ b/src/python/txtai/pipeline/data/segmentation.py@@ -42,7 +42,7 @@ class Segmentation(Pipeline):join: joins tokenized sections back together if True, defaults to Falsesections: tokenizes text into sections if True, defaults to False. Splits using section or page breaks, depending on what's availablecleantext: apply text cleaning rules, defaults to True- chunker: creates a chonkie chunker to tokenize text if set, defaults to None+ chunker: creates a third-party chunker to tokenize text if set, defaults to Nonekwargs: additional keyword arguments"""@@ -60,7 +60,7 @@ class Segmentation(Pipeline):self.join = joinself.cleantext = cleantext- # Create a chonkie chunker, if applicable+ # Create a third-party chunker, if applicableself.chunker = self.createchunker(chunker, **kwargs) if chunker else Nonedef __call__(self, text):@@ -163,16 +163,16 @@ class Segmentation(Pipeline):def createchunker(self, chunker, **kwargs):"""- Creates a new Chonkie chunker+ Creates a new third-party chunkerArgs:- chunker: name of chonkie chunker to create+ chunker: name of chunker to createkwargs: additional keyword argumentsReturns:new chunker"""- # Resolve and create Chonkie chunker+ # Resolve and create a third-party chunkerchunker = f"{chunker[0].upper() + chunker[1:]}Chunker"return getattr(chonkie, chunker)(**kwargs)