💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Consistency changes
davidmezzetti83f625cc3235
diff --git a/src/python/txtai/pipeline/summary.py b/src/python/txtai/pipeline/summary.pyindex 4e0cdd2c..d9d20262 100644--- a/src/python/txtai/pipeline/summary.py+++ b/src/python/txtai/pipeline/summary.py@@ -2,6 +2,8 @@Summary module"""+import re+from .hfpipeline import HFPipeline@@ -43,6 +45,22 @@ class Summary(HFPipeline):results = [results]# Pull out summary text- results = [x["summary_text"].strip() for x in results]+ results = [self.clean(x["summary_text"]) for x in results]return results[0] if isinstance(text, str) else results++ def clean(self, text):+ """+ Applies a series of rules to clean extracted text.++ Args:+ text: input text++ Returns:+ clean text+ """++ text = re.sub(r"\s*\.\s*", ". ", text)+ text = text.strip()++ return textdiff --git a/src/python/txtai/pipeline/textractor.py b/src/python/txtai/pipeline/textractor.pyindex 9f2f34c9..2a3a83f5 100644--- a/src/python/txtai/pipeline/textractor.py+++ b/src/python/txtai/pipeline/textractor.py@@ -35,6 +35,10 @@ class Textractor(Pipeline):"""Extracts text from a file at path.+ This method supports files as a string or a list. If the input is a string, the return+ type is text|list for the file. If files is a list, a list of returned, this could be a+ list of text or a list of lists depending on the tokenization strategy.+Args:files: text|list@@ -42,18 +46,20 @@ class Textractor(Pipeline):extracted text from files"""- files = [files] if not isinstance(files, list) else files+ # Get inputs+ values = [files] if not isinstance(files, list) else files- content = []- for path in files:+ # Extract text for each input file+ results = []+ for path in values:parsed = parser.from_file(path)text = parsed["content"]if text:# Parse and add extracted results- content.append(self.parse(text))+ results.append(self.parse(text))- return content+ return results[0] if isinstance(files, str) else resultsdef parse(self, text):"""