Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Consistency changes

davidmezzetti83f625cc3235
diff --git a/src/python/txtai/pipeline/summary.py b/src/python/txtai/pipeline/summary.py
index 4e0cdd2c..d9d20262 100644
--- a/src/python/txtai/pipeline/summary.py
+++ b/src/python/txtai/pipeline/summary.py
@@ -2,6 +2,8 @@
Summary module
"""
+import re
+
from .hfpipeline import HFPipeline
@@ -43,6 +45,22 @@ class Summary(HFPipeline):
results = [results]
# Pull out summary text
- results = [x["summary_text"].strip() for x in results]
+ results = [self.clean(x["summary_text"]) for x in results]
return results[0] if isinstance(text, str) else results
+
+ def clean(self, text):
+ """
+ Applies a series of rules to clean extracted text.
+
+ Args:
+ text: input text
+
+ Returns:
+ clean text
+ """
+
+ text = re.sub(r"\s*\.\s*", ". ", text)
+ text = text.strip()
+
+ return text
diff --git a/src/python/txtai/pipeline/textractor.py b/src/python/txtai/pipeline/textractor.py
index 9f2f34c9..2a3a83f5 100644
--- a/src/python/txtai/pipeline/textractor.py
+++ b/src/python/txtai/pipeline/textractor.py
@@ -35,6 +35,10 @@ class Textractor(Pipeline):
"""
Extracts text from a file at path.
+ This method supports files as a string or a list. If the input is a string, the return
+ type is text|list for the file. If files is a list, a list of returned, this could be a
+ list of text or a list of lists depending on the tokenization strategy.
+
Args:
files: text|list
@@ -42,18 +46,20 @@ class Textractor(Pipeline):
extracted text from files
"""
- files = [files] if not isinstance(files, list) else files
+ # Get inputs
+ values = [files] if not isinstance(files, list) else files
- content = []
- for path in files:
+ # Extract text for each input file
+ results = []
+ for path in values:
parsed = parser.from_file(path)
text = parsed["content"]
if text:
# Parse and add extracted results
- content.append(self.parse(text))
+ results.append(self.parse(text))
- return content
+ return results[0] if isinstance(files, str) else results
def parse(self, text):
"""