Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add text segmentation pipeline, closes #81

davidmezzetti983fe9f2c978
diff --git a/examples/workflows.py b/examples/workflows.py
index 8078c65f..f60b5436 100644
--- a/examples/workflows.py
+++ b/examples/workflows.py
@@ -11,7 +11,7 @@ import pandas as pd
import streamlit as st
from txtai.embeddings import Documents, Embeddings
-from txtai.pipeline import Summary, Textractor, Transcription, Translation
+from txtai.pipeline import Segmentation, Summary, Textractor, Transcription, Translation
from txtai.workflow import Workflow, FileTask, Task
@@ -73,8 +73,12 @@ class Application:
options["minlength"] = self.number("Min length")
options["maxlength"] = self.number("Max length")
- elif component == "textract":
- st.sidebar.markdown("**Textractor** \n*Extract text from documents*")
+ elif component in ("segment", "textract"):
+ if component == "segment":
+ st.sidebar.markdown("**Segment** \n*Split text into semantic units*")
+ else:
+ st.sidebar.markdown("**Textractor** \n*Extract text from documents*")
+
options["sentences"] = st.sidebar.checkbox("Split sentences")
options["lines"] = st.sidebar.checkbox("Split lines")
options["paragraphs"] = st.sidebar.checkbox("Split paragraphs")
@@ -116,6 +120,10 @@ class Application:
self.pipelines[wtype] = Summary(component.pop("path"))
tasks.append(Task(lambda x: self.pipelines["summary"](x, **self.components["summary"])))
+ elif wtype == "segment":
+ self.pipelines[wtype] = Segmentation(**self.components["segment"])
+ tasks.append(Task(self.pipelines["segment"]))
+
elif wtype == "textract":
self.pipelines[wtype] = Textractor(**self.components["textract"])
tasks.append(FileTask(self.pipelines["textract"]))
@@ -198,7 +206,7 @@ class Application:
st.sidebar.markdown("# Workflow builder \n*Build and apply workflows to data* ")
# Get selected components
- selected = st.sidebar.multiselect("Select components", ["embeddings", "summary", "textract", "transcribe", "translate"])
+ selected = st.sidebar.multiselect("Select components", ["embeddings", "segment", "summary", "textract", "transcribe", "translate"])
# Get selected options
components = [self.options(component) for component in selected]
@@ -212,10 +220,9 @@ class Application:
with st.beta_expander("Data", expanded=not self.data):
data = st.text_area("Input", height=10)
- split = st.checkbox("Split input")
# Parse text items
- data = data.split("\n") if "file://" in data or split else [data]
+ data = [x for x in data.split("\n") if x] if "file://" in data else [data]
# Process current action
self.process(data)
diff --git a/src/python/txtai/api/application.py b/src/python/txtai/api/application.py
index 6b9d1da9..ba773a16 100644
--- a/src/python/txtai/api/application.py
+++ b/src/python/txtai/api/application.py
@@ -12,7 +12,7 @@ from .base import API
from .factory import Factory
# pylint: disable=R0401
-from .routers import embeddings, extractor, labels, similarity, summary, textractor, transcription, translation
+from .routers import embeddings, extractor, labels, segmentation, similarity, summary, textractor, transcription, translation
# API instance
app = FastAPI()
@@ -57,6 +57,7 @@ def start():
("embeddings", embeddings.router),
("extractor", extractor.router),
("labels", labels.router),
+ ("segmentation", segmentation.router),
("similarity", similarity.router),
("summary", summary.router),
("textractor", textractor.router),
@@ -65,8 +66,8 @@ def start():
]
# Conditionally add routes based on configuration
- for route, router in routers:
- if route in config:
+ for name, router in routers:
+ if name in config:
app.include_router(router)
# Special case to add similarity instance for embeddings
diff --git a/src/python/txtai/api/base.py b/src/python/txtai/api/base.py
index e0191104..df43ea11 100644
--- a/src/python/txtai/api/base.py
+++ b/src/python/txtai/api/base.py
@@ -33,7 +33,7 @@ class API:
self.pipelines = {}
# Default pipelines
- pipelines = ["extractor", "labels", "similarity", "summary", "textractor", "transcription", "translation"]
+ pipelines = ["extractor", "labels", "segmentation", "similarity", "summary", "textractor", "transcription", "translation"]
# Add custom pipelines
for key in self.config:
diff --git a/src/python/txtai/api/routers/__init__.py b/src/python/txtai/api/routers/__init__.py
index 0f2e3f0b..09ef8cb8 100644
--- a/src/python/txtai/api/routers/__init__.py
+++ b/src/python/txtai/api/routers/__init__.py
@@ -4,6 +4,7 @@ Router imports
from . import embeddings
from . import extractor
+from . import segmentation
from . import similarity
from . import summary
from . import textractor
diff --git a/src/python/txtai/api/routers/segmentation.py b/src/python/txtai/api/routers/segmentation.py
new file mode 100644
index 00000000..f93cba2d
--- /dev/null
+++ b/src/python/txtai/api/routers/segmentation.py
@@ -0,0 +1,41 @@
+"""
+Defines API paths for segmentation endpoints.
+"""
+
+from typing import List
+
+from fastapi import APIRouter, Body
+
+from .. import application
+
+router = APIRouter()
+
+
[email protected]("/segment")
+def textract(text: str):
+ """
+ Segments input text.
+
+ Args:
+ text: input text
+
+ Returns:
+ segmented text
+ """
+
+ return application.get().pipeline("segmentation", (text,))
+
+
[email protected]("/batchsegment")
+def batchsegment(texts: List[str] = Body(...)):
+ """
+ Extracts text from a file at path.
+
+ Args:
+ text: list of texts to segment
+
+ Returns:
+ list of segmented text
+ """
+
+ return application.get().pipeline("segmentation", (texts,))
diff --git a/src/python/txtai/pipeline/__init__.py b/src/python/txtai/pipeline/__init__.py
index 8006b0a8..2f628e4c 100644
--- a/src/python/txtai/pipeline/__init__.py
+++ b/src/python/txtai/pipeline/__init__.py
@@ -9,6 +9,7 @@ from .hfmodel import HFModel
from .hfpipeline import HFPipeline
from .labels import Labels
from .questions import Questions
+from .segmentation import Segmentation
from .similarity import Similarity
from .summary import Summary
from .tensors import Tensors
diff --git a/src/python/txtai/pipeline/segmentation.py b/src/python/txtai/pipeline/segmentation.py
new file mode 100644
index 00000000..5c051813
--- /dev/null
+++ b/src/python/txtai/pipeline/segmentation.py
@@ -0,0 +1,119 @@
+"""
+Segmentation module
+"""
+
+import re
+
+from nltk import sent_tokenize
+
+from .base import Pipeline
+
+
+class Segmentation(Pipeline):
+ """
+ Segments text into logical units.
+ """
+
+ def __init__(self, sentences=False, lines=False, paragraphs=False, minlength=None, join=False):
+ """
+ Creates a new Segmentation pipeline.
+
+ Args:
+ sentences: tokenize text into sentences if True, defaults to False
+ lines: tokenizes text into lines if True, defaults to False
+ paragraphs: tokenizes text into paragraphs if True, defaults to False
+ minlength: require at least minlength characters per text element, defaults to None
+ join: joins tokenized sections back together if True, defaults to False
+ """
+
+ self.sentences = sentences
+ self.lines = lines
+ self.paragraphs = paragraphs
+ self.minlength = minlength
+ self.join = join
+
+ def __call__(self, text):
+ """
+ Segments text into semantic units.
+
+ This method supports text as a string or a list. If the input is a string, the return
+ type is text|list. If text is a list, a list of returned, this could be a
+ list of text or a list of lists depending on the tokenization strategy.
+
+ Args:
+ text: text|list
+
+ Returns:
+ segmented text
+ """
+
+ # Get inputs
+ texts = [text] if not isinstance(text, list) else text
+
+ # Extract text for each input file
+ results = []
+ for value in texts:
+ # Get text
+ value = self.text(value)
+
+ # Parse and add extracted results
+ results.append(self.parse(value))
+
+ return results[0] if isinstance(text, str) else results
+
+ def text(self, text):
+ """
+ Hook to allow extracting text out of input text object.
+
+ Args:
+ text: object to extract text from
+ """
+
+ return text
+
+ def parse(self, text):
+ """
+ Splits and cleans text based on the current parameters.
+
+ Args:
+ text: input text
+
+ Returns:
+ parsed and clean content
+ """
+
+ content = None
+
+ if self.sentences:
+ content = [self.clean(x) for x in sent_tokenize(text)]
+ elif self.lines:
+ content = [self.clean(x) for x in text.split("\n")]
+ elif self.paragraphs:
+ content = [self.clean(x) for x in text.split("\n\n")]
+ else:
+ content = [self.clean(text)]
+
+ # Remove empty strings
+ content = [x for x in content if x]
+
+ if self.sentences or self.lines or self.paragraphs:
+ return " ".join(content) if self.join else content
+
+ return content[0] if content else content
+
+ def clean(self, text):
+ """
+ Applies a series of rules to clean text.
+
+ Args:
+ text: input text
+
+ Returns:
+ clean text
+ """
+
+ text = text.replace("\n", " ")
+ text = re.sub(r"\s+", " ", text)
+ text = text.strip()
+
+ return text if not self.minlength or len(text) >= self.minlength else None
diff --git a/src/python/txtai/pipeline/textractor.py b/src/python/txtai/pipeline/textractor.py
index 1c4fc652..89c88b00 100644
--- a/src/python/txtai/pipeline/textractor.py
+++ b/src/python/txtai/pipeline/textractor.py
@@ -2,110 +2,17 @@
Textractor module
"""
-import re
-
-from nltk import sent_tokenize
from tika import parser
-from .base import Pipeline
+from .segmentation import Segmentation
-class Textractor(Pipeline):
+class Textractor(Segmentation):
"""
Extracts text from files.
"""
- def __init__(self, sentences=False, lines=False, paragraphs=False, minlength=None, join=False):
- """
- Creates a new Textractor.
-
- Args:
- sentences: tokenize text into sentences if True, defaults to False
- lines: tokenizes text into lines if True, defaults to False
- paragraphs: tokenizes text into paragraphs if True, defaults to False
- minlength: require at least minlength characters per text element, defaults to None
- join: joins tokenized sections back together if True, defaults to False
- """
-
- self.sentences = sentences
- self.lines = lines
- self.paragraphs = paragraphs
- self.minlength = minlength
- self.join = join
-
- def __call__(self, files):
- """
- Extracts text from a file at path.
-
- This method supports files as a string or a list. If the input is a string, the return
- type is text|list for the file. If files is a list, a list of returned, this could be a
- list of text or a list of lists depending on the tokenization strategy.
-
- Args:
- files: text|list
-
- Returns:
- extracted text from files
- """
-
- # Get inputs
- values = [files] if not isinstance(files, list) else files
-
- # Extract text for each input file
- results = []
- for path in values:
- parsed = parser.from_file(path)
- text = parsed["content"]
-
- if text:
- # Parse and add extracted results
- results.append(self.parse(text))
-
- return results[0] if isinstance(files, str) else results
-
- def parse(self, text):
- """
- Splits and cleans text based on the current parameters.
-
- Args:
- text: input text
-
- Returns:
- parsed and clean content
- """
-
- content = None
-
- if self.sentences:
- content = [self.clean(x) for x in sent_tokenize(text)]
- elif self.lines:
- content = [self.clean(x) for x in text.split("\n")]
- elif self.paragraphs:
- content = [self.clean(x) for x in text.split("\n\n")]
- else:
- content = [self.clean(text)]
-
- # Remove empty strings
- content = [x for x in content if x]
-
- if self.sentences or self.lines or self.paragraphs:
- return " ".join(content) if self.join else content
-
- return content[0]
-
- def clean(self, text):
- """
- Applies a series of rules to clean extracted text.
-
- Args:
- text: input text
-
- Returns:
- clean text
- """
-
- text = text.replace("\n", " ")
- text = re.sub(r"\s+", " ", text)
- text = text.strip()
-
- return text if not self.minlength or len(text) >= self.minlength else None
+ def text(self, text):
+ # text is a path to a file
+ parsed = parser.from_file(text)
+ return parsed["content"]
diff --git a/test/python/testapi/testpipelines.py b/test/python/testapi/testpipelines.py
index 1954e63d..7e0ff9b1 100644
--- a/test/python/testapi/testpipelines.py
+++ b/test/python/testapi/testpipelines.py
@@ -22,6 +22,10 @@ PIPELINES = """
labels:
path: prajjwal1/bert-medium-mnli
+# Text segmentation
+segmentation:
+ sentences: true
+
# Enable pipeline similarity backed by zero shot classifier
similarity:
@@ -122,6 +126,28 @@ class TestPipelines(unittest.TestCase):
results = [l[0]["id"] for l in labels]
self.assertEqual(results, [0, 1])
+ def testSegment(self):
+ """
+ Test segmentation via API
+ """
+
+ text = self.client.get("segment?text=This is a test. And another test.").json()
+
+ # Check array length is 2
+ self.assertEqual(len(text), 2)
+
+ def testSegmentBatch(self):
+ """
+ Test batch segmentation via API
+ """
+
+ text = "This is a test. And another test."
+ texts = self.client.post("batchsegment", json=[text, text]).json()
+
+ # Check array length is 2 and first element length is 2
+ self.assertEqual(len(texts), 2)
+ self.assertEqual(len(texts[0]), 2)
+
def testSimilarity(self):
"""
Test similarity via API