💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add text segmentation pipeline, closes #81
davidmezzetti983fe9f2c978
diff --git a/examples/workflows.py b/examples/workflows.pyindex 8078c65f..f60b5436 100644--- a/examples/workflows.py+++ b/examples/workflows.py@@ -11,7 +11,7 @@ import pandas as pdimport streamlit as stfrom txtai.embeddings import Documents, Embeddings-from txtai.pipeline import Summary, Textractor, Transcription, Translation+from txtai.pipeline import Segmentation, Summary, Textractor, Transcription, Translationfrom txtai.workflow import Workflow, FileTask, Task@@ -73,8 +73,12 @@ class Application:options["minlength"] = self.number("Min length")options["maxlength"] = self.number("Max length")- elif component == "textract":- st.sidebar.markdown("**Textractor** \n*Extract text from documents*")+ elif component in ("segment", "textract"):+ if component == "segment":+ st.sidebar.markdown("**Segment** \n*Split text into semantic units*")+ else:+ st.sidebar.markdown("**Textractor** \n*Extract text from documents*")+options["sentences"] = st.sidebar.checkbox("Split sentences")options["lines"] = st.sidebar.checkbox("Split lines")options["paragraphs"] = st.sidebar.checkbox("Split paragraphs")@@ -116,6 +120,10 @@ class Application:self.pipelines[wtype] = Summary(component.pop("path"))tasks.append(Task(lambda x: self.pipelines["summary"](x, **self.components["summary"])))+ elif wtype == "segment":+ self.pipelines[wtype] = Segmentation(**self.components["segment"])+ tasks.append(Task(self.pipelines["segment"]))+elif wtype == "textract":self.pipelines[wtype] = Textractor(**self.components["textract"])tasks.append(FileTask(self.pipelines["textract"]))@@ -198,7 +206,7 @@ class Application:st.sidebar.markdown("# Workflow builder \n*Build and apply workflows to data* ")# Get selected components- selected = st.sidebar.multiselect("Select components", ["embeddings", "summary", "textract", "transcribe", "translate"])+ selected = st.sidebar.multiselect("Select components", ["embeddings", "segment", "summary", "textract", "transcribe", "translate"])# Get selected optionscomponents = [self.options(component) for component in selected]@@ -212,10 +220,9 @@ class Application:with st.beta_expander("Data", expanded=not self.data):data = st.text_area("Input", height=10)- split = st.checkbox("Split input")# Parse text items- data = data.split("\n") if "file://" in data or split else [data]+ data = [x for x in data.split("\n") if x] if "file://" in data else [data]# Process current actionself.process(data)diff --git a/src/python/txtai/api/application.py b/src/python/txtai/api/application.pyindex 6b9d1da9..ba773a16 100644--- a/src/python/txtai/api/application.py+++ b/src/python/txtai/api/application.py@@ -12,7 +12,7 @@ from .base import APIfrom .factory import Factory# pylint: disable=R0401-from .routers import embeddings, extractor, labels, similarity, summary, textractor, transcription, translation+from .routers import embeddings, extractor, labels, segmentation, similarity, summary, textractor, transcription, translation# API instanceapp = FastAPI()@@ -57,6 +57,7 @@ def start():("embeddings", embeddings.router),("extractor", extractor.router),("labels", labels.router),+ ("segmentation", segmentation.router),("similarity", similarity.router),("summary", summary.router),("textractor", textractor.router),@@ -65,8 +66,8 @@ def start():]# Conditionally add routes based on configuration- for route, router in routers:- if route in config:+ for name, router in routers:+ if name in config:app.include_router(router)# Special case to add similarity instance for embeddingsdiff --git a/src/python/txtai/api/base.py b/src/python/txtai/api/base.pyindex e0191104..df43ea11 100644--- a/src/python/txtai/api/base.py+++ b/src/python/txtai/api/base.py@@ -33,7 +33,7 @@ class API:self.pipelines = {}# Default pipelines- pipelines = ["extractor", "labels", "similarity", "summary", "textractor", "transcription", "translation"]+ pipelines = ["extractor", "labels", "segmentation", "similarity", "summary", "textractor", "transcription", "translation"]# Add custom pipelinesfor key in self.config:diff --git a/src/python/txtai/api/routers/__init__.py b/src/python/txtai/api/routers/__init__.pyindex 0f2e3f0b..09ef8cb8 100644--- a/src/python/txtai/api/routers/__init__.py+++ b/src/python/txtai/api/routers/__init__.py@@ -4,6 +4,7 @@ Router importsfrom . import embeddingsfrom . import extractor+from . import segmentationfrom . import similarityfrom . import summaryfrom . import textractordiff --git a/src/python/txtai/api/routers/segmentation.py b/src/python/txtai/api/routers/segmentation.pynew file mode 100644index 00000000..f93cba2d--- /dev/null+++ b/src/python/txtai/api/routers/segmentation.py@@ -0,0 +1,41 @@+"""+Defines API paths for segmentation endpoints.+"""++from typing import List++from fastapi import APIRouter, Body++from .. import application++router = APIRouter()++[email protected]("/segment")+def textract(text: str):+ """+ Segments input text.++ Args:+ text: input text++ Returns:+ segmented text+ """++ return application.get().pipeline("segmentation", (text,))++[email protected]("/batchsegment")+def batchsegment(texts: List[str] = Body(...)):+ """+ Extracts text from a file at path.++ Args:+ text: list of texts to segment++ Returns:+ list of segmented text+ """++ return application.get().pipeline("segmentation", (texts,))diff --git a/src/python/txtai/pipeline/__init__.py b/src/python/txtai/pipeline/__init__.pyindex 8006b0a8..2f628e4c 100644--- a/src/python/txtai/pipeline/__init__.py+++ b/src/python/txtai/pipeline/__init__.py@@ -9,6 +9,7 @@ from .hfmodel import HFModelfrom .hfpipeline import HFPipelinefrom .labels import Labelsfrom .questions import Questions+from .segmentation import Segmentationfrom .similarity import Similarityfrom .summary import Summaryfrom .tensors import Tensorsdiff --git a/src/python/txtai/pipeline/segmentation.py b/src/python/txtai/pipeline/segmentation.pynew file mode 100644index 00000000..5c051813--- /dev/null+++ b/src/python/txtai/pipeline/segmentation.py@@ -0,0 +1,119 @@+"""+Segmentation module+"""++import re++from nltk import sent_tokenize++from .base import Pipeline+++class Segmentation(Pipeline):+ """+ Segments text into logical units.+ """++ def __init__(self, sentences=False, lines=False, paragraphs=False, minlength=None, join=False):+ """+ Creates a new Segmentation pipeline.++ Args:+ sentences: tokenize text into sentences if True, defaults to False+ lines: tokenizes text into lines if True, defaults to False+ paragraphs: tokenizes text into paragraphs if True, defaults to False+ minlength: require at least minlength characters per text element, defaults to None+ join: joins tokenized sections back together if True, defaults to False+ """++ self.sentences = sentences+ self.lines = lines+ self.paragraphs = paragraphs+ self.minlength = minlength+ self.join = join++ def __call__(self, text):+ """+ Segments text into semantic units.++ This method supports text as a string or a list. If the input is a string, the return+ type is text|list. If text is a list, a list of returned, this could be a+ list of text or a list of lists depending on the tokenization strategy.++ Args:+ text: text|list++ Returns:+ segmented text+ """++ # Get inputs+ texts = [text] if not isinstance(text, list) else text++ # Extract text for each input file+ results = []+ for value in texts:+ # Get text+ value = self.text(value)++ # Parse and add extracted results+ results.append(self.parse(value))++ return results[0] if isinstance(text, str) else results++ def text(self, text):+ """+ Hook to allow extracting text out of input text object.++ Args:+ text: object to extract text from+ """++ return text++ def parse(self, text):+ """+ Splits and cleans text based on the current parameters.++ Args:+ text: input text++ Returns:+ parsed and clean content+ """++ content = None++ if self.sentences:+ content = [self.clean(x) for x in sent_tokenize(text)]+ elif self.lines:+ content = [self.clean(x) for x in text.split("\n")]+ elif self.paragraphs:+ content = [self.clean(x) for x in text.split("\n\n")]+ else:+ content = [self.clean(text)]++ # Remove empty strings+ content = [x for x in content if x]++ if self.sentences or self.lines or self.paragraphs:+ return " ".join(content) if self.join else content++ return content[0] if content else content++ def clean(self, text):+ """+ Applies a series of rules to clean text.++ Args:+ text: input text++ Returns:+ clean text+ """++ text = text.replace("\n", " ")+ text = re.sub(r"\s+", " ", text)+ text = text.strip()++ return text if not self.minlength or len(text) >= self.minlength else Nonediff --git a/src/python/txtai/pipeline/textractor.py b/src/python/txtai/pipeline/textractor.pyindex 1c4fc652..89c88b00 100644--- a/src/python/txtai/pipeline/textractor.py+++ b/src/python/txtai/pipeline/textractor.py@@ -2,110 +2,17 @@Textractor module"""-import re--from nltk import sent_tokenizefrom tika import parser-from .base import Pipeline+from .segmentation import Segmentation-class Textractor(Pipeline):+class Textractor(Segmentation):"""Extracts text from files."""- def __init__(self, sentences=False, lines=False, paragraphs=False, minlength=None, join=False):- """- Creates a new Textractor.-- Args:- sentences: tokenize text into sentences if True, defaults to False- lines: tokenizes text into lines if True, defaults to False- paragraphs: tokenizes text into paragraphs if True, defaults to False- minlength: require at least minlength characters per text element, defaults to None- join: joins tokenized sections back together if True, defaults to False- """-- self.sentences = sentences- self.lines = lines- self.paragraphs = paragraphs- self.minlength = minlength- self.join = join-- def __call__(self, files):- """- Extracts text from a file at path.-- This method supports files as a string or a list. If the input is a string, the return- type is text|list for the file. If files is a list, a list of returned, this could be a- list of text or a list of lists depending on the tokenization strategy.-- Args:- files: text|list-- Returns:- extracted text from files- """-- # Get inputs- values = [files] if not isinstance(files, list) else files-- # Extract text for each input file- results = []- for path in values:- parsed = parser.from_file(path)- text = parsed["content"]-- if text:- # Parse and add extracted results- results.append(self.parse(text))-- return results[0] if isinstance(files, str) else results-- def parse(self, text):- """- Splits and cleans text based on the current parameters.-- Args:- text: input text-- Returns:- parsed and clean content- """-- content = None-- if self.sentences:- content = [self.clean(x) for x in sent_tokenize(text)]- elif self.lines:- content = [self.clean(x) for x in text.split("\n")]- elif self.paragraphs:- content = [self.clean(x) for x in text.split("\n\n")]- else:- content = [self.clean(text)]-- # Remove empty strings- content = [x for x in content if x]-- if self.sentences or self.lines or self.paragraphs:- return " ".join(content) if self.join else content-- return content[0]-- def clean(self, text):- """- Applies a series of rules to clean extracted text.-- Args:- text: input text-- Returns:- clean text- """-- text = text.replace("\n", " ")- text = re.sub(r"\s+", " ", text)- text = text.strip()-- return text if not self.minlength or len(text) >= self.minlength else None+ def text(self, text):+ # text is a path to a file+ parsed = parser.from_file(text)+ return parsed["content"]diff --git a/test/python/testapi/testpipelines.py b/test/python/testapi/testpipelines.pyindex 1954e63d..7e0ff9b1 100644--- a/test/python/testapi/testpipelines.py+++ b/test/python/testapi/testpipelines.py@@ -22,6 +22,10 @@ PIPELINES = """labels:path: prajjwal1/bert-medium-mnli+# Text segmentation+segmentation:+ sentences: true+# Enable pipeline similarity backed by zero shot classifiersimilarity:@@ -122,6 +126,28 @@ class TestPipelines(unittest.TestCase):results = [l[0]["id"] for l in labels]self.assertEqual(results, [0, 1])+ def testSegment(self):+ """+ Test segmentation via API+ """++ text = self.client.get("segment?text=This is a test. And another test.").json()++ # Check array length is 2+ self.assertEqual(len(text), 2)++ def testSegmentBatch(self):+ """+ Test batch segmentation via API+ """++ text = "This is a test. And another test."+ texts = self.client.post("batchsegment", json=[text, text]).json()++ # Check array length is 2 and first element length is 2+ self.assertEqual(len(texts), 2)+ self.assertEqual(len(texts[0]), 2)+def testSimilarity(self):"""Test similarity via API