Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add objects and caption pipelines to API, closes #161

davidmezzettidbdb68144b27
diff --git a/setup.py b/setup.py
index 325113e4..00d7343b 100644
--- a/setup.py
+++ b/setup.py
@@ -29,6 +29,7 @@ extras["pipeline"] = [
"onnxmltools>=1.9.1",
"onnxruntime>=1.8.1",
"pandas>=1.1.0",
+ "pillow>=7.2.0",
"sentencepiece>=0.1.91",
"soundfile>=0.10.3.post1",
"tika>=1.24",
diff --git a/src/python/txtai/api/application.py b/src/python/txtai/api/application.py
index a8cdbcfc..80ecd1ca 100644
--- a/src/python/txtai/api/application.py
+++ b/src/python/txtai/api/application.py
@@ -11,8 +11,8 @@ from fastapi import FastAPI
from .base import API
from .factory import Factory
-# pylint: disable=R0401
-from .routers import embeddings, extractor, labels, segmentation, similarity, summary, tabular, textractor, transcription, translation, workflow
+# pylint: disable=R0401,W0401
+from .routers import *
# API instance
app = FastAPI()
@@ -52,9 +52,11 @@ def start():
# Router definitions
routers = [
+ ("caption", caption.router),
("embeddings", embeddings.router),
("extractor", extractor.router),
("labels", labels.router),
+ ("objects", objects.router),
("segmentation", segmentation.router),
("similarity", similarity.router),
("summary", summary.router),
diff --git a/src/python/txtai/api/base.py b/src/python/txtai/api/base.py
index 04f04638..ea6ad7a3 100644
--- a/src/python/txtai/api/base.py
+++ b/src/python/txtai/api/base.py
@@ -58,7 +58,20 @@ class API:
self.pipelines = {}
# Default pipelines
- pipelines = ["extractor", "labels", "nop", "segmentation", "similarity", "summary", "tabular", "textractor", "transcription", "translation"]
+ pipelines = [
+ "caption",
+ "extractor",
+ "labels",
+ "nop",
+ "objects",
+ "segmentation",
+ "similarity",
+ "summary",
+ "tabular",
+ "textractor",
+ "transcription",
+ "translation",
+ ]
# Add custom pipelines
for key in self.config:
diff --git a/src/python/txtai/api/routers/__init__.py b/src/python/txtai/api/routers/__init__.py
index 2ab87bfa..3a8830d1 100644
--- a/src/python/txtai/api/routers/__init__.py
+++ b/src/python/txtai/api/routers/__init__.py
@@ -2,8 +2,11 @@
Router imports
"""
+from . import caption
from . import embeddings
from . import extractor
+from . import labels
+from . import objects
from . import segmentation
from . import similarity
from . import summary
diff --git a/src/python/txtai/api/routers/caption.py b/src/python/txtai/api/routers/caption.py
new file mode 100644
index 00000000..6dbea869
--- /dev/null
+++ b/src/python/txtai/api/routers/caption.py
@@ -0,0 +1,41 @@
+"""
+Defines API paths for caption endpoints.
+"""
+
+from typing import List
+
+from fastapi import APIRouter, Body
+
+from .. import application
+
+router = APIRouter()
+
+
[email protected]("/caption")
+def caption(file: str):
+ """
+ Builds captions for images.
+
+ Args:
+ file: file to process
+
+ Returns:
+ list of captions
+ """
+
+ return application.get().pipeline("caption", (file,))
+
+
[email protected]("/batchcaption")
+def batchcaption(files: List[str] = Body(...)):
+ """
+ Builds captions for images.
+
+ Args:
+ files: list of files to process
+
+ Returns:
+ list of captions
+ """
+
+ return application.get().pipeline("caption", (files,))
diff --git a/src/python/txtai/api/routers/objects.py b/src/python/txtai/api/routers/objects.py
new file mode 100644
index 00000000..2ee85213
--- /dev/null
+++ b/src/python/txtai/api/routers/objects.py
@@ -0,0 +1,41 @@
+"""
+Defines API paths for objects endpoints.
+"""
+
+from typing import List
+
+from fastapi import APIRouter, Body
+
+from .. import application
+
+router = APIRouter()
+
+
[email protected]("/objects")
+def objects(file: str):
+ """
+ Applies object detection/image classification models to images.
+
+ Args:
+ file: file to process
+
+ Returns:
+ list of (label, score) elements
+ """
+
+ return application.get().pipeline("objects", (file,))
+
+
[email protected]("/batchobjects")
+def batchobjects(files: List[str] = Body(...)):
+ """
+ Applies object detection/image classification models to images.
+
+ Args:
+ files: list of files to process
+
+ Returns:
+ list of (label, score) elements
+ """
+
+ return application.get().pipeline("objects", (files,))
diff --git a/src/python/txtai/pipeline/image/caption.py b/src/python/txtai/pipeline/image/caption.py
index 0e9976b5..021c6cd4 100644
--- a/src/python/txtai/pipeline/image/caption.py
+++ b/src/python/txtai/pipeline/image/caption.py
@@ -2,6 +2,14 @@
Caption module
"""
+# Conditional import
+try:
+ from PIL import Image
+
+ PIL = True
+except ImportError:
+ PIL = False
+
import torch
from transformers import AutoTokenizer, VisionEncoderDecoderModel, ViTFeatureExtractor
@@ -26,6 +34,9 @@ class Caption(HFModel):
batch: batch size used to incrementally process content
"""
+ if not PIL:
+ raise ImportError('Captions pipeline is not available - install "pipeline" extra to enable')
+
# Call parent constructor
super().__init__(path, quantize, gpu, batch)
@@ -54,6 +65,9 @@ class Caption(HFModel):
# Convert single element to list
values = [images] if not isinstance(images, list) else images
+ # Open images if file strings
+ values = [Image.open(image) if isinstance(image, str) else image for image in values]
+
# Feature extraction
pixels = self.extractor(images=values, return_tensors="pt").pixel_values
pixels = pixels.to(self.device)
diff --git a/src/python/txtai/pipeline/image/objects.py b/src/python/txtai/pipeline/image/objects.py
index a48e7e08..bf1475a7 100644
--- a/src/python/txtai/pipeline/image/objects.py
+++ b/src/python/txtai/pipeline/image/objects.py
@@ -2,6 +2,14 @@
Objects module
"""
+# Conditional import
+try:
+ from PIL import Image
+
+ PIL = True
+except ImportError:
+ PIL = False
+
from ..hfpipeline import HFPipeline
@@ -11,6 +19,9 @@ class Objects(HFPipeline):
"""
def __init__(self, path=None, quantize=False, gpu=True, model=None, classification=False, threshold=0.9):
+ if not PIL:
+ raise ImportError('Objects pipeline is not available - install "pipeline" extra to enable')
+
super().__init__("image-classification" if classification else "object-detection", path, quantize, gpu, model)
self.classification = classification
@@ -36,6 +47,9 @@ class Objects(HFPipeline):
# Convert single element to list
values = [images] if not isinstance(images, list) else images
+ # Open images if file strings
+ values = [Image.open(image) if isinstance(image, str) else image for image in values]
+
# Run pipeline
results = (
self.pipeline(values, num_workers=workers)
diff --git a/src/python/txtai/workflow/task/image.py b/src/python/txtai/workflow/task/image.py
index ee14d332..29cccf86 100644
--- a/src/python/txtai/workflow/task/image.py
+++ b/src/python/txtai/workflow/task/image.py
@@ -21,7 +21,6 @@ class ImageTask(FileTask):
"""
def __init__(self, action=None, select=None, unpack=True, column=None, merge="hstack", initialize=None, finalize=None):
-
if not PIL:
raise ImportError('ImageTask is not available - install "workflow" extra to enable')
diff --git a/test/python/testapi/testpipelines.py b/test/python/testapi/testpipelines.py
index a6b3bb28..8b6e85d4 100644
--- a/test/python/testapi/testpipelines.py
+++ b/test/python/testapi/testpipelines.py
@@ -18,10 +18,16 @@ from utils import Utils
# Configuration for pipelines
PIPELINES = """
-# Labels settings
+# Image captions
+caption:
+
+# Label settings
labels:
path: prajjwal1/bert-medium-mnli
+# Image objects
+objects:
+
# Text segmentation
segmentation:
sentences: true
@@ -47,6 +53,7 @@ translation:
"""
+# pylint: disable=R0904
class TestPipelines(unittest.TestCase):
"""
API tests for pipelines
@@ -95,6 +102,25 @@ class TestPipelines(unittest.TestCase):
"that enables Natural Language Understanding (NLU) based search in any application."
)
+ def testCaption(self):
+ """
+ Test caption via API
+ """
+
+ caption = self.client.get(f"caption?file={Utils.PATH}/books.jpg").json()
+
+ self.assertEqual(caption, "a book shelf filled with many books")
+
+ def testCaptionBatch(self):
+ """
+ Test batch caption via API
+ """
+
+ path = Utils.PATH + "/books.jpg"
+
+ captions = self.client.post("batchcaption", json=[path, path]).json()
+ self.assertEqual(captions, ["a book shelf filled with many books"] * 2)
+
def testEmpty(self):
"""
Test empty API configuration
@@ -126,6 +152,25 @@ class TestPipelines(unittest.TestCase):
results = [l[0]["id"] for l in labels]
self.assertEqual(results, [0, 1])
+ def testObjects(self):
+ """
+ Test objects via API
+ """
+
+ objects = self.client.get(f"objects?file={Utils.PATH}/books.jpg").json()
+
+ self.assertEqual(objects[0][0], "book")
+
+ def testObjectsBatch(self):
+ """
+ Test batch objects via API
+ """
+
+ path = Utils.PATH + "/books.jpg"
+
+ objects = self.client.post("batchobjects", json=[path, path]).json()
+ self.assertEqual([o[0][0] for o in objects], ["book"] * 2)
+
def testSegment(self):
"""
Test segmentation via API
@@ -180,15 +225,15 @@ class TestPipelines(unittest.TestCase):
Test batch summary via API
"""
- summaries = self.client.post("batchsummary", json={"texts": [self.text, self.text], "maxlength": 10}).json()
- self.assertEqual(len(summaries), 2)
+ summaries = self.client.post("batchsummary", json={"texts": [self.text, self.text], "minlength": 15, "maxlength": 15}).json()
+ self.assertEqual(summaries, ["txtai is an AI-powered search engine that"] * 2)
def testTabular(self):
"""
Test tabular via API
"""
- results = self.client.get(f"tabular?file={Utils.PATH}" + "/tabular.csv").json()
+ results = self.client.get(f"tabular?file={Utils.PATH}/tabular.csv").json()
# Check length of results is as expected
self.assertEqual(len(results), 6)
@@ -200,15 +245,15 @@ class TestPipelines(unittest.TestCase):
path = Utils.PATH + "/tabular.csv"
- texts = self.client.post("batchtabular", json=[path, path]).json()
- self.assertEqual(len(texts), 2)
+ results = self.client.post("batchtabular", json=[path, path]).json()
+ self.assertEqual((len(results[0]), len(results[1])), (6, 6))
def testTextractor(self):
"""
Test textractor via API
"""
- text = self.client.get(f"textract?file={Utils.PATH}" + "/article.pdf").json()
+ text = self.client.get(f"textract?file={Utils.PATH}/article.pdf").json()
# Check length of text is as expected
self.assertEqual(len(text), 2301)
@@ -221,14 +266,14 @@ class TestPipelines(unittest.TestCase):
path = Utils.PATH + "/article.pdf"
texts = self.client.post("batchtextract", json=[path, path]).json()
- self.assertEqual(len(texts), 2)
+ self.assertEqual((len(texts[0]), len(texts[1])), (2301, 2301))
def testTranscribe(self):
"""
Test transcribe via API
"""
- text = self.client.get(f"transcribe?file={Utils.PATH}" + "/Make_huge_profits.wav").json()
+ text = self.client.get(f"transcribe?file={Utils.PATH}/Make_huge_profits.wav").json()
# Check length of text is as expected
self.assertEqual(text, "Make huge profits without working make up to one hundred thousand dollars a day")
@@ -241,7 +286,7 @@ class TestPipelines(unittest.TestCase):
path = Utils.PATH + "/Make_huge_profits.wav"
texts = self.client.post("batchtranscribe", json=[path, path]).json()
- self.assertEqual(len(texts), 2)
+ self.assertEqual(texts, ["Make huge profits without working make up to one hundred thousand dollars a day"] * 2)
def testTranslate(self):
"""
@@ -258,4 +303,4 @@ class TestPipelines(unittest.TestCase):
text = "This is a test translation into Spanish"
translations = self.client.post("batchtranslate", json={"texts": [text, text], "target": "es"}).json()
- self.assertEqual(len(translations), 2)
+ self.assertEqual(translations, ["Esta es una traducción de prueba al español"] * 2)
diff --git a/test/python/testoptional.py b/test/python/testoptional.py
index 891e6074..e265833d 100644
--- a/test/python/testoptional.py
+++ b/test/python/testoptional.py
@@ -99,7 +99,10 @@ class TestOptional(unittest.TestCase):
Test missing pipeline dependencies
"""
- from txtai.pipeline import HFOnnx, MLOnnx, Segmentation, Tabular, Textractor, Transcription, Translation
+ from txtai.pipeline import Caption, HFOnnx, MLOnnx, Objects, Segmentation, Tabular, Textractor, Transcription, Translation
+
+ with self.assertRaises(ImportError):
+ Caption()
with self.assertRaises(ImportError):
HFOnnx()("google/bert_uncased_L-2_H-128_A-2", quantize=True)
@@ -107,6 +110,9 @@ class TestOptional(unittest.TestCase):
with self.assertRaises(ImportError):
MLOnnx()
+ with self.assertRaises(ImportError):
+ Objects()
+
with self.assertRaises(ImportError):
Segmentation()