💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add objects and caption pipelines to API, closes #161
davidmezzettidbdb68144b27
diff --git a/setup.py b/setup.pyindex 325113e4..00d7343b 100644--- a/setup.py+++ b/setup.py@@ -29,6 +29,7 @@ extras["pipeline"] = ["onnxmltools>=1.9.1","onnxruntime>=1.8.1","pandas>=1.1.0",+ "pillow>=7.2.0","sentencepiece>=0.1.91","soundfile>=0.10.3.post1","tika>=1.24",diff --git a/src/python/txtai/api/application.py b/src/python/txtai/api/application.pyindex a8cdbcfc..80ecd1ca 100644--- a/src/python/txtai/api/application.py+++ b/src/python/txtai/api/application.py@@ -11,8 +11,8 @@ from fastapi import FastAPIfrom .base import APIfrom .factory import Factory-# pylint: disable=R0401-from .routers import embeddings, extractor, labels, segmentation, similarity, summary, tabular, textractor, transcription, translation, workflow+# pylint: disable=R0401,W0401+from .routers import *# API instanceapp = FastAPI()@@ -52,9 +52,11 @@ def start():# Router definitionsrouters = [+ ("caption", caption.router),("embeddings", embeddings.router),("extractor", extractor.router),("labels", labels.router),+ ("objects", objects.router),("segmentation", segmentation.router),("similarity", similarity.router),("summary", summary.router),diff --git a/src/python/txtai/api/base.py b/src/python/txtai/api/base.pyindex 04f04638..ea6ad7a3 100644--- a/src/python/txtai/api/base.py+++ b/src/python/txtai/api/base.py@@ -58,7 +58,20 @@ class API:self.pipelines = {}# Default pipelines- pipelines = ["extractor", "labels", "nop", "segmentation", "similarity", "summary", "tabular", "textractor", "transcription", "translation"]+ pipelines = [+ "caption",+ "extractor",+ "labels",+ "nop",+ "objects",+ "segmentation",+ "similarity",+ "summary",+ "tabular",+ "textractor",+ "transcription",+ "translation",+ ]# Add custom pipelinesfor key in self.config:diff --git a/src/python/txtai/api/routers/__init__.py b/src/python/txtai/api/routers/__init__.pyindex 2ab87bfa..3a8830d1 100644--- a/src/python/txtai/api/routers/__init__.py+++ b/src/python/txtai/api/routers/__init__.py@@ -2,8 +2,11 @@Router imports"""+from . import captionfrom . import embeddingsfrom . import extractor+from . import labels+from . import objectsfrom . import segmentationfrom . import similarityfrom . import summarydiff --git a/src/python/txtai/api/routers/caption.py b/src/python/txtai/api/routers/caption.pynew file mode 100644index 00000000..6dbea869--- /dev/null+++ b/src/python/txtai/api/routers/caption.py@@ -0,0 +1,41 @@+"""+Defines API paths for caption endpoints.+"""++from typing import List++from fastapi import APIRouter, Body++from .. import application++router = APIRouter()++[email protected]("/caption")+def caption(file: str):+ """+ Builds captions for images.++ Args:+ file: file to process++ Returns:+ list of captions+ """++ return application.get().pipeline("caption", (file,))++[email protected]("/batchcaption")+def batchcaption(files: List[str] = Body(...)):+ """+ Builds captions for images.++ Args:+ files: list of files to process++ Returns:+ list of captions+ """++ return application.get().pipeline("caption", (files,))diff --git a/src/python/txtai/api/routers/objects.py b/src/python/txtai/api/routers/objects.pynew file mode 100644index 00000000..2ee85213--- /dev/null+++ b/src/python/txtai/api/routers/objects.py@@ -0,0 +1,41 @@+"""+Defines API paths for objects endpoints.+"""++from typing import List++from fastapi import APIRouter, Body++from .. import application++router = APIRouter()++[email protected]("/objects")+def objects(file: str):+ """+ Applies object detection/image classification models to images.++ Args:+ file: file to process++ Returns:+ list of (label, score) elements+ """++ return application.get().pipeline("objects", (file,))++[email protected]("/batchobjects")+def batchobjects(files: List[str] = Body(...)):+ """+ Applies object detection/image classification models to images.++ Args:+ files: list of files to process++ Returns:+ list of (label, score) elements+ """++ return application.get().pipeline("objects", (files,))diff --git a/src/python/txtai/pipeline/image/caption.py b/src/python/txtai/pipeline/image/caption.pyindex 0e9976b5..021c6cd4 100644--- a/src/python/txtai/pipeline/image/caption.py+++ b/src/python/txtai/pipeline/image/caption.py@@ -2,6 +2,14 @@Caption module"""+# Conditional import+try:+ from PIL import Image++ PIL = True+except ImportError:+ PIL = False+import torchfrom transformers import AutoTokenizer, VisionEncoderDecoderModel, ViTFeatureExtractor@@ -26,6 +34,9 @@ class Caption(HFModel):batch: batch size used to incrementally process content"""+ if not PIL:+ raise ImportError('Captions pipeline is not available - install "pipeline" extra to enable')+# Call parent constructorsuper().__init__(path, quantize, gpu, batch)@@ -54,6 +65,9 @@ class Caption(HFModel):# Convert single element to listvalues = [images] if not isinstance(images, list) else images+ # Open images if file strings+ values = [Image.open(image) if isinstance(image, str) else image for image in values]+# Feature extractionpixels = self.extractor(images=values, return_tensors="pt").pixel_valuespixels = pixels.to(self.device)diff --git a/src/python/txtai/pipeline/image/objects.py b/src/python/txtai/pipeline/image/objects.pyindex a48e7e08..bf1475a7 100644--- a/src/python/txtai/pipeline/image/objects.py+++ b/src/python/txtai/pipeline/image/objects.py@@ -2,6 +2,14 @@Objects module"""+# Conditional import+try:+ from PIL import Image++ PIL = True+except ImportError:+ PIL = False+from ..hfpipeline import HFPipeline@@ -11,6 +19,9 @@ class Objects(HFPipeline):"""def __init__(self, path=None, quantize=False, gpu=True, model=None, classification=False, threshold=0.9):+ if not PIL:+ raise ImportError('Objects pipeline is not available - install "pipeline" extra to enable')+super().__init__("image-classification" if classification else "object-detection", path, quantize, gpu, model)self.classification = classification@@ -36,6 +47,9 @@ class Objects(HFPipeline):# Convert single element to listvalues = [images] if not isinstance(images, list) else images+ # Open images if file strings+ values = [Image.open(image) if isinstance(image, str) else image for image in values]+# Run pipelineresults = (self.pipeline(values, num_workers=workers)diff --git a/src/python/txtai/workflow/task/image.py b/src/python/txtai/workflow/task/image.pyindex ee14d332..29cccf86 100644--- a/src/python/txtai/workflow/task/image.py+++ b/src/python/txtai/workflow/task/image.py@@ -21,7 +21,6 @@ class ImageTask(FileTask):"""def __init__(self, action=None, select=None, unpack=True, column=None, merge="hstack", initialize=None, finalize=None):-if not PIL:raise ImportError('ImageTask is not available - install "workflow" extra to enable')diff --git a/test/python/testapi/testpipelines.py b/test/python/testapi/testpipelines.pyindex a6b3bb28..8b6e85d4 100644--- a/test/python/testapi/testpipelines.py+++ b/test/python/testapi/testpipelines.py@@ -18,10 +18,16 @@ from utils import Utils# Configuration for pipelinesPIPELINES = """-# Labels settings+# Image captions+caption:++# Label settingslabels:path: prajjwal1/bert-medium-mnli+# Image objects+objects:+# Text segmentationsegmentation:sentences: true@@ -47,6 +53,7 @@ translation:"""+# pylint: disable=R0904class TestPipelines(unittest.TestCase):"""API tests for pipelines@@ -95,6 +102,25 @@ class TestPipelines(unittest.TestCase):"that enables Natural Language Understanding (NLU) based search in any application.")+ def testCaption(self):+ """+ Test caption via API+ """++ caption = self.client.get(f"caption?file={Utils.PATH}/books.jpg").json()++ self.assertEqual(caption, "a book shelf filled with many books")++ def testCaptionBatch(self):+ """+ Test batch caption via API+ """++ path = Utils.PATH + "/books.jpg"++ captions = self.client.post("batchcaption", json=[path, path]).json()+ self.assertEqual(captions, ["a book shelf filled with many books"] * 2)+def testEmpty(self):"""Test empty API configuration@@ -126,6 +152,25 @@ class TestPipelines(unittest.TestCase):results = [l[0]["id"] for l in labels]self.assertEqual(results, [0, 1])+ def testObjects(self):+ """+ Test objects via API+ """++ objects = self.client.get(f"objects?file={Utils.PATH}/books.jpg").json()++ self.assertEqual(objects[0][0], "book")++ def testObjectsBatch(self):+ """+ Test batch objects via API+ """++ path = Utils.PATH + "/books.jpg"++ objects = self.client.post("batchobjects", json=[path, path]).json()+ self.assertEqual([o[0][0] for o in objects], ["book"] * 2)+def testSegment(self):"""Test segmentation via API@@ -180,15 +225,15 @@ class TestPipelines(unittest.TestCase):Test batch summary via API"""- summaries = self.client.post("batchsummary", json={"texts": [self.text, self.text], "maxlength": 10}).json()- self.assertEqual(len(summaries), 2)+ summaries = self.client.post("batchsummary", json={"texts": [self.text, self.text], "minlength": 15, "maxlength": 15}).json()+ self.assertEqual(summaries, ["txtai is an AI-powered search engine that"] * 2)def testTabular(self):"""Test tabular via API"""- results = self.client.get(f"tabular?file={Utils.PATH}" + "/tabular.csv").json()+ results = self.client.get(f"tabular?file={Utils.PATH}/tabular.csv").json()# Check length of results is as expectedself.assertEqual(len(results), 6)@@ -200,15 +245,15 @@ class TestPipelines(unittest.TestCase):path = Utils.PATH + "/tabular.csv"- texts = self.client.post("batchtabular", json=[path, path]).json()- self.assertEqual(len(texts), 2)+ results = self.client.post("batchtabular", json=[path, path]).json()+ self.assertEqual((len(results[0]), len(results[1])), (6, 6))def testTextractor(self):"""Test textractor via API"""- text = self.client.get(f"textract?file={Utils.PATH}" + "/article.pdf").json()+ text = self.client.get(f"textract?file={Utils.PATH}/article.pdf").json()# Check length of text is as expectedself.assertEqual(len(text), 2301)@@ -221,14 +266,14 @@ class TestPipelines(unittest.TestCase):path = Utils.PATH + "/article.pdf"texts = self.client.post("batchtextract", json=[path, path]).json()- self.assertEqual(len(texts), 2)+ self.assertEqual((len(texts[0]), len(texts[1])), (2301, 2301))def testTranscribe(self):"""Test transcribe via API"""- text = self.client.get(f"transcribe?file={Utils.PATH}" + "/Make_huge_profits.wav").json()+ text = self.client.get(f"transcribe?file={Utils.PATH}/Make_huge_profits.wav").json()# Check length of text is as expectedself.assertEqual(text, "Make huge profits without working make up to one hundred thousand dollars a day")@@ -241,7 +286,7 @@ class TestPipelines(unittest.TestCase):path = Utils.PATH + "/Make_huge_profits.wav"texts = self.client.post("batchtranscribe", json=[path, path]).json()- self.assertEqual(len(texts), 2)+ self.assertEqual(texts, ["Make huge profits without working make up to one hundred thousand dollars a day"] * 2)def testTranslate(self):"""@@ -258,4 +303,4 @@ class TestPipelines(unittest.TestCase):text = "This is a test translation into Spanish"translations = self.client.post("batchtranslate", json={"texts": [text, text], "target": "es"}).json()- self.assertEqual(len(translations), 2)+ self.assertEqual(translations, ["Esta es una traducción de prueba al español"] * 2)diff --git a/test/python/testoptional.py b/test/python/testoptional.pyindex 891e6074..e265833d 100644--- a/test/python/testoptional.py+++ b/test/python/testoptional.py@@ -99,7 +99,10 @@ class TestOptional(unittest.TestCase):Test missing pipeline dependencies"""- from txtai.pipeline import HFOnnx, MLOnnx, Segmentation, Tabular, Textractor, Transcription, Translation+ from txtai.pipeline import Caption, HFOnnx, MLOnnx, Objects, Segmentation, Tabular, Textractor, Transcription, Translation++ with self.assertRaises(ImportError):+ Caption()with self.assertRaises(ImportError):HFOnnx()("google/bert_uncased_L-2_H-128_A-2", quantize=True)@@ -107,6 +110,9 @@ class TestOptional(unittest.TestCase):with self.assertRaises(ImportError):MLOnnx()+ with self.assertRaises(ImportError):+ Objects()+with self.assertRaises(ImportError):Segmentation()