💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Update caption pipeline to call image-to-text pipeline, closes #361
davidmezzettif3a185c33967
diff --git a/src/python/txtai/pipeline/image/caption.py b/src/python/txtai/pipeline/image/caption.pyindex 021c6cd4..3206a377 100644--- a/src/python/txtai/pipeline/image/caption.py+++ b/src/python/txtai/pipeline/image/caption.py@@ -10,43 +10,20 @@ try:except ImportError:PIL = False-import torch+from ..hfpipeline import HFPipeline-from transformers import AutoTokenizer, VisionEncoderDecoderModel, ViTFeatureExtractor-from ..hfmodel import HFModel---class Caption(HFModel):+class Caption(HFPipeline):"""Constructs captions for images."""- def __init__(self, path="ydshieh/vit-gpt2-coco-en", quantize=False, gpu=True, batch=64):- """- Constructs a new caption pipeline.-- Args:- path: optional path to model, accepts Hugging Face model hub id or local path,- uses default model for task if not provided- quantize: if model should be quantized, defaults to False- gpu: True/False if GPU should be enabled, also supports a GPU device id- batch: batch size used to incrementally process content- """-+ def __init__(self, path=None, quantize=False, gpu=True, model=None):if not PIL:raise ImportError('Captions pipeline is not available - install "pipeline" extra to enable')# Call parent constructor- super().__init__(path, quantize, gpu, batch)-- # load model and processor- self.model = VisionEncoderDecoderModel.from_pretrained(self.path)- self.tokenizer = AutoTokenizer.from_pretrained(self.path)- self.extractor = ViTFeatureExtractor.from_pretrained(self.path)-- # Move model to device- self.model.to(self.device)+ super().__init__("image-to-text", path, quantize, gpu, model)def __call__(self, images):"""@@ -68,17 +45,11 @@ class Caption(HFModel):# Open images if file stringsvalues = [Image.open(image) if isinstance(image, str) else image for image in values]- # Feature extraction- pixels = self.extractor(images=values, return_tensors="pt").pixel_values- pixels = pixels.to(self.device)-- # Run model- with torch.no_grad():- outputs = self.model.generate(pixels, max_length=16, num_beams=4, return_dict_in_generate=True).sequences-- # Tokenize outputs into text results- captions = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)- captions = [caption.strip() for caption in captions]+ # Get and clean captions+ captions = []+ for result in self.pipeline(values):+ text = " ".join([r["generated_text"] for r in result]).strip()+ captions.append(text)# Return single element if single element passed inreturn captions[0] if not isinstance(images, list) else captionsdiff --git a/test/python/testapi/testpipelines.py b/test/python/testapi/testpipelines.pyindex 91d58175..e7c571bd 100644--- a/test/python/testapi/testpipelines.py+++ b/test/python/testapi/testpipelines.py@@ -113,7 +113,7 @@ class TestPipelines(unittest.TestCase):caption = self.client.get(f"caption?file={Utils.PATH}/books.jpg").json()- self.assertEqual(caption, "a book shelf filled with many books")+ self.assertEqual(caption, "a book shelf filled with books and a stack of books")def testCaptionBatch(self):"""@@ -123,7 +123,7 @@ class TestPipelines(unittest.TestCase):path = Utils.PATH + "/books.jpg"captions = self.client.post("batchcaption", json=[path, path]).json()- self.assertEqual(captions, ["a book shelf filled with many books"] * 2)+ self.assertEqual(captions, ["a book shelf filled with books and a stack of books"] * 2)def testEntity(self):"""diff --git a/test/python/testpipeline/testcaption.py b/test/python/testpipeline/testcaption.pyindex a75f44bc..e9ba6fa6 100644--- a/test/python/testpipeline/testcaption.py+++ b/test/python/testpipeline/testcaption.py@@ -23,4 +23,4 @@ class TestCaption(unittest.TestCase):"""caption = Caption()- self.assertEqual(caption(Image.open(Utils.PATH + "/books.jpg")), "a book shelf filled with many books")+ self.assertEqual(caption(Image.open(Utils.PATH + "/books.jpg")), "a book shelf filled with books and a stack of books")