Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Update caption pipeline to call image-to-text pipeline, closes #361

davidmezzettif3a185c33967
diff --git a/src/python/txtai/pipeline/image/caption.py b/src/python/txtai/pipeline/image/caption.py
index 021c6cd4..3206a377 100644
--- a/src/python/txtai/pipeline/image/caption.py
+++ b/src/python/txtai/pipeline/image/caption.py
@@ -10,43 +10,20 @@ try:
except ImportError:
PIL = False
-import torch
+from ..hfpipeline import HFPipeline
-from transformers import AutoTokenizer, VisionEncoderDecoderModel, ViTFeatureExtractor
-from ..hfmodel import HFModel
-
-
-class Caption(HFModel):
+class Caption(HFPipeline):
"""
Constructs captions for images.
"""
- def __init__(self, path="ydshieh/vit-gpt2-coco-en", quantize=False, gpu=True, batch=64):
- """
- Constructs a new caption pipeline.
-
- Args:
- path: optional path to model, accepts Hugging Face model hub id or local path,
- uses default model for task if not provided
- quantize: if model should be quantized, defaults to False
- gpu: True/False if GPU should be enabled, also supports a GPU device id
- batch: batch size used to incrementally process content
- """
-
+ def __init__(self, path=None, quantize=False, gpu=True, model=None):
if not PIL:
raise ImportError('Captions pipeline is not available - install "pipeline" extra to enable')
# Call parent constructor
- super().__init__(path, quantize, gpu, batch)
-
- # load model and processor
- self.model = VisionEncoderDecoderModel.from_pretrained(self.path)
- self.tokenizer = AutoTokenizer.from_pretrained(self.path)
- self.extractor = ViTFeatureExtractor.from_pretrained(self.path)
-
- # Move model to device
- self.model.to(self.device)
+ super().__init__("image-to-text", path, quantize, gpu, model)
def __call__(self, images):
"""
@@ -68,17 +45,11 @@ class Caption(HFModel):
# Open images if file strings
values = [Image.open(image) if isinstance(image, str) else image for image in values]
- # Feature extraction
- pixels = self.extractor(images=values, return_tensors="pt").pixel_values
- pixels = pixels.to(self.device)
-
- # Run model
- with torch.no_grad():
- outputs = self.model.generate(pixels, max_length=16, num_beams=4, return_dict_in_generate=True).sequences
-
- # Tokenize outputs into text results
- captions = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)
- captions = [caption.strip() for caption in captions]
+ # Get and clean captions
+ captions = []
+ for result in self.pipeline(values):
+ text = " ".join([r["generated_text"] for r in result]).strip()
+ captions.append(text)
# Return single element if single element passed in
return captions[0] if not isinstance(images, list) else captions
diff --git a/test/python/testapi/testpipelines.py b/test/python/testapi/testpipelines.py
index 91d58175..e7c571bd 100644
--- a/test/python/testapi/testpipelines.py
+++ b/test/python/testapi/testpipelines.py
@@ -113,7 +113,7 @@ class TestPipelines(unittest.TestCase):
caption = self.client.get(f"caption?file={Utils.PATH}/books.jpg").json()
- self.assertEqual(caption, "a book shelf filled with many books")
+ self.assertEqual(caption, "a book shelf filled with books and a stack of books")
def testCaptionBatch(self):
"""
@@ -123,7 +123,7 @@ class TestPipelines(unittest.TestCase):
path = Utils.PATH + "/books.jpg"
captions = self.client.post("batchcaption", json=[path, path]).json()
- self.assertEqual(captions, ["a book shelf filled with many books"] * 2)
+ self.assertEqual(captions, ["a book shelf filled with books and a stack of books"] * 2)
def testEntity(self):
"""
diff --git a/test/python/testpipeline/testcaption.py b/test/python/testpipeline/testcaption.py
index a75f44bc..e9ba6fa6 100644
--- a/test/python/testpipeline/testcaption.py
+++ b/test/python/testpipeline/testcaption.py
@@ -23,4 +23,4 @@ class TestCaption(unittest.TestCase):
"""
caption = Caption()
- self.assertEqual(caption(Image.open(Utils.PATH + "/books.jpg")), "a book shelf filled with many books")
+ self.assertEqual(caption(Image.open(Utils.PATH + "/books.jpg")), "a book shelf filled with books and a stack of books")