💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Pad batched inputs in the Summary pipeline, closes #1294
bob2e89af084554
diff --git a/src/python/txtai/pipeline/text/summary.py b/src/python/txtai/pipeline/text/summary.pyindex bf7ba2f2..b313c5ec 100644--- a/src/python/txtai/pipeline/text/summary.py+++ b/src/python/txtai/pipeline/text/summary.py@@ -91,14 +91,20 @@ class Summary(HFModel):if prefix:inputs = [prefix + x for x in inputs]- # Tokenize inputs- tokens = self.tokenizer(inputs, truncation=kwargs.pop("truncation", False), return_tensors="pt").to(self.device)+ truncation = kwargs.pop("truncation", False)- # Generate outputs- outputs = self.model.generate(**tokens, **kwargs)+ results = []+ for batch in self.batch(inputs, self.batchsize):+ # Tokenize inputs, padding texts of different lengths within a batch+ tokens = self.tokenizer(batch, padding=True, truncation=truncation, return_tensors="pt").to(self.device)- # Decode and return- return [self.tokenizer.decode(x, skip_special_tokens=True) for x in outputs]+ # Generate outputs+ outputs = self.model.generate(**tokens, **kwargs)++ # Decode outputs+ results.extend(self.tokenizer.decode(x, skip_special_tokens=True) for x in outputs)++ return resultsdef clean(self, text):"""diff --git a/test/python/testpipeline/testtext/testsummary.py b/test/python/testpipeline/testtext/testsummary.pyindex c3f74a50..d1638946 100644--- a/test/python/testpipeline/testtext/testsummary.py+++ b/test/python/testpipeline/testtext/testsummary.py@@ -44,6 +44,24 @@ class TestSummary(unittest.TestCase):summaries = self.summary([self.text, self.text], maxlength=15)self.assertEqual(len(summaries), 2)+ def testSummaryBatchLengths(self):+ """+ Test batch summarization of texts with different lengths+ """++ texts = [self.text, self.text + " " + self.text, self.text]+ summaries = self.summary(texts, maxlength=15)+ self.assertEqual(len(summaries), 3)+ self.assertTrue(all(summary and summary not in texts for summary in summaries))++ # Inputs that span multiple batches keep their order+ batchsize = self.summary.batchsize+ try:+ self.summary.batchsize = 2+ self.assertEqual(self.summary(texts, maxlength=15), summaries)+ finally:+ self.summary.batchsize = batchsize+def testSummaryNoLength(self):"""Test summary with no max length set