Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Improve default max length logic, closes #364

davidmezzetticd43c47fc9c6
diff --git a/src/python/txtai/models/models.py b/src/python/txtai/models/models.py
index ea0358ea..c43a772c 100644
--- a/src/python/txtai/models/models.py
+++ b/src/python/txtai/models/models.py
@@ -24,8 +24,8 @@ class Models:
with older tokenizers.
Args:
- config: Transformers config
- tokenizer: Transformers tokenizer
+ config: transformers config
+ tokenizer: transformers tokenizer
"""
# Unpack nested config, handles passing model directly
@@ -40,6 +40,27 @@ class Models:
):
tokenizer.model_max_length = config.max_position_embeddings
+ @staticmethod
+ def maxlength(config, tokenizer):
+ """
+ Gets the best max length to use for generate calls. This method will return config.max_length if it's set. Otherwise, it will return
+ tokenizer.model_max_length.
+
+ Args:
+ config: transformers config
+ tokenizer: transformers tokenizer
+ """
+
+ # Unpack nested config, handles passing model directly
+ if hasattr(config, "config"):
+ config = config.config
+
+ # Get non-defaulted fields
+ keys = config.to_diff_dict()
+
+ # Use config.max_length if not set to default value, else use tokenizer.model_max_length if available
+ return config.max_length if "max_length" in keys or not hasattr(tokenizer, "model_max_length") else tokenizer.model_max_length
+
@staticmethod
def deviceid(gpu):
"""
diff --git a/src/python/txtai/pipeline/hfpipeline.py b/src/python/txtai/pipeline/hfpipeline.py
index 171a8fec..3794521d 100644
--- a/src/python/txtai/pipeline/hfpipeline.py
+++ b/src/python/txtai/pipeline/hfpipeline.py
@@ -50,3 +50,13 @@ class HFPipeline(Tensors):
# Detect unbounded tokenizer typically found in older models
Models.checklength(self.pipeline.model, self.pipeline.tokenizer)
+
+ def maxlength(self):
+ """
+ Gets the max length to use for generate calls.
+
+ Returns:
+ max length
+ """
+
+ return Models.maxlength(self.pipeline.model, self.pipeline.tokenizer)
diff --git a/src/python/txtai/pipeline/text/summary.py b/src/python/txtai/pipeline/text/summary.py
index fe03e7eb..3b0bfd1d 100644
--- a/src/python/txtai/pipeline/text/summary.py
+++ b/src/python/txtai/pipeline/text/summary.py
@@ -33,7 +33,7 @@ class Summary(HFPipeline):
"""
# Validate text length greater than max length
- check = maxlength if maxlength else self.pipeline.model.config.max_length
+ check = maxlength if maxlength else self.maxlength()
# Skip text shorter than max length
texts = text if isinstance(text, list) else [text]
diff --git a/src/python/txtai/pipeline/text/translation.py b/src/python/txtai/pipeline/text/translation.py
index e1d1a4b6..895b10b9 100644
--- a/src/python/txtai/pipeline/text/translation.py
+++ b/src/python/txtai/pipeline/text/translation.py
@@ -14,6 +14,7 @@ from huggingface_hub import cached_download
from huggingface_hub.hf_api import HfApi
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+from ...models import Models
from ..hfmodel import HFModel
@@ -154,6 +155,7 @@ class Translation(HFModel):
model.to(self.device)
indices = None
+ maxlength = Models.maxlength(model, tokenizer)
with self.context():
if hasattr(tokenizer, "lang_code_to_id"):
@@ -163,10 +165,10 @@ class Translation(HFModel):
tokenizer.src_lang = source
tokens, indices = self.tokenize(tokenizer, texts)
- translated = model.generate(**tokens, forced_bos_token_id=tokenizer.lang_code_to_id[target], max_length=model.config.max_length)
+ translated = model.generate(**tokens, forced_bos_token_id=tokenizer.lang_code_to_id[target], max_length=maxlength)
else:
tokens, indices = self.tokenize(tokenizer, texts)
- translated = model.generate(**tokens, max_length=model.config.max_length)
+ translated = model.generate(**tokens, max_length=maxlength)
# Decode translations
translated = tokenizer.batch_decode(translated, skip_special_tokens=True)
diff --git a/test/python/testpipeline/testsummary.py b/test/python/testpipeline/testsummary.py
index a73abdd5..f0561a02 100644
--- a/test/python/testpipeline/testsummary.py
+++ b/test/python/testpipeline/testsummary.py
@@ -36,13 +36,6 @@ class TestSummary(unittest.TestCase):
self.assertEqual(self.summary(self.text, minlength=15, maxlength=15), "txtai is an AI-powered search engine that")
- def testSummaryShort(self):
- """
- Test that summarization is skipped
- """
-
- self.assertEqual(self.summary("Text", maxlength=15), "Text")
-
def testSummaryBatch(self):
"""
Test batch summarization of text
@@ -50,3 +43,22 @@ class TestSummary(unittest.TestCase):
summaries = self.summary([self.text, self.text], maxlength=15)
self.assertEqual(len(summaries), 2)
+
+ def testSummaryNoLength(self):
+ """
+ Test summary with no max length set
+ """
+
+ self.assertEqual(
+ self.summary(self.text + self.text),
+ "search is the base of many applications. Once data starts to pile up, users want to be able to find it. "
+ + "Large-scale general language models are an exciting new capability allowing us to add amazing functionality quickly "
+ + "with limited compute and people.",
+ )
+
+ def testSummaryShort(self):
+ """
+ Test that summarization is skipped
+ """
+
+ self.assertEqual(self.summary("Text", maxlength=15), "Text")