💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Improve default max length logic, closes #364
davidmezzetticd43c47fc9c6
diff --git a/src/python/txtai/models/models.py b/src/python/txtai/models/models.pyindex ea0358ea..c43a772c 100644--- a/src/python/txtai/models/models.py+++ b/src/python/txtai/models/models.py@@ -24,8 +24,8 @@ class Models:with older tokenizers.Args:- config: Transformers config- tokenizer: Transformers tokenizer+ config: transformers config+ tokenizer: transformers tokenizer"""# Unpack nested config, handles passing model directly@@ -40,6 +40,27 @@ class Models:):tokenizer.model_max_length = config.max_position_embeddings+ @staticmethod+ def maxlength(config, tokenizer):+ """+ Gets the best max length to use for generate calls. This method will return config.max_length if it's set. Otherwise, it will return+ tokenizer.model_max_length.++ Args:+ config: transformers config+ tokenizer: transformers tokenizer+ """++ # Unpack nested config, handles passing model directly+ if hasattr(config, "config"):+ config = config.config++ # Get non-defaulted fields+ keys = config.to_diff_dict()++ # Use config.max_length if not set to default value, else use tokenizer.model_max_length if available+ return config.max_length if "max_length" in keys or not hasattr(tokenizer, "model_max_length") else tokenizer.model_max_length+@staticmethoddef deviceid(gpu):"""diff --git a/src/python/txtai/pipeline/hfpipeline.py b/src/python/txtai/pipeline/hfpipeline.pyindex 171a8fec..3794521d 100644--- a/src/python/txtai/pipeline/hfpipeline.py+++ b/src/python/txtai/pipeline/hfpipeline.py@@ -50,3 +50,13 @@ class HFPipeline(Tensors):# Detect unbounded tokenizer typically found in older modelsModels.checklength(self.pipeline.model, self.pipeline.tokenizer)++ def maxlength(self):+ """+ Gets the max length to use for generate calls.++ Returns:+ max length+ """++ return Models.maxlength(self.pipeline.model, self.pipeline.tokenizer)diff --git a/src/python/txtai/pipeline/text/summary.py b/src/python/txtai/pipeline/text/summary.pyindex fe03e7eb..3b0bfd1d 100644--- a/src/python/txtai/pipeline/text/summary.py+++ b/src/python/txtai/pipeline/text/summary.py@@ -33,7 +33,7 @@ class Summary(HFPipeline):"""# Validate text length greater than max length- check = maxlength if maxlength else self.pipeline.model.config.max_length+ check = maxlength if maxlength else self.maxlength()# Skip text shorter than max lengthtexts = text if isinstance(text, list) else [text]diff --git a/src/python/txtai/pipeline/text/translation.py b/src/python/txtai/pipeline/text/translation.pyindex e1d1a4b6..895b10b9 100644--- a/src/python/txtai/pipeline/text/translation.py+++ b/src/python/txtai/pipeline/text/translation.py@@ -14,6 +14,7 @@ from huggingface_hub import cached_downloadfrom huggingface_hub.hf_api import HfApifrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer+from ...models import Modelsfrom ..hfmodel import HFModel@@ -154,6 +155,7 @@ class Translation(HFModel):model.to(self.device)indices = None+ maxlength = Models.maxlength(model, tokenizer)with self.context():if hasattr(tokenizer, "lang_code_to_id"):@@ -163,10 +165,10 @@ class Translation(HFModel):tokenizer.src_lang = sourcetokens, indices = self.tokenize(tokenizer, texts)- translated = model.generate(**tokens, forced_bos_token_id=tokenizer.lang_code_to_id[target], max_length=model.config.max_length)+ translated = model.generate(**tokens, forced_bos_token_id=tokenizer.lang_code_to_id[target], max_length=maxlength)else:tokens, indices = self.tokenize(tokenizer, texts)- translated = model.generate(**tokens, max_length=model.config.max_length)+ translated = model.generate(**tokens, max_length=maxlength)# Decode translationstranslated = tokenizer.batch_decode(translated, skip_special_tokens=True)diff --git a/test/python/testpipeline/testsummary.py b/test/python/testpipeline/testsummary.pyindex a73abdd5..f0561a02 100644--- a/test/python/testpipeline/testsummary.py+++ b/test/python/testpipeline/testsummary.py@@ -36,13 +36,6 @@ class TestSummary(unittest.TestCase):self.assertEqual(self.summary(self.text, minlength=15, maxlength=15), "txtai is an AI-powered search engine that")- def testSummaryShort(self):- """- Test that summarization is skipped- """-- self.assertEqual(self.summary("Text", maxlength=15), "Text")-def testSummaryBatch(self):"""Test batch summarization of text@@ -50,3 +43,22 @@ class TestSummary(unittest.TestCase):summaries = self.summary([self.text, self.text], maxlength=15)self.assertEqual(len(summaries), 2)++ def testSummaryNoLength(self):+ """+ Test summary with no max length set+ """++ self.assertEqual(+ self.summary(self.text + self.text),+ "search is the base of many applications. Once data starts to pile up, users want to be able to find it. "+ + "Large-scale general language models are an exciting new capability allowing us to add amazing functionality quickly "+ + "with limited compute and people.",+ )++ def testSummaryShort(self):+ """+ Test that summarization is skipped+ """++ self.assertEqual(self.summary("Text", maxlength=15), "Text")