💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Remove unnecessary new test file and consolidate test into existing class
davidmezzetti45d334f4cea9
diff --git a/test/python/testpipeline/testtrain/testtexts.py b/test/python/testpipeline/testtrain/testtexts.pydeleted file mode 100644index b2cf29d2..00000000--- a/test/python/testpipeline/testtrain/testtexts.py+++ /dev/null@@ -1,39 +0,0 @@-"""-Texts module tests-"""--import unittest--from transformers import AutoTokenizer--from txtai.data import Texts---class TestTexts(unittest.TestCase):- """- Texts tests.- """-- @classmethod- def setUpClass(cls):- """- Create tokenizer.- """-- cls.tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2")-- def testPack(self):- """- Test packing rows into chunks up to maxlength- """-- rows = ["a b c d", "e f g h", "i j k l"]- length = len(self.tokenizer(rows[0])["input_ids"])-- # Two rows fill maxlength exactly and are packed into one chunk- packed = Texts(self.tokenizer, None, length * 2, "pack").process({"text": list(rows)})- self.assertEqual([len(chunk) for chunk in packed["input_ids"]], [length * 2, length])-- # Rows are never split across chunks- packed = Texts(self.tokenizer, None, length * 2 - 1, "pack").process({"text": list(rows)})- self.assertEqual([len(chunk) for chunk in packed["input_ids"]], [length] * 3)diff --git a/test/python/testpipeline/testtrain/testtrainer.py b/test/python/testpipeline/testtrain/testtrainer.pyindex 279512e9..638f81ea 100644--- a/test/python/testpipeline/testtrain/testtrainer.py+++ b/test/python/testpipeline/testtrain/testtrainer.py@@ -15,7 +15,7 @@ import torchfrom transformers import AutoTokenizer, AutoModelForSequenceClassification-from txtai.data import Data, Questions as QuestionsData+from txtai.data import Data, Questions as QuestionsData, Textsfrom txtai.models import Lemur, Models, PoolingFactoryfrom txtai.pipeline import HFTrainer, Labels, LemurTrainer, Questions, Sequences@@ -691,6 +691,24 @@ class TestTrainer(unittest.TestCase):labels = Labels((model, tokenizer), dynamic=False)self.assertEqual(labels("cat")[0][0], 1)+ def testPack(self):+ """+ Test packing rows into chunks up to maxlength+ """++ tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2")++ rows = ["a b c d", "e f g h", "i j k l"]+ length = len(tokenizer(rows[0])["input_ids"])++ # Two rows fill maxlength exactly and are packed into one chunk+ packed = Texts(tokenizer, None, length * 2, "pack").process({"text": list(rows)})+ self.assertEqual([len(chunk) for chunk in packed["input_ids"]], [length * 2, length])++ # Rows are never split across chunks+ packed = Texts(tokenizer, None, length * 2 - 1, "pack").process({"text": list(rows)})+ self.assertEqual([len(chunk) for chunk in packed["input_ids"]], [length] * 3)+def testPEFT(self):"""Test training a model with causal language modeling and PEFT