💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Update HF LLM to reduce noisy warnings, closes #752
davidmezzettibd94ebab75c8
diff --git a/src/python/txtai/pipeline/llm/huggingface.py b/src/python/txtai/pipeline/llm/huggingface.pyindex d3eee164..53c7148e 100644--- a/src/python/txtai/pipeline/llm/huggingface.py+++ b/src/python/txtai/pipeline/llm/huggingface.py@@ -68,13 +68,7 @@ class HFLLM(HFPipeline):texts = [f"{prefix}{x}" for x in texts]# Combine all keyword arguments- kwargs = {- **{- "max_length": maxlength,- "num_workers": workers,- },- **kwargs,- }+ kwargs = self.parameters(maxlength, workers, **kwargs)# Stream responseif stream:@@ -88,6 +82,41 @@ class HFLLM(HFPipeline):return results[0] if isinstance(text, str) else results+ def parameters(self, maxlength, workers, **kwargs):+ """+ Builds a combined map of parameters.++ Args:+ maxlength: maximum sequence length+ workers: number of concurrent workers to use for processing data, defaults to None+ kwargs: additional generation keyword arguments++ Returns:+ dict of parameters+ """++ # Default parameters+ defaults = {+ "max_length": maxlength,+ "num_workers": workers,+ }++ # Add pad token if it's missing from model config+ model = self.pipeline.model+ if not model.config.pad_token_id:+ tokenid = model.config.eos_token_id+ tokenid = tokenid[0] if isinstance(tokenid, list) else tokenid++ # Set pad_token_id parameter+ defaults["pad_token_id"] = tokenid++ # Update tokenizer for batching+ if "batch_size" in kwargs and self.pipeline.tokenizer.pad_token_id is None:+ self.pipeline.tokenizer.pad_token_id = tokenid+ self.pipeline.tokenizer.padding_side = "left"++ return {**defaults, **kwargs}+def extract(self, result):"""Extracts generated text from a pipeline result.diff --git a/test/python/testpipeline/testllm.py b/test/python/testpipeline/testllm.pyindex 54a6b4e9..0a1e878d 100644--- a/test/python/testpipeline/testllm.py+++ b/test/python/testpipeline/testllm.py@@ -30,6 +30,14 @@ class TestLLM(unittest.TestCase):model = LLM("hf-internal-testing/tiny-random-gpt2", task="language-generation", torch_dtype=torch.float32)self.assertIsNotNone(model(start))+ def testBatchSize(self):+ """+ Test batch size+ """++ model = LLM("sshleifer/tiny-gpt2")+ self.assertIsNotNone(model(["Hello, how are"] * 2, batch_size=2))+def testCustom(self):"""Test custom LLM framework