Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Update HF LLM to reduce noisy warnings, closes #752

davidmezzettibd94ebab75c8
diff --git a/src/python/txtai/pipeline/llm/huggingface.py b/src/python/txtai/pipeline/llm/huggingface.py
index d3eee164..53c7148e 100644
--- a/src/python/txtai/pipeline/llm/huggingface.py
+++ b/src/python/txtai/pipeline/llm/huggingface.py
@@ -68,13 +68,7 @@ class HFLLM(HFPipeline):
texts = [f"{prefix}{x}" for x in texts]
# Combine all keyword arguments
- kwargs = {
- **{
- "max_length": maxlength,
- "num_workers": workers,
- },
- **kwargs,
- }
+ kwargs = self.parameters(maxlength, workers, **kwargs)
# Stream response
if stream:
@@ -88,6 +82,41 @@ class HFLLM(HFPipeline):
return results[0] if isinstance(text, str) else results
+ def parameters(self, maxlength, workers, **kwargs):
+ """
+ Builds a combined map of parameters.
+
+ Args:
+ maxlength: maximum sequence length
+ workers: number of concurrent workers to use for processing data, defaults to None
+ kwargs: additional generation keyword arguments
+
+ Returns:
+ dict of parameters
+ """
+
+ # Default parameters
+ defaults = {
+ "max_length": maxlength,
+ "num_workers": workers,
+ }
+
+ # Add pad token if it's missing from model config
+ model = self.pipeline.model
+ if not model.config.pad_token_id:
+ tokenid = model.config.eos_token_id
+ tokenid = tokenid[0] if isinstance(tokenid, list) else tokenid
+
+ # Set pad_token_id parameter
+ defaults["pad_token_id"] = tokenid
+
+ # Update tokenizer for batching
+ if "batch_size" in kwargs and self.pipeline.tokenizer.pad_token_id is None:
+ self.pipeline.tokenizer.pad_token_id = tokenid
+ self.pipeline.tokenizer.padding_side = "left"
+
+ return {**defaults, **kwargs}
+
def extract(self, result):
"""
Extracts generated text from a pipeline result.
diff --git a/test/python/testpipeline/testllm.py b/test/python/testpipeline/testllm.py
index 54a6b4e9..0a1e878d 100644
--- a/test/python/testpipeline/testllm.py
+++ b/test/python/testpipeline/testllm.py
@@ -30,6 +30,14 @@ class TestLLM(unittest.TestCase):
model = LLM("hf-internal-testing/tiny-random-gpt2", task="language-generation", torch_dtype=torch.float32)
self.assertIsNotNone(model(start))
+ def testBatchSize(self):
+ """
+ Test batch size
+ """
+
+ model = LLM("sshleifer/tiny-gpt2")
+ self.assertIsNotNone(model(["Hello, how are"] * 2, batch_size=2))
+
def testCustom(self):
"""
Test custom LLM framework