Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Add Support for Multi-GPU Inference

Rick Battle210d8d1f3042
diff --git a/dsp/modules/hf.py b/dsp/modules/hf.py
index e7942af2c..a26d92fb1 100644
--- a/dsp/modules/hf.py
+++ b/dsp/modules/hf.py
@@ -30,10 +30,10 @@ class HFModel(LM):
self.is_client = is_client
if not self.is_client:
self.model = AutoModelForSeq2SeqLM.from_pretrained(
- model if checkpoint is None else checkpoint
- ).to("cuda")
+ model if checkpoint is None else checkpoint,
+ device_map="auto"
+ )
self.tokenizer = AutoTokenizer.from_pretrained(model)
- self.tokenizer.add_tokens(AddedToken("\n", normalized=False))
self.history = []