DSPy: The framework for programming—not prompting—language models
Python
Add Support for Multi-GPU Inference
Rick Battle210d8d1f3042
diff --git a/dsp/modules/hf.py b/dsp/modules/hf.pyindex e7942af2c..a26d92fb1 100644--- a/dsp/modules/hf.py+++ b/dsp/modules/hf.py@@ -30,10 +30,10 @@ class HFModel(LM):self.is_client = is_clientif not self.is_client:self.model = AutoModelForSeq2SeqLM.from_pretrained(- model if checkpoint is None else checkpoint- ).to("cuda")+ model if checkpoint is None else checkpoint,+ device_map="auto"+ )self.tokenizer = AutoTokenizer.from_pretrained(model)- self.tokenizer.add_tokens(AddedToken("\n", normalized=False))self.history = []