DSPy: The framework for programming—not prompting—language models
Python
minor device usage fix
Sri Vardhamanan158e494bec98
diff --git a/dsp/modules/hf.py b/dsp/modules/hf.pyindex a47884374..3d7fa2462 100644--- a/dsp/modules/hf.py+++ b/dsp/modules/hf.py@@ -35,6 +35,7 @@ class HFModel(LM):"""try:from transformers import AutoModelForSeq2SeqLM, AutoModelForCausalLM, AutoTokenizer+ import torchexcept ImportError as exc:raise ModuleNotFoundError("You need to install Hugging Face transformers library to use HF models."@@ -43,6 +44,7 @@ class HFModel(LM):self.provider = "hf"self.is_client = is_clientself.device_map = hf_device_map+ self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")if not self.is_client:try:self.model = AutoModelForSeq2SeqLM.from_pretrained(@@ -78,7 +80,7 @@ class HFModel(LM):assert not self.is_client# TODO: Add cachingkwargs = {**openai_to_hf(**self.kwargs), **openai_to_hf(**kwargs)}- inputs = self.tokenizer(prompt, return_tensors="pt")+ inputs = self.tokenizer(prompt, return_tensors="pt").to(self.device)outputs = self.model.generate(**inputs, **kwargs)if self.drop_prompt_from_output:input_length = inputs.input_ids.shape[1]