Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Merge remote-tracking branch 'upstream/main' into model-kwargs-hf

Ben Steelb249146ff22b
diff --cc dsp/modules/hf.py
index 13c7ad6d4,7407b3f47..3eae1bc00
--- a/dsp/modules/hf.py
+++ b/dsp/modules/hf.py
@@@ -29,38 -28,62 +28,65 @@@ def openai_to_hf(**kwargs)
class HFModel(LM):
- def __init__(self, model: str, checkpoint: Optional[str] = None, is_client: bool = False,
- hf_device_map: Literal["auto", "balanced", "balanced_low_0", "sequential"] = "auto",
- model_kwargs: Optional[dict] = {}):
+ def __init__(
+ self,
+ model: str,
+ checkpoint: Optional[str] = None,
+ is_client: bool = False,
+ hf_device_map: Literal[
+ "auto",
+ "balanced",
+ "balanced_low_0",
+ "sequential",
+ ] = "auto",
+ token: Optional[str] = None,
++ model_kwargs: Optional[dict] = {},
+ ):
"""wrapper for Hugging Face models
Args:
model (str): HF model identifier to load and use
checkpoint (str, optional): load specific checkpoints of the model. Defaults to None.
is_client (bool, optional): whether to access models via client. Defaults to False.
- hf_device_map (str, optional): HF config strategy to load the model.
+ hf_device_map (str, optional): HF config strategy to load the model.
Recommeded to use "auto", which will help loading large models using accelerate. Defaults to "auto".
- model_kwargs (dict, optional): additional kwargs to pass to the model. Defaults to None.
++ model_kwargs (dict, optional): additional kwargs to pass to the model constructor. Defaults to empty dict.
"""
- try:
- from transformers import AutoModelForSeq2SeqLM, AutoModelForCausalLM, AutoTokenizer, AutoConfig
- import torch
- except ImportError as exc:
- raise ModuleNotFoundError(
- "You need to install Hugging Face transformers library to use HF models."
- ) from exc
+
super().__init__(model)
self.provider = "hf"
self.is_client = is_client
self.device_map = hf_device_map
- self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+ hf_autoconfig_kwargs = dict(token=token or os.environ.get("HF_TOKEN"))
+ hf_autotokenizer_kwargs = hf_autoconfig_kwargs.copy()
+ hf_automodel_kwargs = hf_autoconfig_kwargs.copy()
++ hf_automodel_kwargs.update(model_kwargs)
if not self.is_client:
try:
- architecture = AutoConfig.from_pretrained(model).__dict__["architectures"][0]
- self.encoder_decoder_model = ("ConditionalGeneration" in architecture) or ("T5WithLMHeadModel" in architecture)
+ import torch
+ from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoTokenizer
+ except ImportError as exc:
+ raise ModuleNotFoundError(
+ "You need to install Hugging Face transformers library to use HF models.",
+ ) from exc
+ self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ try:
+ architecture = AutoConfig.from_pretrained(
+ model,
+ **hf_autoconfig_kwargs,
+ ).__dict__["architectures"][0]
+ self.encoder_decoder_model = ("ConditionalGeneration" in architecture) or (
+ "T5WithLMHeadModel" in architecture
+ )
self.decoder_only_model = ("CausalLM" in architecture) or ("GPT2LMHeadModel" in architecture)
- assert self.encoder_decoder_model or self.decoder_only_model, f"Unknown HuggingFace model class: {model}"
- self.tokenizer = AutoTokenizer.from_pretrained(model if checkpoint is None else checkpoint)
+ assert (
+ self.encoder_decoder_model or self.decoder_only_model
+ ), f"Unknown HuggingFace model class: {model}"
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ model if checkpoint is None else checkpoint,
+ **hf_autotokenizer_kwargs,
+ )
self.rationale = True
AutoModelClass = AutoModelForSeq2SeqLM if self.encoder_decoder_model else AutoModelForCausalLM