DSPy: The framework for programming—not prompting—language models
Python
Merge remote-tracking branch 'upstream/main' into model-kwargs-hf
Ben Steelb249146ff22b
diff --cc dsp/modules/hf.pyindex 13c7ad6d4,7407b3f47..3eae1bc00--- a/dsp/modules/hf.py+++ b/dsp/modules/hf.py@@@ -29,38 -28,62 +28,65 @@@ def openai_to_hf(**kwargs)class HFModel(LM):- def __init__(self, model: str, checkpoint: Optional[str] = None, is_client: bool = False,- hf_device_map: Literal["auto", "balanced", "balanced_low_0", "sequential"] = "auto",- model_kwargs: Optional[dict] = {}):+ def __init__(+ self,+ model: str,+ checkpoint: Optional[str] = None,+ is_client: bool = False,+ hf_device_map: Literal[+ "auto",+ "balanced",+ "balanced_low_0",+ "sequential",+ ] = "auto",+ token: Optional[str] = None,++ model_kwargs: Optional[dict] = {},+ ):"""wrapper for Hugging Face modelsArgs:model (str): HF model identifier to load and usecheckpoint (str, optional): load specific checkpoints of the model. Defaults to None.is_client (bool, optional): whether to access models via client. Defaults to False.- hf_device_map (str, optional): HF config strategy to load the model.+ hf_device_map (str, optional): HF config strategy to load the model.Recommeded to use "auto", which will help loading large models using accelerate. Defaults to "auto".- model_kwargs (dict, optional): additional kwargs to pass to the model. Defaults to None.++ model_kwargs (dict, optional): additional kwargs to pass to the model constructor. Defaults to empty dict."""- try:- from transformers import AutoModelForSeq2SeqLM, AutoModelForCausalLM, AutoTokenizer, AutoConfig- import torch- except ImportError as exc:- raise ModuleNotFoundError(- "You need to install Hugging Face transformers library to use HF models."- ) from exc+super().__init__(model)self.provider = "hf"self.is_client = is_clientself.device_map = hf_device_map- self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")++ hf_autoconfig_kwargs = dict(token=token or os.environ.get("HF_TOKEN"))+ hf_autotokenizer_kwargs = hf_autoconfig_kwargs.copy()+ hf_automodel_kwargs = hf_autoconfig_kwargs.copy()++ hf_automodel_kwargs.update(model_kwargs)if not self.is_client:try:- architecture = AutoConfig.from_pretrained(model).__dict__["architectures"][0]- self.encoder_decoder_model = ("ConditionalGeneration" in architecture) or ("T5WithLMHeadModel" in architecture)+ import torch+ from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoTokenizer+ except ImportError as exc:+ raise ModuleNotFoundError(+ "You need to install Hugging Face transformers library to use HF models.",+ ) from exc+ self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")+ try:+ architecture = AutoConfig.from_pretrained(+ model,+ **hf_autoconfig_kwargs,+ ).__dict__["architectures"][0]+ self.encoder_decoder_model = ("ConditionalGeneration" in architecture) or (+ "T5WithLMHeadModel" in architecture+ )self.decoder_only_model = ("CausalLM" in architecture) or ("GPT2LMHeadModel" in architecture)- assert self.encoder_decoder_model or self.decoder_only_model, f"Unknown HuggingFace model class: {model}"- self.tokenizer = AutoTokenizer.from_pretrained(model if checkpoint is None else checkpoint)+ assert (+ self.encoder_decoder_model or self.decoder_only_model+ ), f"Unknown HuggingFace model class: {model}"+ self.tokenizer = AutoTokenizer.from_pretrained(+ model if checkpoint is None else checkpoint,+ **hf_autotokenizer_kwargs,+ )self.rationale = TrueAutoModelClass = AutoModelForSeq2SeqLM if self.encoder_decoder_model else AutoModelForCausalLM