Skip to content

The agent engineering platform.

langchain_huggingface: Fix multiple GPU usage bug in from_model_id function (#23628)

Kwan Kin Chan6d2a76ac05e1
diff --git a/libs/partners/huggingface/langchain_huggingface/llms/huggingface_pipeline.py b/libs/partners/huggingface/langchain_huggingface/llms/huggingface_pipeline.py
index 42754d4698..9c8d5f7145 100644
--- a/libs/partners/huggingface/langchain_huggingface/llms/huggingface_pipeline.py
+++ b/libs/partners/huggingface/langchain_huggingface/llms/huggingface_pipeline.py
@@ -74,7 +74,7 @@ class HuggingFacePipeline(BaseLLM):
model_id: str,
task: str,
backend: str = "default",
- device: Optional[int] = -1,
+ device: Optional[int] = None,
device_map: Optional[str] = None,
model_kwargs: Optional[dict] = None,
pipeline_kwargs: Optional[dict] = None,
@@ -96,7 +96,21 @@ class HuggingFacePipeline(BaseLLM):
"Please install it with `pip install transformers`."
)
- _model_kwargs = model_kwargs or {}
+ _model_kwargs = model_kwargs.copy() if model_kwargs else {}
+ if device_map is not None:
+ if device is not None:
+ raise ValueError(
+ "Both `device` and `device_map` are specified. "
+ "`device` will override `device_map`. "
+ "You will most likely encounter unexpected behavior."
+ "Please remove `device` and keep "
+ "`device_map`."
+ )
+
+ if "device_map" in _model_kwargs:
+ raise ValueError("`device_map` is already specified in `model_kwargs`.")
+
+ _model_kwargs["device_map"] = device_map
tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs)
try:
@@ -218,7 +232,6 @@ class HuggingFacePipeline(BaseLLM):
model=model,
tokenizer=tokenizer,
device=device,
- device_map=device_map,
batch_size=batch_size,
model_kwargs=_model_kwargs,
**_pipeline_kwargs,