Skip to content

The agent engineering platform.

feat(huggingface): use torch.accelerator for device-agnostic device count detection (#39640)

wanhakim443154df9db5
diff --git a/libs/partners/huggingface/langchain_huggingface/llms/huggingface_pipeline.py b/libs/partners/huggingface/langchain_huggingface/llms/huggingface_pipeline.py
index a354191c1f..e27a56c5b1 100644
--- a/libs/partners/huggingface/langchain_huggingface/llms/huggingface_pipeline.py
+++ b/libs/partners/huggingface/langchain_huggingface/llms/huggingface_pipeline.py
@@ -244,22 +244,28 @@ class HuggingFacePipeline(BaseLLM):
):
import torch
- cuda_device_count = torch.cuda.device_count()
- if device < -1 or (device >= cuda_device_count):
+ # `torch.accelerator` (added in PyTorch 2.6) is device-agnostic and
+ # detects CUDA, XPU, and other accelerator backends.
+ if hasattr(torch, "accelerator"):
+ accelerator_device_count = torch.accelerator.device_count()
+ else:
+ accelerator_device_count = torch.cuda.device_count()
+ if device < -1 or (device >= accelerator_device_count):
msg = (
f"Got device=={device}, "
- f"device is required to be within [-1, {cuda_device_count})"
+ f"device is required to be within [-1, {accelerator_device_count})"
)
raise ValueError(msg)
if device_map is not None and device < 0:
device = None
- if device is not None and device < 0 and cuda_device_count > 0:
+ if device is not None and device < 0 and accelerator_device_count > 0:
logger.warning(
- "Device has %d GPUs available. "
+ "Device has %d accelerator(s) available. "
"Provide device={deviceId} to `from_model_id` to use available"
- "GPUs for execution. deviceId is -1 (default) for CPU and "
- "can be a positive integer associated with CUDA device id.",
- cuda_device_count,
+ "accelerators for execution. deviceId is -1 (default) for CPU "
+ "and can be a positive integer associated with the accelerator "
+ "device id.",
+ accelerator_device_count,
)
if device is not None and device_map is not None and backend == "openvino":
logger.warning("Please set device for OpenVINO through: `model_kwargs`")
diff --git a/libs/partners/huggingface/tests/integration_tests/test_llms.py b/libs/partners/huggingface/tests/integration_tests/test_llms.py
index 147c6aefe6..e341c0b9bc 100644
--- a/libs/partners/huggingface/tests/integration_tests/test_llms.py
+++ b/libs/partners/huggingface/tests/integration_tests/test_llms.py
@@ -1,5 +1,8 @@
+import os
from collections.abc import Generator
+import pytest
+
from langchain_huggingface.llms import HuggingFacePipeline
@@ -18,3 +21,32 @@ def test_huggingface_pipeline_streaming() -> None:
assert isinstance(chunk, str)
stream_results_string = chunk
assert len(stream_results_string.strip()) > 0
+
+
+ os.getenv("HF_TEST_DEVICE") is None,
+ reason="Set HF_TEST_DEVICE=<index> on an accelerator (XPU/CUDA) box to run.",
+)
+def test_from_model_id_runs_on_accelerator() -> None:
+ """Load a model on an accelerator and confirm placement is not CPU."""
+ device = int(os.environ["HF_TEST_DEVICE"])
+ model_id = os.getenv("HF_TEST_MODEL_ID", "openai-community/gpt2")
+
+ llm = HuggingFacePipeline.from_model_id(
+ model_id=model_id,
+ task="text-generation",
+ device=device,
+ pipeline_kwargs={"max_new_tokens": 10},
+ )
+
+ result = llm.invoke("Hello")
+ assert isinstance(result, str)
+ assert result.strip()
+
+ # Confirm the model is actually on the accelerator, not silently on CPU —
+ # a green run with a CPU fallback would prove nothing about XPU coverage.
+ model_device = llm.pipeline.model.device
+ assert model_device.type != "cpu", (
+ f"Expected model on accelerator, but it loaded on {model_device}. "
+ "Check torch device ordering, HF_TEST_BACKEND, or device index."
+ )
diff --git a/libs/partners/huggingface/tests/unit_tests/test_huggingface_pipeline.py b/libs/partners/huggingface/tests/unit_tests/test_huggingface_pipeline.py
index 00a8a60673..224e6eb3e8 100644
--- a/libs/partners/huggingface/tests/unit_tests/test_huggingface_pipeline.py
+++ b/libs/partners/huggingface/tests/unit_tests/test_huggingface_pipeline.py
@@ -1,5 +1,7 @@
from unittest.mock import MagicMock, patch
+import pytest
+
from langchain_huggingface import HuggingFacePipeline
DEFAULT_MODEL_ID = "gpt2"
@@ -45,3 +47,117 @@ def test_initialization_with_from_model_id(
)
assert llm.model_id == "mock-model-id"
+
+
+@patch("transformers.AutoTokenizer.from_pretrained")
+@patch("transformers.AutoModelForCausalLM.from_pretrained")
+@patch("transformers.pipeline")
+def test_from_model_id_uses_torch_accelerator_device_count(
+ mock_pipeline: MagicMock, mock_model: MagicMock, mock_tokenizer: MagicMock
+) -> None:
+ """Test `from_model_id` prefers the device-agnostic `torch.accelerator` API.
+
+ Available in `torch>=2.6`, this is preferred over `torch.cuda` so that
+ non-CUDA accelerators, such as XPU, are recognized.
+ """
+ mock_tokenizer.return_value = MagicMock(pad_token_id=0)
+ mock_model.return_value = MagicMock(
+ is_loaded_in_4bit=False, is_loaded_in_8bit=False
+ )
+
+ mock_pipe = MagicMock()
+ mock_pipe.task = "text-generation"
+ mock_pipe.model = mock_model.return_value
+ mock_pipeline.return_value = mock_pipe
+
+ mock_accelerator = MagicMock()
+ mock_accelerator.device_count.return_value = 2
+
+ with (
+ patch("torch.accelerator", mock_accelerator),
+ patch("torch.cuda.device_count") as mock_cuda_device_count,
+ ):
+ HuggingFacePipeline.from_model_id(
+ model_id="mock-model-id",
+ task="text-generation",
+ device=0,
+ )
+
+ mock_accelerator.device_count.assert_called_once()
+ mock_cuda_device_count.assert_not_called()
+
+
+@patch("transformers.AutoTokenizer.from_pretrained")
+@patch("transformers.AutoModelForCausalLM.from_pretrained")
+@patch("transformers.pipeline")
+def test_from_model_id_falls_back_to_torch_cuda_device_count(
+ mock_pipeline: MagicMock,
+ mock_model: MagicMock,
+ mock_tokenizer: MagicMock,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Test `from_model_id` falls back to `torch.cuda.device_count`.
+
+ This happens when the installed `torch` version predates the
+ `torch.accelerator` API.
+ """
+ mock_tokenizer.return_value = MagicMock(pad_token_id=0)
+ mock_model.return_value = MagicMock(
+ is_loaded_in_4bit=False, is_loaded_in_8bit=False
+ )
+
+ mock_pipe = MagicMock()
+ mock_pipe.task = "text-generation"
+ mock_pipe.model = mock_model.return_value
+ mock_pipeline.return_value = mock_pipe
+
+ import torch
+
+ # Simulate a `torch` version predating the `torch.accelerator` API.
+ monkeypatch.delattr(torch, "accelerator", raising=False)
+
+ with patch("torch.cuda.device_count", return_value=1) as mock_cuda_device_count:
+ HuggingFacePipeline.from_model_id(
+ model_id="mock-model-id",
+ task="text-generation",
+ device=0,
+ )
+
+ mock_cuda_device_count.assert_called_once()
+
+
+@patch("transformers.AutoTokenizer.from_pretrained")
+@patch("transformers.AutoModelForCausalLM.from_pretrained")
+@patch("transformers.pipeline")
+def test_from_model_id_raises_for_out_of_range_device(
+ mock_pipeline: MagicMock, mock_model: MagicMock, mock_tokenizer: MagicMock
+) -> None:
+ """Test `from_model_id` raises when `device` is out of range.
+
+ This is the range reported by the accelerator device count.
+ """
+ mock_tokenizer.return_value = MagicMock(pad_token_id=0)
+ mock_model.return_value = MagicMock(
+ is_loaded_in_4bit=False, is_loaded_in_8bit=False
+ )
+
+ mock_pipe = MagicMock()
+ mock_pipe.task = "text-generation"
+ mock_pipe.model = mock_model.return_value
+ mock_pipeline.return_value = mock_pipe
+
+ mock_accelerator = MagicMock()
+ mock_accelerator.device_count.return_value = 1
+
+ with patch("torch.accelerator", mock_accelerator):
+ try:
+ HuggingFacePipeline.from_model_id(
+ model_id="mock-model-id",
+ task="text-generation",
+ device=5,
+ )
+ except ValueError as exc:
+ assert "device is required to be within" in str(exc)
+ else:
+ msg = "Expected ValueError to be raised for out-of-range device"
+ raise AssertionError(msg)