The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
feat(huggingface): use torch.accelerator for device-agnostic device count detection (#39640)
wanhakim443154df9db5
diff --git a/libs/partners/huggingface/langchain_huggingface/llms/huggingface_pipeline.py b/libs/partners/huggingface/langchain_huggingface/llms/huggingface_pipeline.pyindex a354191c1f..e27a56c5b1 100644--- a/libs/partners/huggingface/langchain_huggingface/llms/huggingface_pipeline.py+++ b/libs/partners/huggingface/langchain_huggingface/llms/huggingface_pipeline.py@@ -244,22 +244,28 @@ class HuggingFacePipeline(BaseLLM):):import torch- cuda_device_count = torch.cuda.device_count()- if device < -1 or (device >= cuda_device_count):+ # `torch.accelerator` (added in PyTorch 2.6) is device-agnostic and+ # detects CUDA, XPU, and other accelerator backends.+ if hasattr(torch, "accelerator"):+ accelerator_device_count = torch.accelerator.device_count()+ else:+ accelerator_device_count = torch.cuda.device_count()+ if device < -1 or (device >= accelerator_device_count):msg = (f"Got device=={device}, "- f"device is required to be within [-1, {cuda_device_count})"+ f"device is required to be within [-1, {accelerator_device_count})")raise ValueError(msg)if device_map is not None and device < 0:device = None- if device is not None and device < 0 and cuda_device_count > 0:+ if device is not None and device < 0 and accelerator_device_count > 0:logger.warning(- "Device has %d GPUs available. "+ "Device has %d accelerator(s) available. ""Provide device={deviceId} to `from_model_id` to use available"- "GPUs for execution. deviceId is -1 (default) for CPU and "- "can be a positive integer associated with CUDA device id.",- cuda_device_count,+ "accelerators for execution. deviceId is -1 (default) for CPU "+ "and can be a positive integer associated with the accelerator "+ "device id.",+ accelerator_device_count,)if device is not None and device_map is not None and backend == "openvino":logger.warning("Please set device for OpenVINO through: `model_kwargs`")diff --git a/libs/partners/huggingface/tests/integration_tests/test_llms.py b/libs/partners/huggingface/tests/integration_tests/test_llms.pyindex 147c6aefe6..e341c0b9bc 100644--- a/libs/partners/huggingface/tests/integration_tests/test_llms.py+++ b/libs/partners/huggingface/tests/integration_tests/test_llms.py@@ -1,5 +1,8 @@+import osfrom collections.abc import Generator+import pytest+from langchain_huggingface.llms import HuggingFacePipeline@@ -18,3 +21,32 @@ def test_huggingface_pipeline_streaming() -> None:assert isinstance(chunk, str)stream_results_string = chunkassert len(stream_results_string.strip()) > 0+++ os.getenv("HF_TEST_DEVICE") is None,+ reason="Set HF_TEST_DEVICE=<index> on an accelerator (XPU/CUDA) box to run.",+)+def test_from_model_id_runs_on_accelerator() -> None:+ """Load a model on an accelerator and confirm placement is not CPU."""+ device = int(os.environ["HF_TEST_DEVICE"])+ model_id = os.getenv("HF_TEST_MODEL_ID", "openai-community/gpt2")++ llm = HuggingFacePipeline.from_model_id(+ model_id=model_id,+ task="text-generation",+ device=device,+ pipeline_kwargs={"max_new_tokens": 10},+ )++ result = llm.invoke("Hello")+ assert isinstance(result, str)+ assert result.strip()++ # Confirm the model is actually on the accelerator, not silently on CPU —+ # a green run with a CPU fallback would prove nothing about XPU coverage.+ model_device = llm.pipeline.model.device+ assert model_device.type != "cpu", (+ f"Expected model on accelerator, but it loaded on {model_device}. "+ "Check torch device ordering, HF_TEST_BACKEND, or device index."+ )diff --git a/libs/partners/huggingface/tests/unit_tests/test_huggingface_pipeline.py b/libs/partners/huggingface/tests/unit_tests/test_huggingface_pipeline.pyindex 00a8a60673..224e6eb3e8 100644--- a/libs/partners/huggingface/tests/unit_tests/test_huggingface_pipeline.py+++ b/libs/partners/huggingface/tests/unit_tests/test_huggingface_pipeline.py@@ -1,5 +1,7 @@from unittest.mock import MagicMock, patch+import pytest+from langchain_huggingface import HuggingFacePipelineDEFAULT_MODEL_ID = "gpt2"@@ -45,3 +47,117 @@ def test_initialization_with_from_model_id()assert llm.model_id == "mock-model-id"+++@patch("transformers.AutoTokenizer.from_pretrained")+@patch("transformers.AutoModelForCausalLM.from_pretrained")+@patch("transformers.pipeline")+def test_from_model_id_uses_torch_accelerator_device_count(+ mock_pipeline: MagicMock, mock_model: MagicMock, mock_tokenizer: MagicMock+) -> None:+ """Test `from_model_id` prefers the device-agnostic `torch.accelerator` API.++ Available in `torch>=2.6`, this is preferred over `torch.cuda` so that+ non-CUDA accelerators, such as XPU, are recognized.+ """+ mock_tokenizer.return_value = MagicMock(pad_token_id=0)+ mock_model.return_value = MagicMock(+ is_loaded_in_4bit=False, is_loaded_in_8bit=False+ )++ mock_pipe = MagicMock()+ mock_pipe.task = "text-generation"+ mock_pipe.model = mock_model.return_value+ mock_pipeline.return_value = mock_pipe++ mock_accelerator = MagicMock()+ mock_accelerator.device_count.return_value = 2++ with (+ patch("torch.accelerator", mock_accelerator),+ patch("torch.cuda.device_count") as mock_cuda_device_count,+ ):+ HuggingFacePipeline.from_model_id(+ model_id="mock-model-id",+ task="text-generation",+ device=0,+ )++ mock_accelerator.device_count.assert_called_once()+ mock_cuda_device_count.assert_not_called()+++@patch("transformers.AutoTokenizer.from_pretrained")+@patch("transformers.AutoModelForCausalLM.from_pretrained")+@patch("transformers.pipeline")+def test_from_model_id_falls_back_to_torch_cuda_device_count(+ mock_pipeline: MagicMock,+ mock_model: MagicMock,+ mock_tokenizer: MagicMock,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Test `from_model_id` falls back to `torch.cuda.device_count`.++ This happens when the installed `torch` version predates the+ `torch.accelerator` API.+ """+ mock_tokenizer.return_value = MagicMock(pad_token_id=0)+ mock_model.return_value = MagicMock(+ is_loaded_in_4bit=False, is_loaded_in_8bit=False+ )++ mock_pipe = MagicMock()+ mock_pipe.task = "text-generation"+ mock_pipe.model = mock_model.return_value+ mock_pipeline.return_value = mock_pipe++ import torch++ # Simulate a `torch` version predating the `torch.accelerator` API.+ monkeypatch.delattr(torch, "accelerator", raising=False)++ with patch("torch.cuda.device_count", return_value=1) as mock_cuda_device_count:+ HuggingFacePipeline.from_model_id(+ model_id="mock-model-id",+ task="text-generation",+ device=0,+ )++ mock_cuda_device_count.assert_called_once()+++@patch("transformers.AutoTokenizer.from_pretrained")+@patch("transformers.AutoModelForCausalLM.from_pretrained")+@patch("transformers.pipeline")+def test_from_model_id_raises_for_out_of_range_device(+ mock_pipeline: MagicMock, mock_model: MagicMock, mock_tokenizer: MagicMock+) -> None:+ """Test `from_model_id` raises when `device` is out of range.++ This is the range reported by the accelerator device count.+ """+ mock_tokenizer.return_value = MagicMock(pad_token_id=0)+ mock_model.return_value = MagicMock(+ is_loaded_in_4bit=False, is_loaded_in_8bit=False+ )++ mock_pipe = MagicMock()+ mock_pipe.task = "text-generation"+ mock_pipe.model = mock_model.return_value+ mock_pipeline.return_value = mock_pipe++ mock_accelerator = MagicMock()+ mock_accelerator.device_count.return_value = 1++ with patch("torch.accelerator", mock_accelerator):+ try:+ HuggingFacePipeline.from_model_id(+ model_id="mock-model-id",+ task="text-generation",+ device=5,+ )+ except ValueError as exc:+ assert "device is required to be within" in str(exc)+ else:+ msg = "Expected ValueError to be raised for out-of-range device"+ raise AssertionError(msg)