Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

feat: add GPU device selection via environment variables

Andy Lee76193ace3d55
diff --git a/README.md b/README.md
index d338bdd..c503923 100755
--- a/README.md
+++ b/README.md
@@ -356,6 +356,10 @@ LEANN provides flexible parameters for embedding models, search strategies, and
All RAG examples share these common parameters. **Interactive mode** is available in all examples - simply run without `--query` to start a continuous Q&A session where you can ask multiple questions. Type 'quit' to exit.
```bash
+# Environment Variables (GPU Device Selection)
+LEANN_EMBEDDING_DEVICE # GPU for embedding model (e.g., cuda:0, cuda:1, cpu)
+LEANN_LLM_DEVICE # GPU for HFChat LLM (e.g., cuda:1, or "cuda" for multi-GPU auto)
+
# Core Parameters (General preprocessing for all examples)
--index-dir DIR # Directory to store the index (default: current directory)
--query "YOUR QUESTION" # Single query mode. Omit for interactive chat (type 'quit' to exit), and now you can play with your index interactively
diff --git a/packages/leann-core/src/leann/chat.py b/packages/leann-core/src/leann/chat.py
index 6b6e042..72e414c 100644
--- a/packages/leann-core/src/leann/chat.py
+++ b/packages/leann-core/src/leann/chat.py
@@ -589,8 +589,12 @@ class HFChat(LLMInterface):
"The 'transformers' and 'torch' libraries are required for Hugging Face models. Please install them with 'pip install transformers torch'."
)
- # Auto-detect device
- if torch.cuda.is_available():
+ # Auto-detect device (check environment variable first)
+ env_device = os.getenv("LEANN_LLM_DEVICE")
+ if env_device:
+ self.device = env_device
+ logger.info(f"Using device from LEANN_LLM_DEVICE: {self.device}")
+ elif torch.cuda.is_available():
self.device = "cuda"
logger.info("CUDA is available. Using GPU.")
elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
@@ -618,10 +622,20 @@ class HFChat(LLMInterface):
)
logger.info(f"Loading model {model_name}...")
+ # Determine device_map based on device setting
+ if self.device == "cpu":
+ device_map = None
+ elif self.device.startswith("cuda:"):
+ # Specific GPU requested, use it exclusively
+ device_map = {"": self.device}
+ else:
+ # Auto mode: let HuggingFace distribute across available GPUs
+ device_map = "auto"
+
self.model = AutoModelForCausalLM.from_pretrained(
model_name,
torch_dtype=torch.float16 if self.device != "cpu" else torch.float32,
- device_map="auto" if self.device != "cpu" else None,
+ device_map=device_map,
trust_remote_code=self.trust_remote_code,
)
logger.info(f"Successfully loaded {model_name}")
diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.py
index eb2a1be..fc4fe58 100644
--- a/packages/leann-core/src/leann/embedding_compute.py
+++ b/packages/leann-core/src/leann/embedding_compute.py
@@ -402,7 +402,12 @@ def compute_embeddings_sentence_transformers(
# Auto-detect device
if device == "auto":
- if torch.cuda.is_available():
+ # Check environment variable first
+ env_device = os.getenv("LEANN_EMBEDDING_DEVICE")
+ if env_device:
+ device = env_device
+ logger.info(f"Using device from LEANN_EMBEDDING_DEVICE: {device}")
+ elif torch.cuda.is_available():
device = "cuda"
elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
device = "mps"