Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

perf: lazy-import torch and tiktoken in embedding_compute / chat (#323)

Abic33837b2d380
diff --git a/packages/leann-core/src/leann/chat.py b/packages/leann-core/src/leann/chat.py
index 52426d1..a36383f 100644
--- a/packages/leann-core/src/leann/chat.py
+++ b/packages/leann-core/src/leann/chat.py
@@ -10,8 +10,6 @@ import os
from abc import ABC, abstractmethod
from typing import Any, Optional, cast
-import torch
-
from .settings import (
resolve_anthropic_api_key,
resolve_anthropic_base_url,
@@ -720,6 +718,8 @@ class HFChat(LLMInterface):
logger.info(f"Generating with HuggingFace model, config: {generation_config}")
# Generate
+ import torch
+
with torch.no_grad():
outputs = self.model.generate(**inputs, **generation_config)
diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.py
index 34c02e5..7e51d4f 100644
--- a/packages/leann-core/src/leann/embedding_compute.py
+++ b/packages/leann-core/src/leann/embedding_compute.py
@@ -12,11 +12,13 @@ import time
from typing import Any, Optional, Protocol, cast
import numpy as np
-import tiktoken
-import torch
from .settings import resolve_ollama_host, resolve_openai_api_key, resolve_openai_base_url
+# torch and tiktoken are imported lazily inside the functions that use them, so
+# `import leann` (e.g. for MCP search over an existing index, BM25-only flows,
+# or non-embedding utilities) doesn't pull torch's ~1 GB of state into memory.
+
# Set up logger with proper level
logger = logging.getLogger(__name__)
LOG_LEVEL = os.getenv("LEANN_LOG_LEVEL", "WARNING").upper()
@@ -145,6 +147,8 @@ def truncate_to_token_limit(texts: list[str], token_limit: int) -> list[str]:
if not texts:
return []
+ import tiktoken
+
# Use tiktoken with cl100k_base encoding
enc = tiktoken.get_encoding("cl100k_base")
@@ -419,6 +423,8 @@ def compute_embeddings_sentence_transformers(
is_build: Whether this is a build operation (shows progress bar)
adaptive_optimization: Whether to use adaptive optimization based on batch size
"""
+ import torch
+
outer_start_time = time.time()
# Handle empty input
if not texts: