[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
perf: lazy-import torch and tiktoken in embedding_compute / chat (#323)
Abic33837b2d380
diff --git a/packages/leann-core/src/leann/chat.py b/packages/leann-core/src/leann/chat.pyindex 52426d1..a36383f 100644--- a/packages/leann-core/src/leann/chat.py+++ b/packages/leann-core/src/leann/chat.py@@ -10,8 +10,6 @@ import osfrom abc import ABC, abstractmethodfrom typing import Any, Optional, cast-import torch-from .settings import (resolve_anthropic_api_key,resolve_anthropic_base_url,@@ -720,6 +718,8 @@ class HFChat(LLMInterface):logger.info(f"Generating with HuggingFace model, config: {generation_config}")# Generate+ import torch+with torch.no_grad():outputs = self.model.generate(**inputs, **generation_config)diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.pyindex 34c02e5..7e51d4f 100644--- a/packages/leann-core/src/leann/embedding_compute.py+++ b/packages/leann-core/src/leann/embedding_compute.py@@ -12,11 +12,13 @@ import timefrom typing import Any, Optional, Protocol, castimport numpy as np-import tiktoken-import torchfrom .settings import resolve_ollama_host, resolve_openai_api_key, resolve_openai_base_url+# torch and tiktoken are imported lazily inside the functions that use them, so+# `import leann` (e.g. for MCP search over an existing index, BM25-only flows,+# or non-embedding utilities) doesn't pull torch's ~1 GB of state into memory.+# Set up logger with proper levellogger = logging.getLogger(__name__)LOG_LEVEL = os.getenv("LEANN_LOG_LEVEL", "WARNING").upper()@@ -145,6 +147,8 @@ def truncate_to_token_limit(texts: list[str], token_limit: int) -> list[str]:if not texts:return []+ import tiktoken+# Use tiktoken with cl100k_base encodingenc = tiktoken.get_encoding("cl100k_base")@@ -419,6 +423,8 @@ def compute_embeddings_sentence_transformers(is_build: Whether this is a build operation (shows progress bar)adaptive_optimization: Whether to use adaptive optimization based on batch size"""+ import torch+outer_start_time = time.time()# Handle empty inputif not texts: