[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
fix(core): refuse to load a custom encoder as a vanilla BertModel (#445)
Aakash Suresh0463b761916d
diff --git a/benchmarks/contextbench/scripts/batch_run_random.py b/benchmarks/contextbench/scripts/batch_run_random.pyindex 6371703..e5736ba 100644--- a/benchmarks/contextbench/scripts/batch_run_random.py+++ b/benchmarks/contextbench/scripts/batch_run_random.py@@ -5,9 +5,7 @@ import subprocessimport timefrom pathlib import Path-# run_single_task is used by the task-running block below, which ships-# commented out so this script selects and prefetches without executing.-from auto_run import prefetch_task_repositories, run_single_task # noqa: F401+from auto_run import prefetch_task_repositories, run_single_taskfrom datasets import load_datasetROOT = Path(__file__).resolve().parents[1]@@ -90,9 +88,8 @@ def main():else:print("⏭️ PREFETCH_REPOS=0; skipping prefetch step.")- # Incremented by the commented-out task-running block below.- success_count = 0 # noqa: F841- failure_count = 0 # noqa: F841+ success_count = 0+ failure_count = 0for i, task in enumerate(selected_tasks):instance_id = task["instance_id"]repo_url = task["repo_url"]@@ -101,45 +98,46 @@ def main():print(f"📦 [{i + 1}/{len(selected_tasks)}] Running: {instance_id}")print(f" repo: {repo_url} source: {task.get('source', '?')}")- # If want to run the task, uncomment the following code- # try:- # patch, elapsed, traj_data, usage = run_single_task(- # instance_id=instance_id,- # repo_url=repo_url,- # work_root=WORK_ROOT,- # mitm_script_path=str(MITM_SCRIPT),- # trace_dir=TRACE_DIR,- # model=MODEL,- # task=task,- # )-- # result_entry = {- # "instance_id": instance_id,- # "model_patch": patch if patch else "",- # "model_name_or_path": "claude-code-cli",- # "elapsed_seconds": round(elapsed, 1),- # "traj_data": traj_data,- # "token_usage": usage,- # }-- # with open(OUTPUT_FILE, "a", encoding="utf-8") as f:- # f.write(json.dumps(result_entry) + "\n")- # print(f"✅ Result saved for {instance_id}")- # success_count += 1-- # except Exception as e:- # print(f"❌ Error processing {instance_id}: {e}")- # failure_count += 1- # finally:- # cleanup_residuals()- # print("💤 Cooldown...")- # time.sleep(20)-- # print(- # f"\n✅ Finished {len(selected_tasks)} random tasks: "- # f"{success_count} succeeded, {failure_count} failed. "- # f"Results in {OUTPUT_FILE}"- # )+ try:+ patch, elapsed, agent_seconds, traj_data, usage = run_single_task(+ instance_id=instance_id,+ repo_url=repo_url,+ work_root=WORK_ROOT,+ mitm_script_path=str(MITM_SCRIPT),+ trace_dir=TRACE_DIR,+ model=MODEL,+ task=task,+ )++ result_entry = {+ "instance_id": instance_id,+ "model_patch": patch if patch else "",+ "model_name_or_path": "claude-code-cli",+ "elapsed_seconds": round(elapsed, 1),+ "latency_seconds": round(elapsed, 1),+ "agent_seconds": round(agent_seconds, 1) if agent_seconds is not None else None,+ "traj_data": traj_data,+ "token_usage": usage,+ }++ with open(OUTPUT_FILE, "a", encoding="utf-8") as f:+ f.write(json.dumps(result_entry) + "\n")+ print(f"✅ Result saved for {instance_id}")+ success_count += 1++ except Exception as e:+ print(f"❌ Error processing {instance_id}: {e}")+ failure_count += 1+ finally:+ cleanup_residuals()+ print("💤 Cooldown...")+ time.sleep(20)++ print(+ f"\n✅ Finished {len(selected_tasks)} random tasks: "+ f"{success_count} succeeded, {failure_count} failed. "+ f"Results in {OUTPUT_FILE}"+ )if __name__ == "__main__":diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.mdindex 8a69061..8265070 100644--- a/docs/CHANGELOG.md+++ b/docs/CHANGELOG.md@@ -88,3 +88,31 @@ fixes). Newest entries at the bottom.- `get_model_token_limit()` reads `max_seq_length` from Hugging Face sentence-transformers repos instead of assuming 2048 tokens for them.- `create_text_chunks(max_tokens_per_chunk=...)` now only shrinks `chunk_size`; it previously replaced it, growing 256-token chunks to ~1700 for unknown models.- Existing indexes are unchanged; rebuild with `--force` to re-chunk with the new sizes.++## 2026-10-23: Refuse to load a custom encoder as a vanilla BertModel++- `jinaai/jina-embeddings-v2-base-code` (and any encoder that declares its+ architecture through `config.auto_map` while keeping `model_type="bert"`) was+ being loaded without `trust_remote_code`. Transformers silently built a plain+ `BertModel` instead, discarded every weight it could not map, randomly+ initialised ~70 tensors (all 12 layers' feed-forward and output LayerNorm+ weights, plus position embeddings), and returned one constant vector for every+ input. Nothing raised: the index built, search returned results, and the+ ranking was noise. Measured on an 8-chunk retrieval probe, pairwise chunk+ cosine similarity was 1.0000 (mean and max) and recall@1 was 0.000; with the+ model's own code loaded, cosine fell to 0.064 mean / 0.239 max and recall@1+ reached 1.000.+- `embedding_compute.py` now checks `config.auto_map` before loading and raises+ with the model name and the fix, rather than producing a degenerate index.+ Opt in with `LEANN_TRUST_REMOTE_CODE=1`; it stays off by default because+ `trust_remote_code` executes arbitrary Python from the model repository, which+ matches the existing default in `chat.py`.+- The flag is now threaded through all five `SentenceTransformer(...)` call+ sites in the loader cascade and included in the model cache key.+- `benchmarks/contextbench/scripts/batch_run_random.py` executes tasks again.+ The block had been commented out, and the commented code unpacked four values+ from `run_single_task`, which has since returned five+ (`patch, latency_seconds, agent_seconds, traj_data, usage`) — so uncommenting+ it would have failed on the unpack. The restored block matches+ `batch_run_selected.py`, which was already on the current contract, and+ records `agent_seconds` and `latency_seconds` in each result row.diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.pyindex 62da2d4..dc92459 100644--- a/packages/leann-core/src/leann/embedding_compute.py+++ b/packages/leann-core/src/leann/embedding_compute.py@@ -531,6 +531,46 @@ def compute_embeddings(raise ValueError(f"Unsupported embedding mode: {mode}")+def _resolve_trust_remote_code() -> bool:+ """Whether to execute model-repository code when loading an embedding model.++ Off by default: trust_remote_code runs arbitrary Python from the model repo.+ Set LEANN_TRUST_REMOTE_CODE=1 for a model that needs it;+ _assert_remote_code_available names the variable when it hits one.+ """+ return os.getenv("LEANN_TRUST_REMOTE_CODE", "").strip().lower() in {"1", "true", "yes"}+++def _assert_remote_code_available(model_name: str, trust_remote_code: bool) -> None:+ """Refuse to load a model whose real architecture would be silently discarded.++ jinaai/jina-embeddings-v2-base-code and other custom encoders declare their+ architecture through config.auto_map while keeping model_type="bert". Loaded+ without trust_remote_code, transformers builds a vanilla BertModel instead,+ drops every weight it cannot map, randomly initialises the rest, and returns+ one constant vector for every input. Nothing raises: the index builds, search+ returns results, and the ranking is noise. Raise here instead.+ """+ if trust_remote_code:+ return+ try:+ from transformers import AutoConfig++ auto_map = getattr(AutoConfig.from_pretrained(model_name), "auto_map", None)+ except Exception as exc:+ # Offline, a private repo, or not a transformers model: nothing to assert.+ logger.debug(f"Could not read the config for {model_name}: {exc}")+ return+ if auto_map:+ raise ValueError(+ f"{model_name} ships its own modeling code (config.auto_map: "+ f"{sorted(auto_map)}). Loading it without trust_remote_code produces a "+ "randomly initialised model that returns an identical embedding for every "+ "input, so the index would rank by noise. Set LEANN_TRUST_REMOTE_CODE=1 to "+ "run the model repository's code, or pick a model that does not need it."+ )++def compute_embeddings_sentence_transformers(texts: list[str],model_name: str,@@ -588,8 +628,13 @@ def compute_embeddings_sentence_transformers(if adaptive_optimization:batch_size = _resolve_adaptive_batch_size(device, model_name)+ trust_remote_code = _resolve_trust_remote_code()+ _assert_remote_code_available(model_name, trust_remote_code)+# Create cache key- cache_key = f"sentence_transformers_{model_name}_{device}_{use_fp16}_optimized"+ cache_key = (+ f"sentence_transformers_{model_name}_{device}_{use_fp16}_{trust_remote_code}_optimized"+ )pre_model_init_end_time = time.time()logger.debug(@@ -653,6 +698,7 @@ def compute_embeddings_sentence_transformers(model = SentenceTransformer(model_name,device=device,+ trust_remote_code=trust_remote_code,model_kwargs=local_model_kwargs,tokenizer_kwargs=local_tokenizer_kwargs,local_files_only=True,@@ -668,6 +714,7 @@ def compute_embeddings_sentence_transformers(model = SentenceTransformer(model_name,device=device,+ trust_remote_code=trust_remote_code,local_files_only=True,)logger.info("Model loaded successfully! (local + basic)")@@ -676,6 +723,7 @@ def compute_embeddings_sentence_transformers(model = SentenceTransformer(model_name,device=device,+ trust_remote_code=trust_remote_code,local_files_only=False,)logger.info("Model loaded successfully! (network + basic)")@@ -693,6 +741,7 @@ def compute_embeddings_sentence_transformers(model = SentenceTransformer(model_name,device=device,+ trust_remote_code=trust_remote_code,model_kwargs=network_model_kwargs,tokenizer_kwargs=network_tokenizer_kwargs,local_files_only=False,@@ -706,6 +755,7 @@ def compute_embeddings_sentence_transformers(model = SentenceTransformer(model_name,device=device,+ trust_remote_code=trust_remote_code,local_files_only=False,)logger.info("Model loaded successfully! (network + basic)")diff --git a/tests/test_trust_remote_code.py b/tests/test_trust_remote_code.pynew file mode 100644index 0000000..9d2478a--- /dev/null+++ b/tests/test_trust_remote_code.py@@ -0,0 +1,75 @@+"""Guards against silently loading a custom encoder as a vanilla BertModel.++jinaai/jina-embeddings-v2-base-code declares its architecture through+config.auto_map while keeping model_type="bert". Loaded without+trust_remote_code, transformers builds a plain BertModel, discards the weights+it cannot map, randomly initialises ~70 tensors, and returns one constant+vector for every input. Nothing raises, so an index builds and searches+normally while ranking by noise. These tests pin the refusal and the opt-in.+"""++import unittest+from unittest.mock import patch++from leann.embedding_compute import _assert_remote_code_available, _resolve_trust_remote_code++JINA_AUTO_MAP = {+ "AutoConfig": "jinaai/jina-bert-v2-qk-post-norm--configuration_bert.JinaBertConfig",+ "AutoModel": "jinaai/jina-bert-v2-qk-post-norm--modeling_bert.JinaBertModel",+}+++class _Config:+ def __init__(self, auto_map=None):+ if auto_map is not None:+ self.auto_map = auto_map+++class TestResolveTrustRemoteCode(unittest.TestCase):+ def test_absent_env_is_off(self):+ with patch.dict("os.environ", {}, clear=True):+ self.assertFalse(_resolve_trust_remote_code())++ def test_truthy_values_enable_it(self):+ for value in ("1", "true", "TRUE", "yes", " Yes "):+ with patch.dict("os.environ", {"LEANN_TRUST_REMOTE_CODE": value}, clear=True):+ self.assertTrue(_resolve_trust_remote_code(), value)++ def test_other_values_leave_it_off(self):+ for value in ("0", "false", "no", "", "maybe"):+ with patch.dict("os.environ", {"LEANN_TRUST_REMOTE_CODE": value}, clear=True):+ self.assertFalse(_resolve_trust_remote_code(), value)+++class TestAssertRemoteCodeAvailable(unittest.TestCase):+ def _with_config(self, config):+ return patch("transformers.AutoConfig.from_pretrained", return_value=config)++ def test_rejects_auto_map_model_when_not_trusted(self):+ with self._with_config(_Config(JINA_AUTO_MAP)):+ with self.assertRaises(ValueError) as ctx:+ _assert_remote_code_available("jinaai/jina-embeddings-v2-base-code", False)+ message = str(ctx.exception)+ self.assertIn("LEANN_TRUST_REMOTE_CODE", message)+ self.assertIn("identical embedding", message)++ def test_allows_auto_map_model_when_trusted(self):+ with self._with_config(_Config(JINA_AUTO_MAP)):+ _assert_remote_code_available("jinaai/jina-embeddings-v2-base-code", True)++ def test_allows_ordinary_model(self):+ with self._with_config(_Config()):+ _assert_remote_code_available("sentence-transformers/all-MiniLM-L6-v2", False)++ def test_allows_empty_auto_map(self):+ with self._with_config(_Config({})):+ _assert_remote_code_available("some/model", False)++ def test_unreadable_config_does_not_block_loading(self):+ # Offline, a private repo, or a non-transformers model: nothing to assert.+ with patch("transformers.AutoConfig.from_pretrained", side_effect=OSError("offline")):+ _assert_remote_code_available("some/model", False)+++if __name__ == "__main__":+ unittest.main()