Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

fix(core): refuse to load a custom encoder as a vanilla BertModel (#445)

Aakash Suresh0463b761916d
diff --git a/benchmarks/contextbench/scripts/batch_run_random.py b/benchmarks/contextbench/scripts/batch_run_random.py
index 6371703..e5736ba 100644
--- a/benchmarks/contextbench/scripts/batch_run_random.py
+++ b/benchmarks/contextbench/scripts/batch_run_random.py
@@ -5,9 +5,7 @@ import subprocess
import time
from pathlib import Path
-# run_single_task is used by the task-running block below, which ships
-# commented out so this script selects and prefetches without executing.
-from auto_run import prefetch_task_repositories, run_single_task # noqa: F401
+from auto_run import prefetch_task_repositories, run_single_task
from datasets import load_dataset
ROOT = Path(__file__).resolve().parents[1]
@@ -90,9 +88,8 @@ def main():
else:
print("⏭️ PREFETCH_REPOS=0; skipping prefetch step.")
- # Incremented by the commented-out task-running block below.
- success_count = 0 # noqa: F841
- failure_count = 0 # noqa: F841
+ success_count = 0
+ failure_count = 0
for i, task in enumerate(selected_tasks):
instance_id = task["instance_id"]
repo_url = task["repo_url"]
@@ -101,45 +98,46 @@ def main():
print(f"📦 [{i + 1}/{len(selected_tasks)}] Running: {instance_id}")
print(f" repo: {repo_url} source: {task.get('source', '?')}")
- # If want to run the task, uncomment the following code
- # try:
- # patch, elapsed, traj_data, usage = run_single_task(
- # instance_id=instance_id,
- # repo_url=repo_url,
- # work_root=WORK_ROOT,
- # mitm_script_path=str(MITM_SCRIPT),
- # trace_dir=TRACE_DIR,
- # model=MODEL,
- # task=task,
- # )
-
- # result_entry = {
- # "instance_id": instance_id,
- # "model_patch": patch if patch else "",
- # "model_name_or_path": "claude-code-cli",
- # "elapsed_seconds": round(elapsed, 1),
- # "traj_data": traj_data,
- # "token_usage": usage,
- # }
-
- # with open(OUTPUT_FILE, "a", encoding="utf-8") as f:
- # f.write(json.dumps(result_entry) + "\n")
- # print(f"✅ Result saved for {instance_id}")
- # success_count += 1
-
- # except Exception as e:
- # print(f"❌ Error processing {instance_id}: {e}")
- # failure_count += 1
- # finally:
- # cleanup_residuals()
- # print("💤 Cooldown...")
- # time.sleep(20)
-
- # print(
- # f"\n✅ Finished {len(selected_tasks)} random tasks: "
- # f"{success_count} succeeded, {failure_count} failed. "
- # f"Results in {OUTPUT_FILE}"
- # )
+ try:
+ patch, elapsed, agent_seconds, traj_data, usage = run_single_task(
+ instance_id=instance_id,
+ repo_url=repo_url,
+ work_root=WORK_ROOT,
+ mitm_script_path=str(MITM_SCRIPT),
+ trace_dir=TRACE_DIR,
+ model=MODEL,
+ task=task,
+ )
+
+ result_entry = {
+ "instance_id": instance_id,
+ "model_patch": patch if patch else "",
+ "model_name_or_path": "claude-code-cli",
+ "elapsed_seconds": round(elapsed, 1),
+ "latency_seconds": round(elapsed, 1),
+ "agent_seconds": round(agent_seconds, 1) if agent_seconds is not None else None,
+ "traj_data": traj_data,
+ "token_usage": usage,
+ }
+
+ with open(OUTPUT_FILE, "a", encoding="utf-8") as f:
+ f.write(json.dumps(result_entry) + "\n")
+ print(f"✅ Result saved for {instance_id}")
+ success_count += 1
+
+ except Exception as e:
+ print(f"❌ Error processing {instance_id}: {e}")
+ failure_count += 1
+ finally:
+ cleanup_residuals()
+ print("💤 Cooldown...")
+ time.sleep(20)
+
+ print(
+ f"\n✅ Finished {len(selected_tasks)} random tasks: "
+ f"{success_count} succeeded, {failure_count} failed. "
+ f"Results in {OUTPUT_FILE}"
+ )
if __name__ == "__main__":
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 8a69061..8265070 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -88,3 +88,31 @@ fixes). Newest entries at the bottom.
- `get_model_token_limit()` reads `max_seq_length` from Hugging Face sentence-transformers repos instead of assuming 2048 tokens for them.
- `create_text_chunks(max_tokens_per_chunk=...)` now only shrinks `chunk_size`; it previously replaced it, growing 256-token chunks to ~1700 for unknown models.
- Existing indexes are unchanged; rebuild with `--force` to re-chunk with the new sizes.
+
+## 2026-10-23: Refuse to load a custom encoder as a vanilla BertModel
+
+- `jinaai/jina-embeddings-v2-base-code` (and any encoder that declares its
+ architecture through `config.auto_map` while keeping `model_type="bert"`) was
+ being loaded without `trust_remote_code`. Transformers silently built a plain
+ `BertModel` instead, discarded every weight it could not map, randomly
+ initialised ~70 tensors (all 12 layers' feed-forward and output LayerNorm
+ weights, plus position embeddings), and returned one constant vector for every
+ input. Nothing raised: the index built, search returned results, and the
+ ranking was noise. Measured on an 8-chunk retrieval probe, pairwise chunk
+ cosine similarity was 1.0000 (mean and max) and recall@1 was 0.000; with the
+ model's own code loaded, cosine fell to 0.064 mean / 0.239 max and recall@1
+ reached 1.000.
+- `embedding_compute.py` now checks `config.auto_map` before loading and raises
+ with the model name and the fix, rather than producing a degenerate index.
+ Opt in with `LEANN_TRUST_REMOTE_CODE=1`; it stays off by default because
+ `trust_remote_code` executes arbitrary Python from the model repository, which
+ matches the existing default in `chat.py`.
+- The flag is now threaded through all five `SentenceTransformer(...)` call
+ sites in the loader cascade and included in the model cache key.
+- `benchmarks/contextbench/scripts/batch_run_random.py` executes tasks again.
+ The block had been commented out, and the commented code unpacked four values
+ from `run_single_task`, which has since returned five
+ (`patch, latency_seconds, agent_seconds, traj_data, usage`) — so uncommenting
+ it would have failed on the unpack. The restored block matches
+ `batch_run_selected.py`, which was already on the current contract, and
+ records `agent_seconds` and `latency_seconds` in each result row.
diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.py
index 62da2d4..dc92459 100644
--- a/packages/leann-core/src/leann/embedding_compute.py
+++ b/packages/leann-core/src/leann/embedding_compute.py
@@ -531,6 +531,46 @@ def compute_embeddings(
raise ValueError(f"Unsupported embedding mode: {mode}")
+def _resolve_trust_remote_code() -> bool:
+ """Whether to execute model-repository code when loading an embedding model.
+
+ Off by default: trust_remote_code runs arbitrary Python from the model repo.
+ Set LEANN_TRUST_REMOTE_CODE=1 for a model that needs it;
+ _assert_remote_code_available names the variable when it hits one.
+ """
+ return os.getenv("LEANN_TRUST_REMOTE_CODE", "").strip().lower() in {"1", "true", "yes"}
+
+
+def _assert_remote_code_available(model_name: str, trust_remote_code: bool) -> None:
+ """Refuse to load a model whose real architecture would be silently discarded.
+
+ jinaai/jina-embeddings-v2-base-code and other custom encoders declare their
+ architecture through config.auto_map while keeping model_type="bert". Loaded
+ without trust_remote_code, transformers builds a vanilla BertModel instead,
+ drops every weight it cannot map, randomly initialises the rest, and returns
+ one constant vector for every input. Nothing raises: the index builds, search
+ returns results, and the ranking is noise. Raise here instead.
+ """
+ if trust_remote_code:
+ return
+ try:
+ from transformers import AutoConfig
+
+ auto_map = getattr(AutoConfig.from_pretrained(model_name), "auto_map", None)
+ except Exception as exc:
+ # Offline, a private repo, or not a transformers model: nothing to assert.
+ logger.debug(f"Could not read the config for {model_name}: {exc}")
+ return
+ if auto_map:
+ raise ValueError(
+ f"{model_name} ships its own modeling code (config.auto_map: "
+ f"{sorted(auto_map)}). Loading it without trust_remote_code produces a "
+ "randomly initialised model that returns an identical embedding for every "
+ "input, so the index would rank by noise. Set LEANN_TRUST_REMOTE_CODE=1 to "
+ "run the model repository's code, or pick a model that does not need it."
+ )
+
+
def compute_embeddings_sentence_transformers(
texts: list[str],
model_name: str,
@@ -588,8 +628,13 @@ def compute_embeddings_sentence_transformers(
if adaptive_optimization:
batch_size = _resolve_adaptive_batch_size(device, model_name)
+ trust_remote_code = _resolve_trust_remote_code()
+ _assert_remote_code_available(model_name, trust_remote_code)
+
# Create cache key
- cache_key = f"sentence_transformers_{model_name}_{device}_{use_fp16}_optimized"
+ cache_key = (
+ f"sentence_transformers_{model_name}_{device}_{use_fp16}_{trust_remote_code}_optimized"
+ )
pre_model_init_end_time = time.time()
logger.debug(
@@ -653,6 +698,7 @@ def compute_embeddings_sentence_transformers(
model = SentenceTransformer(
model_name,
device=device,
+ trust_remote_code=trust_remote_code,
model_kwargs=local_model_kwargs,
tokenizer_kwargs=local_tokenizer_kwargs,
local_files_only=True,
@@ -668,6 +714,7 @@ def compute_embeddings_sentence_transformers(
model = SentenceTransformer(
model_name,
device=device,
+ trust_remote_code=trust_remote_code,
local_files_only=True,
)
logger.info("Model loaded successfully! (local + basic)")
@@ -676,6 +723,7 @@ def compute_embeddings_sentence_transformers(
model = SentenceTransformer(
model_name,
device=device,
+ trust_remote_code=trust_remote_code,
local_files_only=False,
)
logger.info("Model loaded successfully! (network + basic)")
@@ -693,6 +741,7 @@ def compute_embeddings_sentence_transformers(
model = SentenceTransformer(
model_name,
device=device,
+ trust_remote_code=trust_remote_code,
model_kwargs=network_model_kwargs,
tokenizer_kwargs=network_tokenizer_kwargs,
local_files_only=False,
@@ -706,6 +755,7 @@ def compute_embeddings_sentence_transformers(
model = SentenceTransformer(
model_name,
device=device,
+ trust_remote_code=trust_remote_code,
local_files_only=False,
)
logger.info("Model loaded successfully! (network + basic)")
diff --git a/tests/test_trust_remote_code.py b/tests/test_trust_remote_code.py
new file mode 100644
index 0000000..9d2478a
--- /dev/null
+++ b/tests/test_trust_remote_code.py
@@ -0,0 +1,75 @@
+"""Guards against silently loading a custom encoder as a vanilla BertModel.
+
+jinaai/jina-embeddings-v2-base-code declares its architecture through
+config.auto_map while keeping model_type="bert". Loaded without
+trust_remote_code, transformers builds a plain BertModel, discards the weights
+it cannot map, randomly initialises ~70 tensors, and returns one constant
+vector for every input. Nothing raises, so an index builds and searches
+normally while ranking by noise. These tests pin the refusal and the opt-in.
+"""
+
+import unittest
+from unittest.mock import patch
+
+from leann.embedding_compute import _assert_remote_code_available, _resolve_trust_remote_code
+
+JINA_AUTO_MAP = {
+ "AutoConfig": "jinaai/jina-bert-v2-qk-post-norm--configuration_bert.JinaBertConfig",
+ "AutoModel": "jinaai/jina-bert-v2-qk-post-norm--modeling_bert.JinaBertModel",
+}
+
+
+class _Config:
+ def __init__(self, auto_map=None):
+ if auto_map is not None:
+ self.auto_map = auto_map
+
+
+class TestResolveTrustRemoteCode(unittest.TestCase):
+ def test_absent_env_is_off(self):
+ with patch.dict("os.environ", {}, clear=True):
+ self.assertFalse(_resolve_trust_remote_code())
+
+ def test_truthy_values_enable_it(self):
+ for value in ("1", "true", "TRUE", "yes", " Yes "):
+ with patch.dict("os.environ", {"LEANN_TRUST_REMOTE_CODE": value}, clear=True):
+ self.assertTrue(_resolve_trust_remote_code(), value)
+
+ def test_other_values_leave_it_off(self):
+ for value in ("0", "false", "no", "", "maybe"):
+ with patch.dict("os.environ", {"LEANN_TRUST_REMOTE_CODE": value}, clear=True):
+ self.assertFalse(_resolve_trust_remote_code(), value)
+
+
+class TestAssertRemoteCodeAvailable(unittest.TestCase):
+ def _with_config(self, config):
+ return patch("transformers.AutoConfig.from_pretrained", return_value=config)
+
+ def test_rejects_auto_map_model_when_not_trusted(self):
+ with self._with_config(_Config(JINA_AUTO_MAP)):
+ with self.assertRaises(ValueError) as ctx:
+ _assert_remote_code_available("jinaai/jina-embeddings-v2-base-code", False)
+ message = str(ctx.exception)
+ self.assertIn("LEANN_TRUST_REMOTE_CODE", message)
+ self.assertIn("identical embedding", message)
+
+ def test_allows_auto_map_model_when_trusted(self):
+ with self._with_config(_Config(JINA_AUTO_MAP)):
+ _assert_remote_code_available("jinaai/jina-embeddings-v2-base-code", True)
+
+ def test_allows_ordinary_model(self):
+ with self._with_config(_Config()):
+ _assert_remote_code_available("sentence-transformers/all-MiniLM-L6-v2", False)
+
+ def test_allows_empty_auto_map(self):
+ with self._with_config(_Config({})):
+ _assert_remote_code_available("some/model", False)
+
+ def test_unreadable_config_does_not_block_loading(self):
+ # Offline, a private repo, or a non-transformers model: nothing to assert.
+ with patch("transformers.AutoConfig.from_pretrained", side_effect=OSError("offline")):
+ _assert_remote_code_available("some/model", False)
+
+
+if __name__ == "__main__":
+ unittest.main()