[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
Add ColQwen2.5 model support and improve model selection (#183)
Yichuan Wanga0bbf831dbb8
diff --git a/apps/multimodal/vision-based-pdf-multi-vector/leann_multi_vector.py b/apps/multimodal/vision-based-pdf-multi-vector/leann_multi_vector.pyindex 2ea933f..075bb3d 100644--- a/apps/multimodal/vision-based-pdf-multi-vector/leann_multi_vector.py+++ b/apps/multimodal/vision-based-pdf-multi-vector/leann_multi_vector.py@@ -152,20 +152,65 @@ def _select_device_and_dtype():def _load_colvision(model_choice: str):import torch- from colpali_engine.models import ColPali, ColQwen2, ColQwen2Processor+ from colpali_engine.models import (+ ColPali,+ ColQwen2,+ ColQwen2_5,+ ColQwen2_5_Processor,+ ColQwen2Processor,+ )from colpali_engine.models.paligemma.colpali.processing_colpali import ColPaliProcessorfrom transformers.utils.import_utils import is_flash_attn_2_availabledevice_str, device, dtype = _select_device_and_dtype()+ # Determine model name and type+ # IMPORTANT: Check colqwen2.5 BEFORE colqwen2 to avoid false matches+ model_choice_lower = model_choice.lower()if model_choice == "colqwen2":model_name = "vidore/colqwen2-v1.0"- # On CPU/MPS we must avoid flash-attn and stay eager; on CUDA prefer flash-attn if available- attn_implementation = (- "flash_attention_2"- if (device_str == "cuda" and is_flash_attn_2_available())- else "eager"- )+ model_type = "colqwen2"+ elif model_choice == "colqwen2.5" or model_choice == "colqwen25":+ model_name = "vidore/colqwen2.5-v0.2"+ model_type = "colqwen2.5"+ elif model_choice == "colpali":+ model_name = "vidore/colpali-v1.2"+ model_type = "colpali"+ elif (+ "colqwen2.5" in model_choice_lower+ or "colqwen25" in model_choice_lower+ or "colqwen2_5" in model_choice_lower+ ):+ # Handle HuggingFace model names like "vidore/colqwen2.5-v0.2"+ model_name = model_choice+ model_type = "colqwen2.5"+ elif "colqwen2" in model_choice_lower and "colqwen2-v1.0" in model_choice_lower:+ # Handle HuggingFace model names like "vidore/colqwen2-v1.0" (but not colqwen2.5)+ model_name = model_choice+ model_type = "colqwen2"+ elif "colpali" in model_choice_lower:+ # Handle HuggingFace model names like "vidore/colpali-v1.2"+ model_name = model_choice+ model_type = "colpali"+ else:+ # Default to colpali for backward compatibility+ model_name = "vidore/colpali-v1.2"+ model_type = "colpali"++ # Load model based on type+ attn_implementation = (+ "flash_attention_2" if (device_str == "cuda" and is_flash_attn_2_available()) else "eager"+ )++ if model_type == "colqwen2.5":+ model = ColQwen2_5.from_pretrained(+ model_name,+ torch_dtype=torch.bfloat16,+ device_map=device,+ attn_implementation=attn_implementation,+ ).eval()+ processor = ColQwen2_5_Processor.from_pretrained(model_name)+ elif model_type == "colqwen2":model = ColQwen2.from_pretrained(model_name,torch_dtype=torch.bfloat16,@@ -173,8 +218,7 @@ def _load_colvision(model_choice: str):attn_implementation=attn_implementation,).eval()processor = ColQwen2Processor.from_pretrained(model_name)- else:- model_name = "vidore/colpali-v1.2"+ else: # colpalimodel = ColPali.from_pretrained(model_name,torch_dtype=torch.bfloat16,diff --git a/apps/multimodal/vision-based-pdf-multi-vector/vidore_v1_benchmark.py b/apps/multimodal/vision-based-pdf-multi-vector/vidore_v1_benchmark.pyindex e68a689..79472df 100644--- a/apps/multimodal/vision-based-pdf-multi-vector/vidore_v1_benchmark.py+++ b/apps/multimodal/vision-based-pdf-multi-vector/vidore_v1_benchmark.py@@ -90,6 +90,51 @@ VIDORE_V1_TASKS = {},}+# Task name aliases (short names -> full names)+TASK_ALIASES = {+ "arxivqa": "VidoreArxivQARetrieval",+ "docvqa": "VidoreDocVQARetrieval",+ "infovqa": "VidoreInfoVQARetrieval",+ "tabfquad": "VidoreTabfquadRetrieval",+ "tatdqa": "VidoreTatdqaRetrieval",+ "shiftproject": "VidoreShiftProjectRetrieval",+ "syntheticdocqa_ai": "VidoreSyntheticDocQAAIRetrieval",+ "syntheticdocqa_energy": "VidoreSyntheticDocQAEnergyRetrieval",+ "syntheticdocqa_government": "VidoreSyntheticDocQAGovernmentReportsRetrieval",+ "syntheticdocqa_healthcare": "VidoreSyntheticDocQAHealthcareIndustryRetrieval",+}+++def normalize_task_name(task_name: str) -> str:+ """Normalize task name (handle aliases)."""+ task_name_lower = task_name.lower()+ if task_name in VIDORE_V1_TASKS:+ return task_name+ if task_name_lower in TASK_ALIASES:+ return TASK_ALIASES[task_name_lower]+ # Try partial match+ for alias, full_name in TASK_ALIASES.items():+ if alias in task_name_lower or task_name_lower in alias:+ return full_name+ return task_name+++def get_safe_model_name(model_name: str) -> str:+ """Get a safe model name for use in file paths."""+ import hashlib+ import os++ # If it's a path, use basename or hash+ if os.path.exists(model_name) and os.path.isdir(model_name):+ # Use basename if it's reasonable, otherwise use hash+ basename = os.path.basename(model_name.rstrip("/"))+ if basename and len(basename) < 100 and not basename.startswith("."):+ return basename+ # Use hash for very long or problematic paths+ return hashlib.md5(model_name.encode()).hexdigest()[:16]+ # For HuggingFace model names, replace / with _+ return model_name.replace("/", "_").replace(":", "_")+def load_vidore_v1_data(dataset_path: str,@@ -181,6 +226,9 @@ def evaluate_task(print(f"Evaluating task: {task_name}")print(f"{'=' * 80}")+ # Normalize task name (handle aliases)+ task_name = normalize_task_name(task_name)+# Get task configif task_name not in VIDORE_V1_TASKS:raise ValueError(f"Unknown task: {task_name}. Available: {list(VIDORE_V1_TASKS.keys())}")@@ -223,11 +271,13 @@ def evaluate_task()# Build or load index+ # Use safe model name for index path (different models need different indexes)+ safe_model_name = get_safe_model_name(model_name)index_path_full = index_path if not use_fast_plaid else fast_plaid_index_pathif index_path_full is None:- index_path_full = f"./indexes/{task_name}_{model_name}"+ index_path_full = f"./indexes/{task_name}_{safe_model_name}"if use_fast_plaid:- index_path_full = f"./indexes/{task_name}_{model_name}_fastplaid"+ index_path_full = f"./indexes/{task_name}_{safe_model_name}_fastplaid"index_or_retriever, corpus_ids_ordered = evaluator.build_index_from_corpus(corpus=corpus,@@ -281,8 +331,7 @@ def main():"--model",type=str,default="colqwen2",- choices=["colqwen2", "colpali"],- help="Model to use",+ help="Model to use: 'colqwen2', 'colpali', or path to a model directory (supports LoRA adapters)",)parser.add_argument("--task",@@ -350,11 +399,11 @@ def main():# Determine tasks to evaluateif args.task:- tasks_to_eval = [args.task]+ tasks_to_eval = [normalize_task_name(args.task)]elif args.tasks.lower() == "all":tasks_to_eval = list(VIDORE_V1_TASKS.keys())else:- tasks_to_eval = [t.strip() for t in args.tasks.split(",")]+ tasks_to_eval = [normalize_task_name(t.strip()) for t in args.tasks.split(",")]print(f"Tasks to evaluate: {tasks_to_eval}")