Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

WIP: use rust model

Daoud Clarke3dec49acfa22
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.py
index 81a806e..b75e0a4 100644
--- a/mwmbl/search_setup.py
+++ b/mwmbl/search_setup.py
@@ -10,6 +10,7 @@ from mwmbl.models import DomainSubmission
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.tinysearchengine.completer import Completer
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
+from mwmbl.tinysearchengine.ltr import RustXGBPipeline
from mwmbl.tinysearchengine.ltr_rank import LTRRanker
from mwmbl.tinysearchengine.rank import DomainLimitingRanker, HeuristicAndWikiRanker
@@ -32,25 +33,7 @@ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
tiny_index = TinyIndex(item_factory=Document, index_path=index_path)
tiny_index.__enter__()
-# To use the Rust XGBoost model, set MWMBL_LTR_MODEL_PATH to the path of a trained model file.
-# The model must have been trained with RustXGBPipeline.save_model().
-# If the env var is not set, fall back to the heuristic ranker.
-_ltr_model_path = os.environ.get("MWMBL_LTR_MODEL_PATH")
-
-if _ltr_model_path and Path(_ltr_model_path).exists():
- try:
- from mwmbl.tinysearchengine.ltr import RustXGBPipeline
- _model = RustXGBPipeline.from_model_path(_ltr_model_path)
- ranker = DomainLimitingRanker(LTRRanker(tiny_index, completer, _model, include_wiki=True, num_wiki_results=3))
- ranker = LTRRanker(tiny_index, completer, _model, top_n=1000, include_wiki=True, num_wiki_results=5)
- except ImportError:
- import logging
- logging.getLogger(__name__).warning(
- "mwmbl_rank Rust extension not available; falling back to heuristic ranker. "
- "Run 'maturin develop' to build the Rust extension."
- )
- ranker = HeuristicAndWikiRanker(tiny_index, completer)
-else:
- ranker = HeuristicAndWikiRanker(tiny_index, completer)
+_model = RustXGBPipeline.from_model_path(str(settings.RUST_MODEL_PATH))
+ranker = DomainLimitingRanker(LTRRanker(tiny_index, completer, _model, include_wiki=True, num_wiki_results=3))
batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index 7092143..ca4ca9e 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -229,3 +229,5 @@ REDIS_URL = os.environ.get("REDIS_URL", "redis://127.0.0.1:6379")
INTERNAL_IPS = [
"127.0.0.1",
]
+
+RUST_MODEL_PATH = Path(__file__).parent.parent / 'devdata' / 'rankeval-2026-04' / "model.xgb"