An open source, non-profit web search engine
WIP: use rust model
Daoud Clarke3dec49acfa22
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex 81a806e..b75e0a4 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -10,6 +10,7 @@ from mwmbl.models import DomainSubmissionfrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document+from mwmbl.tinysearchengine.ltr import RustXGBPipelinefrom mwmbl.tinysearchengine.ltr_rank import LTRRankerfrom mwmbl.tinysearchengine.rank import DomainLimitingRanker, HeuristicAndWikiRanker@@ -32,25 +33,7 @@ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAMEtiny_index = TinyIndex(item_factory=Document, index_path=index_path)tiny_index.__enter__()-# To use the Rust XGBoost model, set MWMBL_LTR_MODEL_PATH to the path of a trained model file.-# The model must have been trained with RustXGBPipeline.save_model().-# If the env var is not set, fall back to the heuristic ranker.-_ltr_model_path = os.environ.get("MWMBL_LTR_MODEL_PATH")--if _ltr_model_path and Path(_ltr_model_path).exists():- try:- from mwmbl.tinysearchengine.ltr import RustXGBPipeline- _model = RustXGBPipeline.from_model_path(_ltr_model_path)- ranker = DomainLimitingRanker(LTRRanker(tiny_index, completer, _model, include_wiki=True, num_wiki_results=3))- ranker = LTRRanker(tiny_index, completer, _model, top_n=1000, include_wiki=True, num_wiki_results=5)- except ImportError:- import logging- logging.getLogger(__name__).warning(- "mwmbl_rank Rust extension not available; falling back to heuristic ranker. "- "Run 'maturin develop' to build the Rust extension."- )- ranker = HeuristicAndWikiRanker(tiny_index, completer)-else:- ranker = HeuristicAndWikiRanker(tiny_index, completer)+_model = RustXGBPipeline.from_model_path(str(settings.RUST_MODEL_PATH))+ranker = DomainLimitingRanker(LTRRanker(tiny_index, completer, _model, include_wiki=True, num_wiki_results=3))batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex 7092143..ca4ca9e 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -229,3 +229,5 @@ REDIS_URL = os.environ.get("REDIS_URL", "redis://127.0.0.1:6379")INTERNAL_IPS = ["127.0.0.1",]++RUST_MODEL_PATH = Path(__file__).parent.parent / 'devdata' / 'rankeval-2026-04' / "model.xgb"