An open source, non-profit web search engine
Evaluate a remote model
Daoud Clarke93fdf1cf00a2
diff --git a/mwmbl/rankeval/evaluation/evaluate.py b/mwmbl/rankeval/evaluation/evaluate.pyindex 332c21b..c27b724 100644--- a/mwmbl/rankeval/evaluation/evaluate.py+++ b/mwmbl/rankeval/evaluation/evaluate.py@@ -16,6 +16,9 @@ CLICK_PROPORTIONS = [0.285, 0.157, 0.110, 0.080, 0.072, 0.051, 0.040, 0.032, 0.0NUM_RESULTS_FOR_EVAL = len(CLICK_PROPORTIONS)+random = np.random.default_rng(42)++class RankingModel(ABC):@abstractmethoddef predict(self, query: str) -> list[str]:@@ -25,7 +28,7 @@ class RankingModel(ABC):pass-def evaluate(ranking_model: RankingModel):+def evaluate(ranking_model: RankingModel, fraction: float = 1.0):# TODO:# - output feature importances from XGBoost# - experiment with more features@@ -33,7 +36,19 @@ def evaluate(ranking_model: RankingModel):dataset = pd.read_csv(RANKINGS_DATASET_TEST_PATH)ndcg_scores = []proportions = []++ queries = dataset['query'].unique()+ if fraction < 1.0:+ num_queries = int(fraction * len(queries))+ print("Num queries", num_queries)+ random_queries = set(random.choice(queries, num_queries, replace=False))+ else:+ random_queries = set(queries)+for query, rankings in dataset.groupby('query'):+ if query not in random_queries:+ continue+top_ranked = rankings[['url']].iloc[:NUM_RESULTS_FOR_EVAL]top_ranked['score'] = CLICK_PROPORTIONS[:len(top_ranked)]scores = top_ranked.set_index('url')['score'].to_dict()diff --git a/mwmbl/rankeval/evaluation/evaluate_remote.py b/mwmbl/rankeval/evaluation/evaluate_remote.pynew file mode 100644index 0000000..dfbd60e--- /dev/null+++ b/mwmbl/rankeval/evaluation/evaluate_remote.py@@ -0,0 +1,40 @@+"""+Evaluate against queries on a remote search engine.+"""++from argparse import ArgumentParser++import requests+from joblib import Memory++from mwmbl.tinysearchengine.indexer import TinyIndex, Document+from mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker++from mwmbl.rankeval.evaluation.evaluate import RankingModel, evaluate+++memory = Memory(location="devdata/cache")+++def fetch_results(url: str, query: str):+ results = requests.get(url + "/api/v1/search/?s=" + query).json()+ return results+++class RemoteRankingModel(RankingModel):+ def __init__(self, url: str = "https://mwmbl.org"):+ self.url = url++ def predict(self, query: str) -> list[str]:+ results = fetch_results(self.url, query)+ return [x['url'] for x in results]+++def run():+ model = RemoteRankingModel()+ evaluate(model, fraction=1.0)+++if __name__ == '__main__':+ run()