Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Evaluate a remote model

Daoud Clarke93fdf1cf00a2
diff --git a/mwmbl/rankeval/evaluation/evaluate.py b/mwmbl/rankeval/evaluation/evaluate.py
index 332c21b..c27b724 100644
--- a/mwmbl/rankeval/evaluation/evaluate.py
+++ b/mwmbl/rankeval/evaluation/evaluate.py
@@ -16,6 +16,9 @@ CLICK_PROPORTIONS = [0.285, 0.157, 0.110, 0.080, 0.072, 0.051, 0.040, 0.032, 0.0
NUM_RESULTS_FOR_EVAL = len(CLICK_PROPORTIONS)
+random = np.random.default_rng(42)
+
+
class RankingModel(ABC):
@abstractmethod
def predict(self, query: str) -> list[str]:
@@ -25,7 +28,7 @@ class RankingModel(ABC):
pass
-def evaluate(ranking_model: RankingModel):
+def evaluate(ranking_model: RankingModel, fraction: float = 1.0):
# TODO:
# - output feature importances from XGBoost
# - experiment with more features
@@ -33,7 +36,19 @@ def evaluate(ranking_model: RankingModel):
dataset = pd.read_csv(RANKINGS_DATASET_TEST_PATH)
ndcg_scores = []
proportions = []
+
+ queries = dataset['query'].unique()
+ if fraction < 1.0:
+ num_queries = int(fraction * len(queries))
+ print("Num queries", num_queries)
+ random_queries = set(random.choice(queries, num_queries, replace=False))
+ else:
+ random_queries = set(queries)
+
for query, rankings in dataset.groupby('query'):
+ if query not in random_queries:
+ continue
+
top_ranked = rankings[['url']].iloc[:NUM_RESULTS_FOR_EVAL]
top_ranked['score'] = CLICK_PROPORTIONS[:len(top_ranked)]
scores = top_ranked.set_index('url')['score'].to_dict()
diff --git a/mwmbl/rankeval/evaluation/evaluate_remote.py b/mwmbl/rankeval/evaluation/evaluate_remote.py
new file mode 100644
index 0000000..dfbd60e
--- /dev/null
+++ b/mwmbl/rankeval/evaluation/evaluate_remote.py
@@ -0,0 +1,40 @@
+"""
+Evaluate against queries on a remote search engine.
+"""
+
+from argparse import ArgumentParser
+
+import requests
+from joblib import Memory
+
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document
+from mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker
+
+from mwmbl.rankeval.evaluation.evaluate import RankingModel, evaluate
+
+
+memory = Memory(location="devdata/cache")
+
+
+def fetch_results(url: str, query: str):
+ results = requests.get(url + "/api/v1/search/?s=" + query).json()
+ return results
+
+
+class RemoteRankingModel(RankingModel):
+ def __init__(self, url: str = "https://mwmbl.org"):
+ self.url = url
+
+ def predict(self, query: str) -> list[str]:
+ results = fetch_results(self.url, query)
+ return [x['url'] for x in results]
+
+
+def run():
+ model = RemoteRankingModel()
+ evaluate(model, fraction=1.0)
+
+
+if __name__ == '__main__':
+ run()