Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Use remote index so we can tweak the algorithm

Daoud Clarkebee015bb336e
diff --git a/mwmbl/rankeval/evaluation/evaluate.py b/mwmbl/rankeval/evaluation/evaluate.py
index c27b724..781ae9b 100644
--- a/mwmbl/rankeval/evaluation/evaluate.py
+++ b/mwmbl/rankeval/evaluation/evaluate.py
@@ -10,7 +10,7 @@ from sklearn.metrics import ndcg_score
# Sourced from https://www.searchenginejournal.com/google-first-page-clicks/374516/
-from mwmbl.rankeval.paths import RANKINGS_DATASET_TEST_PATH
+from mwmbl.rankeval.paths import RANKINGS_DATASET_TEST_PATH, RANKINGS_DATASET_TRAIN_PATH
CLICK_PROPORTIONS = [0.285, 0.157, 0.110, 0.080, 0.072, 0.051, 0.040, 0.032, 0.028, 0.025]
NUM_RESULTS_FOR_EVAL = len(CLICK_PROPORTIONS)
@@ -28,12 +28,16 @@ class RankingModel(ABC):
pass
-def evaluate(ranking_model: RankingModel, fraction: float = 1.0):
+def evaluate(ranking_model: RankingModel, fraction: float = 1.0, use_test=False):
# TODO:
# - output feature importances from XGBoost
# - experiment with more features
- dataset = pd.read_csv(RANKINGS_DATASET_TEST_PATH)
+ if use_test:
+ dataset = pd.read_csv(RANKINGS_DATASET_TEST_PATH)
+ else:
+ dataset = pd.read_csv(RANKINGS_DATASET_TRAIN_PATH)
+
ndcg_scores = []
proportions = []
@@ -74,3 +78,7 @@ def evaluate(ranking_model: RankingModel, fraction: float = 1.0):
print()
print("proportion_score_mean:", np.mean(proportions))
print("proportion_score_sem:", sem(proportions))
+
+ print("Dataset\tNDCG\tSEM\tProportion\tSEM")
+ print(f"{'Test' if use_test else 'Train'}\t{np.mean(ndcg_scores)}\t{sem(ndcg_scores)}\t{np.mean(proportions)}\t"
+ f"{sem(proportions)}")
diff --git a/mwmbl/rankeval/evaluation/evaluate_ranker.py b/mwmbl/rankeval/evaluation/evaluate_ranker.py
index 52a6613..6b7ffa3 100644
--- a/mwmbl/rankeval/evaluation/evaluate_ranker.py
+++ b/mwmbl/rankeval/evaluation/evaluate_ranker.py
@@ -11,8 +11,8 @@ class MwmblRankingModel(RankingModel):
self.ranker = ranker
def predict(self, query: str) -> list[str]:
- results = self.ranker.search(query)
- return [x['url'] for x in results]
+ results = self.ranker.search(query, [])
+ return [x.url for x in results]
class DummyCompleter:
diff --git a/mwmbl/rankeval/evaluation/evaluate_remote.py b/mwmbl/rankeval/evaluation/evaluate_remote.py
index dfbd60e..0d3329d 100644
--- a/mwmbl/rankeval/evaluation/evaluate_remote.py
+++ b/mwmbl/rankeval/evaluation/evaluate_remote.py
@@ -7,6 +7,8 @@ from argparse import ArgumentParser
import requests
from joblib import Memory
+from mwmbl.rankeval.evaluation.evaluate_ranker import DummyCompleter, MwmblRankingModel
+from mwmbl.rankeval.evaluation.remote_index import RemoteIndex
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
from mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker
@@ -22,17 +24,9 @@ def fetch_results(url: str, query: str):
return results
-class RemoteRankingModel(RankingModel):
- def __init__(self, url: str = "https://mwmbl.org"):
- self.url = url
-
- def predict(self, query: str) -> list[str]:
- results = fetch_results(self.url, query)
- return [x['url'] for x in results]
-
-
def run():
- model = RemoteRankingModel()
+ ranker = HeuristicRanker(RemoteIndex(), DummyCompleter())
+ model = MwmblRankingModel(ranker)
evaluate(model, fraction=1.0)
diff --git a/mwmbl/rankeval/evaluation/remote_index.py b/mwmbl/rankeval/evaluation/remote_index.py
new file mode 100644
index 0000000..3b4de29
--- /dev/null
+++ b/mwmbl/rankeval/evaluation/remote_index.py
@@ -0,0 +1,25 @@
+"""
+Pretend to be an index but retrieve results from a remote index.
+"""
+import requests
+from joblib import Memory
+
+from mwmbl.tinysearchengine.indexer import Document
+
+memory = Memory(location='devdata/cache')
+
+
+def retrieve_url(url: str):
+ response = requests.get(url)
+ print(f"Retrieved {url}", response.content)
+ return response.json()
+
+
+class RemoteIndex:
+ url = "https://mwmbl.org/api/v1/search/raw?s="
+
+ def retrieve(self, query: str):
+ url = self.url + query
+ results = retrieve_url(url)
+ return [Document(**result) for result in results["results"]]