An open source, non-profit web search engine
Use remote index so we can tweak the algorithm
Daoud Clarkebee015bb336e
diff --git a/mwmbl/rankeval/evaluation/evaluate.py b/mwmbl/rankeval/evaluation/evaluate.pyindex c27b724..781ae9b 100644--- a/mwmbl/rankeval/evaluation/evaluate.py+++ b/mwmbl/rankeval/evaluation/evaluate.py@@ -10,7 +10,7 @@ from sklearn.metrics import ndcg_score# Sourced from https://www.searchenginejournal.com/google-first-page-clicks/374516/-from mwmbl.rankeval.paths import RANKINGS_DATASET_TEST_PATH+from mwmbl.rankeval.paths import RANKINGS_DATASET_TEST_PATH, RANKINGS_DATASET_TRAIN_PATHCLICK_PROPORTIONS = [0.285, 0.157, 0.110, 0.080, 0.072, 0.051, 0.040, 0.032, 0.028, 0.025]NUM_RESULTS_FOR_EVAL = len(CLICK_PROPORTIONS)@@ -28,12 +28,16 @@ class RankingModel(ABC):pass-def evaluate(ranking_model: RankingModel, fraction: float = 1.0):+def evaluate(ranking_model: RankingModel, fraction: float = 1.0, use_test=False):# TODO:# - output feature importances from XGBoost# - experiment with more features- dataset = pd.read_csv(RANKINGS_DATASET_TEST_PATH)+ if use_test:+ dataset = pd.read_csv(RANKINGS_DATASET_TEST_PATH)+ else:+ dataset = pd.read_csv(RANKINGS_DATASET_TRAIN_PATH)+ndcg_scores = []proportions = []@@ -74,3 +78,7 @@ def evaluate(ranking_model: RankingModel, fraction: float = 1.0):print()print("proportion_score_mean:", np.mean(proportions))print("proportion_score_sem:", sem(proportions))++ print("Dataset\tNDCG\tSEM\tProportion\tSEM")+ print(f"{'Test' if use_test else 'Train'}\t{np.mean(ndcg_scores)}\t{sem(ndcg_scores)}\t{np.mean(proportions)}\t"+ f"{sem(proportions)}")diff --git a/mwmbl/rankeval/evaluation/evaluate_ranker.py b/mwmbl/rankeval/evaluation/evaluate_ranker.pyindex 52a6613..6b7ffa3 100644--- a/mwmbl/rankeval/evaluation/evaluate_ranker.py+++ b/mwmbl/rankeval/evaluation/evaluate_ranker.py@@ -11,8 +11,8 @@ class MwmblRankingModel(RankingModel):self.ranker = rankerdef predict(self, query: str) -> list[str]:- results = self.ranker.search(query)- return [x['url'] for x in results]+ results = self.ranker.search(query, [])+ return [x.url for x in results]class DummyCompleter:diff --git a/mwmbl/rankeval/evaluation/evaluate_remote.py b/mwmbl/rankeval/evaluation/evaluate_remote.pyindex dfbd60e..0d3329d 100644--- a/mwmbl/rankeval/evaluation/evaluate_remote.py+++ b/mwmbl/rankeval/evaluation/evaluate_remote.py@@ -7,6 +7,8 @@ from argparse import ArgumentParserimport requestsfrom joblib import Memory+from mwmbl.rankeval.evaluation.evaluate_ranker import DummyCompleter, MwmblRankingModel+from mwmbl.rankeval.evaluation.remote_index import RemoteIndexfrom mwmbl.tinysearchengine.indexer import TinyIndex, Documentfrom mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker@@ -22,17 +24,9 @@ def fetch_results(url: str, query: str):return results-class RemoteRankingModel(RankingModel):- def __init__(self, url: str = "https://mwmbl.org"):- self.url = url-- def predict(self, query: str) -> list[str]:- results = fetch_results(self.url, query)- return [x['url'] for x in results]--def run():- model = RemoteRankingModel()+ ranker = HeuristicRanker(RemoteIndex(), DummyCompleter())+ model = MwmblRankingModel(ranker)evaluate(model, fraction=1.0)diff --git a/mwmbl/rankeval/evaluation/remote_index.py b/mwmbl/rankeval/evaluation/remote_index.pynew file mode 100644index 0000000..3b4de29--- /dev/null+++ b/mwmbl/rankeval/evaluation/remote_index.py@@ -0,0 +1,25 @@+"""+Pretend to be an index but retrieve results from a remote index.+"""+import requests+from joblib import Memory++from mwmbl.tinysearchengine.indexer import Document++memory = Memory(location='devdata/cache')+++def retrieve_url(url: str):+ response = requests.get(url)+ print(f"Retrieved {url}", response.content)+ return response.json()+++class RemoteIndex:+ url = "https://mwmbl.org/api/v1/search/raw?s="++ def retrieve(self, query: str):+ url = self.url + query+ results = retrieve_url(url)+ return [Document(**result) for result in results["results"]]