Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Cache Bing queries

Daoud Clarkeef84bdcd9a25
diff --git a/mwmbl/rankeval/dataset/get_gold_rankings.py b/mwmbl/rankeval/dataset/get_gold_rankings.py
index ca2fc5c..2ab3961 100644
--- a/mwmbl/rankeval/dataset/get_gold_rankings.py
+++ b/mwmbl/rankeval/dataset/get_gold_rankings.py
@@ -42,12 +42,14 @@ def run():
# Get one suggestion for each query
# Use this method: https://stackoverflow.com/a/46660098
# Shuffle, then take the top item from each group
- queries = query_dataset.sample(frac=1.0)\
+ queries = query_dataset.sample(frac=1.0, random_state=1)\
.groupby('query')\
.head()['suggestion']\
.to_list()
for i, path in enumerate([RANKINGS_DATASET_TRAIN_PATH, RANKINGS_DATASET_TEST_PATH]):
+ if i == 0:
+ continue
rankings = get_query_rankings(queries[NUM_QUERIES * i:NUM_QUERIES * (i+1)])
rankings.to_csv(path)
diff --git a/mwmbl/rankeval/dataset/search_api.py b/mwmbl/rankeval/dataset/search_api.py
index 48cd3ac..850e5d8 100644
--- a/mwmbl/rankeval/dataset/search_api.py
+++ b/mwmbl/rankeval/dataset/search_api.py
@@ -5,6 +5,7 @@ import os
from time import sleep
import requests
+from joblib import Memory
BING_API_SUBSCRIPTION_KEY = os.environ['BING_API_SUBSCRIPTION_KEY']
BING_SEARCH_API_ENDPOINT = "https://api.bing.microsoft.com/v7.0/search"
@@ -12,7 +13,12 @@ BING_SUGGEST_API_ENDPOINT = "https://api.bing.microsoft.com/v7.0/Suggestions"
SLEEP_TIME = 0.01
+memory = Memory(location='devdata/cache')
+
+
def retrieve_from_endpoint(query, endpoint, **args):
+ print(f"Retrieving query {query} from endpoint {endpoint}")
headers = {"Ocp-Apim-Subscription-Key": BING_API_SUBSCRIPTION_KEY}
params = {"q": query, "mkt": "en-US", "textDecorations": False, "textFormat": "HTML"}
params.update(args)
diff --git a/mwmbl/rankeval/paths.py b/mwmbl/rankeval/paths.py
index cfd1364..7f204f2 100644
--- a/mwmbl/rankeval/paths.py
+++ b/mwmbl/rankeval/paths.py
@@ -4,7 +4,7 @@ from pathlib import Path
DATA_DIR = Path(__file__).parent.parent.parent / 'devdata' / 'rankeval-2024-06'
QUERIES_DATASET_PATH = DATA_DIR / 'queries.csv'
-REMOTE_DATA_DIR = Path(__file__).parent.parent / 'remote-datasets'
+REMOTE_DATA_DIR = DATA_DIR / 'remote-datasets'
RANKINGS_DATASET_TRAIN_PATH = REMOTE_DATA_DIR / 'rankings-train.csv'
RANKINGS_DATASET_TEST_PATH = REMOTE_DATA_DIR / 'rankings-test.csv'