An open source, non-profit web search engine
Cache Bing queries
Daoud Clarkeef84bdcd9a25
diff --git a/mwmbl/rankeval/dataset/get_gold_rankings.py b/mwmbl/rankeval/dataset/get_gold_rankings.pyindex ca2fc5c..2ab3961 100644--- a/mwmbl/rankeval/dataset/get_gold_rankings.py+++ b/mwmbl/rankeval/dataset/get_gold_rankings.py@@ -42,12 +42,14 @@ def run():# Get one suggestion for each query# Use this method: https://stackoverflow.com/a/46660098# Shuffle, then take the top item from each group- queries = query_dataset.sample(frac=1.0)\+ queries = query_dataset.sample(frac=1.0, random_state=1)\.groupby('query')\.head()['suggestion']\.to_list()for i, path in enumerate([RANKINGS_DATASET_TRAIN_PATH, RANKINGS_DATASET_TEST_PATH]):+ if i == 0:+ continuerankings = get_query_rankings(queries[NUM_QUERIES * i:NUM_QUERIES * (i+1)])rankings.to_csv(path)diff --git a/mwmbl/rankeval/dataset/search_api.py b/mwmbl/rankeval/dataset/search_api.pyindex 48cd3ac..850e5d8 100644--- a/mwmbl/rankeval/dataset/search_api.py+++ b/mwmbl/rankeval/dataset/search_api.py@@ -5,6 +5,7 @@ import osfrom time import sleepimport requests+from joblib import MemoryBING_API_SUBSCRIPTION_KEY = os.environ['BING_API_SUBSCRIPTION_KEY']BING_SEARCH_API_ENDPOINT = "https://api.bing.microsoft.com/v7.0/search"@@ -12,7 +13,12 @@ BING_SUGGEST_API_ENDPOINT = "https://api.bing.microsoft.com/v7.0/Suggestions"SLEEP_TIME = 0.01+memory = Memory(location='devdata/cache')++def retrieve_from_endpoint(query, endpoint, **args):+ print(f"Retrieving query {query} from endpoint {endpoint}")headers = {"Ocp-Apim-Subscription-Key": BING_API_SUBSCRIPTION_KEY}params = {"q": query, "mkt": "en-US", "textDecorations": False, "textFormat": "HTML"}params.update(args)diff --git a/mwmbl/rankeval/paths.py b/mwmbl/rankeval/paths.pyindex cfd1364..7f204f2 100644--- a/mwmbl/rankeval/paths.py+++ b/mwmbl/rankeval/paths.py@@ -4,7 +4,7 @@ from pathlib import PathDATA_DIR = Path(__file__).parent.parent.parent / 'devdata' / 'rankeval-2024-06'QUERIES_DATASET_PATH = DATA_DIR / 'queries.csv'-REMOTE_DATA_DIR = Path(__file__).parent.parent / 'remote-datasets'+REMOTE_DATA_DIR = DATA_DIR / 'remote-datasets'RANKINGS_DATASET_TRAIN_PATH = REMOTE_DATA_DIR / 'rankings-train.csv'RANKINGS_DATASET_TEST_PATH = REMOTE_DATA_DIR / 'rankings-test.csv'