An open source, non-profit web search engine
Build a new LTR dataset
Daoud Clarkecbab8f974486
diff --git a/mwmbl/rankeval/evaluation/remote_index.py b/mwmbl/rankeval/evaluation/remote_index.pyindex 3b4de29..33a840b 100644--- a/mwmbl/rankeval/evaluation/remote_index.py+++ b/mwmbl/rankeval/evaluation/remote_index.py@@ -1,19 +1,12 @@"""Pretend to be an index but retrieve results from a remote index."""+import time+import requests-from joblib import Memoryfrom mwmbl.tinysearchengine.indexer import Document--memory = Memory(location='devdata/cache')---def retrieve_url(url: str):- response = requests.get(url)- print(f"Retrieved {url}", response.content)- return response.json()+from mwmbl.utils import request_cacheclass RemoteIndex:@@ -21,5 +14,17 @@ class RemoteIndex:def retrieve(self, query: str):url = self.url + query- results = retrieve_url(url)+ response = None+ with request_cache() as session:+ for i in range(3):+ try:+ response = session.get(url, timeout=15)+ break+ except requests.exceptions.Timeout:+ print(f"Timeout fetching {url}, sleeping")+ time.sleep(1)+ if response is None:+ raise ValueError(f"Failed to fetch {url}")+ print(f"Retrieved {url}", response.content)+ results = response.json()return [Document(**result) for result in results["results"]]diff --git a/mwmbl/rankeval/ltr/dataset.py b/mwmbl/rankeval/ltr/dataset.pyindex cc4aad2..6a2f6e6 100644--- a/mwmbl/rankeval/ltr/dataset.py+++ b/mwmbl/rankeval/ltr/dataset.py@@ -4,52 +4,55 @@ Generate a learn-to-rank dataset.import sysimport pandas as pd-from mwmbl.tinysearchengine.indexer import TinyIndex, Document-from mwmbl.tinysearchengine.rank import HeuristicRanker++from mwmbl.rankeval.evaluation.remote_index import RemoteIndex+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentState+from mwmbl.tinysearchengine.rank import HeuristicRanker, HeuristicAndWikiRankerfrom mwmbl.rankeval.evaluation.evaluate_ranker import DummyCompleterfrom mwmbl.rankeval.paths import RANKINGS_DATASET_TRAIN_PATH, LEARNING_TO_RANK_DATASET_PATHdef run():- index_path = sys.argv[1]-completer = DummyCompleter()- dataset = get_dataset(completer, index_path)- print("Dataset", dataset)- pd.DataFrame(dataset).to_csv(LEARNING_TO_RANK_DATASET_PATH)---def get_dataset(completer, index_path):- with TinyIndex(item_factory=Document, index_path=index_path) as tiny_index:- ranker = HeuristicRanker(tiny_index, completer)- gold_standard = pd.read_csv(RANKINGS_DATASET_TRAIN_PATH, index_col=0)- dataset = []- for query, rankings in gold_standard.groupby('query'):- print("Query", query)- gold_standard = dict(zip(rankings['url'].tolist(), rankings.index.tolist()))-- predicted, terms = ranker.get_results(query + ' ')- if len(predicted) == 0:- continue-- new_items = []- found_gold = False- for item in predicted:- in_gold_standard = item.url in gold_standard- new_items.append({- 'gold_standard_rank': gold_standard.get(item.url),- 'query': query,- 'url': item.url,- 'title': item.title,- 'extract': item.extract,- 'score': item.score,- })- if in_gold_standard:- found_gold = True- if found_gold:- dataset += new_items+ dataset = get_dataset(completer)+ pd.DataFrame(dataset).to_csv(LEARNING_TO_RANK_DATASET_PATH, errors="replace")+++def get_dataset(completer):+ index = RemoteIndex()+ ranker = HeuristicAndWikiRanker(index, completer, return_none_if_no_mwmbl_results=True)+ gold_standard = pd.read_csv(RANKINGS_DATASET_TRAIN_PATH, index_col=0)+ dataset = []+ for query, rankings in gold_standard.groupby('query'):+ print("Query", query)+ gold_standard = dict(zip(rankings['url'].tolist(), rankings.index.tolist()))++ predicted = ranker.search(query + ' ', [])++ if len(predicted) == 0:+ continue++ print("Found results", len(predicted))++ new_items = []+ found_gold = False+ for item in predicted:+ in_gold_standard = item.url in gold_standard+ new_items.append({+ 'gold_standard_rank': gold_standard.get(item.url),+ 'query': query,+ 'url': item.url,+ 'title': item.title,+ 'extract': item.extract,+ 'state': item.state,+ 'score': item.score,+ })+ if in_gold_standard:+ found_gold = True+ if found_gold:+ dataset += new_itemsreturn datasetdiff --git a/mwmbl/rankeval/ltr/evaluate.py b/mwmbl/rankeval/ltr/evaluate.pyindex 42d4f90..0bfbfb4 100644--- a/mwmbl/rankeval/ltr/evaluate.py+++ b/mwmbl/rankeval/ltr/evaluate.py@@ -47,7 +47,7 @@ def run():predictor = PREDICTORS[args.predictor]- dataset = pd.read_csv(LEARNING_TO_RANK_DATASET_PATH)+ dataset = pd.read_csv(LEARNING_TO_RANK_DATASET_PATH, lineterminator='\n')dataset['gold_discount'] = dataset['gold_standard_rank'].apply(get_discount)print("Gold standard", dataset['gold_discount'])diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex c0186ba..74a8ddb 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -131,8 +131,7 @@ def order_results(terms: list[str], results: list[Document], is_complete: bool)if len(results) == 0:return []- wiki_results = [result for result in results if result.state == DocumentState.FROM_WIKI- and get_wiki_score(result.url) > 0.000001]+ wiki_results = [result for result in results if result.state == DocumentState.FROM_WIKI]wiki_urls = {result.url for result in wiki_results}other_results = [result for result in results if result.url not in wiki_urls]results_and_scores = [(score_result(terms, result, is_complete), result) for result in other_results]@@ -282,14 +281,24 @@ def clean_html(s: str):class HeuristicAndWikiRanker(HeuristicRanker):max_wiki_results = 5+ def __init__(self, tiny_index: TinyIndex, completer: Completer, return_none_if_no_mwmbl_results=False):+ super().__init__(tiny_index, completer)+ self.return_none_if_no_mwmbl_results = return_none_if_no_mwmbl_results+def search(self, s: str, additional_results: list[Document]) -> list[Document]:+ results = super().search(s, additional_results)++ if len(results) == 0 and self.return_none_if_no_mwmbl_results:+ return []+escaped_query = urllib.parse.quote(s, safe='')with request_cache(timedelta(weeks=10)) as session:wiki_response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query)).json()wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),- 0.0, s, state=DocumentState.FROM_WIKI)- for result in wiki_response['query']['search'][:self.max_wiki_results]]+ self.max_wiki_results + 1 - i, s, state=DocumentState.FROM_WIKI)+ for i, result in enumerate(wiki_response['query']['search'])]++ return wiki_results + results- return super().search(s, wiki_results + additional_results)diff --git a/mwmbl/utils.py b/mwmbl/utils.pyindex 4554ed5..cf6d268 100644--- a/mwmbl/utils.py+++ b/mwmbl/utils.py@@ -1,11 +1,12 @@import refrom dataclasses import dataclassfrom datetime import timedelta-from typing import Sequence+from typing import Sequence, Optionalfrom django.conf import settingsfrom django.core.exceptions import ValidationErrorfrom requests_cache import CachedSession+from requests.adapters import Retry, HTTPAdapterfrom mwmbl.indexer.index import tokenize_documentfrom mwmbl.tinysearchengine.indexer import Document, TinyIndex@@ -85,5 +86,5 @@ def validate_domain(domain_or_url: str):raise ValidationError("Invalid domain: %(domain)s", params={"domain": domain_or_url})-def request_cache(expire_after: timedelta) -> CachedSession:+def request_cache(expire_after: Optional[timedelta] = None) -> CachedSession:return CachedSession(expire_after=expire_after, backend="sqlite", cache_name=settings.REQUEST_CACHE_PATH)diff --git a/poetry.lock b/poetry.lockindex 2e44154..409a1e6 100644--- a/poetry.lock+++ b/poetry.lock@@ -1286,6 +1286,16 @@ files = [{file = "numpy-1.23.2.tar.gz", hash = "sha256:b78d00e48261fbbd04aa0d7427cf78d18401ee0abd89c7559bbf422e5b1c7d01"},]+[[package]]+name = "nvidia-nccl-cu12"+version = "2.22.3"+description = "NVIDIA Collective Communication Library (NCCL) Runtime"+optional = false+python-versions = ">=3"+files = [+ {file = "nvidia_nccl_cu12-2.22.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f9f5e03c00269dee2cd1aa57019f9a024478a74ae6e9b32d5341c849fe6f6302"},+]+[[package]]name = "oauthlib"version = "3.2.2"@@ -2651,6 +2661,36 @@ files = [{file = "wasabi-0.10.1.tar.gz", hash = "sha256:c8e372781be19272942382b14d99314d175518d7822057cb7a97010c4259d249"},]+[[package]]+name = "xgboost"+version = "2.1.0"+description = "XGBoost Python Package"+optional = false+python-versions = ">=3.8"+files = [+ {file = "xgboost-2.1.0-py3-none-macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64.whl", hash = "sha256:19d145eb847b070c32342b1bf2d7331c102783e07a484f8b13b7d759d707c6b0"},+ {file = "xgboost-2.1.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:840a0c6e2119d8c8f260a5dace996ea064a267f62b301a25d7d452488a7ac860"},+ {file = "xgboost-2.1.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:73673c9bb85927db7fe2e3aed6df6d35dba708cfd6767cc63d4ea11dda2dede5"},+ {file = "xgboost-2.1.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:43b16205689249d7509daf7a6ab00ad0e6c570b3a9c263cb32b26e39d9477bb3"},+ {file = "xgboost-2.1.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:cedc2e386e686795735448fd4597533acacc5ba6fb47dd910c204c468b80bb96"},+ {file = "xgboost-2.1.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:b2a456eb0f3d3e8fd8ab37e44ac288292bf8ea8744c294be9fd88713d27af810"},+ {file = "xgboost-2.1.0-py3-none-win_amd64.whl", hash = "sha256:74904b91c42524a6c32147fe5718569e78fb65911ff4499b053f81d0964514d4"},+ {file = "xgboost-2.1.0.tar.gz", hash = "sha256:7144980923e76ce741c7b03a14d3bd7514db6de5c7cabe96ba95b229d274f5ca"},+]++[package.dependencies]+numpy = "*"+nvidia-nccl-cu12 = {version = "*", markers = "platform_system == \"Linux\" and platform_machine != \"aarch64\""}+scipy = "*"++[package.extras]+dask = ["dask", "distributed", "pandas"]+datatable = ["datatable"]+pandas = ["pandas"]+plotting = ["graphviz", "matplotlib"]+pyspark = ["cloudpickle", "pyspark", "scikit-learn"]+scikit-learn = ["scikit-learn"]+[[package]]name = "zstandard"version = "0.16.0"@@ -2716,4 +2756,4 @@ indexer = ["Levenshtein", "beautifulsoup4", "idna", "langdetect", "pyarrow", "py[metadata]lock-version = "2.0"python-versions = ">=3.10,<3.11"-content-hash = "45fc86862c2dba353d6bd23111d063993023cef0f77474371333902d92b8e312"+content-hash = "42840ae2874152edd09a9488588b062135c63807e6d0e2863b987bcc41ef30c0"diff --git a/pyproject.toml b/pyproject.tomlindex 84dc93b..f311a84 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -46,6 +46,7 @@ lxml = "^5.2.0"numpy = "1.23.2"objgraph = "^3.6.1"psutil = "^6.0.0"+xgboost = "^2.1.0"[tool.poetry.extras]indexer = ["ujson",