Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Build a new LTR dataset

Daoud Clarkecbab8f974486
diff --git a/mwmbl/rankeval/evaluation/remote_index.py b/mwmbl/rankeval/evaluation/remote_index.py
index 3b4de29..33a840b 100644
--- a/mwmbl/rankeval/evaluation/remote_index.py
+++ b/mwmbl/rankeval/evaluation/remote_index.py
@@ -1,19 +1,12 @@
"""
Pretend to be an index but retrieve results from a remote index.
"""
+import time
+
import requests
-from joblib import Memory
from mwmbl.tinysearchengine.indexer import Document
-
-memory = Memory(location='devdata/cache')
-
-
-def retrieve_url(url: str):
- response = requests.get(url)
- print(f"Retrieved {url}", response.content)
- return response.json()
+from mwmbl.utils import request_cache
class RemoteIndex:
@@ -21,5 +14,17 @@ class RemoteIndex:
def retrieve(self, query: str):
url = self.url + query
- results = retrieve_url(url)
+ response = None
+ with request_cache() as session:
+ for i in range(3):
+ try:
+ response = session.get(url, timeout=15)
+ break
+ except requests.exceptions.Timeout:
+ print(f"Timeout fetching {url}, sleeping")
+ time.sleep(1)
+ if response is None:
+ raise ValueError(f"Failed to fetch {url}")
+ print(f"Retrieved {url}", response.content)
+ results = response.json()
return [Document(**result) for result in results["results"]]
diff --git a/mwmbl/rankeval/ltr/dataset.py b/mwmbl/rankeval/ltr/dataset.py
index cc4aad2..6a2f6e6 100644
--- a/mwmbl/rankeval/ltr/dataset.py
+++ b/mwmbl/rankeval/ltr/dataset.py
@@ -4,52 +4,55 @@ Generate a learn-to-rank dataset.
import sys
import pandas as pd
-from mwmbl.tinysearchengine.indexer import TinyIndex, Document
-from mwmbl.tinysearchengine.rank import HeuristicRanker
+
+from mwmbl.rankeval.evaluation.remote_index import RemoteIndex
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentState
+from mwmbl.tinysearchengine.rank import HeuristicRanker, HeuristicAndWikiRanker
from mwmbl.rankeval.evaluation.evaluate_ranker import DummyCompleter
from mwmbl.rankeval.paths import RANKINGS_DATASET_TRAIN_PATH, LEARNING_TO_RANK_DATASET_PATH
def run():
- index_path = sys.argv[1]
-
completer = DummyCompleter()
- dataset = get_dataset(completer, index_path)
- print("Dataset", dataset)
- pd.DataFrame(dataset).to_csv(LEARNING_TO_RANK_DATASET_PATH)
-
-
-def get_dataset(completer, index_path):
- with TinyIndex(item_factory=Document, index_path=index_path) as tiny_index:
- ranker = HeuristicRanker(tiny_index, completer)
- gold_standard = pd.read_csv(RANKINGS_DATASET_TRAIN_PATH, index_col=0)
- dataset = []
- for query, rankings in gold_standard.groupby('query'):
- print("Query", query)
- gold_standard = dict(zip(rankings['url'].tolist(), rankings.index.tolist()))
-
- predicted, terms = ranker.get_results(query + ' ')
- if len(predicted) == 0:
- continue
-
- new_items = []
- found_gold = False
- for item in predicted:
- in_gold_standard = item.url in gold_standard
- new_items.append({
- 'gold_standard_rank': gold_standard.get(item.url),
- 'query': query,
- 'url': item.url,
- 'title': item.title,
- 'extract': item.extract,
- 'score': item.score,
- })
- if in_gold_standard:
- found_gold = True
- if found_gold:
- dataset += new_items
+ dataset = get_dataset(completer)
+ pd.DataFrame(dataset).to_csv(LEARNING_TO_RANK_DATASET_PATH, errors="replace")
+
+
+def get_dataset(completer):
+ index = RemoteIndex()
+ ranker = HeuristicAndWikiRanker(index, completer, return_none_if_no_mwmbl_results=True)
+ gold_standard = pd.read_csv(RANKINGS_DATASET_TRAIN_PATH, index_col=0)
+ dataset = []
+ for query, rankings in gold_standard.groupby('query'):
+ print("Query", query)
+ gold_standard = dict(zip(rankings['url'].tolist(), rankings.index.tolist()))
+
+ predicted = ranker.search(query + ' ', [])
+
+ if len(predicted) == 0:
+ continue
+
+ print("Found results", len(predicted))
+
+ new_items = []
+ found_gold = False
+ for item in predicted:
+ in_gold_standard = item.url in gold_standard
+ new_items.append({
+ 'gold_standard_rank': gold_standard.get(item.url),
+ 'query': query,
+ 'url': item.url,
+ 'title': item.title,
+ 'extract': item.extract,
+ 'state': item.state,
+ 'score': item.score,
+ })
+ if in_gold_standard:
+ found_gold = True
+ if found_gold:
+ dataset += new_items
return dataset
diff --git a/mwmbl/rankeval/ltr/evaluate.py b/mwmbl/rankeval/ltr/evaluate.py
index 42d4f90..0bfbfb4 100644
--- a/mwmbl/rankeval/ltr/evaluate.py
+++ b/mwmbl/rankeval/ltr/evaluate.py
@@ -47,7 +47,7 @@ def run():
predictor = PREDICTORS[args.predictor]
- dataset = pd.read_csv(LEARNING_TO_RANK_DATASET_PATH)
+ dataset = pd.read_csv(LEARNING_TO_RANK_DATASET_PATH, lineterminator='\n')
dataset['gold_discount'] = dataset['gold_standard_rank'].apply(get_discount)
print("Gold standard", dataset['gold_discount'])
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index c0186ba..74a8ddb 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -131,8 +131,7 @@ def order_results(terms: list[str], results: list[Document], is_complete: bool)
if len(results) == 0:
return []
- wiki_results = [result for result in results if result.state == DocumentState.FROM_WIKI
- and get_wiki_score(result.url) > 0.000001]
+ wiki_results = [result for result in results if result.state == DocumentState.FROM_WIKI]
wiki_urls = {result.url for result in wiki_results}
other_results = [result for result in results if result.url not in wiki_urls]
results_and_scores = [(score_result(terms, result, is_complete), result) for result in other_results]
@@ -282,14 +281,24 @@ def clean_html(s: str):
class HeuristicAndWikiRanker(HeuristicRanker):
max_wiki_results = 5
+ def __init__(self, tiny_index: TinyIndex, completer: Completer, return_none_if_no_mwmbl_results=False):
+ super().__init__(tiny_index, completer)
+ self.return_none_if_no_mwmbl_results = return_none_if_no_mwmbl_results
+
def search(self, s: str, additional_results: list[Document]) -> list[Document]:
+ results = super().search(s, additional_results)
+
+ if len(results) == 0 and self.return_none_if_no_mwmbl_results:
+ return []
+
escaped_query = urllib.parse.quote(s, safe='')
with request_cache(timedelta(weeks=10)) as session:
wiki_response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query)).json()
wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),
- 0.0, s, state=DocumentState.FROM_WIKI)
- for result in wiki_response['query']['search'][:self.max_wiki_results]]
+ self.max_wiki_results + 1 - i, s, state=DocumentState.FROM_WIKI)
+ for i, result in enumerate(wiki_response['query']['search'])]
+
+ return wiki_results + results
- return super().search(s, wiki_results + additional_results)
diff --git a/mwmbl/utils.py b/mwmbl/utils.py
index 4554ed5..cf6d268 100644
--- a/mwmbl/utils.py
+++ b/mwmbl/utils.py
@@ -1,11 +1,12 @@
import re
from dataclasses import dataclass
from datetime import timedelta
-from typing import Sequence
+from typing import Sequence, Optional
from django.conf import settings
from django.core.exceptions import ValidationError
from requests_cache import CachedSession
+from requests.adapters import Retry, HTTPAdapter
from mwmbl.indexer.index import tokenize_document
from mwmbl.tinysearchengine.indexer import Document, TinyIndex
@@ -85,5 +86,5 @@ def validate_domain(domain_or_url: str):
raise ValidationError("Invalid domain: %(domain)s", params={"domain": domain_or_url})
-def request_cache(expire_after: timedelta) -> CachedSession:
+def request_cache(expire_after: Optional[timedelta] = None) -> CachedSession:
return CachedSession(expire_after=expire_after, backend="sqlite", cache_name=settings.REQUEST_CACHE_PATH)
diff --git a/poetry.lock b/poetry.lock
index 2e44154..409a1e6 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1286,6 +1286,16 @@ files = [
{file = "numpy-1.23.2.tar.gz", hash = "sha256:b78d00e48261fbbd04aa0d7427cf78d18401ee0abd89c7559bbf422e5b1c7d01"},
]
+[[package]]
+name = "nvidia-nccl-cu12"
+version = "2.22.3"
+description = "NVIDIA Collective Communication Library (NCCL) Runtime"
+optional = false
+python-versions = ">=3"
+files = [
+ {file = "nvidia_nccl_cu12-2.22.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f9f5e03c00269dee2cd1aa57019f9a024478a74ae6e9b32d5341c849fe6f6302"},
+]
+
[[package]]
name = "oauthlib"
version = "3.2.2"
@@ -2651,6 +2661,36 @@ files = [
{file = "wasabi-0.10.1.tar.gz", hash = "sha256:c8e372781be19272942382b14d99314d175518d7822057cb7a97010c4259d249"},
]
+[[package]]
+name = "xgboost"
+version = "2.1.0"
+description = "XGBoost Python Package"
+optional = false
+python-versions = ">=3.8"
+files = [
+ {file = "xgboost-2.1.0-py3-none-macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64.whl", hash = "sha256:19d145eb847b070c32342b1bf2d7331c102783e07a484f8b13b7d759d707c6b0"},
+ {file = "xgboost-2.1.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:840a0c6e2119d8c8f260a5dace996ea064a267f62b301a25d7d452488a7ac860"},
+ {file = "xgboost-2.1.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:73673c9bb85927db7fe2e3aed6df6d35dba708cfd6767cc63d4ea11dda2dede5"},
+ {file = "xgboost-2.1.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:43b16205689249d7509daf7a6ab00ad0e6c570b3a9c263cb32b26e39d9477bb3"},
+ {file = "xgboost-2.1.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:cedc2e386e686795735448fd4597533acacc5ba6fb47dd910c204c468b80bb96"},
+ {file = "xgboost-2.1.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:b2a456eb0f3d3e8fd8ab37e44ac288292bf8ea8744c294be9fd88713d27af810"},
+ {file = "xgboost-2.1.0-py3-none-win_amd64.whl", hash = "sha256:74904b91c42524a6c32147fe5718569e78fb65911ff4499b053f81d0964514d4"},
+ {file = "xgboost-2.1.0.tar.gz", hash = "sha256:7144980923e76ce741c7b03a14d3bd7514db6de5c7cabe96ba95b229d274f5ca"},
+]
+
+[package.dependencies]
+numpy = "*"
+nvidia-nccl-cu12 = {version = "*", markers = "platform_system == \"Linux\" and platform_machine != \"aarch64\""}
+scipy = "*"
+
+[package.extras]
+dask = ["dask", "distributed", "pandas"]
+datatable = ["datatable"]
+pandas = ["pandas"]
+plotting = ["graphviz", "matplotlib"]
+pyspark = ["cloudpickle", "pyspark", "scikit-learn"]
+scikit-learn = ["scikit-learn"]
+
[[package]]
name = "zstandard"
version = "0.16.0"
@@ -2716,4 +2756,4 @@ indexer = ["Levenshtein", "beautifulsoup4", "idna", "langdetect", "pyarrow", "py
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.11"
-content-hash = "45fc86862c2dba353d6bd23111d063993023cef0f77474371333902d92b8e312"
+content-hash = "42840ae2874152edd09a9488588b062135c63807e6d0e2863b987bcc41ef30c0"
diff --git a/pyproject.toml b/pyproject.toml
index 84dc93b..f311a84 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,6 +46,7 @@ lxml = "^5.2.0"
numpy = "1.23.2"
objgraph = "^3.6.1"
psutil = "^6.0.0"
+xgboost = "^2.1.0"
[tool.poetry.extras]
indexer = [
"ujson",