Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Refactor LTR ranker to preserve curations

Daoud Clarkee5991e3740bc
diff --git a/devdata/index-v2.tinysearch b/devdata/index-v2.tinysearch
index a8c5ccf..59e8d4d 100644
Binary files a/devdata/index-v2.tinysearch and b/devdata/index-v2.tinysearch differ
diff --git a/mwmbl/rankeval/evaluation/evaluate_remote.py b/mwmbl/rankeval/evaluation/evaluate_remote.py
index e11ae94..af43fe4 100644
--- a/mwmbl/rankeval/evaluation/evaluate_remote.py
+++ b/mwmbl/rankeval/evaluation/evaluate_remote.py
@@ -27,10 +27,10 @@ def fetch_results(url: str, query: str):
def run():
- ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter())
+ # ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter())
model = pickle.load(open(MODEL_PATH, 'rb'))
- ranker = LTRRanker(ranker, model, 1000)
+ ranker = LTRRanker(RemoteIndex(), DummyCompleter(), model, 1000, True, 5)
# ranker = HeuristicRanker(RemoteIndex(), DummyCompleter())
model = MwmblRankingModel(ranker)
evaluate(model, fraction=0.01, use_test=True)
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.py
index 681374e..337417c 100644
--- a/mwmbl/search_setup.py
+++ b/mwmbl/search_setup.py
@@ -1,4 +1,5 @@
import os
+import pickle
from pathlib import Path
from django.conf import settings
@@ -6,9 +7,11 @@ from redis import Redis
from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.models import DomainSubmission
+from mwmbl.rankeval.paths import MODEL_PATH
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.tinysearchengine.completer import Completer
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
+from mwmbl.tinysearchengine.ltr_rank import LTRRanker
from mwmbl.tinysearchengine.rank import HeuristicRanker, HeuristicAndWikiRanker
@@ -24,5 +27,7 @@ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
tiny_index = TinyIndex(item_factory=Document, index_path=index_path)
tiny_index.__enter__()
-ranker = HeuristicAndWikiRanker(tiny_index, completer)
+model = pickle.load(open(MODEL_PATH, 'rb'))
+ranker = LTRRanker(tiny_index, completer, model, 1000, True, 5)
+
batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)
diff --git a/mwmbl/tinysearchengine/ltr_rank.py b/mwmbl/tinysearchengine/ltr_rank.py
index d8253a4..f4d2939 100644
--- a/mwmbl/tinysearchengine/ltr_rank.py
+++ b/mwmbl/tinysearchengine/ltr_rank.py
@@ -5,23 +5,25 @@ from sklearn.base import BaseEstimator
from mwmbl.tinysearchengine.completer import Completer
from mwmbl.tinysearchengine.indexer import Document, TinyIndex
from mwmbl.tinysearchengine.ltr import FeatureExtractor
-from mwmbl.tinysearchengine.rank import Ranker
+from mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker, get_wiki_results
from mwmbl.tokenizer import tokenize
-class LTRRanker:
- def __init__(self, base_ranker: Ranker, model: BaseEstimator, top_n: int = 20):
- self.base_ranker = base_ranker
+class LTRRanker(HeuristicRanker):
+ def __init__(self, tiny_index: TinyIndex, completer: Completer, model: BaseEstimator,
+ top_n: int, include_wiki: bool = True, num_wiki_results: int = 5):
+ super().__init__(tiny_index, completer)
self.model = model
self.top_n = top_n
+ self.include_wiki = include_wiki
+ self.num_wiki_results = num_wiki_results
- def search(self, query: str, additional_results: list[Document]) -> list[Document]:
- pages = self.base_ranker.search(query, additional_results)
- if len(pages) == 0:
+ def order_results(self, terms: list[str], results: list[Document], is_complete: bool) -> list[Document]:
+ if len(results) == 0:
return []
- top_pages = pages[:self.top_n]
-
+ query = ' '.join(terms)
+ top_pages = results[:self.top_n]
data = {
'query': [query] * len(top_pages),
'url': [page.url for page in top_pages],
@@ -31,10 +33,13 @@ class LTRRanker:
}
dataframe = DataFrame(data)
- # feature_extractor = FeatureExtractor()
- # features = feature_extractor.transform(dataframe)
print("Ordering results", dataframe)
predictions = self.model.predict(dataframe)
indexes = np.argsort(predictions)[::-1]
return [top_pages[i] for i in indexes]
+
+ def external_search(self, query: str) -> list[Document]:
+ if self.include_wiki:
+ return get_wiki_results(query, self.num_wiki_results)
+ return []
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index 787adbd..f2ba59e 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -244,11 +244,15 @@ class Ranker:
if items is not None:
pages += items
- ordered_results = self.order_results(terms, pages + additional_results, is_complete)
+ external_search_items = self.external_search(q)
+ ordered_results = self.order_results(terms, pages + additional_results + external_search_items, is_complete)
deduplicated_results = deduplicate(curated_items + ordered_results, set())
state_fixed = [fix_document_state(result) for result in deduplicated_results]
return state_fixed, terms, completions
+ def external_search(self, q: str):
+ return []
+
def get_raw_results(self, query: str):
tokens = tokenize(query)
term = ' '.join(tokens)
@@ -275,6 +279,16 @@ def clean_html(s: str):
return html.unescape(HTML_TAG_REGEX.sub('', s))
+def get_wiki_results(s: str, max_wiki_results: int):
+ escaped_query = urllib.parse.quote(s, safe='')
+ with request_cache(timedelta(weeks=10)) as session:
+ wiki_response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query)).json()
+ wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),
+ max_wiki_results + 1 - i, s, state=DocumentState.FROM_WIKI)
+ for i, result in enumerate(wiki_response['query']['search'])]
+ return wiki_results
+
+
class HeuristicAndWikiRanker(HeuristicRanker):
max_wiki_results = 5
@@ -288,13 +302,7 @@ class HeuristicAndWikiRanker(HeuristicRanker):
if len(results) == 0 and self.return_none_if_no_mwmbl_results:
return []
- escaped_query = urllib.parse.quote(s, safe='')
- with request_cache(timedelta(weeks=10)) as session:
- wiki_response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query)).json()
-
- wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),
- self.max_wiki_results + 1 - i, s, state=DocumentState.FROM_WIKI)
- for i, result in enumerate(wiki_response['query']['search'])]
+ wiki_results = get_wiki_results(s, self.max_wiki_results)
return wiki_results + results