An open source, non-profit web search engine
Refactor LTR ranker to preserve curations
Daoud Clarkee5991e3740bc
diff --git a/devdata/index-v2.tinysearch b/devdata/index-v2.tinysearchindex a8c5ccf..59e8d4d 100644Binary files a/devdata/index-v2.tinysearch and b/devdata/index-v2.tinysearch differdiff --git a/mwmbl/rankeval/evaluation/evaluate_remote.py b/mwmbl/rankeval/evaluation/evaluate_remote.pyindex e11ae94..af43fe4 100644--- a/mwmbl/rankeval/evaluation/evaluate_remote.py+++ b/mwmbl/rankeval/evaluation/evaluate_remote.py@@ -27,10 +27,10 @@ def fetch_results(url: str, query: str):def run():- ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter())+ # ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter())model = pickle.load(open(MODEL_PATH, 'rb'))- ranker = LTRRanker(ranker, model, 1000)+ ranker = LTRRanker(RemoteIndex(), DummyCompleter(), model, 1000, True, 5)# ranker = HeuristicRanker(RemoteIndex(), DummyCompleter())model = MwmblRankingModel(ranker)evaluate(model, fraction=0.01, use_test=True)diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex 681374e..337417c 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -1,4 +1,5 @@import os+import picklefrom pathlib import Pathfrom django.conf import settings@@ -6,9 +7,11 @@ from redis import Redisfrom mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.models import DomainSubmission+from mwmbl.rankeval.paths import MODEL_PATHfrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document+from mwmbl.tinysearchengine.ltr_rank import LTRRankerfrom mwmbl.tinysearchengine.rank import HeuristicRanker, HeuristicAndWikiRanker@@ -24,5 +27,7 @@ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAMEtiny_index = TinyIndex(item_factory=Document, index_path=index_path)tiny_index.__enter__()-ranker = HeuristicAndWikiRanker(tiny_index, completer)+model = pickle.load(open(MODEL_PATH, 'rb'))+ranker = LTRRanker(tiny_index, completer, model, 1000, True, 5)+batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)diff --git a/mwmbl/tinysearchengine/ltr_rank.py b/mwmbl/tinysearchengine/ltr_rank.pyindex d8253a4..f4d2939 100644--- a/mwmbl/tinysearchengine/ltr_rank.py+++ b/mwmbl/tinysearchengine/ltr_rank.py@@ -5,23 +5,25 @@ from sklearn.base import BaseEstimatorfrom mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import Document, TinyIndexfrom mwmbl.tinysearchengine.ltr import FeatureExtractor-from mwmbl.tinysearchengine.rank import Ranker+from mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker, get_wiki_resultsfrom mwmbl.tokenizer import tokenize-class LTRRanker:- def __init__(self, base_ranker: Ranker, model: BaseEstimator, top_n: int = 20):- self.base_ranker = base_ranker+class LTRRanker(HeuristicRanker):+ def __init__(self, tiny_index: TinyIndex, completer: Completer, model: BaseEstimator,+ top_n: int, include_wiki: bool = True, num_wiki_results: int = 5):+ super().__init__(tiny_index, completer)self.model = modelself.top_n = top_n+ self.include_wiki = include_wiki+ self.num_wiki_results = num_wiki_results- def search(self, query: str, additional_results: list[Document]) -> list[Document]:- pages = self.base_ranker.search(query, additional_results)- if len(pages) == 0:+ def order_results(self, terms: list[str], results: list[Document], is_complete: bool) -> list[Document]:+ if len(results) == 0:return []- top_pages = pages[:self.top_n]-+ query = ' '.join(terms)+ top_pages = results[:self.top_n]data = {'query': [query] * len(top_pages),'url': [page.url for page in top_pages],@@ -31,10 +33,13 @@ class LTRRanker:}dataframe = DataFrame(data)- # feature_extractor = FeatureExtractor()- # features = feature_extractor.transform(dataframe)print("Ordering results", dataframe)predictions = self.model.predict(dataframe)indexes = np.argsort(predictions)[::-1]return [top_pages[i] for i in indexes]++ def external_search(self, query: str) -> list[Document]:+ if self.include_wiki:+ return get_wiki_results(query, self.num_wiki_results)+ return []diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 787adbd..f2ba59e 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -244,11 +244,15 @@ class Ranker:if items is not None:pages += items- ordered_results = self.order_results(terms, pages + additional_results, is_complete)+ external_search_items = self.external_search(q)+ ordered_results = self.order_results(terms, pages + additional_results + external_search_items, is_complete)deduplicated_results = deduplicate(curated_items + ordered_results, set())state_fixed = [fix_document_state(result) for result in deduplicated_results]return state_fixed, terms, completions+ def external_search(self, q: str):+ return []+def get_raw_results(self, query: str):tokens = tokenize(query)term = ' '.join(tokens)@@ -275,6 +279,16 @@ def clean_html(s: str):return html.unescape(HTML_TAG_REGEX.sub('', s))+def get_wiki_results(s: str, max_wiki_results: int):+ escaped_query = urllib.parse.quote(s, safe='')+ with request_cache(timedelta(weeks=10)) as session:+ wiki_response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query)).json()+ wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),+ max_wiki_results + 1 - i, s, state=DocumentState.FROM_WIKI)+ for i, result in enumerate(wiki_response['query']['search'])]+ return wiki_results++class HeuristicAndWikiRanker(HeuristicRanker):max_wiki_results = 5@@ -288,13 +302,7 @@ class HeuristicAndWikiRanker(HeuristicRanker):if len(results) == 0 and self.return_none_if_no_mwmbl_results:return []- escaped_query = urllib.parse.quote(s, safe='')- with request_cache(timedelta(weeks=10)) as session:- wiki_response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query)).json()-- wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),- self.max_wiki_results + 1 - i, s, state=DocumentState.FROM_WIKI)- for i, result in enumerate(wiki_response['query']['search'])]+ wiki_results = get_wiki_results(s, self.max_wiki_results)return wiki_results + results