An open source, non-profit web search engine
Add wiki results
Daoud Clarkeda4a05142c35
diff --git a/mwmbl/format.py b/mwmbl/format.pyindex dbd646e..f4c6902 100644--- a/mwmbl/format.py+++ b/mwmbl/format.py@@ -10,6 +10,7 @@ DOCUMENT_SOURCES = {DocumentState.ORGANIC_APPROVED: 'mwmbl',DocumentState.FROM_GOOGLE_APPROVED: 'google',DocumentState.FROM_USER_APPROVED: 'user',+ DocumentState.FROM_WIKI: 'wikipedia',}diff --git a/mwmbl/rankeval/evaluation/evaluate_remote.py b/mwmbl/rankeval/evaluation/evaluate_remote.pyindex ef454a8..261d32d 100644--- a/mwmbl/rankeval/evaluation/evaluate_remote.py+++ b/mwmbl/rankeval/evaluation/evaluate_remote.py@@ -10,7 +10,7 @@ from joblib import Memoryfrom mwmbl.rankeval.evaluation.evaluate_ranker import DummyCompleter, MwmblRankingModelfrom mwmbl.rankeval.evaluation.remote_index import RemoteIndexfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document-from mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker+from mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker, HeuristicAndWikiRankerfrom mwmbl.rankeval.evaluation.evaluate import RankingModel, evaluate@@ -25,13 +25,13 @@ def fetch_results(url: str, query: str):def run():- ranker = HeuristicRanker(RemoteIndex(), DummyCompleter())+ ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter())model = MwmblRankingModel(ranker)evaluate(model, fraction=0.01)def single_query(query: str):- ranker = HeuristicRanker(RemoteIndex(), DummyCompleter())+ ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter())results = ranker.search(query, [])for result in results:print(result)diff --git a/mwmbl/rankeval/evaluation/evaluate_wiki.py b/mwmbl/rankeval/evaluation/evaluate_wiki.pyindex e0b8f4f..365ad44 100644--- a/mwmbl/rankeval/evaluation/evaluate_wiki.py+++ b/mwmbl/rankeval/evaluation/evaluate_wiki.py@@ -8,18 +8,11 @@ from joblib import Memoryfrom mwmbl.rankeval.evaluation.evaluate import evaluatefrom mwmbl.tinysearchengine.indexer import Document--WIKI_SEARCH_API_URL = "https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={query}&format=json"-WIKI_URL_FORMAT = "https://en.wikipedia.org/wiki/{title}"-+from mwmbl.tinysearchengine.rank import WIKI_SEARCH_API_URL, get_wiki_urlmemory = Memory(location="devdata/cache")-def get_wiki_url(title: str):- return WIKI_URL_FORMAT.format(title=title.replace(" ", "_"))--@memory.cachedef fetch_results(query: str) -> list[str]:safe_query = urllib.parse.quote(query, safe="")diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex d8cfc44..681374e 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -9,7 +9,7 @@ from mwmbl.models import DomainSubmissionfrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document-from mwmbl.tinysearchengine.rank import HeuristicRanker+from mwmbl.tinysearchengine.rank import HeuristicRanker, HeuristicAndWikiRankerdef get_curated_domains() -> set[str]:@@ -24,5 +24,5 @@ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAMEtiny_index = TinyIndex(item_factory=Document, index_path=index_path)tiny_index.__enter__()-ranker = HeuristicRanker(tiny_index, completer)+ranker = HeuristicAndWikiRanker(tiny_index, completer)batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.pyindex 160ca0c..67f9294 100644--- a/mwmbl/tinysearchengine/indexer.py+++ b/mwmbl/tinysearchengine/indexer.py@@ -40,6 +40,7 @@ class DocumentState(IntEnum):ORGANIC_APPROVED = 7FROM_USER_APPROVED = 8FROM_GOOGLE_APPROVED = 9+ FROM_WIKI = 10@dataclassdiff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex a251e99..c76793d 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -1,11 +1,16 @@import mathimport re+import urllibfrom abc import abstractmethod+from datetime import timedeltafrom logging import getLoggerfrom operator import itemgetterfrom typing import Optionalfrom urllib.parse import urlparse+import requests+from requests_cache import CachedSession+from mwmbl.format import format_result_with_pattern, get_query_regexfrom mwmbl.tokenizer import tokenize, get_bigramsfrom mwmbl.tinysearchengine.completer import Completer@@ -19,7 +24,7 @@ MATCH_SCORE_THRESHOLD = 0.0SCORE_THRESHOLD = 0.0LENGTH_PENALTY = 0.04MATCH_EXPONENT = 2-DOMAIN_SCORE_SMOOTHING = 50+DOMAIN_SCORE_SMOOTHING = 0.1HTTPS_STRING = 'https://'@@ -77,10 +82,16 @@ def get_features(terms, title, url, extract, score, is_complete):return features+DOMAIN_MAX_SCORE = max(DOMAINS.values())+DOMAIN_MIN_SCORE = min(DOMAINS.values())++def get_domain_score(url):domain = urlparse(url).netloc- domain_score = DOMAINS.get(domain, 0.0)- return domain_score+ if domain in DOMAINS:+ normalised_score = (DOMAINS[domain] - DOMAIN_MIN_SCORE) / (DOMAIN_MAX_SCORE - DOMAIN_MIN_SCORE)+ return normalised_score * normalised_score+ return 0.0def get_match_features(terms, result_string, is_complete, is_url):@@ -234,3 +245,33 @@ class Ranker:class HeuristicRanker(Ranker):def order_results(self, terms, pages, is_complete):return order_results(terms, pages, is_complete)+++WIKI_SEARCH_API_URL = "https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={query}&format=json"+WIKI_URL_FORMAT = "https://en.wikipedia.org/wiki/{title}"+++def get_wiki_url(title: str):+ return WIKI_URL_FORMAT.format(title=title.replace(" ", "_"))+++HTML_TAG_REGEX = re.compile(r'<[^>]+>')+++def clean_html(s: str):+ return HTML_TAG_REGEX.sub('', s)+++class HeuristicAndWikiRanker(HeuristicRanker):+ max_wiki_results = 5++ def search(self, s: str, additional_results: list[Document]) -> list[Document]:+ with CachedSession(expire_after=timedelta(weeks=12)) as session:+ escaped_query = urllib.parse.quote(s, safe='')+ wiki_response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query)).json()+ wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),+ 0.0, s, state=DocumentState.FROM_WIKI)+ for result in wiki_response['query']['search'][:self.max_wiki_results]]++ return super().search(s, wiki_results + additional_results)+