Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add wiki results

Daoud Clarkeda4a05142c35
diff --git a/mwmbl/format.py b/mwmbl/format.py
index dbd646e..f4c6902 100644
--- a/mwmbl/format.py
+++ b/mwmbl/format.py
@@ -10,6 +10,7 @@ DOCUMENT_SOURCES = {
DocumentState.ORGANIC_APPROVED: 'mwmbl',
DocumentState.FROM_GOOGLE_APPROVED: 'google',
DocumentState.FROM_USER_APPROVED: 'user',
+ DocumentState.FROM_WIKI: 'wikipedia',
}
diff --git a/mwmbl/rankeval/evaluation/evaluate_remote.py b/mwmbl/rankeval/evaluation/evaluate_remote.py
index ef454a8..261d32d 100644
--- a/mwmbl/rankeval/evaluation/evaluate_remote.py
+++ b/mwmbl/rankeval/evaluation/evaluate_remote.py
@@ -10,7 +10,7 @@ from joblib import Memory
from mwmbl.rankeval.evaluation.evaluate_ranker import DummyCompleter, MwmblRankingModel
from mwmbl.rankeval.evaluation.remote_index import RemoteIndex
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
-from mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker
+from mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker, HeuristicAndWikiRanker
from mwmbl.rankeval.evaluation.evaluate import RankingModel, evaluate
@@ -25,13 +25,13 @@ def fetch_results(url: str, query: str):
def run():
- ranker = HeuristicRanker(RemoteIndex(), DummyCompleter())
+ ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter())
model = MwmblRankingModel(ranker)
evaluate(model, fraction=0.01)
def single_query(query: str):
- ranker = HeuristicRanker(RemoteIndex(), DummyCompleter())
+ ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter())
results = ranker.search(query, [])
for result in results:
print(result)
diff --git a/mwmbl/rankeval/evaluation/evaluate_wiki.py b/mwmbl/rankeval/evaluation/evaluate_wiki.py
index e0b8f4f..365ad44 100644
--- a/mwmbl/rankeval/evaluation/evaluate_wiki.py
+++ b/mwmbl/rankeval/evaluation/evaluate_wiki.py
@@ -8,18 +8,11 @@ from joblib import Memory
from mwmbl.rankeval.evaluation.evaluate import evaluate
from mwmbl.tinysearchengine.indexer import Document
-
-WIKI_SEARCH_API_URL = "https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={query}&format=json"
-WIKI_URL_FORMAT = "https://en.wikipedia.org/wiki/{title}"
-
+from mwmbl.tinysearchengine.rank import WIKI_SEARCH_API_URL, get_wiki_url
memory = Memory(location="devdata/cache")
-def get_wiki_url(title: str):
- return WIKI_URL_FORMAT.format(title=title.replace(" ", "_"))
-
-
@memory.cache
def fetch_results(query: str) -> list[str]:
safe_query = urllib.parse.quote(query, safe="")
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.py
index d8cfc44..681374e 100644
--- a/mwmbl/search_setup.py
+++ b/mwmbl/search_setup.py
@@ -9,7 +9,7 @@ from mwmbl.models import DomainSubmission
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.tinysearchengine.completer import Completer
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
-from mwmbl.tinysearchengine.rank import HeuristicRanker
+from mwmbl.tinysearchengine.rank import HeuristicRanker, HeuristicAndWikiRanker
def get_curated_domains() -> set[str]:
@@ -24,5 +24,5 @@ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
tiny_index = TinyIndex(item_factory=Document, index_path=index_path)
tiny_index.__enter__()
-ranker = HeuristicRanker(tiny_index, completer)
+ranker = HeuristicAndWikiRanker(tiny_index, completer)
batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)
diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.py
index 160ca0c..67f9294 100644
--- a/mwmbl/tinysearchengine/indexer.py
+++ b/mwmbl/tinysearchengine/indexer.py
@@ -40,6 +40,7 @@ class DocumentState(IntEnum):
ORGANIC_APPROVED = 7
FROM_USER_APPROVED = 8
FROM_GOOGLE_APPROVED = 9
+ FROM_WIKI = 10
@dataclass
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index a251e99..c76793d 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -1,11 +1,16 @@
import math
import re
+import urllib
from abc import abstractmethod
+from datetime import timedelta
from logging import getLogger
from operator import itemgetter
from typing import Optional
from urllib.parse import urlparse
+import requests
+from requests_cache import CachedSession
+
from mwmbl.format import format_result_with_pattern, get_query_regex
from mwmbl.tokenizer import tokenize, get_bigrams
from mwmbl.tinysearchengine.completer import Completer
@@ -19,7 +24,7 @@ MATCH_SCORE_THRESHOLD = 0.0
SCORE_THRESHOLD = 0.0
LENGTH_PENALTY = 0.04
MATCH_EXPONENT = 2
-DOMAIN_SCORE_SMOOTHING = 50
+DOMAIN_SCORE_SMOOTHING = 0.1
HTTPS_STRING = 'https://'
@@ -77,10 +82,16 @@ def get_features(terms, title, url, extract, score, is_complete):
return features
+DOMAIN_MAX_SCORE = max(DOMAINS.values())
+DOMAIN_MIN_SCORE = min(DOMAINS.values())
+
+
def get_domain_score(url):
domain = urlparse(url).netloc
- domain_score = DOMAINS.get(domain, 0.0)
- return domain_score
+ if domain in DOMAINS:
+ normalised_score = (DOMAINS[domain] - DOMAIN_MIN_SCORE) / (DOMAIN_MAX_SCORE - DOMAIN_MIN_SCORE)
+ return normalised_score * normalised_score
+ return 0.0
def get_match_features(terms, result_string, is_complete, is_url):
@@ -234,3 +245,33 @@ class Ranker:
class HeuristicRanker(Ranker):
def order_results(self, terms, pages, is_complete):
return order_results(terms, pages, is_complete)
+
+
+WIKI_SEARCH_API_URL = "https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={query}&format=json"
+WIKI_URL_FORMAT = "https://en.wikipedia.org/wiki/{title}"
+
+
+def get_wiki_url(title: str):
+ return WIKI_URL_FORMAT.format(title=title.replace(" ", "_"))
+
+
+HTML_TAG_REGEX = re.compile(r'<[^>]+>')
+
+
+def clean_html(s: str):
+ return HTML_TAG_REGEX.sub('', s)
+
+
+class HeuristicAndWikiRanker(HeuristicRanker):
+ max_wiki_results = 5
+
+ def search(self, s: str, additional_results: list[Document]) -> list[Document]:
+ with CachedSession(expire_after=timedelta(weeks=12)) as session:
+ escaped_query = urllib.parse.quote(s, safe='')
+ wiki_response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query)).json()
+ wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),
+ 0.0, s, state=DocumentState.FROM_WIKI)
+ for result in wiki_response['query']['search'][:self.max_wiki_results]]
+
+ return super().search(s, wiki_results + additional_results)
+