Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

New LTR model trained on more data

Daoud Clarkec1d361c0a0b8
diff --git a/analyse/search.py b/analyse/search.py
new file mode 100644
index 0000000..3e5a57c
--- /dev/null
+++ b/analyse/search.py
@@ -0,0 +1,23 @@
+import logging
+import sys
+
+from mwmbl.indexer.paths import INDEX_PATH
+from mwmbl.tinysearchengine.completer import Completer
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document
+from mwmbl.tinysearchengine.rank import HeuristicRanker
+
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+
+
+def run():
+ with TinyIndex(Document, INDEX_PATH) as tiny_index:
+ completer = Completer()
+ ranker = HeuristicRanker(tiny_index, completer)
+ items = ranker.search('jasper fforde')
+ if items:
+ for item in items:
+ print("Items", item)
+
+
+if __name__ == '__main__':
+ run()
diff --git a/mwmbl/main.py b/mwmbl/main.py
index 79b908f..7c4287e 100644
--- a/mwmbl/main.py
+++ b/mwmbl/main.py
@@ -1,13 +1,13 @@
import argparse
import logging
import os
+import pickle
import sys
from multiprocessing import Process, Queue
from pathlib import Path
import uvicorn
from fastapi import FastAPI
-
from mwmbl import background
from mwmbl.crawler import app as crawler
from mwmbl.indexer.batch_cache import BatchCache
@@ -15,11 +15,14 @@ from mwmbl.indexer.paths import INDEX_NAME, BATCH_DIR_NAME
from mwmbl.tinysearchengine import search
from mwmbl.tinysearchengine.completer import Completer
from mwmbl.tinysearchengine.indexer import TinyIndex, Document, NUM_PAGES, PAGE_SIZE
-from mwmbl.tinysearchengine.rank import HeuristicRanker
+from mwmbl.tinysearchengine.ltr_rank import LTRRanker
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+MODEL_PATH = Path(__file__).parent / 'resources' / 'model.pickle'
+
+
def setup_args():
parser = argparse.ArgumentParser(description="mwmbl-tinysearchengine")
parser.add_argument("--data", help="Path to the tinysearchengine index file", default="/app/storage/")
@@ -55,7 +58,9 @@ def run():
completer = Completer()
with TinyIndex(item_factory=Document, index_path=index_path) as tiny_index:
- ranker = HeuristicRanker(tiny_index, completer)
+ # ranker = HeuristicRanker(tiny_index, completer)
+ model = pickle.load(open(MODEL_PATH, 'rb'))
+ ranker = LTRRanker(model, tiny_index, completer)
# Initialize FastApi instance
app = FastAPI()
diff --git a/mwmbl/resources/model.pickle b/mwmbl/resources/model.pickle
new file mode 100644
index 0000000..4f8a137
Binary files /dev/null and b/mwmbl/resources/model.pickle differ
diff --git a/mwmbl/tinysearchengine/ltr.py b/mwmbl/tinysearchengine/ltr.py
index 77afbd8..5ee6c3f 100644
--- a/mwmbl/tinysearchengine/ltr.py
+++ b/mwmbl/tinysearchengine/ltr.py
@@ -37,6 +37,7 @@ def get_match_features_as_series(item: Series):
features['num_terms'] = len(terms)
features['num_chars'] = len(' '.join(terms))
features['domain_score'] = get_domain_score(item['url'])
+ features['url_length'] = len(item['url'])
features['item_score'] = item['score']
return Series(features)
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index 04130ce..54a1753 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -13,7 +13,8 @@ logger = getLogger(__name__)
SCORE_THRESHOLD = 0.0
-LENGTH_PENALTY=0.01
+LENGTH_PENALTY = 0.01
+MATCH_EXPONENT = 1.5
def _get_query_regex(terms, is_complete, is_url):
@@ -37,11 +38,13 @@ def _score_result(terms, result: Document, is_complete: bool):
domain = parsed_url.netloc
path = parsed_url.path
string_scores = []
+ logger.debug(f"Item: {result}")
for result_string, is_url in [(result.title, False), (result.extract, False), (domain, True), (domain, False), (path, True)]:
last_match_char, match_length, total_possible_match_length = get_match_features(
terms, result_string, is_complete, is_url)
new_score = score_match(last_match_char, match_length, total_possible_match_length)
+ logger.debug(f"Item score: {new_score}, result {result_string}")
string_scores.append(new_score)
title_score, extract_score, domain_score, domain_split_score, path_score = string_scores
@@ -55,7 +58,8 @@ def _score_result(terms, result: Document, is_complete: bool):
def score_match(last_match_char, match_length, total_possible_match_length):
- return (match_length + 1. / last_match_char) / (total_possible_match_length + 1)
+ # return (match_length + 1. / last_match_char) / (total_possible_match_length + 1)
+ return MATCH_EXPONENT ** (match_length - total_possible_match_length) / last_match_char
def get_domain_score(url):
@@ -66,6 +70,7 @@ def get_domain_score(url):
def get_match_features(terms, result_string, is_complete, is_url):
query_regex = _get_query_regex(terms, is_complete, is_url)
+ print("Result string", result_string)
matches = list(re.finditer(query_regex, result_string, flags=re.IGNORECASE))
match_strings = {x.group(0).lower() for x in matches}
match_length = sum(len(x) for x in match_strings)
@@ -138,9 +143,9 @@ class Ranker:
terms = [x.lower() for x in q.replace('.', ' ').split()]
is_complete = q.endswith(' ')
if len(terms) > 0 and not is_complete:
- retrieval_terms = terms + self.completer.complete(terms[-1])
+ retrieval_terms = set(terms + self.completer.complete(terms[-1]))
else:
- retrieval_terms = terms
+ retrieval_terms = set(terms)
pages = []
seen_items = set()