An open source, non-profit web search engine
Implement retrieval
Daoud Clarkefdb5cbbf3c07
diff --git a/app.py b/app.pyindex 1a3492d..5614b98 100644--- a/app.py+++ b/app.py@@ -7,9 +7,11 @@ from fastapi import FastAPIfrom fastapi.staticfiles import StaticFilesfrom starlette.responses import FileResponse, RedirectResponse+from index import TinyIndex, PAGE_SIZE, NUM_PAGESfrom paths import INDEX_PATHapp = FastAPI()+tiny_index = TinyIndex(INDEX_PATH, NUM_PAGES, PAGE_SIZE)@app.get("/search")@@ -43,28 +45,21 @@ def complete_term(term):def complete(q: str):terms = [x.lower() for x in q.split()]- completed = complete_term(terms[-1])- terms = terms[:-1] + [completed]+ # completed = complete_term(terms[-1])+ # terms = terms[:-1] + [completed]- con = sqlite3.connect(INDEX_PATH)- in_part = ','.join('?'*len(terms))- query = f"""- SELECT title, url- FROM terms INNER JOIN pages- ON terms.page_id = pages.id- WHERE term IN ({in_part})- GROUP BY title, url- ORDER BY count(*) DESC, length(title)- LIMIT 5- """-- data = con.execute(query, terms).fetchall()+ pages = []+ for term in terms:+ page = tiny_index.retrieve(term)+ if page is not None:+ pages += pageresults = [title.replace("\n", "") + ' — ' +- url.replace("\n", "") for title, url in data]+ url.replace("\n", "") for title, url in pages]if len(results) == 0:+ # print("No results")return []- # print("Results", results_list)+ # print("Results", results)return [q, results]diff --git a/index.py b/index.pyindex fdd9395..d265822 100644--- a/index.py+++ b/index.py@@ -8,7 +8,7 @@ import sqlite3from dataclasses import dataclassfrom glob import globfrom itertools import chain, count, islice-from mmap import mmap+from mmap import mmap, PROT_READfrom typing import List, Iteratorfrom urllib.parse import unquote@@ -68,6 +68,14 @@ class TinyIndexBase:self.decompressor = ZstdDecompressor()self.mmap = None+ def retrieve(self, token):+ index = self._get_token_page_index(token)+ return self._get_page(index)++ def _get_token_page_index(self, token):+ token_hash = mmh3.hash(token, signed=False)+ return token_hash % self.num_pages+def _get_page(self, i):"""Get the page at index i, decompress and deserialise it using JSON@@ -84,8 +92,8 @@ class TinyIndex(TinyIndexBase):def __init__(self, index_path, num_pages, page_size):super().__init__(num_pages, page_size)self.index_path = index_path- self.index_file = None- self.mmap = None+ self.index_file = open(self.index_path, 'rb')+ self.mmap = mmap(self.index_file.fileno(), 0, prot=PROT_READ)class TinyIndexer(TinyIndexBase):@@ -95,6 +103,7 @@ class TinyIndexer(TinyIndexBase):self.compressor = ZstdCompressor()self.decompressor = ZstdDecompressor()self.index_file = None+ self.mmap = Nonedef __enter__(self):self.create_if_not_exists()@@ -122,10 +131,6 @@ class TinyIndexer(TinyIndexBase):except ValueError:pass- def _get_token_page_index(self, token):- token_hash = mmh3.hash(token, signed=False)- return token_hash % self.num_pages-def _write_page(self, data, i):"""Serialise the data using JSON, compress it and store it at index i.diff --git a/performance.py b/performance.pyindex 4181c7a..7997262 100644--- a/performance.py+++ b/performance.py@@ -15,7 +15,7 @@ from paths import TEST_INDEX_PATHfrom wiki import get_wiki_titles_and_urls-NUM_DOCUMENTS = 10000+NUM_DOCUMENTS = 500def query_test():@@ -29,9 +29,9 @@ def query_test():result = client.get('/complete', params={'q': title})assert result.status_code == 200data = result.content.decode('utf8')- # data = json.dumps(complete(title))+ # print("Data", data, url, sep='\n')- if url in data:+ if title in data:hits += 1end = datetime.now()@@ -61,7 +61,7 @@ def performance_test():print("Index size", index_size)# print("Num tokens", indexer.get_num_tokens())- # query_test()+ query_test()if __name__ == '__main__':