An open source, non-profit web search engine
WIP: implement term completer
Daoud Clarke3c75dd1a74ff
diff --git a/mwmbl/indexer/paths.py b/mwmbl/indexer/paths.pyindex 75e81e8..c372021 100644--- a/mwmbl/indexer/paths.py+++ b/mwmbl/indexer/paths.py@@ -5,7 +5,6 @@ HOME = os.getenv('HOME')DATA_DIR = Path(os.environ['HOME']) / 'data' / 'tinysearch'COMMON_CRAWL_TERMS_PATH = DATA_DIR / 'common-craw-terms.csv'-MWMBL_CRAWL_TERMS_PATH = DATA_DIR / 'mwmbl-craw-terms.csv'HN_TOP_PATH = os.path.join(DATA_DIR, 'hn-top.csv')CRAWL_PREFIX = 'crawl_'@@ -20,6 +19,8 @@ DOMAINS_QUEUE_NAME = 'domains-queue-fs'DOMAINS_TITLES_QUEUE_NAME = 'domains-title-queue-fs'DOMAINS_PATH = os.path.join(DATA_DIR, 'top10milliondomains.csv.gz')-INDEX_PATH = Path(__file__).parent.parent.parent / 'data' / 'index.tinysearch'+LOCAL_DATA_DIR = Path(__file__).parent.parent.parent / 'data'+INDEX_PATH = LOCAL_DATA_DIR / 'index.tinysearch'+MWMBL_CRAWL_TERMS_PATH = LOCAL_DATA_DIR / 'mwmbl-crawl-terms.csv'TOP_DOMAINS_JSON_PATH = DATA_DIR / 'hn-top-domains.json'diff --git a/mwmbl/tinysearchengine/completer.py b/mwmbl/tinysearchengine/completer.pynew file mode 100644index 0000000..eb88a80--- /dev/null+++ b/mwmbl/tinysearchengine/completer.py@@ -0,0 +1,27 @@+from bisect import bisect_left, bisect_right++import pandas as pd+from pandas import DataFrame+++class Completer:+ def __init__(self, terms: DataFrame):+ terms_dict = terms.sort_values('term').set_index('term')['count'].to_dict()+ self.terms = list(terms_dict.keys())+ self.counts = list(terms_dict.values())+ print("Terms", self.terms[:100], self.counts[:100])++ def complete(self, term):+ term_length = len(term)+ start = bisect_left(self.terms, term, key=lambda x: x[:term_length])+ end = bisect_right(self.terms, term, key=lambda x: x[:term_length])++ print("Start", self.terms[start])+ print("End", self.terms[end])++++if __name__ == '__main__':+ data = pd.read_csv('data/mwmbl-crawl-terms.csv')+ completer = Completer(data)+ completer.complete('yo')diff --git a/mwmbl/tinysearchengine/config.py b/mwmbl/tinysearchengine/config.pyindex c506117..f0dd3d3 100644--- a/mwmbl/tinysearchengine/config.py+++ b/mwmbl/tinysearchengine/config.py@@ -11,6 +11,7 @@ class ServerConfigModel(BaseModel):class IndexConfigModel(BaseModel):index_path: StrictStr = "data/index.tinysearch"+ terms_path: StrictStr = "data/mwmbl-crawl-terms.csv"num_pages: StrictInt = 25600page_size: StrictInt = 4096