An open source, non-profit web search engine
Use a custom tokenizer
Daoud Clarke4779371cf377
diff --git a/analyse/index_local.py b/analyse/index_local.pyindex 9db9dbf..24628a5 100644--- a/analyse/index_local.py+++ b/analyse/index_local.py@@ -8,6 +8,7 @@ import loggingimport osimport sysfrom pathlib import Path+from datetime import datetimeimport spacy@@ -41,10 +42,16 @@ def run():TinyIndex.create(item_factory=Document, index_path=EVALUATE_INDEX_PATH, num_pages=NUM_PAGES, page_size=PAGE_SIZE)batches = get_batches()++ start = datetime.now()with Database() as db:nlp = spacy.load("en_core_web_sm")url_db = URLDatabase(db.connection)index_batches(batches, EVALUATE_INDEX_PATH, nlp, url_db)+ end = datetime.now()++ total_time = (end - start).total_seconds()+ print("total_seconds:", total_time)if __name__ == '__main__':diff --git a/mwmbl/indexer/index.py b/mwmbl/indexer/index.pyindex 45793ee..67ae164 100644--- a/mwmbl/indexer/index.py+++ b/mwmbl/indexer/index.py@@ -16,20 +16,35 @@ HTTPS_START = 'https://'BATCH_SIZE = 100-def is_content_token(nlp, token):- lexeme = nlp.vocab[token.orth]- return (lexeme.is_alpha or lexeme.is_digit) and not token.is_stop---def tokenize(nlp, input_text):+STOPWORDS = set("0,1,2,3,4,5,6,7,8,9,a,A,about,above,across,after,again,against,all,almost,alone,along,already,also," \+ "although,always,am,among,an,and,another,any,anyone,anything,anywhere,are,aren't,around,as,at,b,B,back," \+ "be,became,because,become,becomes,been,before,behind,being,below,between,both,but,by,c,C,can,cannot,can't," \+ "could,couldn't,d,D,did,didn't,do,does,doesn't,doing,done,don't,down,during,e,E,each,either,enough,even," \+ "ever,every,everyone,everything,everywhere,f,F,few,find,first,for,four,from,full,further,g,G,get,give,go," \+ "h,H,had,hadn't,has,hasn't,have,haven't,having,he,he'd,he'll,her,here,here's,hers,herself,he's,him," \+ "himself,his,how,however,how's,i,I,i'd,if,i'll,i'm,in,interest,into,is,isn't,it,it's,its,itself,i've," \+ "j,J,k,K,keep,l,L,last,least,less,let's,m,M,made,many,may,me,might,more,most,mostly,much,must,mustn't," \+ "my,myself,n,N,never,next,no,nobody,noone,nor,not,nothing,now,nowhere,o,O,of,off,often,on,once,one,only," \+ "or,other,others,ought,our,ours,ourselves,out,over,own,p,P,part,per,perhaps,put,q,Q,r,R,rather,s,S,same," \+ "see,seem,seemed,seeming,seems,several,shan't,she,she'd,she'll,she's,should,shouldn't,show,side,since,so," \+ "some,someone,something,somewhere,still,such,t,T,take,than,that,that's,the,their,theirs,them,themselves," \+ "then,there,therefore,there's,these,they,they'd,they'll,they're,they've,this,those,though,three,through," \+ "thus,to,together,too,toward,two,u,U,under,until,up,upon,us,v,V,very,w,W,was,wasn't,we,we'd,we'll,well," \+ "we're,were,weren't,we've,what,what's,when,when's,where,where's,whether,which,while,who,whole,whom,who's," \+ "whose,why,why's,will,with,within,without,won't,would,wouldn't,x,X,y,Y,yet,you,you'd,you'll,your,you're," \+ "yours,yourself,yourselves,you've,z,Z".split(','))+++def tokenize(input_text):cleaned_text = input_text.encode('utf8', 'replace').decode('utf8')- tokens = nlp.tokenizer(cleaned_text)+ tokens = cleaned_text.lower().replace('.', ' ').replace(',', ' ').split()+ # tokens = nlp.tokenizer(cleaned_text)if input_text.endswith('…'):# Discard the last two tokens since there will likely be a word cut in twotokens = tokens[:-2]- content_tokens = [token for token in tokens if is_content_token(nlp, token)]- lowered = {nlp.vocab[token.orth].text.lower() for token in content_tokens}- return lowered+ content_tokens = [token for token in tokens if not token in STOPWORDS]+ # lowered = {nlp.vocab[token.orth].text.lower() for token in content_tokens}+ return content_tokensdef prepare_url_for_tokenizing(url: str):@@ -53,12 +68,12 @@ def get_pages(nlp, titles_urls_and_extracts, link_counts) -> Iterable[TokenizedDdef tokenize_document(url, title_cleaned, extract, score, nlp):- title_tokens = tokenize(nlp, title_cleaned)+ title_tokens = tokenize(title_cleaned)prepared_url = prepare_url_for_tokenizing(unquote(url))- url_tokens = tokenize(nlp, prepared_url)- extract_tokens = tokenize(nlp, extract)+ url_tokens = tokenize(prepared_url)+ extract_tokens = tokenize(extract)# print("Extract tokens", extract_tokens)- tokens = title_tokens | url_tokens | extract_tokens+ tokens = set(title_tokens) | set(url_tokens) | set(extract_tokens)document = TokenizedDocument(tokens=list(tokens), url=url, title=title_cleaned, extract=extract, score=score)return documentdiff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.pyindex 405cc17..253faab 100644--- a/mwmbl/tinysearchengine/indexer.py+++ b/mwmbl/tinysearchengine/indexer.py@@ -1,6 +1,6 @@import jsonimport os-from dataclasses import astuple, dataclass, asdict+from dataclasses import dataclass, asdictfrom io import UnsupportedOperation, BytesIOfrom logging import getLoggerfrom mmap import mmap, PROT_READ, PROT_WRITE@@ -20,6 +20,10 @@ PAGE_SIZE = 4096logger = getLogger(__name__)+def astuple(dc):+ return tuple(dc.__dict__.values())++@dataclassclass Document:title: strdiff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 75246d8..68562d7 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -5,6 +5,7 @@ from logging import getLoggerfrom operator import itemgetterfrom urllib.parse import urlparse+from mwmbl.indexer.index import tokenizefrom mwmbl.tinysearchengine.completer import Completerfrom mwmbl.hn_top_domains_filtered import DOMAINSfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document@@ -171,7 +172,7 @@ class Ranker:return [q, urls + completed]def get_results(self, q):- terms = [x.lower() for x in q.replace('.', ' ').split()]+ terms = tokenize(q)is_complete = q.endswith(' ')if len(terms) > 0 and not is_complete:completions = self.completer.complete(terms[-1])