An open source, non-profit web search engine
Merge remote-tracking branch 'origin/main' into crawler-disk-usage
Daoud Clarke73deadd2cd09
diff --cc mwmbl/crawl.pyindex 2eea80c,1b04f59..8f7b71f--- a/mwmbl/crawl.py+++ b/mwmbl/crawl.py@@@ -48,8 -48,6 +48,7 @@@ from mwmbl.rankeval.evaluation.remote_ifrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.tinysearchengine.indexer import Document, TinyIndexfrom mwmbl.tinysearchengine.rank import score_result- from mwmbl.tokenizer import tokenize+from mwmbl.utils import prune_request_cacheBATCH_QUEUE_KEY = "batch-queue"REMOTE_SERVER = "https://api.mwmbl.org"