Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Merge remote-tracking branch 'origin/main' into crawler-disk-usage

Daoud Clarke73deadd2cd09
diff --cc mwmbl/crawl.py
index 2eea80c,1b04f59..8f7b71f
--- a/mwmbl/crawl.py
+++ b/mwmbl/crawl.py
@@@ -48,8 -48,6 +48,7 @@@ from mwmbl.rankeval.evaluation.remote_i
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.tinysearchengine.indexer import Document, TinyIndex
from mwmbl.tinysearchengine.rank import score_result
- from mwmbl.tokenizer import tokenize
+from mwmbl.utils import prune_request_cache
BATCH_QUEUE_KEY = "batch-queue"
REMOTE_SERVER = "https://api.mwmbl.org"