An open source, non-profit web search engine
More batches when indexing
Daoud Clarke03dcfbf0463f
diff --git a/crawl.py b/crawl.pyindex 2f9a60c..1ac4953 100644--- a/crawl.py+++ b/crawl.py@@ -87,7 +87,7 @@ def run_indexing_continuously():def run_indexing():index_path = data_path / settings.INDEX_NAME- batch_jsons = redis.lpop(BATCH_QUEUE_KEY, 1)+ batch_jsons = redis.lpop(BATCH_QUEUE_KEY, 10)if batch_jsons is None:logger.info("No more batches to index. Sleeping for 10 seconds.")time.sleep(10)