Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Process more batches when updating URLs

Daoud Clarkefbf412ec2980
diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.py
index 5db50b0..a383f03 100644
--- a/mwmbl/indexer/update_urls.py
+++ b/mwmbl/indexer/update_urls.py
@@ -35,7 +35,7 @@ def update_urls_continuously(data_path: str, new_item_queue: RedisURLQueue):
def run(batch_cache: BatchCache, new_item_queue: RedisURLQueue):
- process_batch.run(batch_cache, BatchStatus.LOCAL, BatchStatus.URLS_UPDATED, record_urls_in_database, 100,
+ process_batch.run(batch_cache, BatchStatus.LOCAL, BatchStatus.URLS_UPDATED, record_urls_in_database, 1000,
new_item_queue)