An open source, non-profit web search engine
Catch exceptions when updating URL queue
Daoud Clarkea585709d0f76
diff --git a/mwmbl/url_queue.py b/mwmbl/url_queue.pyindex 63f7787..5614730 100644--- a/mwmbl/url_queue.py+++ b/mwmbl/url_queue.py@@ -143,7 +143,13 @@ def _add_urls(domains: Union[set[str], KeysView], domain_urls: dict[str, dict[stdef update_queue_continuously(new_item_queue: Queue, queued_batches: Queue):queue = URLQueue(new_item_queue, queued_batches)while True:- num_processed = queue.update()+ # noinspection PyBroadException+ try:+ num_processed = queue.update()+ except Exception:+ logger.exception("Error updating URL queue")+ time.sleep(10)+ continuelogger.info(f"Queue update, num processed: {num_processed}, queue size: {queue.num_queued_batches}, num top "f"domains: {queue.num_top_domains}")if num_processed == 0: