An open source, non-profit web search engine
Catch errors parsing URLs
Daoud Clarke09e03427c1cb
diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.pyindex fc8c2d0..da67351 100644--- a/mwmbl/indexer/update_urls.py+++ b/mwmbl/indexer/update_urls.py@@ -84,7 +84,12 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Qudef process_link(user_id_hash, crawled_page_domain, link, unknown_domain_multiplier, timestamp, url_scores, url_timestamps, url_users, is_extra: bool, blacklist_domains):- parsed_link = urlparse(link)+ try:+ parsed_link = urlparse(link)+ except ValueError:+ logger.debug(f"Couldn't parse link: {link}")+ return+if is_domain_blacklisted(parsed_link.netloc, blacklist_domains):logger.debug(f"Excluding link for blacklisted domain: {parsed_link}")return