Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Handle a bad batch

Daoud Clarke91269d510029
diff --git a/mwmbl/indexer/batch_cache.py b/mwmbl/indexer/batch_cache.py
index 3001e81..e7af6db 100644
--- a/mwmbl/indexer/batch_cache.py
+++ b/mwmbl/indexer/batch_cache.py
@@ -39,7 +39,11 @@ class BatchCache:
except FileNotFoundError:
logger.exception(f"Missing batch file: {path}")
continue
- batch = HashedBatch.parse_raw(data)
+ try:
+ batch = HashedBatch.parse_raw(data)
+ except ValidationError:
+ logger.exception(f"Unable to parse batch, skipping: '{data}'")
+ continue
batches[url] = batch
return batches