An open source, non-profit web search engine
Replace all invalid bytes when parsing XML
Daoud Clarke5bca3b9487e5
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.pyindex e9a9e19..54ba2c4 100644--- a/mwmbl/crawl.py+++ b/mwmbl/crawl.py@@ -18,6 +18,10 @@ from mwmbl.tinysearchengine.indexer import TinyIndex, Documentfrom mwmbl.tinysearchengine.rank import score_resultfrom mwmbl.tokenizer import tokenize+FORMAT = "%(process)d:%(levelname)s:%(name)s:%(message)s"+logging.basicConfig(level=logging.INFO, format=FORMAT)+logger = logging.getLogger(__name__)+os.environ["DJANGO_SETTINGS_MODULE"] = "mwmbl.settings_crawler"data_path = Path(settings.DATA_PATH)@@ -33,10 +37,6 @@ from mwmbl.crawler.retrieve import crawl_batch, crawl_url, CRAWLER_VERSIONfrom mwmbl.crawler.batch import HashedBatch, Result, Resultsfrom mwmbl.indexer.index_batches import index_batches, index_pages-logger = logging.getLogger(__name__)-FORMAT = "%(process)d:%(levelname)s:%(name)s:%(message)s"-logging.basicConfig(level=logging.INFO, format=FORMAT)-BATCH_QUEUE_KEY = "batch-queue"REMOTE_SERVER = "https://api.mwmbl.org"diff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.pyindex 92052d3..9cbef7a 100644--- a/mwmbl/crawler/retrieve.py+++ b/mwmbl/crawler/retrieve.py@@ -194,6 +194,8 @@ def crawl_url(url):}}+ content = re.sub(rb'[\x00-\x08\x0b\x0c\x0e-\x1f]', b'', content)+try:dom = html_to_dom(content, DEFAULT_ENCODING, None, DEFAULT_ENC_ERRORS)except Exception as e:@@ -222,7 +224,8 @@ def crawl_url(url):try:paragraphs = core.justext_from_dom(dom, utils.get_stoplist("English"))except Exception as e:- logger.exception("Error parsing paragraphs")+ bad_bytes = sorted({b for b in content if b < 0x20 and b not in (0x09, 0x0A, 0x0D)})+ logger.exception("Error parsing paragraphs - offending control bytes: %s", bad_bytes)return {'url': url,'status': status_code,