Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Replace all invalid bytes when parsing XML

Daoud Clarke5bca3b9487e5
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.py
index e9a9e19..54ba2c4 100644
--- a/mwmbl/crawl.py
+++ b/mwmbl/crawl.py
@@ -18,6 +18,10 @@ from mwmbl.tinysearchengine.indexer import TinyIndex, Document
from mwmbl.tinysearchengine.rank import score_result
from mwmbl.tokenizer import tokenize
+FORMAT = "%(process)d:%(levelname)s:%(name)s:%(message)s"
+logging.basicConfig(level=logging.INFO, format=FORMAT)
+logger = logging.getLogger(__name__)
+
os.environ["DJANGO_SETTINGS_MODULE"] = "mwmbl.settings_crawler"
data_path = Path(settings.DATA_PATH)
@@ -33,10 +37,6 @@ from mwmbl.crawler.retrieve import crawl_batch, crawl_url, CRAWLER_VERSION
from mwmbl.crawler.batch import HashedBatch, Result, Results
from mwmbl.indexer.index_batches import index_batches, index_pages
-logger = logging.getLogger(__name__)
-FORMAT = "%(process)d:%(levelname)s:%(name)s:%(message)s"
-logging.basicConfig(level=logging.INFO, format=FORMAT)
-
BATCH_QUEUE_KEY = "batch-queue"
REMOTE_SERVER = "https://api.mwmbl.org"
diff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.py
index 92052d3..9cbef7a 100644
--- a/mwmbl/crawler/retrieve.py
+++ b/mwmbl/crawler/retrieve.py
@@ -194,6 +194,8 @@ def crawl_url(url):
}
}
+ content = re.sub(rb'[\x00-\x08\x0b\x0c\x0e-\x1f]', b'', content)
+
try:
dom = html_to_dom(content, DEFAULT_ENCODING, None, DEFAULT_ENC_ERRORS)
except Exception as e:
@@ -222,7 +224,8 @@ def crawl_url(url):
try:
paragraphs = core.justext_from_dom(dom, utils.get_stoplist("English"))
except Exception as e:
- logger.exception("Error parsing paragraphs")
+ bad_bytes = sorted({b for b in content if b < 0x20 and b not in (0x09, 0x0A, 0x0D)})
+ logger.exception("Error parsing paragraphs - offending control bytes: %s", bad_bytes)
return {
'url': url,
'status': status_code,