An open source, non-profit web search engine
Merge branch 'add-user-agent' into enh_crawler_chunk_1
thiswillbeyourgithub042a088441bf
diff --cc mwmbl/crawler/retrieve.pyindex dc8ca08,0bbf036..8640c14--- a/mwmbl/crawler/retrieve.py+++ b/mwmbl/crawler/retrieve.py@@@ -13,8 -13,9 +13,10 @@@ from urllib3.exceptions import NewConnefrom mwmbl.justext.core import html_to_domfrom mwmbl.justext.paragraph import Paragraph+from mwmbl.crawler.env_vars import TIMEOUT_SECONDS, MAX_FETCH_SIZE, MAX_NEW_LINKS, MAX_EXTRA_LINKS, MAX_SITE_URLS++ VERSION = "2.0"ALLOWED_EXCEPTIONS = (ValueError, ConnectionError, ReadTimeout, TimeoutError,OSError, NewConnectionError, MaxRetryError, SSLCertVerificationError)