Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Merge branch 'add-user-agent' into enh_crawler_chunk_1

thiswillbeyourgithub042a088441bf
diff --cc mwmbl/crawler/retrieve.py
index dc8ca08,0bbf036..8640c14
--- a/mwmbl/crawler/retrieve.py
+++ b/mwmbl/crawler/retrieve.py
@@@ -13,8 -13,9 +13,10 @@@ from urllib3.exceptions import NewConne
from mwmbl.justext.core import html_to_dom
from mwmbl.justext.paragraph import Paragraph
+from mwmbl.crawler.env_vars import TIMEOUT_SECONDS, MAX_FETCH_SIZE, MAX_NEW_LINKS, MAX_EXTRA_LINKS, MAX_SITE_URLS
+
+ VERSION = "2.0"
ALLOWED_EXCEPTIONS = (ValueError, ConnectionError, ReadTimeout, TimeoutError,
OSError, NewConnectionError, MaxRetryError, SSLCertVerificationError)