An open source, non-profit web search engine
Add user agent
Daoud Clarke8f995de18534
diff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.pyindex 4c3e05f..0bbf036 100644--- a/mwmbl/crawler/retrieve.py+++ b/mwmbl/crawler/retrieve.py@@ -14,6 +14,8 @@ from urllib3.exceptions import NewConnectionError, MaxRetryErrorfrom mwmbl.justext.core import html_to_domfrom mwmbl.justext.paragraph import Paragraph++VERSION = "2.0"ALLOWED_EXCEPTIONS = (ValueError, ConnectionError, ReadTimeout, TimeoutError,OSError, NewConnectionError, MaxRetryError, SSLCertVerificationError)@@ -31,7 +33,7 @@ NUM_EXTRACT_CHARS = 155DEFAULT_ENCODING = 'utf8'DEFAULT_ENC_ERRORS = 'replace'MAX_SITE_URLS = 100-+HEADERS = {"User-Agent": f"mwmbl/{VERSION} (https://mwmbl.org)"}logger = getLogger(__name__)@@ -43,7 +45,7 @@ def fetch(url):https://stackoverflow.com/a/22347526"""- r = requests.get(url, stream=True, timeout=TIMEOUT_SECONDS)+ r = requests.get(url, stream=True, timeout=TIMEOUT_SECONDS, headers=HEADERS)size = 0start = time.time()