Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add user agent

Daoud Clarke8f995de18534
diff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.py
index 4c3e05f..0bbf036 100644
--- a/mwmbl/crawler/retrieve.py
+++ b/mwmbl/crawler/retrieve.py
@@ -14,6 +14,8 @@ from urllib3.exceptions import NewConnectionError, MaxRetryError
from mwmbl.justext.core import html_to_dom
from mwmbl.justext.paragraph import Paragraph
+
+VERSION = "2.0"
ALLOWED_EXCEPTIONS = (ValueError, ConnectionError, ReadTimeout, TimeoutError,
OSError, NewConnectionError, MaxRetryError, SSLCertVerificationError)
@@ -31,7 +33,7 @@ NUM_EXTRACT_CHARS = 155
DEFAULT_ENCODING = 'utf8'
DEFAULT_ENC_ERRORS = 'replace'
MAX_SITE_URLS = 100
-
+HEADERS = {"User-Agent": f"mwmbl/{VERSION} (https://mwmbl.org)"}
logger = getLogger(__name__)
@@ -43,7 +45,7 @@ def fetch(url):
https://stackoverflow.com/a/22347526
"""
- r = requests.get(url, stream=True, timeout=TIMEOUT_SECONDS)
+ r = requests.get(url, stream=True, timeout=TIMEOUT_SECONDS, headers=HEADERS)
size = 0
start = time.time()