An open source, non-profit web search engine
refactor: move crawler configuration to env_vars with environment variable support
thiswillbeyourgithub (aider)fde29c43e89d
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex f1a9149..198e94e 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -14,6 +14,7 @@ from ninja import NinjaAPI, Schema, HTTPExceptionfrom redis import Redisfrom mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch, Results, PostResultsResponse, Error+from mwmbl.crawler.env_vars import REDIS_URLfrom mwmbl.crawler.stats import MwmblStats, StatsManagerfrom mwmbl.database import Databasefrom mwmbl.indexer.batch_cache import BatchCache@@ -35,7 +36,7 @@ from mwmbl.settings import (DATE_REGEX)from mwmbl.tinysearchengine.indexer import Document-stats_manager = StatsManager(Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True))+stats_manager = StatsManager(Redis.from_url(REDIS_URL, decode_responses=True))def get_bucket(name):diff --git a/mwmbl/crawler/env_vars.py b/mwmbl/crawler/env_vars.pynew file mode 100644index 0000000..250edd4--- /dev/null+++ b/mwmbl/crawler/env_vars.py@@ -0,0 +1,22 @@+"""+Environment variables configuration for the crawler module.+"""+import os+++# Client has one hour to crawl a URL that has been assigned to them, or it will be reassigned+REASSIGN_MIN_HOURS = int(os.environ.get('REASSIGN_MIN_HOURS', '5'))+BATCH_SIZE = int(os.environ.get('BATCH_SIZE', '100'))+MAX_URLS_PER_TOP_DOMAIN = int(os.environ.get('MAX_URLS_PER_TOP_DOMAIN', '100'))+MAX_TOP_DOMAINS = int(os.environ.get('MAX_TOP_DOMAINS', '500'))+MAX_OTHER_DOMAINS = int(os.environ.get('MAX_OTHER_DOMAINS', '50000'))++# Retrieval configuration+TIMEOUT_SECONDS = int(os.environ.get('TIMEOUT_SECONDS', '3'))+MAX_FETCH_SIZE = int(os.environ.get('MAX_FETCH_SIZE', str(1024*1024)))+MAX_NEW_LINKS = int(os.environ.get('MAX_NEW_LINKS', '50'))+MAX_EXTRA_LINKS = int(os.environ.get('MAX_EXTRA_LINKS', '50'))+MAX_SITE_URLS = int(os.environ.get('MAX_SITE_URLS', '100'))++# Redis configuration+REDIS_URL = os.environ.get('REDIS_URL', 'redis://127.0.0.1:6379')diff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.pyindex 4c3e05f..5ff014b 100644--- a/mwmbl/crawler/retrieve.py+++ b/mwmbl/crawler/retrieve.py@@ -13,6 +13,7 @@ from urllib3.exceptions import NewConnectionError, MaxRetryErrorfrom mwmbl.justext.core import html_to_domfrom mwmbl.justext.paragraph import Paragraph+from mwmbl.crawler.env_vars import TIMEOUT_SECONDS, MAX_FETCH_SIZE, MAX_NEW_LINKS, MAX_EXTRA_LINKS, MAX_SITE_URLSALLOWED_EXCEPTIONS = (ValueError, ConnectionError, ReadTimeout, TimeoutError,OSError, NewConnectionError, MaxRetryError, SSLCertVerificationError)@@ -20,17 +21,12 @@ ALLOWED_EXCEPTIONS = (ValueError, ConnectionError, ReadTimeout, TimeoutError,POST_BATCH_URL = '/api/v1/crawler/batches/'POST_NEW_BATCH_URL = '/api/v1/crawler/batches/new'-TIMEOUT_SECONDS = 3-MAX_FETCH_SIZE = 1024*1024MAX_URL_LENGTH = 150BAD_URL_REGEX = re.compile(r'\/\/localhost\b|\.jpg$|\.png$|\.js$|\.gz$|\.zip$|\.pdf$|\.bz2$|\.ipynb$|\.py$')-MAX_NEW_LINKS = 50-MAX_EXTRA_LINKS = 50NUM_TITLE_CHARS = 65NUM_EXTRACT_CHARS = 155DEFAULT_ENCODING = 'utf8'DEFAULT_ENC_ERRORS = 'replace'-MAX_SITE_URLS = 100logger = getLogger(__name__)diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.pyindex bae07e0..123de2b 100644--- a/mwmbl/crawler/urls.py+++ b/mwmbl/crawler/urls.py@@ -12,15 +12,15 @@ from django.conf import settingsfrom psycopg2.extras import execute_valuesfrom pybloomfilter import BloomFilter-# Client has one hour to crawl a URL that has been assigned to them, or it will be reassigned+from mwmbl.crawler.env_vars import (+ REASSIGN_MIN_HOURS,+ BATCH_SIZE,+ MAX_URLS_PER_TOP_DOMAIN,+ MAX_TOP_DOMAINS,+ MAX_OTHER_DOMAINS,+)from mwmbl.utils import batch-REASSIGN_MIN_HOURS = 5-BATCH_SIZE = 100-MAX_URLS_PER_TOP_DOMAIN = 100-MAX_TOP_DOMAINS = 500-MAX_OTHER_DOMAINS = 50000-logger = getLogger(__name__)