Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

refactor: move crawler configuration to env_vars with environment variable support

thiswillbeyourgithub (aider)fde29c43e89d
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index f1a9149..198e94e 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -14,6 +14,7 @@ from ninja import NinjaAPI, Schema, HTTPException
from redis import Redis
from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch, Results, PostResultsResponse, Error
+from mwmbl.crawler.env_vars import REDIS_URL
from mwmbl.crawler.stats import MwmblStats, StatsManager
from mwmbl.database import Database
from mwmbl.indexer.batch_cache import BatchCache
@@ -35,7 +36,7 @@ from mwmbl.settings import (
DATE_REGEX)
from mwmbl.tinysearchengine.indexer import Document
-stats_manager = StatsManager(Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True))
+stats_manager = StatsManager(Redis.from_url(REDIS_URL, decode_responses=True))
def get_bucket(name):
diff --git a/mwmbl/crawler/env_vars.py b/mwmbl/crawler/env_vars.py
new file mode 100644
index 0000000..250edd4
--- /dev/null
+++ b/mwmbl/crawler/env_vars.py
@@ -0,0 +1,22 @@
+"""
+Environment variables configuration for the crawler module.
+"""
+import os
+
+
+# Client has one hour to crawl a URL that has been assigned to them, or it will be reassigned
+REASSIGN_MIN_HOURS = int(os.environ.get('REASSIGN_MIN_HOURS', '5'))
+BATCH_SIZE = int(os.environ.get('BATCH_SIZE', '100'))
+MAX_URLS_PER_TOP_DOMAIN = int(os.environ.get('MAX_URLS_PER_TOP_DOMAIN', '100'))
+MAX_TOP_DOMAINS = int(os.environ.get('MAX_TOP_DOMAINS', '500'))
+MAX_OTHER_DOMAINS = int(os.environ.get('MAX_OTHER_DOMAINS', '50000'))
+
+# Retrieval configuration
+TIMEOUT_SECONDS = int(os.environ.get('TIMEOUT_SECONDS', '3'))
+MAX_FETCH_SIZE = int(os.environ.get('MAX_FETCH_SIZE', str(1024*1024)))
+MAX_NEW_LINKS = int(os.environ.get('MAX_NEW_LINKS', '50'))
+MAX_EXTRA_LINKS = int(os.environ.get('MAX_EXTRA_LINKS', '50'))
+MAX_SITE_URLS = int(os.environ.get('MAX_SITE_URLS', '100'))
+
+# Redis configuration
+REDIS_URL = os.environ.get('REDIS_URL', 'redis://127.0.0.1:6379')
diff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.py
index 4c3e05f..5ff014b 100644
--- a/mwmbl/crawler/retrieve.py
+++ b/mwmbl/crawler/retrieve.py
@@ -13,6 +13,7 @@ from urllib3.exceptions import NewConnectionError, MaxRetryError
from mwmbl.justext.core import html_to_dom
from mwmbl.justext.paragraph import Paragraph
+from mwmbl.crawler.env_vars import TIMEOUT_SECONDS, MAX_FETCH_SIZE, MAX_NEW_LINKS, MAX_EXTRA_LINKS, MAX_SITE_URLS
ALLOWED_EXCEPTIONS = (ValueError, ConnectionError, ReadTimeout, TimeoutError,
OSError, NewConnectionError, MaxRetryError, SSLCertVerificationError)
@@ -20,17 +21,12 @@ ALLOWED_EXCEPTIONS = (ValueError, ConnectionError, ReadTimeout, TimeoutError,
POST_BATCH_URL = '/api/v1/crawler/batches/'
POST_NEW_BATCH_URL = '/api/v1/crawler/batches/new'
-TIMEOUT_SECONDS = 3
-MAX_FETCH_SIZE = 1024*1024
MAX_URL_LENGTH = 150
BAD_URL_REGEX = re.compile(r'\/\/localhost\b|\.jpg$|\.png$|\.js$|\.gz$|\.zip$|\.pdf$|\.bz2$|\.ipynb$|\.py$')
-MAX_NEW_LINKS = 50
-MAX_EXTRA_LINKS = 50
NUM_TITLE_CHARS = 65
NUM_EXTRACT_CHARS = 155
DEFAULT_ENCODING = 'utf8'
DEFAULT_ENC_ERRORS = 'replace'
-MAX_SITE_URLS = 100
logger = getLogger(__name__)
diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.py
index bae07e0..123de2b 100644
--- a/mwmbl/crawler/urls.py
+++ b/mwmbl/crawler/urls.py
@@ -12,15 +12,15 @@ from django.conf import settings
from psycopg2.extras import execute_values
from pybloomfilter import BloomFilter
-# Client has one hour to crawl a URL that has been assigned to them, or it will be reassigned
+from mwmbl.crawler.env_vars import (
+ REASSIGN_MIN_HOURS,
+ BATCH_SIZE,
+ MAX_URLS_PER_TOP_DOMAIN,
+ MAX_TOP_DOMAINS,
+ MAX_OTHER_DOMAINS,
+)
from mwmbl.utils import batch
-REASSIGN_MIN_HOURS = 5
-BATCH_SIZE = 100
-MAX_URLS_PER_TOP_DOMAIN = 100
-MAX_TOP_DOMAINS = 500
-MAX_OTHER_DOMAINS = 50000
-
logger = getLogger(__name__)