An open source, non-profit web search engine
feat: add configurable threads for crawl_batch with env var
thiswillbeyourgithub (aider)6d2424a1958d
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.pyindex a7301d9..06813a3 100644--- a/mwmbl/crawl.py+++ b/mwmbl/crawl.py@@ -9,7 +9,7 @@ import requestsfrom django.conf import settingsfrom redis import Redis-from mwmbl.crawler.env_vars import CRAWLER_WORKERS+from mwmbl.crawler.env_vars import CRAWLER_WORKERS, CRAWL_THREADSfrom mwmbl.rankeval.evaluation.remote_index import RemoteIndexfrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.tinysearchengine.indexer import TinyIndex, Document@@ -84,7 +84,7 @@ def process_batch():user_id = "test"urls = url_queue.get_batch(user_id)logger.info(f"Processing batch of {len(urls)} URLs")- results = crawl_batch(urls, 20)+ results = crawl_batch(batch=urls, num_threads=CRAWL_THREADS)for result in results:print("Result", result)js_timestamp = int(time.time() * 1000)diff --git a/mwmbl/crawler/README.md b/mwmbl/crawler/README.mdindex 4334821..f0bd12f 100644--- a/mwmbl/crawler/README.md+++ b/mwmbl/crawler/README.md@@ -22,6 +22,7 @@ The crawler can be configured using the following environment variables:### Worker Configuration- `CRAWLER_WORKERS` (default: 10) - Number of parallel crawler worker processes to run+- `CRAWL_THREADS` (default: 20) - Number of threads per worker process for concurrent URL crawling### Redis Configuration- `REDIS_URL` (default: 'redis://127.0.0.1:6379') - Redis connection URL for URL queues and statsdiff --git a/mwmbl/crawler/env_vars.py b/mwmbl/crawler/env_vars.pyindex cdd6a46..9ee02e3 100644--- a/mwmbl/crawler/env_vars.py+++ b/mwmbl/crawler/env_vars.py@@ -20,6 +20,7 @@ MAX_SITE_URLS = int(os.environ.get('MAX_SITE_URLS', '100'))# Crawler worker configurationCRAWLER_WORKERS = int(os.environ.get('CRAWLER_WORKERS', '10'))+CRAWL_THREADS = int(os.environ.get('CRAWL_THREADS', '20'))# Redis configurationREDIS_URL = os.environ.get('REDIS_URL', 'redis://127.0.0.1:6379')