Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

feat: add configurable threads for crawl_batch with env var

thiswillbeyourgithub (aider)6d2424a1958d
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.py
index a7301d9..06813a3 100644
--- a/mwmbl/crawl.py
+++ b/mwmbl/crawl.py
@@ -9,7 +9,7 @@ import requests
from django.conf import settings
from redis import Redis
-from mwmbl.crawler.env_vars import CRAWLER_WORKERS
+from mwmbl.crawler.env_vars import CRAWLER_WORKERS, CRAWL_THREADS
from mwmbl.rankeval.evaluation.remote_index import RemoteIndex
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
@@ -84,7 +84,7 @@ def process_batch():
user_id = "test"
urls = url_queue.get_batch(user_id)
logger.info(f"Processing batch of {len(urls)} URLs")
- results = crawl_batch(urls, 20)
+ results = crawl_batch(batch=urls, num_threads=CRAWL_THREADS)
for result in results:
print("Result", result)
js_timestamp = int(time.time() * 1000)
diff --git a/mwmbl/crawler/README.md b/mwmbl/crawler/README.md
index 4334821..f0bd12f 100644
--- a/mwmbl/crawler/README.md
+++ b/mwmbl/crawler/README.md
@@ -22,6 +22,7 @@ The crawler can be configured using the following environment variables:
### Worker Configuration
- `CRAWLER_WORKERS` (default: 10) - Number of parallel crawler worker processes to run
+- `CRAWL_THREADS` (default: 20) - Number of threads per worker process for concurrent URL crawling
### Redis Configuration
- `REDIS_URL` (default: 'redis://127.0.0.1:6379') - Redis connection URL for URL queues and stats
diff --git a/mwmbl/crawler/env_vars.py b/mwmbl/crawler/env_vars.py
index cdd6a46..9ee02e3 100644
--- a/mwmbl/crawler/env_vars.py
+++ b/mwmbl/crawler/env_vars.py
@@ -20,6 +20,7 @@ MAX_SITE_URLS = int(os.environ.get('MAX_SITE_URLS', '100'))
# Crawler worker configuration
CRAWLER_WORKERS = int(os.environ.get('CRAWLER_WORKERS', '10'))
+CRAWL_THREADS = int(os.environ.get('CRAWL_THREADS', '20'))
# Redis configuration
REDIS_URL = os.environ.get('REDIS_URL', 'redis://127.0.0.1:6379')