An open source, non-profit web search engine
feat: add rate limiting with configurable delay and random fuzz
thiswillbeyourgithub (aider)91889091a497
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.pyindex c8cfa0b..f674ccc 100644--- a/mwmbl/crawl.py+++ b/mwmbl/crawl.py@@ -1,5 +1,6 @@import loggingimport os+import randomimport timefrom multiprocessing import Processfrom pathlib import Path@@ -9,7 +10,7 @@ import requestsfrom django.conf import settingsfrom redis import Redis-from mwmbl.crawler.env_vars import CRAWLER_WORKERS, CRAWL_THREADS+from mwmbl.crawler.env_vars import CRAWLER_WORKERS, CRAWL_THREADS, CRAWL_DELAY_SECONDSfrom mwmbl.rankeval.evaluation.remote_index import RemoteIndexfrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.tinysearchengine.indexer import TinyIndex, Document@@ -26,7 +27,7 @@ django.setup()from mwmbl.indexer.update_urls import record_urls_in_database-from mwmbl.crawler.retrieve import crawl_batch+from mwmbl.crawler.retrieve import crawl_batch, crawl_urlfrom mwmbl.crawler.batch import HashedBatch, Result, Resultsfrom mwmbl.indexer.index_batches import index_batches, index_pages@@ -85,8 +86,17 @@ def process_batch():user_id = "test"urls = url_queue.get_batch(user_id)logger.info(f"Processing batch of {len(urls)} URLs")- results = crawl_batch(batch=urls, num_threads=CRAWL_THREADS)- for result in results:++ # Process URLs sequentially with rate limiting+ results = []+ for i, url in enumerate(urls):+ if i > 0: # Don't delay before the first URL+ # Add delay with 10% random fuzz+ delay = CRAWL_DELAY_SECONDS * (0.9 + 0.2 * random.random())+ time.sleep(delay)++ result = crawl_url(url)+ results.append(result)logger.debug("Result", result)js_timestamp = int(time.time() * 1000)batch = HashedBatch.parse_obj({"user_id_hash": user_id, "timestamp": js_timestamp, "items": results})diff --git a/mwmbl/crawler/README.md b/mwmbl/crawler/README.mdindex 1074c3c..69c2c82 100644--- a/mwmbl/crawler/README.md+++ b/mwmbl/crawler/README.md@@ -26,6 +26,9 @@ The crawler can be configured using the following environment variables:**Process Architecture**: The crawler spawns `CRAWLER_WORKERS` processes for crawling work, each using `CRAWL_THREADS` threads for concurrent URL fetching. Additionally, one separate process handles indexing work. For example, with default settings you get 10 crawler processes × 20 threads = 200 total crawling threads, plus 1 indexing process.+### Rate Limiting Configuration+- `CRAWL_DELAY_SECONDS` (default: 0.0) - Delay in seconds between crawling each URL within a batch. Includes 10% random fuzz (±10%) to avoid synchronized requests across workers. Set to 0 to disable delays.+### Redis Configuration- `REDIS_URL` (default: 'redis://127.0.0.1:6379') - Redis connection URL for URL queues and statsdiff --git a/mwmbl/crawler/env_vars.py b/mwmbl/crawler/env_vars.pyindex 9ee02e3..aae2026 100644--- a/mwmbl/crawler/env_vars.py+++ b/mwmbl/crawler/env_vars.py@@ -22,5 +22,8 @@ MAX_SITE_URLS = int(os.environ.get('MAX_SITE_URLS', '100'))CRAWLER_WORKERS = int(os.environ.get('CRAWLER_WORKERS', '10'))CRAWL_THREADS = int(os.environ.get('CRAWL_THREADS', '20'))+# Rate limiting configuration+CRAWL_DELAY_SECONDS = float(os.environ.get('CRAWL_DELAY_SECONDS', '0.0'))+# Redis configurationREDIS_URL = os.environ.get('REDIS_URL', 'redis://127.0.0.1:6379')