Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

feat: add rate limiting with configurable delay and random fuzz

thiswillbeyourgithub (aider)91889091a497
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.py
index c8cfa0b..f674ccc 100644
--- a/mwmbl/crawl.py
+++ b/mwmbl/crawl.py
@@ -1,5 +1,6 @@
import logging
import os
+import random
import time
from multiprocessing import Process
from pathlib import Path
@@ -9,7 +10,7 @@ import requests
from django.conf import settings
from redis import Redis
-from mwmbl.crawler.env_vars import CRAWLER_WORKERS, CRAWL_THREADS
+from mwmbl.crawler.env_vars import CRAWLER_WORKERS, CRAWL_THREADS, CRAWL_DELAY_SECONDS
from mwmbl.rankeval.evaluation.remote_index import RemoteIndex
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
@@ -26,7 +27,7 @@ django.setup()
from mwmbl.indexer.update_urls import record_urls_in_database
-from mwmbl.crawler.retrieve import crawl_batch
+from mwmbl.crawler.retrieve import crawl_batch, crawl_url
from mwmbl.crawler.batch import HashedBatch, Result, Results
from mwmbl.indexer.index_batches import index_batches, index_pages
@@ -85,8 +86,17 @@ def process_batch():
user_id = "test"
urls = url_queue.get_batch(user_id)
logger.info(f"Processing batch of {len(urls)} URLs")
- results = crawl_batch(batch=urls, num_threads=CRAWL_THREADS)
- for result in results:
+
+ # Process URLs sequentially with rate limiting
+ results = []
+ for i, url in enumerate(urls):
+ if i > 0: # Don't delay before the first URL
+ # Add delay with 10% random fuzz
+ delay = CRAWL_DELAY_SECONDS * (0.9 + 0.2 * random.random())
+ time.sleep(delay)
+
+ result = crawl_url(url)
+ results.append(result)
logger.debug("Result", result)
js_timestamp = int(time.time() * 1000)
batch = HashedBatch.parse_obj({"user_id_hash": user_id, "timestamp": js_timestamp, "items": results})
diff --git a/mwmbl/crawler/README.md b/mwmbl/crawler/README.md
index 1074c3c..69c2c82 100644
--- a/mwmbl/crawler/README.md
+++ b/mwmbl/crawler/README.md
@@ -26,6 +26,9 @@ The crawler can be configured using the following environment variables:
**Process Architecture**: The crawler spawns `CRAWLER_WORKERS` processes for crawling work, each using `CRAWL_THREADS` threads for concurrent URL fetching. Additionally, one separate process handles indexing work. For example, with default settings you get 10 crawler processes × 20 threads = 200 total crawling threads, plus 1 indexing process.
+### Rate Limiting Configuration
+- `CRAWL_DELAY_SECONDS` (default: 0.0) - Delay in seconds between crawling each URL within a batch. Includes 10% random fuzz (±10%) to avoid synchronized requests across workers. Set to 0 to disable delays.
+
### Redis Configuration
- `REDIS_URL` (default: 'redis://127.0.0.1:6379') - Redis connection URL for URL queues and stats
diff --git a/mwmbl/crawler/env_vars.py b/mwmbl/crawler/env_vars.py
index 9ee02e3..aae2026 100644
--- a/mwmbl/crawler/env_vars.py
+++ b/mwmbl/crawler/env_vars.py
@@ -22,5 +22,8 @@ MAX_SITE_URLS = int(os.environ.get('MAX_SITE_URLS', '100'))
CRAWLER_WORKERS = int(os.environ.get('CRAWLER_WORKERS', '10'))
CRAWL_THREADS = int(os.environ.get('CRAWL_THREADS', '20'))
+# Rate limiting configuration
+CRAWL_DELAY_SECONDS = float(os.environ.get('CRAWL_DELAY_SECONDS', '0.0'))
+
# Redis configuration
REDIS_URL = os.environ.get('REDIS_URL', 'redis://127.0.0.1:6379')