Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Rewrite crawl script as a class for easier testing

Daoud Clarke144bc6203084
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.py
index 38dfc50..11fe4a5 100644
--- a/mwmbl/crawl.py
+++ b/mwmbl/crawl.py
@@ -40,222 +40,276 @@ logging.basicConfig(level=logging.INFO, format=FORMAT)
BATCH_QUEUE_KEY = "batch-queue"
-# Initialize Redis connection lazily
-redis = None
-url_queue = None
-
-def get_redis():
- global redis
- if redis is None:
- redis = Redis.from_url(
- settings.REDIS_URL,
- decode_responses=True,
- health_check_interval=30,
- )
- return redis
-
-def get_url_queue():
- global url_queue
- if url_queue is None:
- url_queue = RedisURLQueue(get_redis(), lambda: set())
- return url_queue
-
-def check_redis():
- try:
- get_redis().ping()
- logger.debug("Redis ping successful")
- except ConnectionError as e:
- raise SystemExit(f"Cannot reach Redis at {settings.REDIS_URL}. Make sure your Redis server is running.")
-
-
-def run():
- check_redis()
- workers: int = CRAWLER_WORKERS
- assert workers > 0, f"Invalid value for CRAWLER_WORKERS: {workers}"
-
- batch_processes: list[Process] = []
- for i in range(workers):
- process = Process(target=process_batch_continuously)
- process.start()
- batch_processes.append(process)
- time.sleep(5)
-
- index_process = Process(target=run_indexing_continuously)
- index_process.start()
-
- # Track index process crashes - list of (timestamp, exit_code, pid) tuples
- index_crash_history: list[tuple[datetime, int, int]] = []
-
- while True:
- if not index_process.is_alive():
- crash_time = datetime.now()
- exit_code = index_process.exitcode or -1
- pid = index_process.pid
-
- # Record this crash
- index_crash_history.append((crash_time, exit_code, pid))
-
- # Remove crashes older than 1 hour
- one_hour_ago = crash_time - timedelta(hours=1)
- index_crash_history = [(t, c, p) for t, c, p in index_crash_history if t > one_hour_ago]
-
- # Check if we've exceeded the crash threshold
- if len(index_crash_history) > 5:
- crash_details = []
- for crash_time, exit_code, pid in index_crash_history:
- crash_details.append(f" - {crash_time.isoformat()}: pid={pid}, exit_code={exit_code}")
-
- error_msg = (f"Index process crashed {len(index_crash_history)} times in the last hour "
- f"(threshold: 5). Recent crashes:\n" + "\n".join(crash_details))
- logger.error(error_msg)
- raise RuntimeError(error_msg)
-
- logger.warning(f"Indexing process [pid={pid}] died with exit code {exit_code}, respawning. "
- f"Crash count in last hour: {len(index_crash_history)}")
- index_process = Process(target=run_indexing_continuously)
- index_process.start()
-
+class Crawler:
+ """
+ Main crawler class that manages Redis connections, URL queues, and crawling operations.
+
+ This class encapsulates all the crawler functionality including:
+ - Redis connection management
+ - URL queue operations
+ - Batch processing
+ - Indexing operations
+ """
+
+ def __init__(self, redis_url=None):
+ """
+ Initialize the crawler with Redis connection.
+
+ Args:
+ redis_url: Optional Redis URL. If not provided, uses settings.REDIS_URL
+ """
+ self.redis_url = redis_url or settings.REDIS_URL
+ self._redis = None
+ self._url_queue = None
+
+ @property
+ def redis(self):
+ """Lazy initialization of Redis connection."""
+ if self._redis is None:
+ self._redis = Redis.from_url(
+ self.redis_url,
+ decode_responses=True,
+ health_check_interval=30,
+ )
+ return self._redis
+
+ @property
+ def url_queue(self):
+ """Lazy initialization of URL queue."""
+ if self._url_queue is None:
+ self._url_queue = RedisURLQueue(self.redis, lambda: set())
+ return self._url_queue
+
+ def check_redis(self):
+ """Check Redis connection health."""
+ try:
+ self.redis.ping()
+ logger.debug("Redis ping successful")
+ except ConnectionError as e:
+ raise SystemExit(f"Cannot reach Redis at {self.redis_url}. Make sure your Redis server is running.")
+
+ def process_batch(self):
+ """
+ Process a single batch of URLs by crawling them sequentially with rate limiting.
+
+ This function handles the core crawling workflow:
+ 1. Gets a batch of URLs from the Redis URL queue
+ 2. Crawls each URL sequentially with configurable delay between requests
+ 3. Records crawl results in the database for URL tracking
+ 4. Pushes the completed batch to Redis queue for indexing
+
+ The sequential crawling with delays respects rate limits and reduces load on target servers.
+ Each batch is processed as a HashedBatch object containing metadata and crawl results.
+ """
+ user_id = "test"
+ urls = self.url_queue.get_batch(user_id)
+ logger.info(f"Processing batch of {len(urls)} URLs")
+
+ # Process URLs sequentially with rate limiting
+ results = []
+ for i, url in enumerate(urls):
+ if i > 0: # Don't delay before the first URL
+ # Add delay with 10% random fuzz
+ delay = CRAWL_DELAY_SECONDS * (0.9 + 0.2 * random.random())
+ time.sleep(delay)
+
+ result = crawl_url(url)
+ results.append(result)
+ logger.debug("Result", result)
+
+ js_timestamp = int(time.time() * 1000)
+ batch = HashedBatch.parse_obj({
+ "user_id_hash": user_id,
+ "timestamp": js_timestamp,
+ "items": results,
+ })
+ record_urls_in_database([batch], self.url_queue)
+
+ # Push the batch into the Redis queue
+ batch_json = batch.json()
+ self.redis.rpush(BATCH_QUEUE_KEY, batch_json)
+
+ def run_indexing(self):
+ """
+ Process completed crawl batches and integrate results into the search index.
+
+ This function handles the indexing workflow:
+ 1. Pulls completed crawl batches from Redis queue (up to 10 at once)
+ 2. Indexes batches locally using the tiny search engine indexer
+ 3. For top terms, syncs high-scoring local results with the remote Mwmbl index
+ 4. Downloads updated remote results and merges them back into local index
+
+ The sync process ensures that high-quality local crawl results get submitted
+ to the main Mwmbl search index, while also keeping the local index updated
+ with the latest remote results for better search quality.
+
+ Only results that score higher than existing remote results are submitted,
+ preventing low-quality content from polluting the main index.
+ """
+ index_path = data_path / settings.INDEX_NAME
+ batch_jsons = self.redis.lpop(BATCH_QUEUE_KEY, 10)
+ if batch_jsons is None:
+ logger.info("No more batches to index. Sleeping for 10 seconds.")
+ time.sleep(10)
+ return
+
+ logger.info(f"Got {len(batch_jsons)} batches to index")
+ batches = [HashedBatch.parse_raw(b) for b in batch_jsons]
+ term_new_doc_count = index_batches(batches, index_path)
+ logger.info(f"Indexed, top terms to sync: {term_new_doc_count.most_common(10)}")
+
+ remote_index = RemoteIndex()
+ with TinyIndex(Document, index_path, 'w') as local_index:
+ for term, count in term_new_doc_count.most_common(100):
+ logger.info(f"Syncing term {term} with {count} new local items")
+ remote_items = remote_index.retrieve(term)
+ remote_item_urls = {item.url for item in remote_items}
+ local_items = local_index.retrieve(term)
+ new_items = [item for item in local_items if item.url not in remote_item_urls]
+ logger.info(f"Found {len(new_items)} new items for term {term}")
+
+ terms = tokenize(term)
+ remote_item_scores = [score_result(terms, item, True) for item in remote_items]
+ min_remote_score = min(remote_item_scores, default=0.0)
+ local_scores = [score_result(terms, item, True) for item in new_items]
+ max_local_score = max(local_scores, default=0.0)
+ logger.info(f"Max local score: {max_local_score}, min remote score: {min_remote_score}")
+
+ new_high_score = max_local_score < min_remote_score
+
+ if new_high_score:
+ result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,
+ score=doc.score, term=doc.term, state=doc.state) for doc in new_items]
+ results = Results(api_key=MWMBL_API_KEY, results=result_items, crawler_version=CRAWLER_VERSION)
+ logger.info(f"Posting {len(result_items)} results")
+ response = requests.post(
+ "https://mwmbl.org/api/v1/crawler/results", json=results.dict()
+ )
+ logger.info(f"Response: {response.text}")
+ response.raise_for_status()
+
+ new_remote_items = remote_index.retrieve(term, refresh=True)
+ # Check how many of our items were indexed
+ new_remote_item_urls = {item.url for item in new_remote_items}
+ indexed_items = sum(1 for item in new_items if item.url in new_remote_item_urls)
+ logger.info(f'Indexed items: {indexed_items}/{len(new_items)} for term "{term}"')
+
+ page_index = local_index.get_key_page_index(term)
+ index_pages(index_path, {page_index: new_remote_items}, mark_synced=True)
+ logger.info(f"Completed indexing for term {term}")
+
+ new_page_content = local_index.get_page(page_index)
+ logger.info(f"Page content: {new_page_content}")
+
+ def process_batch_continuously(self):
+ """Continuously process batches with error handling."""
+ while True:
+ self.check_redis()
+ try:
+ self.process_batch()
+ except Exception as err:
+ logger.exception(f"Error processing batch: '{err}'")
+ time.sleep(10)
+
+ def run_indexing_continuously(self):
+ """Continuously run indexing with error handling."""
+ while True:
+ self.check_redis()
+ try:
+ self.run_indexing()
+ except Exception as err:
+ logger.exception(f"Error running indexing: '{err}'")
+ time.sleep(10)
+
+ def run(self):
+ """
+ Main entry point for the crawler.
+
+ Starts multiple worker processes for batch processing and one indexing process.
+ Monitors processes and restarts them if they crash.
+ """
+ self.check_redis()
+ workers: int = CRAWLER_WORKERS
+ assert workers > 0, f"Invalid value for CRAWLER_WORKERS: {workers}"
+
+ batch_processes: list[Process] = []
for i in range(workers):
- if not batch_processes[i].is_alive():
- logger.info(f"Batch process [pid={batch_processes[i].pid}] died, respawning.")
- batch_processes[i] = Process(target=process_batch_continuously)
- batch_processes[i].start()
- time.sleep(5)
+ process = Process(target=self.process_batch_continuously)
+ process.start()
+ batch_processes.append(process)
+ time.sleep(5)
+
+ index_process = Process(target=self.run_indexing_continuously)
+ index_process.start()
+
+ # Track index process crashes - list of (timestamp, exit_code, pid) tuples
+ index_crash_history: list[tuple[datetime, int, int]] = []
+
+ while True:
+ if not index_process.is_alive():
+ crash_time = datetime.now()
+ exit_code = index_process.exitcode or -1
+ pid = index_process.pid
+
+ # Record this crash
+ index_crash_history.append((crash_time, exit_code, pid))
+
+ # Remove crashes older than 1 hour
+ one_hour_ago = crash_time - timedelta(hours=1)
+ index_crash_history = [(t, c, p) for t, c, p in index_crash_history if t > one_hour_ago]
+
+ # Check if we've exceeded the crash threshold
+ if len(index_crash_history) > 5:
+ crash_details = []
+ for crash_time, exit_code, pid in index_crash_history:
+ crash_details.append(f" - {crash_time.isoformat()}: pid={pid}, exit_code={exit_code}")
+
+ error_msg = (f"Index process crashed {len(index_crash_history)} times in the last hour "
+ f"(threshold: 5). Recent crashes:\n" + "\n".join(crash_details))
+ logger.error(error_msg)
+ raise RuntimeError(error_msg)
+
+ logger.warning(f"Indexing process [pid={pid}] died with exit code {exit_code}, respawning. "
+ f"Crash count in last hour: {len(index_crash_history)}")
+ index_process = Process(target=self.run_indexing_continuously)
+ index_process.start()
+
+ for i in range(workers):
+ if not batch_processes[i].is_alive():
+ logger.info(f"Batch process [pid={batch_processes[i].pid}] died, respawning.")
+ batch_processes[i] = Process(target=self.process_batch_continuously)
+ batch_processes[i].start()
+ time.sleep(5)
- time.sleep(10)
-
-
-def process_batch_continuously():
- while True:
- check_redis()
- try:
- process_batch()
- except Exception as err:
- logger.exception(f"Error processing batch: '{err}'")
time.sleep(10)
-def process_batch():
- """
- Process a single batch of URLs by crawling them sequentially with rate limiting.
+# Global instance for backward compatibility
+_default_crawler = None
- This function handles the core crawling workflow:
- 1. Gets a batch of URLs from the Redis URL queue
- 2. Crawls each URL sequentially with configurable delay between requests
- 3. Records crawl results in the database for URL tracking
- 4. Pushes the completed batch to Redis queue for indexing
+def get_default_crawler():
+ """Get the default crawler instance."""
+ global _default_crawler
+ if _default_crawler is None:
+ _default_crawler = Crawler()
+ return _default_crawler
- The sequential crawling with delays respects rate limits and reduces load on target servers.
- Each batch is processed as a HashedBatch object containing metadata and crawl results.
- """
- user_id = "test"
- urls = get_url_queue().get_batch(user_id)
- logger.info(f"Processing batch of {len(urls)} URLs")
-
- # Process URLs sequentially with rate limiting
- results = []
- for i, url in enumerate(urls):
- if i > 0: # Don't delay before the first URL
- # Add delay with 10% random fuzz
- delay = CRAWL_DELAY_SECONDS * (0.9 + 0.2 * random.random())
- time.sleep(delay)
-
- result = crawl_url(url)
- results.append(result)
- logger.debug("Result", result)
- js_timestamp = int(time.time() * 1000)
- batch = HashedBatch.parse_obj({
- "user_id_hash": user_id,
- "timestamp": js_timestamp,
- "items": results,
- })
- record_urls_in_database([batch], get_url_queue())
-
- # Push the batch into the Redis queue
- batch_json = batch.json()
- get_redis().rpush(BATCH_QUEUE_KEY, batch_json)
-
-
-def run_indexing_continuously():
- while True:
- check_redis()
- try:
- run_indexing()
- except Exception as err:
- logger.exception(f"Error running indexing: '{err}'")
- time.sleep(10)
+# Backward compatibility functions
+def check_redis():
+ """Check Redis connection health (backward compatibility)."""
+ return get_default_crawler().check_redis()
+def process_batch():
+ """Process a single batch (backward compatibility)."""
+ return get_default_crawler().process_batch()
def run_indexing():
- """
- Process completed crawl batches and integrate results into the search index.
-
- This function handles the indexing workflow:
- 1. Pulls completed crawl batches from Redis queue (up to 10 at once)
- 2. Indexes batches locally using the tiny search engine indexer
- 3. For top terms, syncs high-scoring local results with the remote Mwmbl index
- 4. Downloads updated remote results and merges them back into local index
+ """Run indexing (backward compatibility)."""
+ return get_default_crawler().run_indexing()
- The sync process ensures that high-quality local crawl results get submitted
- to the main Mwmbl search index, while also keeping the local index updated
- with the latest remote results for better search quality.
-
- Only results that score higher than existing remote results are submitted,
- preventing low-quality content from polluting the main index.
- """
- index_path = data_path / settings.INDEX_NAME
- batch_jsons = get_redis().lpop(BATCH_QUEUE_KEY, 10)
- if batch_jsons is None:
- logger.info("No more batches to index. Sleeping for 10 seconds.")
- time.sleep(10)
- return
- logger.info(f"Got {len(batch_jsons)} batches to index")
- batches = [HashedBatch.parse_raw(b) for b in batch_jsons]
- term_new_doc_count = index_batches(batches, index_path)
- logger.info(f"Indexed, top terms to sync: {term_new_doc_count.most_common(10)}")
-
- remote_index = RemoteIndex()
- with TinyIndex(Document, index_path, 'w') as local_index:
- for term, count in term_new_doc_count.most_common(100):
- logger.info(f"Syncing term {term} with {count} new local items")
- remote_items = remote_index.retrieve(term)
- remote_item_urls = {item.url for item in remote_items}
- local_items = local_index.retrieve(term)
- new_items = [item for item in local_items if item.url not in remote_item_urls]
- logger.info(f"Found {len(new_items)} new items for term {term}")
-
- terms = tokenize(term)
- remote_item_scores = [score_result(terms, item, True) for item in remote_items]
- min_remote_score = min(remote_item_scores, default=0.0)
- local_scores = [score_result(terms, item, True) for item in new_items]
- max_local_score = max(local_scores, default=0.0)
- logger.info(f"Max local score: {max_local_score}, min remote score: {min_remote_score}")
-
- new_high_score = max_local_score < min_remote_score
-
- if new_high_score:
- result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,
- score=doc.score, term=doc.term, state=doc.state) for doc in new_items]
- results = Results(api_key=MWMBL_API_KEY, results=result_items, crawler_version=CRAWLER_VERSION)
- logger.info(f"Posting {len(result_items)} results")
- response = requests.post(
- "https://mwmbl.org/api/v1/crawler/results", json=results.dict()
- )
- logger.info(f"Response: {response.text}")
- response.raise_for_status()
-
- new_remote_items = remote_index.retrieve(term, refresh=True)
- # Check how many of our items were indexed
- new_remote_item_urls = {item.url for item in new_remote_items}
- indexed_items = sum(1 for item in new_items if item.url in new_remote_item_urls)
- logger.info(f'Indexed items: {indexed_items}/{len(new_items)} for term "{term}"')
-
- page_index = local_index.get_key_page_index(term)
- index_pages(index_path, {page_index: new_remote_items}, mark_synced=True)
- logger.info(f"Completed indexing for term {term}")
-
- new_page_content = local_index.get_page(page_index)
- logger.info(f"Page content: {new_page_content}")
+def run():
+ """Main entry point (backward compatibility)."""
+ return get_default_crawler().run()
if __name__ == "__main__":
diff --git a/test/test_crawl_functional.py b/test/test_crawl_functional.py
index ebd3745..f0c3949 100644
--- a/test/test_crawl_functional.py
+++ b/test/test_crawl_functional.py
@@ -104,20 +104,21 @@ class TestCrawlFunctional:
def test_redis_connection_and_health_check(self, fake_redis, mock_settings, mock_environment):
"""Test Redis connection and health check functionality"""
- with patch('mwmbl.crawl.get_redis', return_value=fake_redis):
- # Import after patching to ensure mocks are in place
- from mwmbl.crawl import check_redis
-
- # Should not raise an exception
- check_redis()
-
- # Test with broken Redis
- broken_redis = MagicMock()
- broken_redis.ping.side_effect = ConnectionError("Redis unavailable")
-
- with patch('mwmbl.crawl.get_redis', return_value=broken_redis):
- with pytest.raises(SystemExit):
- check_redis()
+ from mwmbl.crawl import Crawler
+
+ # Test with working Redis
+ crawler = Crawler()
+ crawler._redis = fake_redis # Set the private attribute directly
+ # Should not raise an exception
+ crawler.check_redis()
+
+ # Test with broken Redis
+ broken_redis = MagicMock()
+ broken_redis.ping.side_effect = ConnectionError("Redis unavailable")
+
+ crawler._redis = broken_redis # Set the private attribute directly
+ with pytest.raises(SystemExit):
+ crawler.check_redis()
def test_url_queue_operations(self, fake_redis, sample_found_urls):
"""Test RedisURLQueue operations with fake Redis"""
@@ -136,34 +137,36 @@ class TestCrawlFunctional:
def test_process_batch_workflow(self, fake_redis, mock_settings, mock_environment, mock_crawl_response):
"""Test the complete batch processing workflow"""
- with patch('mwmbl.crawl.get_redis', return_value=fake_redis):
- with patch('mwmbl.crawl.get_url_queue') as mock_get_url_queue:
- with patch('mwmbl.crawl.crawl_url', return_value=mock_crawl_response):
- with patch('mwmbl.crawl.record_urls_in_database'):
- # Setup URL queue mock
- mock_url_queue = MagicMock()
- mock_url_queue.get_batch.return_value = ['https://example.com']
- mock_get_url_queue.return_value = mock_url_queue
-
- # Import and test process_batch
- from mwmbl.crawl import process_batch
-
- # Should complete without errors
- process_batch()
-
- # Verify batch was pushed to Redis
- batch_data = fake_redis.lpop("batch-queue")
- assert batch_data is not None
-
- # Verify batch structure
- batch = json.loads(batch_data)
- assert 'user_id_hash' in batch
- assert 'timestamp' in batch
- assert 'items' in batch
- assert len(batch['items']) == 1
+ from mwmbl.crawl import Crawler
+
+ crawler = Crawler()
+ crawler._redis = fake_redis # Set the private attribute directly
+
+ # Mock the URL queue
+ mock_url_queue = MagicMock()
+ mock_url_queue.get_batch.return_value = ['https://example.com']
+ crawler._url_queue = mock_url_queue
+
+ with patch('mwmbl.crawl.crawl_url', return_value=mock_crawl_response):
+ with patch('mwmbl.crawl.record_urls_in_database'):
+ # Should complete without errors
+ crawler.process_batch()
+
+ # Verify batch was pushed to Redis
+ batch_data = fake_redis.lpop("batch-queue")
+ assert batch_data is not None
+
+ # Verify batch structure
+ batch = json.loads(batch_data)
+ assert 'user_id_hash' in batch
+ assert 'timestamp' in batch
+ assert 'items' in batch
+ assert len(batch['items']) == 1
def test_indexing_workflow(self, fake_redis, mock_settings, mock_environment, temp_data_path):
"""Test the indexing workflow with fake Redis"""
+ from mwmbl.crawl import Crawler
+
# Create a sample batch in Redis
# Create a sample item that matches the Item schema
from mwmbl.crawler.batch import Item, ItemContent
@@ -184,11 +187,13 @@ class TestCrawlFunctional:
)
fake_redis.rpush("batch-queue", sample_batch.json())
- with patch('mwmbl.crawl.get_redis', return_value=fake_redis):
- # Mock Counter object for index_batches return value
- from collections import Counter
- mock_counter = Counter({'example': 1})
- with patch('mwmbl.crawl.index_batches', return_value=mock_counter):
+ crawler = Crawler()
+ crawler._redis = fake_redis # Set the private attribute directly
+
+ # Mock Counter object for index_batches return value
+ from collections import Counter
+ mock_counter = Counter({'example': 1})
+ with patch('mwmbl.crawl.index_batches', return_value=mock_counter):
with patch('mwmbl.crawl.RemoteIndex') as mock_remote_index:
with patch('mwmbl.crawl.TinyIndex') as mock_tiny_index:
with patch('mwmbl.crawl.index_pages') as mock_index_pages:
@@ -207,11 +212,8 @@ class TestCrawlFunctional:
mock_post.return_value.status_code = 200
mock_post.return_value.text = "OK"
- # Import and test run_indexing
- from mwmbl.crawl import run_indexing
-
# Should complete without errors
- run_indexing()
+ crawler.run_indexing()
# Verify batch was consumed from Redis
remaining_batches = fake_redis.llen("batch-queue")