An open source, non-profit web search engine
Rewrite crawl script as a class for easier testing
Daoud Clarke144bc6203084
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.pyindex 38dfc50..11fe4a5 100644--- a/mwmbl/crawl.py+++ b/mwmbl/crawl.py@@ -40,222 +40,276 @@ logging.basicConfig(level=logging.INFO, format=FORMAT)BATCH_QUEUE_KEY = "batch-queue"-# Initialize Redis connection lazily-redis = None-url_queue = None--def get_redis():- global redis- if redis is None:- redis = Redis.from_url(- settings.REDIS_URL,- decode_responses=True,- health_check_interval=30,- )- return redis--def get_url_queue():- global url_queue- if url_queue is None:- url_queue = RedisURLQueue(get_redis(), lambda: set())- return url_queue--def check_redis():- try:- get_redis().ping()- logger.debug("Redis ping successful")- except ConnectionError as e:- raise SystemExit(f"Cannot reach Redis at {settings.REDIS_URL}. Make sure your Redis server is running.")---def run():- check_redis()- workers: int = CRAWLER_WORKERS- assert workers > 0, f"Invalid value for CRAWLER_WORKERS: {workers}"-- batch_processes: list[Process] = []- for i in range(workers):- process = Process(target=process_batch_continuously)- process.start()- batch_processes.append(process)- time.sleep(5)-- index_process = Process(target=run_indexing_continuously)- index_process.start()-- # Track index process crashes - list of (timestamp, exit_code, pid) tuples- index_crash_history: list[tuple[datetime, int, int]] = []-- while True:- if not index_process.is_alive():- crash_time = datetime.now()- exit_code = index_process.exitcode or -1- pid = index_process.pid-- # Record this crash- index_crash_history.append((crash_time, exit_code, pid))-- # Remove crashes older than 1 hour- one_hour_ago = crash_time - timedelta(hours=1)- index_crash_history = [(t, c, p) for t, c, p in index_crash_history if t > one_hour_ago]-- # Check if we've exceeded the crash threshold- if len(index_crash_history) > 5:- crash_details = []- for crash_time, exit_code, pid in index_crash_history:- crash_details.append(f" - {crash_time.isoformat()}: pid={pid}, exit_code={exit_code}")-- error_msg = (f"Index process crashed {len(index_crash_history)} times in the last hour "- f"(threshold: 5). Recent crashes:\n" + "\n".join(crash_details))- logger.error(error_msg)- raise RuntimeError(error_msg)-- logger.warning(f"Indexing process [pid={pid}] died with exit code {exit_code}, respawning. "- f"Crash count in last hour: {len(index_crash_history)}")- index_process = Process(target=run_indexing_continuously)- index_process.start()-+class Crawler:+ """+ Main crawler class that manages Redis connections, URL queues, and crawling operations.++ This class encapsulates all the crawler functionality including:+ - Redis connection management+ - URL queue operations+ - Batch processing+ - Indexing operations+ """++ def __init__(self, redis_url=None):+ """+ Initialize the crawler with Redis connection.++ Args:+ redis_url: Optional Redis URL. If not provided, uses settings.REDIS_URL+ """+ self.redis_url = redis_url or settings.REDIS_URL+ self._redis = None+ self._url_queue = None++ @property+ def redis(self):+ """Lazy initialization of Redis connection."""+ if self._redis is None:+ self._redis = Redis.from_url(+ self.redis_url,+ decode_responses=True,+ health_check_interval=30,+ )+ return self._redis++ @property+ def url_queue(self):+ """Lazy initialization of URL queue."""+ if self._url_queue is None:+ self._url_queue = RedisURLQueue(self.redis, lambda: set())+ return self._url_queue++ def check_redis(self):+ """Check Redis connection health."""+ try:+ self.redis.ping()+ logger.debug("Redis ping successful")+ except ConnectionError as e:+ raise SystemExit(f"Cannot reach Redis at {self.redis_url}. Make sure your Redis server is running.")++ def process_batch(self):+ """+ Process a single batch of URLs by crawling them sequentially with rate limiting.++ This function handles the core crawling workflow:+ 1. Gets a batch of URLs from the Redis URL queue+ 2. Crawls each URL sequentially with configurable delay between requests+ 3. Records crawl results in the database for URL tracking+ 4. Pushes the completed batch to Redis queue for indexing++ The sequential crawling with delays respects rate limits and reduces load on target servers.+ Each batch is processed as a HashedBatch object containing metadata and crawl results.+ """+ user_id = "test"+ urls = self.url_queue.get_batch(user_id)+ logger.info(f"Processing batch of {len(urls)} URLs")++ # Process URLs sequentially with rate limiting+ results = []+ for i, url in enumerate(urls):+ if i > 0: # Don't delay before the first URL+ # Add delay with 10% random fuzz+ delay = CRAWL_DELAY_SECONDS * (0.9 + 0.2 * random.random())+ time.sleep(delay)++ result = crawl_url(url)+ results.append(result)+ logger.debug("Result", result)++ js_timestamp = int(time.time() * 1000)+ batch = HashedBatch.parse_obj({+ "user_id_hash": user_id,+ "timestamp": js_timestamp,+ "items": results,+ })+ record_urls_in_database([batch], self.url_queue)++ # Push the batch into the Redis queue+ batch_json = batch.json()+ self.redis.rpush(BATCH_QUEUE_KEY, batch_json)++ def run_indexing(self):+ """+ Process completed crawl batches and integrate results into the search index.++ This function handles the indexing workflow:+ 1. Pulls completed crawl batches from Redis queue (up to 10 at once)+ 2. Indexes batches locally using the tiny search engine indexer+ 3. For top terms, syncs high-scoring local results with the remote Mwmbl index+ 4. Downloads updated remote results and merges them back into local index++ The sync process ensures that high-quality local crawl results get submitted+ to the main Mwmbl search index, while also keeping the local index updated+ with the latest remote results for better search quality.++ Only results that score higher than existing remote results are submitted,+ preventing low-quality content from polluting the main index.+ """+ index_path = data_path / settings.INDEX_NAME+ batch_jsons = self.redis.lpop(BATCH_QUEUE_KEY, 10)+ if batch_jsons is None:+ logger.info("No more batches to index. Sleeping for 10 seconds.")+ time.sleep(10)+ return++ logger.info(f"Got {len(batch_jsons)} batches to index")+ batches = [HashedBatch.parse_raw(b) for b in batch_jsons]+ term_new_doc_count = index_batches(batches, index_path)+ logger.info(f"Indexed, top terms to sync: {term_new_doc_count.most_common(10)}")++ remote_index = RemoteIndex()+ with TinyIndex(Document, index_path, 'w') as local_index:+ for term, count in term_new_doc_count.most_common(100):+ logger.info(f"Syncing term {term} with {count} new local items")+ remote_items = remote_index.retrieve(term)+ remote_item_urls = {item.url for item in remote_items}+ local_items = local_index.retrieve(term)+ new_items = [item for item in local_items if item.url not in remote_item_urls]+ logger.info(f"Found {len(new_items)} new items for term {term}")++ terms = tokenize(term)+ remote_item_scores = [score_result(terms, item, True) for item in remote_items]+ min_remote_score = min(remote_item_scores, default=0.0)+ local_scores = [score_result(terms, item, True) for item in new_items]+ max_local_score = max(local_scores, default=0.0)+ logger.info(f"Max local score: {max_local_score}, min remote score: {min_remote_score}")++ new_high_score = max_local_score < min_remote_score++ if new_high_score:+ result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,+ score=doc.score, term=doc.term, state=doc.state) for doc in new_items]+ results = Results(api_key=MWMBL_API_KEY, results=result_items, crawler_version=CRAWLER_VERSION)+ logger.info(f"Posting {len(result_items)} results")+ response = requests.post(+ "https://mwmbl.org/api/v1/crawler/results", json=results.dict()+ )+ logger.info(f"Response: {response.text}")+ response.raise_for_status()++ new_remote_items = remote_index.retrieve(term, refresh=True)+ # Check how many of our items were indexed+ new_remote_item_urls = {item.url for item in new_remote_items}+ indexed_items = sum(1 for item in new_items if item.url in new_remote_item_urls)+ logger.info(f'Indexed items: {indexed_items}/{len(new_items)} for term "{term}"')++ page_index = local_index.get_key_page_index(term)+ index_pages(index_path, {page_index: new_remote_items}, mark_synced=True)+ logger.info(f"Completed indexing for term {term}")++ new_page_content = local_index.get_page(page_index)+ logger.info(f"Page content: {new_page_content}")++ def process_batch_continuously(self):+ """Continuously process batches with error handling."""+ while True:+ self.check_redis()+ try:+ self.process_batch()+ except Exception as err:+ logger.exception(f"Error processing batch: '{err}'")+ time.sleep(10)++ def run_indexing_continuously(self):+ """Continuously run indexing with error handling."""+ while True:+ self.check_redis()+ try:+ self.run_indexing()+ except Exception as err:+ logger.exception(f"Error running indexing: '{err}'")+ time.sleep(10)++ def run(self):+ """+ Main entry point for the crawler.++ Starts multiple worker processes for batch processing and one indexing process.+ Monitors processes and restarts them if they crash.+ """+ self.check_redis()+ workers: int = CRAWLER_WORKERS+ assert workers > 0, f"Invalid value for CRAWLER_WORKERS: {workers}"++ batch_processes: list[Process] = []for i in range(workers):- if not batch_processes[i].is_alive():- logger.info(f"Batch process [pid={batch_processes[i].pid}] died, respawning.")- batch_processes[i] = Process(target=process_batch_continuously)- batch_processes[i].start()- time.sleep(5)+ process = Process(target=self.process_batch_continuously)+ process.start()+ batch_processes.append(process)+ time.sleep(5)++ index_process = Process(target=self.run_indexing_continuously)+ index_process.start()++ # Track index process crashes - list of (timestamp, exit_code, pid) tuples+ index_crash_history: list[tuple[datetime, int, int]] = []++ while True:+ if not index_process.is_alive():+ crash_time = datetime.now()+ exit_code = index_process.exitcode or -1+ pid = index_process.pid++ # Record this crash+ index_crash_history.append((crash_time, exit_code, pid))++ # Remove crashes older than 1 hour+ one_hour_ago = crash_time - timedelta(hours=1)+ index_crash_history = [(t, c, p) for t, c, p in index_crash_history if t > one_hour_ago]++ # Check if we've exceeded the crash threshold+ if len(index_crash_history) > 5:+ crash_details = []+ for crash_time, exit_code, pid in index_crash_history:+ crash_details.append(f" - {crash_time.isoformat()}: pid={pid}, exit_code={exit_code}")++ error_msg = (f"Index process crashed {len(index_crash_history)} times in the last hour "+ f"(threshold: 5). Recent crashes:\n" + "\n".join(crash_details))+ logger.error(error_msg)+ raise RuntimeError(error_msg)++ logger.warning(f"Indexing process [pid={pid}] died with exit code {exit_code}, respawning. "+ f"Crash count in last hour: {len(index_crash_history)}")+ index_process = Process(target=self.run_indexing_continuously)+ index_process.start()++ for i in range(workers):+ if not batch_processes[i].is_alive():+ logger.info(f"Batch process [pid={batch_processes[i].pid}] died, respawning.")+ batch_processes[i] = Process(target=self.process_batch_continuously)+ batch_processes[i].start()+ time.sleep(5)- time.sleep(10)---def process_batch_continuously():- while True:- check_redis()- try:- process_batch()- except Exception as err:- logger.exception(f"Error processing batch: '{err}'")time.sleep(10)-def process_batch():- """- Process a single batch of URLs by crawling them sequentially with rate limiting.+# Global instance for backward compatibility+_default_crawler = None- This function handles the core crawling workflow:- 1. Gets a batch of URLs from the Redis URL queue- 2. Crawls each URL sequentially with configurable delay between requests- 3. Records crawl results in the database for URL tracking- 4. Pushes the completed batch to Redis queue for indexing+def get_default_crawler():+ """Get the default crawler instance."""+ global _default_crawler+ if _default_crawler is None:+ _default_crawler = Crawler()+ return _default_crawler- The sequential crawling with delays respects rate limits and reduces load on target servers.- Each batch is processed as a HashedBatch object containing metadata and crawl results.- """- user_id = "test"- urls = get_url_queue().get_batch(user_id)- logger.info(f"Processing batch of {len(urls)} URLs")-- # Process URLs sequentially with rate limiting- results = []- for i, url in enumerate(urls):- if i > 0: # Don't delay before the first URL- # Add delay with 10% random fuzz- delay = CRAWL_DELAY_SECONDS * (0.9 + 0.2 * random.random())- time.sleep(delay)-- result = crawl_url(url)- results.append(result)- logger.debug("Result", result)- js_timestamp = int(time.time() * 1000)- batch = HashedBatch.parse_obj({- "user_id_hash": user_id,- "timestamp": js_timestamp,- "items": results,- })- record_urls_in_database([batch], get_url_queue())-- # Push the batch into the Redis queue- batch_json = batch.json()- get_redis().rpush(BATCH_QUEUE_KEY, batch_json)---def run_indexing_continuously():- while True:- check_redis()- try:- run_indexing()- except Exception as err:- logger.exception(f"Error running indexing: '{err}'")- time.sleep(10)+# Backward compatibility functions+def check_redis():+ """Check Redis connection health (backward compatibility)."""+ return get_default_crawler().check_redis()+def process_batch():+ """Process a single batch (backward compatibility)."""+ return get_default_crawler().process_batch()def run_indexing():- """- Process completed crawl batches and integrate results into the search index.-- This function handles the indexing workflow:- 1. Pulls completed crawl batches from Redis queue (up to 10 at once)- 2. Indexes batches locally using the tiny search engine indexer- 3. For top terms, syncs high-scoring local results with the remote Mwmbl index- 4. Downloads updated remote results and merges them back into local index+ """Run indexing (backward compatibility)."""+ return get_default_crawler().run_indexing()- The sync process ensures that high-quality local crawl results get submitted- to the main Mwmbl search index, while also keeping the local index updated- with the latest remote results for better search quality.-- Only results that score higher than existing remote results are submitted,- preventing low-quality content from polluting the main index.- """- index_path = data_path / settings.INDEX_NAME- batch_jsons = get_redis().lpop(BATCH_QUEUE_KEY, 10)- if batch_jsons is None:- logger.info("No more batches to index. Sleeping for 10 seconds.")- time.sleep(10)- return- logger.info(f"Got {len(batch_jsons)} batches to index")- batches = [HashedBatch.parse_raw(b) for b in batch_jsons]- term_new_doc_count = index_batches(batches, index_path)- logger.info(f"Indexed, top terms to sync: {term_new_doc_count.most_common(10)}")-- remote_index = RemoteIndex()- with TinyIndex(Document, index_path, 'w') as local_index:- for term, count in term_new_doc_count.most_common(100):- logger.info(f"Syncing term {term} with {count} new local items")- remote_items = remote_index.retrieve(term)- remote_item_urls = {item.url for item in remote_items}- local_items = local_index.retrieve(term)- new_items = [item for item in local_items if item.url not in remote_item_urls]- logger.info(f"Found {len(new_items)} new items for term {term}")-- terms = tokenize(term)- remote_item_scores = [score_result(terms, item, True) for item in remote_items]- min_remote_score = min(remote_item_scores, default=0.0)- local_scores = [score_result(terms, item, True) for item in new_items]- max_local_score = max(local_scores, default=0.0)- logger.info(f"Max local score: {max_local_score}, min remote score: {min_remote_score}")-- new_high_score = max_local_score < min_remote_score-- if new_high_score:- result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,- score=doc.score, term=doc.term, state=doc.state) for doc in new_items]- results = Results(api_key=MWMBL_API_KEY, results=result_items, crawler_version=CRAWLER_VERSION)- logger.info(f"Posting {len(result_items)} results")- response = requests.post(- "https://mwmbl.org/api/v1/crawler/results", json=results.dict()- )- logger.info(f"Response: {response.text}")- response.raise_for_status()-- new_remote_items = remote_index.retrieve(term, refresh=True)- # Check how many of our items were indexed- new_remote_item_urls = {item.url for item in new_remote_items}- indexed_items = sum(1 for item in new_items if item.url in new_remote_item_urls)- logger.info(f'Indexed items: {indexed_items}/{len(new_items)} for term "{term}"')-- page_index = local_index.get_key_page_index(term)- index_pages(index_path, {page_index: new_remote_items}, mark_synced=True)- logger.info(f"Completed indexing for term {term}")-- new_page_content = local_index.get_page(page_index)- logger.info(f"Page content: {new_page_content}")+def run():+ """Main entry point (backward compatibility)."""+ return get_default_crawler().run()if __name__ == "__main__":diff --git a/test/test_crawl_functional.py b/test/test_crawl_functional.pyindex ebd3745..f0c3949 100644--- a/test/test_crawl_functional.py+++ b/test/test_crawl_functional.py@@ -104,20 +104,21 @@ class TestCrawlFunctional:def test_redis_connection_and_health_check(self, fake_redis, mock_settings, mock_environment):"""Test Redis connection and health check functionality"""- with patch('mwmbl.crawl.get_redis', return_value=fake_redis):- # Import after patching to ensure mocks are in place- from mwmbl.crawl import check_redis-- # Should not raise an exception- check_redis()-- # Test with broken Redis- broken_redis = MagicMock()- broken_redis.ping.side_effect = ConnectionError("Redis unavailable")-- with patch('mwmbl.crawl.get_redis', return_value=broken_redis):- with pytest.raises(SystemExit):- check_redis()+ from mwmbl.crawl import Crawler++ # Test with working Redis+ crawler = Crawler()+ crawler._redis = fake_redis # Set the private attribute directly+ # Should not raise an exception+ crawler.check_redis()++ # Test with broken Redis+ broken_redis = MagicMock()+ broken_redis.ping.side_effect = ConnectionError("Redis unavailable")++ crawler._redis = broken_redis # Set the private attribute directly+ with pytest.raises(SystemExit):+ crawler.check_redis()def test_url_queue_operations(self, fake_redis, sample_found_urls):"""Test RedisURLQueue operations with fake Redis"""@@ -136,34 +137,36 @@ class TestCrawlFunctional:def test_process_batch_workflow(self, fake_redis, mock_settings, mock_environment, mock_crawl_response):"""Test the complete batch processing workflow"""- with patch('mwmbl.crawl.get_redis', return_value=fake_redis):- with patch('mwmbl.crawl.get_url_queue') as mock_get_url_queue:- with patch('mwmbl.crawl.crawl_url', return_value=mock_crawl_response):- with patch('mwmbl.crawl.record_urls_in_database'):- # Setup URL queue mock- mock_url_queue = MagicMock()- mock_url_queue.get_batch.return_value = ['https://example.com']- mock_get_url_queue.return_value = mock_url_queue-- # Import and test process_batch- from mwmbl.crawl import process_batch-- # Should complete without errors- process_batch()-- # Verify batch was pushed to Redis- batch_data = fake_redis.lpop("batch-queue")- assert batch_data is not None-- # Verify batch structure- batch = json.loads(batch_data)- assert 'user_id_hash' in batch- assert 'timestamp' in batch- assert 'items' in batch- assert len(batch['items']) == 1+ from mwmbl.crawl import Crawler++ crawler = Crawler()+ crawler._redis = fake_redis # Set the private attribute directly++ # Mock the URL queue+ mock_url_queue = MagicMock()+ mock_url_queue.get_batch.return_value = ['https://example.com']+ crawler._url_queue = mock_url_queue++ with patch('mwmbl.crawl.crawl_url', return_value=mock_crawl_response):+ with patch('mwmbl.crawl.record_urls_in_database'):+ # Should complete without errors+ crawler.process_batch()++ # Verify batch was pushed to Redis+ batch_data = fake_redis.lpop("batch-queue")+ assert batch_data is not None++ # Verify batch structure+ batch = json.loads(batch_data)+ assert 'user_id_hash' in batch+ assert 'timestamp' in batch+ assert 'items' in batch+ assert len(batch['items']) == 1def test_indexing_workflow(self, fake_redis, mock_settings, mock_environment, temp_data_path):"""Test the indexing workflow with fake Redis"""+ from mwmbl.crawl import Crawler+# Create a sample batch in Redis# Create a sample item that matches the Item schemafrom mwmbl.crawler.batch import Item, ItemContent@@ -184,11 +187,13 @@ class TestCrawlFunctional:)fake_redis.rpush("batch-queue", sample_batch.json())- with patch('mwmbl.crawl.get_redis', return_value=fake_redis):- # Mock Counter object for index_batches return value- from collections import Counter- mock_counter = Counter({'example': 1})- with patch('mwmbl.crawl.index_batches', return_value=mock_counter):+ crawler = Crawler()+ crawler._redis = fake_redis # Set the private attribute directly++ # Mock Counter object for index_batches return value+ from collections import Counter+ mock_counter = Counter({'example': 1})+ with patch('mwmbl.crawl.index_batches', return_value=mock_counter):with patch('mwmbl.crawl.RemoteIndex') as mock_remote_index:with patch('mwmbl.crawl.TinyIndex') as mock_tiny_index:with patch('mwmbl.crawl.index_pages') as mock_index_pages:@@ -207,11 +212,8 @@ class TestCrawlFunctional:mock_post.return_value.status_code = 200mock_post.return_value.text = "OK"- # Import and test run_indexing- from mwmbl.crawl import run_indexing-# Should complete without errors- run_indexing()+ crawler.run_indexing()# Verify batch was consumed from Redisremaining_batches = fake_redis.llen("batch-queue")