An open source, non-profit web search engine
Don't construct redis on import
Daoud Clarkee35de919039d
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex f8e4664..5126581 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -67,7 +67,7 @@ jobs:#----------------------------------------------# run test suite#----------------------------------------------- - name: Create test directories+ - name: Create test directories and filesrun: |mkdir -p /tmp/test_mwmbl_data- name: Run testsdiff --git a/mwmbl/crawl.py b/mwmbl/crawl.pyindex facb8d4..38dfc50 100644--- a/mwmbl/crawl.py+++ b/mwmbl/crawl.py@@ -40,21 +40,32 @@ logging.basicConfig(level=logging.INFO, format=FORMAT)BATCH_QUEUE_KEY = "batch-queue"-redis = Redis.from_url(- settings.REDIS_URL,- decode_responses=True,- health_check_interval=30,-)+# Initialize Redis connection lazily+redis = None+url_queue = None++def get_redis():+ global redis+ if redis is None:+ redis = Redis.from_url(+ settings.REDIS_URL,+ decode_responses=True,+ health_check_interval=30,+ )+ return redis++def get_url_queue():+ global url_queue+ if url_queue is None:+ url_queue = RedisURLQueue(get_redis(), lambda: set())+ return url_queuedef check_redis():try:- redis.ping()+ get_redis().ping()logger.debug("Redis ping successful")except ConnectionError as e:raise SystemExit(f"Cannot reach Redis at {settings.REDIS_URL}. Make sure your Redis server is running.")-check_redis()--url_queue = RedisURLQueue(redis, lambda: set())def run():@@ -138,7 +149,7 @@ def process_batch():Each batch is processed as a HashedBatch object containing metadata and crawl results."""user_id = "test"- urls = url_queue.get_batch(user_id)+ urls = get_url_queue().get_batch(user_id)logger.info(f"Processing batch of {len(urls)} URLs")# Process URLs sequentially with rate limiting@@ -158,11 +169,11 @@ def process_batch():"timestamp": js_timestamp,"items": results,})- record_urls_in_database([batch], url_queue)+ record_urls_in_database([batch], get_url_queue())# Push the batch into the Redis queuebatch_json = batch.json()- redis.rpush(BATCH_QUEUE_KEY, batch_json)+ get_redis().rpush(BATCH_QUEUE_KEY, batch_json)def run_indexing_continuously():@@ -193,7 +204,7 @@ def run_indexing():preventing low-quality content from polluting the main index."""index_path = data_path / settings.INDEX_NAME- batch_jsons = redis.lpop(BATCH_QUEUE_KEY, 10)+ batch_jsons = get_redis().lpop(BATCH_QUEUE_KEY, 10)if batch_jsons is None:logger.info("No more batches to index. Sleeping for 10 seconds.")time.sleep(10)diff --git a/mwmbl/settings_test.py b/mwmbl/settings_test.pyindex de903f0..d19242d 100644--- a/mwmbl/settings_test.py+++ b/mwmbl/settings_test.py@@ -13,7 +13,7 @@ DATABASES = {# Test-specific settingsDATA_PATH = "/tmp/test_mwmbl_data"-INDEX_NAME = "test-index.tinysearch"+INDEX_NAME = "index-v2.tinysearch"NUM_PAGES = 10# Disable Redis URL requirement for testingdiff --git a/test/test_crawl_functional.py b/test/test_crawl_functional.pyindex 32dedd8..ebd3745 100644--- a/test/test_crawl_functional.py+++ b/test/test_crawl_functional.py@@ -33,7 +33,6 @@ def mock_settings(temp_data_path):"""Mock Django settings for crawler"""with override_settings(DATA_PATH=str(temp_data_path),- INDEX_NAME="test-index.tinysearch",REDIS_URL="redis://localhost:6379",SETUP_DATABASE=False, # Disable database setup for testingNUM_PAGES=10 # Small number for testing@@ -105,21 +104,20 @@ class TestCrawlFunctional:def test_redis_connection_and_health_check(self, fake_redis, mock_settings, mock_environment):"""Test Redis connection and health check functionality"""- with patch('mwmbl.crawl.redis', fake_redis):- with patch('redis.Redis.from_url', return_value=fake_redis):- # Import after patching to ensure mocks are in place- from mwmbl.crawl import check_redis-- # Should not raise an exception- check_redis()-- # Test with broken Redis- broken_redis = MagicMock()- broken_redis.ping.side_effect = ConnectionError("Redis unavailable")-- with patch('mwmbl.crawl.redis', broken_redis):- with pytest.raises(SystemExit):- check_redis()+ with patch('mwmbl.crawl.get_redis', return_value=fake_redis):+ # Import after patching to ensure mocks are in place+ from mwmbl.crawl import check_redis++ # Should not raise an exception+ check_redis()++ # Test with broken Redis+ broken_redis = MagicMock()+ broken_redis.ping.side_effect = ConnectionError("Redis unavailable")++ with patch('mwmbl.crawl.get_redis', return_value=broken_redis):+ with pytest.raises(SystemExit):+ check_redis()def test_url_queue_operations(self, fake_redis, sample_found_urls):"""Test RedisURLQueue operations with fake Redis"""@@ -138,30 +136,31 @@ class TestCrawlFunctional:def test_process_batch_workflow(self, fake_redis, mock_settings, mock_environment, mock_crawl_response):"""Test the complete batch processing workflow"""- with patch('mwmbl.crawl.redis', fake_redis):- with patch('redis.Redis.from_url', return_value=fake_redis):- with patch('mwmbl.crawl.url_queue') as mock_url_queue:- with patch('mwmbl.crawl.crawl_url', return_value=mock_crawl_response):- with patch('mwmbl.crawl.record_urls_in_database'):- # Setup URL queue mock- mock_url_queue.get_batch.return_value = ['https://example.com']-- # Import and test process_batch- from mwmbl.crawl import process_batch-- # Should complete without errors- process_batch()-- # Verify batch was pushed to Redis- batch_data = fake_redis.lpop("batch-queue")- assert batch_data is not None-- # Verify batch structure- batch = json.loads(batch_data)- assert 'user_id_hash' in batch- assert 'timestamp' in batch- assert 'items' in batch- assert len(batch['items']) == 1+ with patch('mwmbl.crawl.get_redis', return_value=fake_redis):+ with patch('mwmbl.crawl.get_url_queue') as mock_get_url_queue:+ with patch('mwmbl.crawl.crawl_url', return_value=mock_crawl_response):+ with patch('mwmbl.crawl.record_urls_in_database'):+ # Setup URL queue mock+ mock_url_queue = MagicMock()+ mock_url_queue.get_batch.return_value = ['https://example.com']+ mock_get_url_queue.return_value = mock_url_queue++ # Import and test process_batch+ from mwmbl.crawl import process_batch++ # Should complete without errors+ process_batch()++ # Verify batch was pushed to Redis+ batch_data = fake_redis.lpop("batch-queue")+ assert batch_data is not None++ # Verify batch structure+ batch = json.loads(batch_data)+ assert 'user_id_hash' in batch+ assert 'timestamp' in batch+ assert 'items' in batch+ assert len(batch['items']) == 1def test_indexing_workflow(self, fake_redis, mock_settings, mock_environment, temp_data_path):"""Test the indexing workflow with fake Redis"""@@ -185,39 +184,38 @@ class TestCrawlFunctional:)fake_redis.rpush("batch-queue", sample_batch.json())- with patch('mwmbl.crawl.redis', fake_redis):- with patch('redis.Redis.from_url', return_value=fake_redis):- # Mock Counter object for index_batches return value- from collections import Counter- mock_counter = Counter({'example': 1})- with patch('mwmbl.crawl.index_batches', return_value=mock_counter):- with patch('mwmbl.crawl.RemoteIndex') as mock_remote_index:- with patch('mwmbl.crawl.TinyIndex') as mock_tiny_index:- with patch('mwmbl.crawl.index_pages') as mock_index_pages:- with patch('requests.post') as mock_post:- # Setup mocks- mock_remote_index_instance = MagicMock()- mock_remote_index.return_value = mock_remote_index_instance- mock_remote_index_instance.retrieve.return_value = []-- mock_tiny_index_instance = MagicMock()- mock_tiny_index.return_value.__enter__.return_value = mock_tiny_index_instance- mock_tiny_index_instance.retrieve.return_value = []- mock_tiny_index_instance.get_key_page_index.return_value = 0- mock_tiny_index_instance.get_page.return_value = "test page content"-- mock_post.return_value.status_code = 200- mock_post.return_value.text = "OK"-- # Import and test run_indexing- from mwmbl.crawl import run_indexing-- # Should complete without errors- run_indexing()-- # Verify batch was consumed from Redis- remaining_batches = fake_redis.llen("batch-queue")- assert remaining_batches == 0+ with patch('mwmbl.crawl.get_redis', return_value=fake_redis):+ # Mock Counter object for index_batches return value+ from collections import Counter+ mock_counter = Counter({'example': 1})+ with patch('mwmbl.crawl.index_batches', return_value=mock_counter):+ with patch('mwmbl.crawl.RemoteIndex') as mock_remote_index:+ with patch('mwmbl.crawl.TinyIndex') as mock_tiny_index:+ with patch('mwmbl.crawl.index_pages') as mock_index_pages:+ with patch('requests.post') as mock_post:+ # Setup mocks+ mock_remote_index_instance = MagicMock()+ mock_remote_index.return_value = mock_remote_index_instance+ mock_remote_index_instance.retrieve.return_value = []++ mock_tiny_index_instance = MagicMock()+ mock_tiny_index.return_value.__enter__.return_value = mock_tiny_index_instance+ mock_tiny_index_instance.retrieve.return_value = []+ mock_tiny_index_instance.get_key_page_index.return_value = 0+ mock_tiny_index_instance.get_page.return_value = "test page content"++ mock_post.return_value.status_code = 200+ mock_post.return_value.text = "OK"++ # Import and test run_indexing+ from mwmbl.crawl import run_indexing++ # Should complete without errors+ run_indexing()++ # Verify batch was consumed from Redis+ remaining_batches = fake_redis.llen("batch-queue")+ assert remaining_batches == 0def test_crawl_url_functionality(self, mock_environment):"""Test individual URL crawling functionality"""