Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Don't construct redis on import

Daoud Clarkee35de919039d
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f8e4664..5126581 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -67,7 +67,7 @@ jobs:
#----------------------------------------------
# run test suite
#----------------------------------------------
- - name: Create test directories
+ - name: Create test directories and files
run: |
mkdir -p /tmp/test_mwmbl_data
- name: Run tests
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.py
index facb8d4..38dfc50 100644
--- a/mwmbl/crawl.py
+++ b/mwmbl/crawl.py
@@ -40,21 +40,32 @@ logging.basicConfig(level=logging.INFO, format=FORMAT)
BATCH_QUEUE_KEY = "batch-queue"
-redis = Redis.from_url(
- settings.REDIS_URL,
- decode_responses=True,
- health_check_interval=30,
-)
+# Initialize Redis connection lazily
+redis = None
+url_queue = None
+
+def get_redis():
+ global redis
+ if redis is None:
+ redis = Redis.from_url(
+ settings.REDIS_URL,
+ decode_responses=True,
+ health_check_interval=30,
+ )
+ return redis
+
+def get_url_queue():
+ global url_queue
+ if url_queue is None:
+ url_queue = RedisURLQueue(get_redis(), lambda: set())
+ return url_queue
def check_redis():
try:
- redis.ping()
+ get_redis().ping()
logger.debug("Redis ping successful")
except ConnectionError as e:
raise SystemExit(f"Cannot reach Redis at {settings.REDIS_URL}. Make sure your Redis server is running.")
-check_redis()
-
-url_queue = RedisURLQueue(redis, lambda: set())
def run():
@@ -138,7 +149,7 @@ def process_batch():
Each batch is processed as a HashedBatch object containing metadata and crawl results.
"""
user_id = "test"
- urls = url_queue.get_batch(user_id)
+ urls = get_url_queue().get_batch(user_id)
logger.info(f"Processing batch of {len(urls)} URLs")
# Process URLs sequentially with rate limiting
@@ -158,11 +169,11 @@ def process_batch():
"timestamp": js_timestamp,
"items": results,
})
- record_urls_in_database([batch], url_queue)
+ record_urls_in_database([batch], get_url_queue())
# Push the batch into the Redis queue
batch_json = batch.json()
- redis.rpush(BATCH_QUEUE_KEY, batch_json)
+ get_redis().rpush(BATCH_QUEUE_KEY, batch_json)
def run_indexing_continuously():
@@ -193,7 +204,7 @@ def run_indexing():
preventing low-quality content from polluting the main index.
"""
index_path = data_path / settings.INDEX_NAME
- batch_jsons = redis.lpop(BATCH_QUEUE_KEY, 10)
+ batch_jsons = get_redis().lpop(BATCH_QUEUE_KEY, 10)
if batch_jsons is None:
logger.info("No more batches to index. Sleeping for 10 seconds.")
time.sleep(10)
diff --git a/mwmbl/settings_test.py b/mwmbl/settings_test.py
index de903f0..d19242d 100644
--- a/mwmbl/settings_test.py
+++ b/mwmbl/settings_test.py
@@ -13,7 +13,7 @@ DATABASES = {
# Test-specific settings
DATA_PATH = "/tmp/test_mwmbl_data"
-INDEX_NAME = "test-index.tinysearch"
+INDEX_NAME = "index-v2.tinysearch"
NUM_PAGES = 10
# Disable Redis URL requirement for testing
diff --git a/test/test_crawl_functional.py b/test/test_crawl_functional.py
index 32dedd8..ebd3745 100644
--- a/test/test_crawl_functional.py
+++ b/test/test_crawl_functional.py
@@ -33,7 +33,6 @@ def mock_settings(temp_data_path):
"""Mock Django settings for crawler"""
with override_settings(
DATA_PATH=str(temp_data_path),
- INDEX_NAME="test-index.tinysearch",
REDIS_URL="redis://localhost:6379",
SETUP_DATABASE=False, # Disable database setup for testing
NUM_PAGES=10 # Small number for testing
@@ -105,21 +104,20 @@ class TestCrawlFunctional:
def test_redis_connection_and_health_check(self, fake_redis, mock_settings, mock_environment):
"""Test Redis connection and health check functionality"""
- with patch('mwmbl.crawl.redis', fake_redis):
- with patch('redis.Redis.from_url', return_value=fake_redis):
- # Import after patching to ensure mocks are in place
- from mwmbl.crawl import check_redis
-
- # Should not raise an exception
- check_redis()
-
- # Test with broken Redis
- broken_redis = MagicMock()
- broken_redis.ping.side_effect = ConnectionError("Redis unavailable")
-
- with patch('mwmbl.crawl.redis', broken_redis):
- with pytest.raises(SystemExit):
- check_redis()
+ with patch('mwmbl.crawl.get_redis', return_value=fake_redis):
+ # Import after patching to ensure mocks are in place
+ from mwmbl.crawl import check_redis
+
+ # Should not raise an exception
+ check_redis()
+
+ # Test with broken Redis
+ broken_redis = MagicMock()
+ broken_redis.ping.side_effect = ConnectionError("Redis unavailable")
+
+ with patch('mwmbl.crawl.get_redis', return_value=broken_redis):
+ with pytest.raises(SystemExit):
+ check_redis()
def test_url_queue_operations(self, fake_redis, sample_found_urls):
"""Test RedisURLQueue operations with fake Redis"""
@@ -138,30 +136,31 @@ class TestCrawlFunctional:
def test_process_batch_workflow(self, fake_redis, mock_settings, mock_environment, mock_crawl_response):
"""Test the complete batch processing workflow"""
- with patch('mwmbl.crawl.redis', fake_redis):
- with patch('redis.Redis.from_url', return_value=fake_redis):
- with patch('mwmbl.crawl.url_queue') as mock_url_queue:
- with patch('mwmbl.crawl.crawl_url', return_value=mock_crawl_response):
- with patch('mwmbl.crawl.record_urls_in_database'):
- # Setup URL queue mock
- mock_url_queue.get_batch.return_value = ['https://example.com']
-
- # Import and test process_batch
- from mwmbl.crawl import process_batch
-
- # Should complete without errors
- process_batch()
-
- # Verify batch was pushed to Redis
- batch_data = fake_redis.lpop("batch-queue")
- assert batch_data is not None
-
- # Verify batch structure
- batch = json.loads(batch_data)
- assert 'user_id_hash' in batch
- assert 'timestamp' in batch
- assert 'items' in batch
- assert len(batch['items']) == 1
+ with patch('mwmbl.crawl.get_redis', return_value=fake_redis):
+ with patch('mwmbl.crawl.get_url_queue') as mock_get_url_queue:
+ with patch('mwmbl.crawl.crawl_url', return_value=mock_crawl_response):
+ with patch('mwmbl.crawl.record_urls_in_database'):
+ # Setup URL queue mock
+ mock_url_queue = MagicMock()
+ mock_url_queue.get_batch.return_value = ['https://example.com']
+ mock_get_url_queue.return_value = mock_url_queue
+
+ # Import and test process_batch
+ from mwmbl.crawl import process_batch
+
+ # Should complete without errors
+ process_batch()
+
+ # Verify batch was pushed to Redis
+ batch_data = fake_redis.lpop("batch-queue")
+ assert batch_data is not None
+
+ # Verify batch structure
+ batch = json.loads(batch_data)
+ assert 'user_id_hash' in batch
+ assert 'timestamp' in batch
+ assert 'items' in batch
+ assert len(batch['items']) == 1
def test_indexing_workflow(self, fake_redis, mock_settings, mock_environment, temp_data_path):
"""Test the indexing workflow with fake Redis"""
@@ -185,39 +184,38 @@ class TestCrawlFunctional:
)
fake_redis.rpush("batch-queue", sample_batch.json())
- with patch('mwmbl.crawl.redis', fake_redis):
- with patch('redis.Redis.from_url', return_value=fake_redis):
- # Mock Counter object for index_batches return value
- from collections import Counter
- mock_counter = Counter({'example': 1})
- with patch('mwmbl.crawl.index_batches', return_value=mock_counter):
- with patch('mwmbl.crawl.RemoteIndex') as mock_remote_index:
- with patch('mwmbl.crawl.TinyIndex') as mock_tiny_index:
- with patch('mwmbl.crawl.index_pages') as mock_index_pages:
- with patch('requests.post') as mock_post:
- # Setup mocks
- mock_remote_index_instance = MagicMock()
- mock_remote_index.return_value = mock_remote_index_instance
- mock_remote_index_instance.retrieve.return_value = []
-
- mock_tiny_index_instance = MagicMock()
- mock_tiny_index.return_value.__enter__.return_value = mock_tiny_index_instance
- mock_tiny_index_instance.retrieve.return_value = []
- mock_tiny_index_instance.get_key_page_index.return_value = 0
- mock_tiny_index_instance.get_page.return_value = "test page content"
-
- mock_post.return_value.status_code = 200
- mock_post.return_value.text = "OK"
-
- # Import and test run_indexing
- from mwmbl.crawl import run_indexing
-
- # Should complete without errors
- run_indexing()
-
- # Verify batch was consumed from Redis
- remaining_batches = fake_redis.llen("batch-queue")
- assert remaining_batches == 0
+ with patch('mwmbl.crawl.get_redis', return_value=fake_redis):
+ # Mock Counter object for index_batches return value
+ from collections import Counter
+ mock_counter = Counter({'example': 1})
+ with patch('mwmbl.crawl.index_batches', return_value=mock_counter):
+ with patch('mwmbl.crawl.RemoteIndex') as mock_remote_index:
+ with patch('mwmbl.crawl.TinyIndex') as mock_tiny_index:
+ with patch('mwmbl.crawl.index_pages') as mock_index_pages:
+ with patch('requests.post') as mock_post:
+ # Setup mocks
+ mock_remote_index_instance = MagicMock()
+ mock_remote_index.return_value = mock_remote_index_instance
+ mock_remote_index_instance.retrieve.return_value = []
+
+ mock_tiny_index_instance = MagicMock()
+ mock_tiny_index.return_value.__enter__.return_value = mock_tiny_index_instance
+ mock_tiny_index_instance.retrieve.return_value = []
+ mock_tiny_index_instance.get_key_page_index.return_value = 0
+ mock_tiny_index_instance.get_page.return_value = "test page content"
+
+ mock_post.return_value.status_code = 200
+ mock_post.return_value.text = "OK"
+
+ # Import and test run_indexing
+ from mwmbl.crawl import run_indexing
+
+ # Should complete without errors
+ run_indexing()
+
+ # Verify batch was consumed from Redis
+ remaining_batches = fake_redis.llen("batch-queue")
+ assert remaining_batches == 0
def test_crawl_url_functionality(self, mock_environment):
"""Test individual URL crawling functionality"""