An open source, non-profit web search engine
Make redis parameter required in crawl_url and robots_allowed
Daoud Clarkec2f399a0d439
diff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.pyindex 84c5c58..71ca35e 100644--- a/mwmbl/crawler/retrieve.py+++ b/mwmbl/crawler/retrieve.py@@ -90,7 +90,7 @@ def fetch(url):raise ValueError(f"Too many redirects for URL {url}")-def robots_allowed(url: str, redis: Redis | None = None) -> bool:+def robots_allowed(url: str, redis: Redis) -> bool:try:parsed_url = urlparse(url)except ValueError:@@ -100,28 +100,25 @@ def robots_allowed(url: str, redis: Redis | None = None) -> bool:domain = parsed_url.netlocrobots_url = urlunsplit((parsed_url.scheme, parsed_url.netloc, 'robots.txt', '', ''))- if redis:- cached_content = _get_robots_from_cache(redis, domain)- if cached_content is not None:- logger.debug(f"Robots cache hit for {domain}")- parse_robots = RobotFileParser()- parse_robots.parse(cached_content)- allowed = parse_robots.can_fetch(USER_AGENT, url)- logger.debug(f"Robots allowed for {url} (cached): {allowed}")- return allowed+ cached_content = _get_robots_from_cache(redis, domain)+ if cached_content is not None:+ logger.debug(f"Robots cache hit for {domain}")+ parse_robots = RobotFileParser()+ parse_robots.parse(cached_content)+ allowed = parse_robots.can_fetch(USER_AGENT, url)+ logger.debug(f"Robots allowed for {url} (cached): {allowed}")+ return allowedtry:status_code, content = fetch(robots_url)except ALLOWED_EXCEPTIONS as e:logger.debug(f"Robots error: {robots_url}, {e}")- if redis:- _cache_robots_content(redis, domain, [], error=True)+ _cache_robots_content(redis, domain, [], error=True)return Trueif status_code != 200:logger.debug(f"Robots status code: {status_code}")- if redis:- _cache_robots_content(redis, domain, [], error=True)+ _cache_robots_content(redis, domain, [], error=True)return Truedecoded = None@@ -134,16 +131,14 @@ def robots_allowed(url: str, redis: Redis | None = None) -> bool:if decoded is None:logger.info(f"Unable to decode robots file {robots_url}")- if redis:- _cache_robots_content(redis, domain, [], error=True)+ _cache_robots_content(redis, domain, [], error=True)return Trueparse_robots = RobotFileParser()parse_robots.parse(decoded)allowed = parse_robots.can_fetch(USER_AGENT, url)- if redis:- _cache_robots_content(redis, domain, decoded, error=False)+ _cache_robots_content(redis, domain, decoded, error=False)logger.debug(f"Robots allowed for {url}: {allowed}")return allowed@@ -271,7 +266,7 @@ def get_og_meta(dom) -> tuple[str, str]:return og_title, og_desc-def crawl_url(url, redis: Redis | None = None):+def crawl_url(url, redis: Redis):logger.info(url)js_timestamp = int(time.time() * 1000)allowed = robots_allowed(url, redis)@@ -400,7 +395,7 @@ def crawl_url(url, redis: Redis | None = None):}-def crawl_batch(batch, num_threads, redis: Redis | None = None):+def crawl_batch(batch, num_threads, redis: Redis):with ThreadPool(num_threads) as pool:result = pool.map(lambda url: crawl_url(url, redis), batch)return resultdiff --git a/test/test_crawl_functional.py b/test/test_crawl_functional.pyindex 5349663..0a60f28 100644--- a/test/test_crawl_functional.py+++ b/test/test_crawl_functional.py@@ -244,7 +244,7 @@ class TestCrawlFunctional:with patch('mwmbl.crawler.retrieve.robots_allowed', return_value=True):mock_fetch.return_value = (200, mock_html_content)- result = crawl_url('https://example.com')+ result = crawl_url('https://example.com', redis=None)assert result['url'] == 'https://example.com'assert result['status'] == 200@@ -261,7 +261,7 @@ class TestCrawlFunctional:# Test connection errormock_fetch.side_effect = ConnectionError("Connection failed")- result = crawl_url('https://example.com')+ result = crawl_url('https://example.com', redis=None)assert result['url'] == 'https://example.com'assert result['status'] is None@@ -272,7 +272,7 @@ class TestCrawlFunctional:def test_robots_denied_handling(self, mock_environment):"""Test robots.txt denial handling"""with patch('mwmbl.crawler.retrieve.robots_allowed', return_value=False):- result = crawl_url('https://example.com')+ result = crawl_url('https://example.com', redis=None)assert result['url'] == 'https://example.com'assert result['status'] is Nonediff --git a/test/test_super_search.py b/test/test_super_search.pyindex 72ad4c3..473c166 100644--- a/test/test_super_search.py+++ b/test/test_super_search.py@@ -179,7 +179,7 @@ def test_promoted_results_use_top_k(client, api_key, monkeypatch):# Scores in descending order — only the top-2 should be promoted._stub_scoring(monkeypatch, [0.9, 0.5, 0.1] + [0.0] * 20)- def fake_crawl(url, redis=None):+ def fake_crawl(url, redis):return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)@@ -221,7 +221,7 @@ def test_heap_replacement_promotes_better_late_doc(client, api_key, monkeypatch)# Best (0.9) beats the minimum and must be promoted._stub_scoring(monkeypatch, [0.1, 0.1, 0.1, 0.9] + [0.0] * 20)- def fake_crawl(url, redis=None):+ def fake_crawl(url, redis):return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)@@ -253,7 +253,7 @@ def test_heap_equal_score_does_not_enter_full_heap(client, api_key, monkeypatch)})_stub_scoring(monkeypatch, [0.5, 0.5, 0.5] + [0.0] * 20)- def fake_crawl(url, redis=None):+ def fake_crawl(url, redis):return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)@@ -279,7 +279,7 @@ def test_final_results_event_emitted(client, api_key, monkeypatch):})_stub_scoring(monkeypatch, [0.5] + [0.0] * 20)- def fake_crawl(url, redis=None):+ def fake_crawl(url, redis):return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)@@ -414,7 +414,7 @@ def test_link_following_adds_followed_docs(client, api_key, monkeypatch):})_stub_scoring(monkeypatch, [0.9] + [0.0] * 20)- def fake_crawl(url, redis=None):+ def fake_crawl(url, redis):if url == "https://parent.example/":return {"url": url, "status": 200, "content": {"title": "Python parent", "extract": "python parent text",@@ -457,7 +457,7 @@ def test_no_duplicate_consecutive_results_frames(client, api_key, monkeypatch):})_stub_scoring(monkeypatch, [0.9, 0.8] + [0.0] * 40)- def fake_crawl(url, redis=None):+ def fake_crawl(url, redis):return {"url": url, "status": 200, "content": None}monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)