Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Make redis parameter required in crawl_url and robots_allowed

Daoud Clarkec2f399a0d439
diff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.py
index 84c5c58..71ca35e 100644
--- a/mwmbl/crawler/retrieve.py
+++ b/mwmbl/crawler/retrieve.py
@@ -90,7 +90,7 @@ def fetch(url):
raise ValueError(f"Too many redirects for URL {url}")
-def robots_allowed(url: str, redis: Redis | None = None) -> bool:
+def robots_allowed(url: str, redis: Redis) -> bool:
try:
parsed_url = urlparse(url)
except ValueError:
@@ -100,28 +100,25 @@ def robots_allowed(url: str, redis: Redis | None = None) -> bool:
domain = parsed_url.netloc
robots_url = urlunsplit((parsed_url.scheme, parsed_url.netloc, 'robots.txt', '', ''))
- if redis:
- cached_content = _get_robots_from_cache(redis, domain)
- if cached_content is not None:
- logger.debug(f"Robots cache hit for {domain}")
- parse_robots = RobotFileParser()
- parse_robots.parse(cached_content)
- allowed = parse_robots.can_fetch(USER_AGENT, url)
- logger.debug(f"Robots allowed for {url} (cached): {allowed}")
- return allowed
+ cached_content = _get_robots_from_cache(redis, domain)
+ if cached_content is not None:
+ logger.debug(f"Robots cache hit for {domain}")
+ parse_robots = RobotFileParser()
+ parse_robots.parse(cached_content)
+ allowed = parse_robots.can_fetch(USER_AGENT, url)
+ logger.debug(f"Robots allowed for {url} (cached): {allowed}")
+ return allowed
try:
status_code, content = fetch(robots_url)
except ALLOWED_EXCEPTIONS as e:
logger.debug(f"Robots error: {robots_url}, {e}")
- if redis:
- _cache_robots_content(redis, domain, [], error=True)
+ _cache_robots_content(redis, domain, [], error=True)
return True
if status_code != 200:
logger.debug(f"Robots status code: {status_code}")
- if redis:
- _cache_robots_content(redis, domain, [], error=True)
+ _cache_robots_content(redis, domain, [], error=True)
return True
decoded = None
@@ -134,16 +131,14 @@ def robots_allowed(url: str, redis: Redis | None = None) -> bool:
if decoded is None:
logger.info(f"Unable to decode robots file {robots_url}")
- if redis:
- _cache_robots_content(redis, domain, [], error=True)
+ _cache_robots_content(redis, domain, [], error=True)
return True
parse_robots = RobotFileParser()
parse_robots.parse(decoded)
allowed = parse_robots.can_fetch(USER_AGENT, url)
- if redis:
- _cache_robots_content(redis, domain, decoded, error=False)
+ _cache_robots_content(redis, domain, decoded, error=False)
logger.debug(f"Robots allowed for {url}: {allowed}")
return allowed
@@ -271,7 +266,7 @@ def get_og_meta(dom) -> tuple[str, str]:
return og_title, og_desc
-def crawl_url(url, redis: Redis | None = None):
+def crawl_url(url, redis: Redis):
logger.info(url)
js_timestamp = int(time.time() * 1000)
allowed = robots_allowed(url, redis)
@@ -400,7 +395,7 @@ def crawl_url(url, redis: Redis | None = None):
}
-def crawl_batch(batch, num_threads, redis: Redis | None = None):
+def crawl_batch(batch, num_threads, redis: Redis):
with ThreadPool(num_threads) as pool:
result = pool.map(lambda url: crawl_url(url, redis), batch)
return result
diff --git a/test/test_crawl_functional.py b/test/test_crawl_functional.py
index 5349663..0a60f28 100644
--- a/test/test_crawl_functional.py
+++ b/test/test_crawl_functional.py
@@ -244,7 +244,7 @@ class TestCrawlFunctional:
with patch('mwmbl.crawler.retrieve.robots_allowed', return_value=True):
mock_fetch.return_value = (200, mock_html_content)
- result = crawl_url('https://example.com')
+ result = crawl_url('https://example.com', redis=None)
assert result['url'] == 'https://example.com'
assert result['status'] == 200
@@ -261,7 +261,7 @@ class TestCrawlFunctional:
# Test connection error
mock_fetch.side_effect = ConnectionError("Connection failed")
- result = crawl_url('https://example.com')
+ result = crawl_url('https://example.com', redis=None)
assert result['url'] == 'https://example.com'
assert result['status'] is None
@@ -272,7 +272,7 @@ class TestCrawlFunctional:
def test_robots_denied_handling(self, mock_environment):
"""Test robots.txt denial handling"""
with patch('mwmbl.crawler.retrieve.robots_allowed', return_value=False):
- result = crawl_url('https://example.com')
+ result = crawl_url('https://example.com', redis=None)
assert result['url'] == 'https://example.com'
assert result['status'] is None
diff --git a/test/test_super_search.py b/test/test_super_search.py
index 72ad4c3..473c166 100644
--- a/test/test_super_search.py
+++ b/test/test_super_search.py
@@ -179,7 +179,7 @@ def test_promoted_results_use_top_k(client, api_key, monkeypatch):
# Scores in descending order — only the top-2 should be promoted.
_stub_scoring(monkeypatch, [0.9, 0.5, 0.1] + [0.0] * 20)
- def fake_crawl(url, redis=None):
+ def fake_crawl(url, redis):
return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}
monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)
@@ -221,7 +221,7 @@ def test_heap_replacement_promotes_better_late_doc(client, api_key, monkeypatch)
# Best (0.9) beats the minimum and must be promoted.
_stub_scoring(monkeypatch, [0.1, 0.1, 0.1, 0.9] + [0.0] * 20)
- def fake_crawl(url, redis=None):
+ def fake_crawl(url, redis):
return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}
monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)
@@ -253,7 +253,7 @@ def test_heap_equal_score_does_not_enter_full_heap(client, api_key, monkeypatch)
})
_stub_scoring(monkeypatch, [0.5, 0.5, 0.5] + [0.0] * 20)
- def fake_crawl(url, redis=None):
+ def fake_crawl(url, redis):
return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}
monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)
@@ -279,7 +279,7 @@ def test_final_results_event_emitted(client, api_key, monkeypatch):
})
_stub_scoring(monkeypatch, [0.5] + [0.0] * 20)
- def fake_crawl(url, redis=None):
+ def fake_crawl(url, redis):
return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}
monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)
@@ -414,7 +414,7 @@ def test_link_following_adds_followed_docs(client, api_key, monkeypatch):
})
_stub_scoring(monkeypatch, [0.9] + [0.0] * 20)
- def fake_crawl(url, redis=None):
+ def fake_crawl(url, redis):
if url == "https://parent.example/":
return {"url": url, "status": 200, "content": {
"title": "Python parent", "extract": "python parent text",
@@ -457,7 +457,7 @@ def test_no_duplicate_consecutive_results_frames(client, api_key, monkeypatch):
})
_stub_scoring(monkeypatch, [0.9, 0.8] + [0.0] * 40)
- def fake_crawl(url, redis=None):
+ def fake_crawl(url, redis):
return {"url": url, "status": 200, "content": None}
monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)