Scrapy, a fast high-level web crawling & scraping framework for Python.
Test local files aren't processed
Cj Malone33b85a9e2a37
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex ac08c6006..fd27e637d 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -214,6 +214,19 @@ Disallow: /some/randome/page.htmlmiddleware.process_request_2(rp, Request("http://site.local/allowed"), None)rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot")+ def test_robotstxt_local_file(self):+ middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())+ assert not middleware.process_request(+ Request("data:text/plain,Hello World data"), None+ )+ assert not middleware.process_request(+ Request("file:///tests/sample_data/test_site/nothinghere.html"), None+ )+ assert isinstance(+ middleware.process_request(Request("http://site.local/allowed"), None),+ Deferred,+ )+def assertNotIgnored(self, request, middleware):spider = None # not actually useddfd = maybeDeferred(middleware.process_request, request, spider)