Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix tests

Laerte Pereira26a16f2c43dc
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 96d43b2b9..6cde4ed8c 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -76,11 +76,11 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def _test_delay(self, total, delay, randomize=False):
- crawl_kwargs = dict(
- maxlatency=delay * 2,
- mockserver=self.mockserver,
- total=total,
- )
+ crawl_kwargs = {
+ "maxlatency": delay * 2,
+ "mockserver": self.mockserver,
+ "total": total,
+ }
tolerance = 1 - (0.6 if randomize else 0.2)
settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize}
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 4a81a638e..425fabcc7 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -320,7 +320,7 @@ class CookiesMiddlewareTest(TestCase):
@pytest.mark.xfail(reason="Cookie header is not currently being processed")
def test_keep_cookie_from_default_request_headers_middleware(self):
- DEFAULT_REQUEST_HEADERS = dict(Cookie="default=value; asdf=qwerty")
+ DEFAULT_REQUEST_HEADERS = {"Cookie": "default=value; asdf=qwerty"}
mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items())
# overwrite with values from 'cookies' request argument
req1 = Request("http://example.org", cookies={"default": "something"})
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index fc110e6cc..500af6536 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -59,7 +59,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase):
self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
def test_auth_already_set(self):
- req = Request("http://example.com/", headers=dict(Authorization="Digest 123"))
+ req = Request("http://example.com/", headers={"Authorization": "Digest 123"})
assert self.mw.process_request(req, self.spider) is None
self.assertEqual(req.headers["Authorization"], b"Digest 123")
@@ -79,6 +79,6 @@ class HttpAuthAnyMiddlewareTest(unittest.TestCase):
self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
def test_auth_already_set(self):
- req = Request("http://example.com/", headers=dict(Authorization="Digest 123"))
+ req = Request("http://example.com/", headers={"Authorization": "Digest 123"})
assert self.mw.process_request(req, self.spider) is None
self.assertEqual(req.headers["Authorization"], b"Digest 123")
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 59b724495..fa9389044 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -152,7 +152,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
def test_nested_item(self):
i1 = self.item_class(name="Joseph", age="22")
- i2 = dict(name="Maria", age=i1)
+ i2 = {"name": "Maria", "age": i1}
i3 = self.item_class(name="Jesus", age=i2)
ie = self._get_exporter()
exported = ie.export_item(i3)
@@ -185,7 +185,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
def test_export_item_dict_list(self):
i1 = self.item_class(name="Joseph", age="22")
- i2 = dict(name="Maria", age=[i1])
+ i2 = {"name": "Maria", "age": [i1]}
i3 = self.item_class(name="Jesus", age=[i2])
ie = self._get_exporter()
exported = ie.export_item(i3)
@@ -373,7 +373,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
def test_join_multivalue_not_strings(self):
self.assertExportResult(
- item=dict(name="John", friends=[4, 8]),
+ item={"name": "John", "friends": [4, 8]},
include_headers_line=False,
expected='"[4, 8]",John\r\n',
)
@@ -388,14 +388,14 @@ class CsvItemExporterTest(BaseItemExporterTest):
def test_errors_default(self):
with self.assertRaises(UnicodeEncodeError):
self.assertExportResult(
- item=dict(text="W\u0275\u200Brd"),
+ item={"text": "W\u0275\u200Brd"},
expected=None,
encoding="windows-1251",
)
def test_errors_xmlcharrefreplace(self):
self.assertExportResult(
- item=dict(text="W\u0275\u200Brd"),
+ item={"text": "W\u0275\u200Brd"},
include_headers_line=False,
expected="Wɵ​rd\r\n",
encoding="windows-1251",
@@ -455,8 +455,8 @@ class XmlItemExporterTest(BaseItemExporterTest):
)
def test_nested_item(self):
- i1 = dict(name="foo\xa3hoo", age="22")
- i2 = dict(name="bar", age=i1)
+ i1 = {"name": "foo\xa3hoo", "age": "22"}
+ i2 = {"name": "bar", "age": i1}
i3 = self.item_class(name="buz", age=i2)
self.assertExportResult(
@@ -478,8 +478,8 @@ class XmlItemExporterTest(BaseItemExporterTest):
)
def test_nested_list_item(self):
- i1 = dict(name="foo")
- i2 = dict(name="bar", v2={"egg": ["spam"]})
+ i1 = {"name": "foo"}
+ i2 = {"name": "bar", "v2": {"egg": ["spam"]}}
i3 = self.item_class(name="buz", age=[i1, i2])
self.assertExportResult(
@@ -534,7 +534,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
def test_nested_item(self):
i1 = self.item_class(name="Joseph", age="22")
- i2 = dict(name="Maria", age=i1)
+ i2 = {"name": "Maria", "age": i1}
i3 = self.item_class(name="Jesus", age=i2)
self.ie.start_exporting()
self.ie.export_item(i3)
@@ -622,9 +622,9 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.assertEqual(exported, [expected])
def test_nested_dict_item(self):
- i1 = dict(name="Joseph\xa3", age="22")
+ i1 = {"name": "Joseph\xa3", "age": "22"}
i2 = self.item_class(name="Maria", age=i1)
- i3 = dict(name="Jesus", age=i2)
+ i3 = {"name": "Jesus", "age": i2}
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 6b4df90d8..217c7a299 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -37,7 +37,7 @@ class Base:
page4_url = "http://example.com/page%204.html"
self.assertEqual(
- [link for link in lx.extract_links(self.response)],
+ list(lx.extract_links(self.response)),
[
Link(url="http://example.com/sample1.html", text=""),
Link(url="http://example.com/sample2.html", text="sample 2"),
@@ -55,7 +55,7 @@ class Base:
def test_extract_filter_allow(self):
lx = self.extractor_cls(allow=("sample",))
self.assertEqual(
- [link for link in lx.extract_links(self.response)],
+ list(lx.extract_links(self.response)),
[
Link(url="http://example.com/sample1.html", text=""),
Link(url="http://example.com/sample2.html", text="sample 2"),
@@ -70,7 +70,7 @@ class Base:
def test_extract_filter_allow_with_duplicates(self):
lx = self.extractor_cls(allow=("sample",), unique=False)
self.assertEqual(
- [link for link in lx.extract_links(self.response)],
+ list(lx.extract_links(self.response)),
[
Link(url="http://example.com/sample1.html", text=""),
Link(url="http://example.com/sample2.html", text="sample 2"),
@@ -93,7 +93,7 @@ class Base:
def test_extract_filter_allow_with_duplicates_canonicalize(self):
lx = self.extractor_cls(allow=("sample",), unique=False, canonicalize=True)
self.assertEqual(
- [link for link in lx.extract_links(self.response)],
+ list(lx.extract_links(self.response)),
[
Link(url="http://example.com/sample1.html", text=""),
Link(url="http://example.com/sample2.html", text="sample 2"),
@@ -116,7 +116,7 @@ class Base:
def test_extract_filter_allow_no_duplicates_canonicalize(self):
lx = self.extractor_cls(allow=("sample",), unique=True, canonicalize=True)
self.assertEqual(
- [link for link in lx.extract_links(self.response)],
+ list(lx.extract_links(self.response)),
[
Link(url="http://example.com/sample1.html", text=""),
Link(url="http://example.com/sample2.html", text="sample 2"),
@@ -127,7 +127,7 @@ class Base:
def test_extract_filter_allow_and_deny(self):
lx = self.extractor_cls(allow=("sample",), deny=("3",))
self.assertEqual(
- [link for link in lx.extract_links(self.response)],
+ list(lx.extract_links(self.response)),
[
Link(url="http://example.com/sample1.html", text=""),
Link(url="http://example.com/sample2.html", text="sample 2"),
@@ -137,7 +137,7 @@ class Base:
def test_extract_filter_allowed_domains(self):
lx = self.extractor_cls(allow_domains=("google.com",))
self.assertEqual(
- [link for link in lx.extract_links(self.response)],
+ list(lx.extract_links(self.response)),
[
Link(url="http://www.google.com/something", text=""),
],
@@ -148,7 +148,7 @@ class Base:
lx = self.extractor_cls(allow="sample")
self.assertEqual(
- [link for link in lx.extract_links(self.response)],
+ list(lx.extract_links(self.response)),
[
Link(url="http://example.com/sample1.html", text=""),
Link(url="http://example.com/sample2.html", text="sample 2"),
@@ -162,7 +162,7 @@ class Base:
lx = self.extractor_cls(allow="sample", deny="3")
self.assertEqual(
- [link for link in lx.extract_links(self.response)],
+ list(lx.extract_links(self.response)),
[
Link(url="http://example.com/sample1.html", text=""),
Link(url="http://example.com/sample2.html", text="sample 2"),
@@ -171,7 +171,7 @@ class Base:
lx = self.extractor_cls(allow_domains="google.com")
self.assertEqual(
- [link for link in lx.extract_links(self.response)],
+ list(lx.extract_links(self.response)),
[
Link(url="http://www.google.com/something", text=""),
],
@@ -179,7 +179,7 @@ class Base:
lx = self.extractor_cls(deny_domains="example.com")
self.assertEqual(
- [link for link in lx.extract_links(self.response)],
+ list(lx.extract_links(self.response)),
[
Link(url="http://www.google.com/something", text=""),
],
@@ -265,7 +265,7 @@ class Base:
def test_restrict_xpaths(self):
lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]',))
self.assertEqual(
- [link for link in lx.extract_links(self.response)],
+ list(lx.extract_links(self.response)),
[
Link(url="http://example.com/sample1.html", text=""),
Link(url="http://example.com/sample2.html", text="sample 2"),
@@ -337,7 +337,7 @@ class Base:
restrict_css=("#subwrapper + a",),
)
self.assertEqual(
- [link for link in lx.extract_links(self.response)],
+ list(lx.extract_links(self.response)),
[
Link(url="http://example.com/sample1.html", text=""),
Link(url="http://example.com/sample2.html", text="sample 2"),
@@ -705,7 +705,7 @@ class Base:
response = HtmlResponse("http://example.org/index.html", body=html)
lx = self.extractor_cls()
self.assertEqual(
- [link for link in lx.extract_links(response)],
+ list(lx.extract_links(response)),
[
Link(
url="http://example.org/item1.html",
@@ -758,7 +758,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
response = HtmlResponse("http://example.org/index.html", body=html)
lx = self.extractor_cls()
self.assertEqual(
- [link for link in lx.extract_links(response)],
+ list(lx.extract_links(response)),
[
Link(
url="http://example.org/item1.html", text="Item 1", nofollow=False
@@ -779,7 +779,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
# Simple text inclusion test
lx = self.extractor_cls(restrict_text="dog")
self.assertEqual(
- [link for link in lx.extract_links(response)],
+ list(lx.extract_links(response)),
[
Link(
url="http://example.org/item2.html",
@@ -791,7 +791,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
# Unique regex test
lx = self.extractor_cls(restrict_text=r"of.*dog")
self.assertEqual(
- [link for link in lx.extract_links(response)],
+ list(lx.extract_links(response)),
[
Link(
url="http://example.org/item2.html",
@@ -803,7 +803,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
# Multiple regex test
lx = self.extractor_cls(restrict_text=[r"of.*dog", r"of.*cat"])
self.assertEqual(
- [link for link in lx.extract_links(response)],
+ list(lx.extract_links(response)),
[
Link(
url="http://example.org/item1.html",
@@ -834,7 +834,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
response = HtmlResponse("http://example.org/index.html", body=html)
lx = self.extractor_cls()
self.assertEqual(
- [link for link in lx.extract_links(response)],
+ list(lx.extract_links(response)),
[
Link(
url="http://example.org/item2.html",
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index 99cdf88d9..528efa142 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -565,37 +565,37 @@ class NoInputReprocessingFromDictTest(unittest.TestCase):
"""
def test_avoid_reprocessing_with_initial_values_single(self):
- il = NoInputReprocessingDictLoader(item=dict(title="foo"))
+ il = NoInputReprocessingDictLoader(item={"title": "foo"})
il_loaded = il.load_item()
- self.assertEqual(il_loaded, dict(title="foo"))
+ self.assertEqual(il_loaded, {"title": "foo"})
self.assertEqual(
- NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo")
+ NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"}
)
def test_avoid_reprocessing_with_initial_values_list(self):
- il = NoInputReprocessingDictLoader(item=dict(title=["foo", "bar"]))
+ il = NoInputReprocessingDictLoader(item={"title": ["foo", "bar"]})
il_loaded = il.load_item()
- self.assertEqual(il_loaded, dict(title="foo"))
+ self.assertEqual(il_loaded, {"title": "foo"})
self.assertEqual(
- NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo")
+ NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"}
)
def test_avoid_reprocessing_without_initial_values_single(self):
il = NoInputReprocessingDictLoader()
il.add_value("title", "foo")
il_loaded = il.load_item()
- self.assertEqual(il_loaded, dict(title="FOO"))
+ self.assertEqual(il_loaded, {"title": "FOO"})
self.assertEqual(
- NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO")
+ NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"}
)
def test_avoid_reprocessing_without_initial_values_list(self):
il = NoInputReprocessingDictLoader()
il.add_value("title", ["foo", "bar"])
il_loaded = il.load_item()
- self.assertEqual(il_loaded, dict(title="FOO"))
+ self.assertEqual(il_loaded, {"title": "FOO"})
self.assertEqual(
- NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO")
+ NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"}
)
diff --git a/tests/test_mail.py b/tests/test_mail.py
index 2535e58db..ff1505397 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -91,7 +91,7 @@ class MailSenderTest(unittest.TestCase):
self.assertEqual(attach.get_payload(decode=True), b"content")
def _catch_mail_sent(self, **kwargs):
- self.catched_msg = dict(**kwargs)
+ self.catched_msg = {**kwargs}
def test_send_utf8(self):
subject = "sübjèçt"
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index be9811980..5a9a217ce 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -140,7 +140,7 @@ class FileDownloadCrawlTestCase(TestCase):
self.assertEqual(logs.count(file_dl_failure), 3)
# check that no files were written to the media store
- self.assertEqual([x for x in self.tmpmediastore.iterdir()], [])
+ self.assertEqual(list(self.tmpmediastore.iterdir()), [])
@defer.inlineCallbacks
def test_download_media(self):
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index e7000e314..0babde4d9 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -221,7 +221,7 @@ class FilesPipelineTestCase(unittest.TestCase):
file_path = CustomFilesPipeline.from_settings(
Settings({"FILES_STORE": self.tempdir})
).file_path
- item = dict(path="path-to-store-file")
+ item = {"path": "path-to-store-file"}
request = Request("http://example.com")
self.assertEqual(file_path(request, item=item), "full/path-to-store-file")
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 2e2e06b89..18a2454b3 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -132,7 +132,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
thumb_path = CustomImagesPipeline.from_settings(
Settings({"IMAGES_STORE": self.tempdir})
).thumb_path
- item = dict(path="path-to-store-file")
+ item = {"path": "path-to-store-file"}
request = Request("http://example.com")
self.assertEqual(
thumb_path(request, "small", item=item), "thumb/small/path-to-store-file"
@@ -433,14 +433,14 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
]
# This should match what is defined in ImagesPipeline.
- default_pipeline_settings = dict(
- MIN_WIDTH=0,
- MIN_HEIGHT=0,
- EXPIRES=90,
- THUMBS={},
- IMAGES_URLS_FIELD="image_urls",
- IMAGES_RESULT_FIELD="images",
- )
+ default_pipeline_settings = {
+ "MIN_WIDTH": 0,
+ "MIN_HEIGHT": 0,
+ "EXPIRES": 90,
+ "THUMBS": {},
+ "IMAGES_URLS_FIELD": "image_urls",
+ "IMAGES_RESULT_FIELD": "images",
+ }
def setUp(self):
self.tempdir = mkdtemp()
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index d477b59be..d4dde4a40 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -59,7 +59,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
assert self.pipe.media_to_download(request, self.info) is None
def test_default_get_media_requests(self):
- item = dict(name="name")
+ item = {"name": "name"}
assert self.pipe.get_media_requests(item, self.info) is None
def test_default_media_downloaded(self):
@@ -73,7 +73,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
assert self.pipe.media_failed(fail, request, self.info) is fail
def test_default_item_completed(self):
- item = dict(name="name")
+ item = {"name": "name"}
assert self.pipe.item_completed([], item, self.info) is item
# Check that failures are logged by default
@@ -98,7 +98,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
@inlineCallbacks
def test_default_process_item(self):
- item = dict(name="name")
+ item = {"name": "name"}
new_item = yield self.pipe.process_item(item, self.spider)
assert new_item is item
@@ -226,11 +226,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
rsp = Response("http://url1")
req = Request(
"http://url1",
- meta=dict(response=rsp),
+ meta={"response": rsp},
callback=self._callback,
errback=self._errback,
)
- item = dict(requests=req)
+ item = {"requests": req}
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item["results"], [(True, rsp)])
self.assertEqual(
@@ -250,11 +250,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
fail = Failure(Exception())
req = Request(
"http://url1",
- meta=dict(response=fail),
+ meta={"response": fail},
callback=self._callback,
errback=self._errback,
)
- item = dict(requests=req)
+ item = {"requests": req}
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item["results"], [(False, fail)])
self.assertEqual(
@@ -272,10 +272,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
def test_mix_of_success_and_failure(self):
self.pipe.LOG_FAILED_RESULTS = False
rsp1 = Response("http://url1")
- req1 = Request("http://url1", meta=dict(response=rsp1))
+ req1 = Request("http://url1", meta={"response": rsp1})
fail = Failure(Exception())
- req2 = Request("http://url2", meta=dict(response=fail))
- item = dict(requests=[req1, req2])
+ req2 = Request("http://url2", meta={"response": fail})
+ item = {"requests": [req1, req2]}
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item["results"], [(True, rsp1), (False, fail)])
m = self.pipe._mockcalled
@@ -294,7 +294,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
def test_get_media_requests(self):
# returns single Request (without callback)
req = Request("http://url")
- item = dict(requests=req) # pass a single item
+ item = {"requests": req} # pass a single item
new_item = yield self.pipe.process_item(item, self.spider)
assert new_item is item
self.assertIn(self.fingerprint(req), self.info.downloaded)
@@ -302,7 +302,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
# returns iterable of Requests
req1 = Request("http://url1")
req2 = Request("http://url2")
- item = dict(requests=iter([req1, req2]))
+ item = {"requests": iter([req1, req2])}
new_item = yield self.pipe.process_item(item, self.spider)
assert new_item is item
assert self.fingerprint(req1) in self.info.downloaded
@@ -311,17 +311,17 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
@inlineCallbacks
def test_results_are_cached_across_multiple_items(self):
rsp1 = Response("http://url1")
- req1 = Request("http://url1", meta=dict(response=rsp1))
- item = dict(requests=req1)
+ req1 = Request("http://url1", meta={"response": rsp1})
+ item = {"requests": req1}
new_item = yield self.pipe.process_item(item, self.spider)
self.assertTrue(new_item is item)
self.assertEqual(new_item["results"], [(True, rsp1)])
# rsp2 is ignored, rsp1 must be in results because request fingerprints are the same
req2 = Request(
- req1.url, meta=dict(response=Response("http://donot.download.me"))
+ req1.url, meta={"response": Response("http://donot.download.me")}
)
- item = dict(requests=req2)
+ item = {"requests": req2}
new_item = yield self.pipe.process_item(item, self.spider)
self.assertTrue(new_item is item)
self.assertEqual(self.fingerprint(req1), self.fingerprint(req2))
@@ -330,11 +330,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
@inlineCallbacks
def test_results_are_cached_for_requests_of_single_item(self):
rsp1 = Response("http://url1")
- req1 = Request("http://url1", meta=dict(response=rsp1))
+ req1 = Request("http://url1", meta={"response": rsp1})
req2 = Request(
- req1.url, meta=dict(response=Response("http://donot.download.me"))
+ req1.url, meta={"response": Response("http://donot.download.me")}
)
- item = dict(requests=[req1, req2])
+ item = {"requests": [req1, req2]}
new_item = yield self.pipe.process_item(item, self.spider)
self.assertTrue(new_item is item)
self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)])
@@ -359,16 +359,16 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
def rsp2_func():
self.fail("it must cache rsp1 result and must not try to redownload")
- req1 = Request("http://url", meta=dict(response=rsp1_func))
- req2 = Request(req1.url, meta=dict(response=rsp2_func))
- item = dict(requests=[req1, req2])
+ req1 = Request("http://url", meta={"response": rsp1_func})
+ req2 = Request(req1.url, meta={"response": rsp2_func})
+ item = {"requests": [req1, req2]}
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)])
@inlineCallbacks
def test_use_media_to_download_result(self):
- req = Request("http://url", meta=dict(result="ITSME", response=self.fail))
- item = dict(requests=req)
+ req = Request("http://url", meta={"result": "ITSME", "response": self.fail})
+ item = {"requests": req}
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item["results"], [(True, "ITSME")])
self.assertEqual(
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index f8465a5ff..37099dae6 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -45,15 +45,15 @@ class MockDownloader:
class MockCrawler(Crawler):
def __init__(self, priority_queue_cls, jobdir):
- settings = dict(
- SCHEDULER_DEBUG=False,
- SCHEDULER_DISK_QUEUE="scrapy.squeues.PickleLifoDiskQueue",
- SCHEDULER_MEMORY_QUEUE="scrapy.squeues.LifoMemoryQueue",
- SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
- JOBDIR=jobdir,
- DUPEFILTER_CLASS="scrapy.dupefilters.BaseDupeFilter",
- REQUEST_FINGERPRINTER_IMPLEMENTATION="2.7",
- )
+ settings = {
+ "SCHEDULER_DEBUG": False,
+ "SCHEDULER_DISK_QUEUE": "scrapy.squeues.PickleLifoDiskQueue",
+ "SCHEDULER_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue",
+ "SCHEDULER_PRIORITY_QUEUE": priority_queue_cls,
+ "JOBDIR": jobdir,
+ "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+ }
super().__init__(Spider, settings)
self.engine = MockEngine(downloader=MockDownloader())
self.stats = load_object(self.settings["STATS_CLASS"])(self)
@@ -338,10 +338,10 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase):
class TestIncompatibility(unittest.TestCase):
def _incompatible(self):
- settings = dict(
- SCHEDULER_PRIORITY_QUEUE="scrapy.pqueues.DownloaderAwarePriorityQueue",
- CONCURRENT_REQUESTS_PER_IP=1,
- )
+ settings = {
+ "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue",
+ "CONCURRENT_REQUESTS_PER_IP": 1,
+ }
crawler = get_crawler(Spider, settings)
scheduler = Scheduler.from_crawler(crawler)
spider = Spider(name="spider")
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index ea45b7698..837f1c2c8 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -16,10 +16,10 @@ class TestOffsiteMiddleware(TestCase):
self.mw.spider_opened(self.spider)
def _get_spiderargs(self):
- return dict(
- name="foo",
- allowed_domains=["scrapytest.org", "scrapy.org", "scrapy.test.org"],
- )
+ return {
+ "name": "foo",
+ "allowed_domains": ["scrapytest.org", "scrapy.org", "scrapy.test.org"],
+ }
def test_process_spider_output(self):
res = Response("http://scrapytest.org")
@@ -50,7 +50,7 @@ class TestOffsiteMiddleware(TestCase):
class TestOffsiteMiddleware2(TestOffsiteMiddleware):
def _get_spiderargs(self):
- return dict(name="foo", allowed_domains=None)
+ return {"name": "foo", "allowed_domains": None}
def test_process_spider_output(self):
res = Response("http://scrapytest.org")
@@ -61,13 +61,16 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):
class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
def _get_spiderargs(self):
- return dict(name="foo")
+ return {"name": "foo"}
class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
def _get_spiderargs(self):
bad_hostname = urlparse("http:////scrapytest.org").hostname
- return dict(name="foo", allowed_domains=["scrapytest.org", None, bad_hostname])
+ return {
+ "name": "foo",
+ "allowed_domains": ["scrapytest.org", None, bad_hostname],
+ }
def test_process_spider_output(self):
res = Response("http://scrapytest.org")
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index ee22e6675..ec377bb19 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -355,7 +355,7 @@ class UtilsCsvTestCase(unittest.TestCase):
response = TextResponse(url="http://example.com/", body=body)
csv = csviter(response)
- result = [row for row in csv]
+ result = list(csv)
self.assertEqual(
result,
[
@@ -377,7 +377,7 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response, delimiter="\t")
self.assertEqual(
- [row for row in csv],
+ list(csv),
[
{"id": "1", "name": "alpha", "value": "foobar"},
{"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
@@ -394,7 +394,7 @@ class UtilsCsvTestCase(unittest.TestCase):
csv1 = csviter(response1, quotechar="'")
self.assertEqual(
- [row for row in csv1],
+ list(csv1),
[
{"id": "1", "name": "alpha", "value": "foobar"},
{"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
@@ -407,7 +407,7 @@ class UtilsCsvTestCase(unittest.TestCase):
csv2 = csviter(response2, delimiter="|", quotechar="'")
self.assertEqual(
- [row for row in csv2],
+ list(csv2),
[
{"id": "1", "name": "alpha", "value": "foobar"},
{"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
@@ -422,7 +422,7 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response)
self.assertEqual(
- [row for row in csv],
+ list(csv),
[
{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"},
{
@@ -441,7 +441,7 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response, delimiter="\t")
self.assertEqual(
- [row for row in csv],
+ list(csv),
[
{"id": "1", "name": "alpha", "value": "foobar"},
{"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
@@ -458,7 +458,7 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response, headers=[h.decode("utf-8") for h in headers])
self.assertEqual(
- [row for row in csv],
+ list(csv),
[
{"id": "1", "name": "alpha", "value": "foobar"},
{"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
@@ -475,7 +475,7 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response)
self.assertEqual(
- [row for row in csv],
+ list(csv),
[
{"id": "1", "name": "alpha", "value": "foobar"},
{"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index cbe80e157..fc42c0d2f 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -16,7 +16,7 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
rmtree(self.tmp_path)
def test_simple_render(self):
- context = dict(project_name="proj", name="spi", classname="TheSpider")
+ context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}
template = "from ${project_name}.spiders.${name} import ${classname}"
rendered = "from proj.spiders.spi import TheSpider"