Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

IOError and other cleanup (#4716)

Marc Hernández5360ba34bc34
diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index 1f270837c..c17b9d511 100644
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -30,7 +30,7 @@ def main():
try:
with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out:
output_lines = out.readlines()
- except IOError:
+ except OSError:
print("linkcheck output not found; please run linkcheck first.")
sys.exit(1)
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 5839dc243..3b8702419 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -63,7 +63,7 @@ class DecompressionMiddleware:
archive = BytesIO(response.body)
try:
body = gzip.GzipFile(fileobj=archive).read()
- except IOError:
+ except OSError:
return
respcls = responsetypes.from_args(body=body)
@@ -72,7 +72,7 @@ class DecompressionMiddleware:
def _is_bzip2(self, response):
try:
body = bz2.decompress(response.body)
- except IOError:
+ except OSError:
return
respcls = responsetypes.from_args(body=body)
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index b9316c43a..ac87d4a4e 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -37,7 +37,7 @@ class HttpCacheMiddleware:
ConnectionLost,
TCPTimedOutError,
ResponseFailed,
- IOError,
+ OSError,
)
def __init__(self, settings: Settings, stats: StatsCollector) -> None:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 89837b4ab..a4cb555bd 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -268,9 +268,9 @@ RETRY_EXCEPTIONS = [
"twisted.internet.error.ConnectionLost",
"twisted.internet.error.TCPTimedOutError",
"twisted.web.client.ResponseFailed",
- # IOError is raised by the HttpCompression middleware when trying to
+ # OSError is raised by the HttpCompression middleware when trying to
# decompress an empty response
- IOError,
+ OSError,
"scrapy.core.downloader.handlers.http11.TunnelError",
]
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index e5df34d2e..77e0197d8 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -15,7 +15,7 @@ def gunzip(data):
try:
chunk = f.read1(8196)
output_list.append(chunk)
- except (IOError, EOFError, struct.error):
+ except (OSError, EOFError, struct.error):
# complete only if there is some data, otherwise re-raise
# see issue 87 about catching struct.error
# some pages are quite small so output_list is empty and f.extrabuf
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 27816c0df..bb5dbebbc 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -291,7 +291,7 @@ def without_none_values(iterable):
try:
return {k: v for k, v in iterable.items() if v is not None}
except AttributeError:
- return type(iterable)((v for v in iterable if v is not None))
+ return type(iterable)(v for v in iterable if v is not None)
def global_object_name(obj):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index fd4176e2f..9731b62c4 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -129,7 +129,7 @@ class FileTestCase(unittest.TestCase):
def test_non_existent(self):
request = Request(f"file://{self.mktemp()}")
d = self.download_request(request, Spider("foo"))
- return self.assertFailure(d, IOError)
+ return self.assertFailure(d, OSError)
class ContentLengthHeaderResource(resource.Resource):
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 060cfe08b..062e8a8b4 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -70,7 +70,7 @@ class DefaultsTest(ManagerTestCase):
In particular when some website returns a 30x response with header
'Content-Encoding: gzip' giving as result the error below:
- exceptions.IOError: Not a gzipped file
+ BadGzipFile: Not a gzipped file (...)
"""
req = Request("http://example.com")
@@ -108,7 +108,7 @@ class DefaultsTest(ManagerTestCase):
"Location": "http://example.com/login",
},
)
- self.assertRaises(IOError, self._download, request=req, response=resp)
+ self.assertRaises(OSError, self._download, request=req, response=resp)
class ResponseFromProcessRequestTest(ManagerTestCase):
diff --git a/tests/test_mail.py b/tests/test_mail.py
index bc7298e9d..504c78486 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -1,5 +1,3 @@
-# coding=utf-8
-
import unittest
from email.charset import Charset
from io import BytesIO
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 8d87a322a..d7a923085 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,4 +1,3 @@
-# coding=utf-8
from twisted.trial import unittest
diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index 6b2a458bc..7b7a25db8 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -28,7 +28,7 @@ class GunzipTest(unittest.TestCase):
def test_gunzip_no_gzip_file_raises(self):
self.assertRaises(
- IOError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes()
+ OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes()
)
def test_gunzip_truncated_short(self):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index faf7d2709..3598fa0bb 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -346,8 +346,8 @@ class UtilsCsvTestCase(unittest.TestCase):
# explicit type check cuz' we no like stinkin' autocasting! yarrr
for result_row in result:
- self.assertTrue(all((isinstance(k, str) for k in result_row.keys())))
- self.assertTrue(all((isinstance(v, str) for v in result_row.values())))
+ self.assertTrue(all(isinstance(k, str) for k in result_row.keys()))
+ self.assertTrue(all(isinstance(v, str) for v in result_row.values()))
def test_csviter_delimiter(self):
body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t")