Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into flake8-max-line-length

Eugenio Lacuesta3eeecb42731d
diff --cc pytest.ini
index f26e274d7,8177fedd3..0e289866c
--- a/pytest.ini
+++ b/pytest.ini
@@@ -20,128 -20,227 +20,25 @@@ addopts
twisted = 1
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
+flake8-max-line-length = 119
flake8-ignore =
W503
-- # Files that are only meant to provide top-level imports are expected not
-- # to use any of their imports:
++
++ # Exclude files that are meant to provide top-level imports
++ # E402: Module level import not at top of file
++ # F401: Module imported but unused
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
++ scrapy/linkextractors/__init__.py E402 F401
++ scrapy/spiders/__init__.py E402 F401
++
# Issues pending a review:
- # extras
- extras/qps-bench-server.py E501
- extras/qpsclient.py E501 E501
-- # scrapy/commands
- scrapy/commands/__init__.py E128
- scrapy/commands/fetch.py E128
- scrapy/commands/genspider.py E128
- scrapy/commands/parse.py E128
- scrapy/commands/settings.py E128
- scrapy/commands/shell.py E128
- scrapy/commands/startproject.py E128
- scrapy/commands/version.py E128
- scrapy/commands/__init__.py E501
- scrapy/commands/check.py E501
- scrapy/commands/crawl.py E501
- scrapy/commands/edit.py E501
- scrapy/commands/fetch.py E501
- scrapy/commands/genspider.py E501
- scrapy/commands/parse.py E501
- scrapy/commands/runspider.py E501
- scrapy/commands/settings.py E501
- scrapy/commands/shell.py E501
- scrapy/commands/startproject.py E501
- scrapy/commands/version.py E501
-- # scrapy/contracts
- scrapy/contracts/default.py E128
- scrapy/contracts/__init__.py E501
-- # scrapy/core
- scrapy/core/engine.py E128
- scrapy/core/scraper.py E128
- scrapy/core/spidermw.py E126
- scrapy/core/downloader/contextfactory.py E128 E126
- scrapy/core/downloader/webclient.py E128 E126
- scrapy/core/downloader/handlers/ftp.py E128
- scrapy/core/downloader/handlers/s3.py E128 E126
- scrapy/core/engine.py E501
- scrapy/core/scheduler.py E501
- scrapy/core/scraper.py E501
- scrapy/core/spidermw.py E501
- scrapy/core/downloader/__init__.py E501
- scrapy/core/downloader/contextfactory.py E501
- scrapy/core/downloader/middleware.py E501
- scrapy/core/downloader/tls.py E501
- scrapy/core/downloader/webclient.py E501
- scrapy/core/downloader/handlers/__init__.py E501
- scrapy/core/downloader/handlers/ftp.py E501
- scrapy/core/downloader/handlers/http10.py E501
- scrapy/core/downloader/handlers/http11.py E501
- scrapy/core/downloader/handlers/s3.py E501
-- # scrapy/downloadermiddlewares
- scrapy/downloadermiddlewares/httpcache.py E126
- scrapy/downloadermiddlewares/httpcompression.py E128
- scrapy/downloadermiddlewares/retry.py E126
- scrapy/downloadermiddlewares/ajaxcrawl.py E501
- scrapy/downloadermiddlewares/decompression.py E501
- scrapy/downloadermiddlewares/defaultheaders.py E501
- scrapy/downloadermiddlewares/httpcache.py E501
- scrapy/downloadermiddlewares/httpcompression.py E501
- scrapy/downloadermiddlewares/httpproxy.py E501
- scrapy/downloadermiddlewares/redirect.py E501
- scrapy/downloadermiddlewares/retry.py E501
- scrapy/downloadermiddlewares/robotstxt.py E501
- scrapy/downloadermiddlewares/stats.py E501
-- # scrapy/extensions
- scrapy/extensions/closespider.py E128
- scrapy/extensions/feedexport.py E128
- scrapy/extensions/httpcache.py E128
- scrapy/extensions/closespider.py E501
- scrapy/extensions/corestats.py E501
- scrapy/extensions/feedexport.py E501
- scrapy/extensions/httpcache.py E501
- scrapy/extensions/memdebug.py E501
- scrapy/extensions/spiderstate.py E501
- scrapy/extensions/telnet.py E501
- scrapy/extensions/throttle.py E501
-- # scrapy/http
- scrapy/http/response/__init__.py E128
- scrapy/http/response/text.py E128 E124
- scrapy/http/common.py E501
- scrapy/http/cookies.py E501
- scrapy/http/request/__init__.py E501
- scrapy/http/request/form.py E501
- scrapy/http/request/json_request.py E501
- scrapy/http/response/__init__.py E501
- scrapy/http/response/text.py E501
-- # scrapy/linkextractors
- scrapy/linkextractors/__init__.py E402
- scrapy/linkextractors/__init__.py E501 E402
- scrapy/linkextractors/lxmlhtml.py E501
-- # scrapy/loader
- scrapy/loader/__init__.py E128
- scrapy/loader/__init__.py E501
- scrapy/loader/processors.py E501
-- # scrapy/pipelines
- scrapy/pipelines/files.py E116
- scrapy/pipelines/__init__.py E501
- scrapy/pipelines/files.py E501
- scrapy/pipelines/images.py E501
- scrapy/pipelines/media.py E501
-- # scrapy/selector
++ scrapy/__init__.py E402
scrapy/selector/__init__.py F403
- scrapy/selector/unified.py E111
- scrapy/selector/unified.py E501
-- # scrapy/settings
- scrapy/settings/default_settings.py E114 E116
- scrapy/settings/__init__.py E501
- scrapy/settings/default_settings.py E501
- scrapy/settings/deprecated.py E501
-- # scrapy/spidermiddlewares
- scrapy/spidermiddlewares/referer.py E129
- scrapy/spidermiddlewares/httperror.py E501
- scrapy/spidermiddlewares/offsite.py E501
- scrapy/spidermiddlewares/referer.py E501
- scrapy/spidermiddlewares/urllength.py E501
-- # scrapy/spiders
- scrapy/spiders/__init__.py E501 E402
- scrapy/spiders/crawl.py E501
- scrapy/spiders/feed.py E501
- scrapy/spiders/sitemap.py E501
- # scrapy/utils
- scrapy/utils/asyncio.py E501
- scrapy/utils/benchserver.py E501
- scrapy/utils/conf.py E402 E501
- scrapy/utils/datatypes.py E501
- scrapy/utils/decorators.py E501
- scrapy/utils/defer.py E501
- scrapy/utils/deprecate.py E501
- scrapy/utils/gz.py E501
+ scrapy/spiders/__init__.py E402
- # scrapy/utils
- scrapy/utils/conf.py E402
- scrapy/utils/defer.py E128
scrapy/utils/http.py F403
- scrapy/utils/log.py E128
- scrapy/utils/httpobj.py E501
- scrapy/utils/iterators.py E501
- scrapy/utils/log.py E501
scrapy/utils/markup.py F403
- scrapy/utils/misc.py E501
scrapy/utils/multipart.py F403
- scrapy/utils/response.py E128
- scrapy/utils/signal.py E128
- scrapy/utils/url.py F403 E128 F405
- scrapy/utils/project.py E501
- scrapy/utils/python.py E501
- scrapy/utils/reactor.py E501
- scrapy/utils/reqser.py E501
- scrapy/utils/request.py E501
- scrapy/utils/response.py E501
- scrapy/utils/signal.py E501
- scrapy/utils/sitemap.py E501
- scrapy/utils/spider.py E501
- scrapy/utils/ssl.py E501
- scrapy/utils/test.py E501
- scrapy/utils/url.py E501 F403 F405
-- # scrapy
- scrapy/__init__.py E402
- scrapy/item.py E128
- scrapy/mail.py E402 E128
- scrapy/middleware.py E128
- scrapy/responsetypes.py E128
- scrapy/spiderloader.py F841 E126
- scrapy/squeues.py E128
- scrapy/__init__.py E402 E501
- scrapy/cmdline.py E501
- scrapy/crawler.py E501
- scrapy/dupefilters.py E501
- scrapy/exceptions.py E501
- scrapy/exporters.py E501
- scrapy/interfaces.py E501
- scrapy/item.py E501
- scrapy/link.py E501
- scrapy/logformatter.py E501
- scrapy/mail.py E402 E501
- scrapy/middleware.py E501
- scrapy/pqueues.py E501
- scrapy/resolver.py E501
- scrapy/responsetypes.py E501
- scrapy/robotstxt.py E501
- scrapy/shell.py E501
- scrapy/signalmanager.py E501
- scrapy/spiderloader.py E501
- scrapy/squeues.py E501
- scrapy/statscollectors.py E501
-- # tests
- tests/__init__.py E402
- tests/mockserver.py E126
- tests/pipelines.py F841
- tests/test_command_parse.py E128
- tests/test_command_shell.py E128
- tests/test_commands.py E128
- tests/test_contracts.py E128
- tests/test_crawl.py E741
- tests/test_crawler.py F841
- tests/test_dependencies.py F841
- tests/test_downloader_handlers.py E124 E128 E126
- tests/test_downloadermiddleware_cookies.py E741 E128 E126
- tests/test_downloadermiddleware_httpcompression.py E126
- tests/test_downloadermiddleware_httpproxy.py E128
- tests/test_downloadermiddleware_redirect.py E128
- tests/test_downloadermiddleware_retry.py E128 E126
- tests/test_dupefilters.py E741 E128 E124
- tests/test_engine.py E128
- tests/test_exporters.py E128 E124
- tests/test_extension_telnet.py F841
- tests/test_feedexport.py F841
- tests/test_http_request.py E402 E128 E128 E126
- tests/test_http_response.py E128
- tests/test_item.py E128 F841
- tests/test_linkextractors.py E128 E124
- tests/test_loader.py E741 E128 E117
- tests/test_logformatter.py E128 E122
- tests/test_mail.py E128
- tests/test_middleware.py E128
- tests/test_pipeline_crawl.py E128 E126
- tests/test_pipeline_images.py F841
- tests/test_pipeline_media.py E741 E128
- tests/test_proxy_connect.py E741
- tests/test_scheduler.py E126
- tests/test_spidermiddleware_httperror.py E128 E121
- tests/test_spidermiddleware_offsite.py E128 E111
- tests/test_spidermiddleware_referer.py F841 E124 E121
- tests/test_squeues.py E741
- tests/test_utils_conf.py E128
- tests/test_utils_datatypes.py E402
- tests/test_utils_defer.py F841
- tests/test_utils_deprecate.py F841
- tests/test_utils_http.py E128
- tests/test_utils_iterators.py E128 E129
- tests/test_utils_log.py E741
- tests/test_utils_reqser.py E128
- tests/test_utils_request.py E128
- tests/test_utils_signal.py E741 F841
- tests/test_utils_sitemap.py E128 E124
- tests/test_utils_url.py E126
- tests/test_webclient.py E128 E122 E402 E126
- tests/test_settings/__init__.py E128
- tests/test_spiderloader/__init__.py E128
- tests/__init__.py E402 E501
- tests/mockserver.py E501
- tests/spiders.py E501
- tests/test_closespider.py E501
- tests/test_command_fetch.py E501
- tests/test_command_parse.py E501
- tests/test_command_shell.py E501
- tests/test_commands.py E501
- tests/test_contracts.py E501
- tests/test_crawl.py E501
- tests/test_crawler.py E501
- tests/test_dependencies.py E501
- tests/test_downloader_handlers.py E501
- tests/test_downloadermiddleware.py E501
- tests/test_downloadermiddleware_ajaxcrawlable.py E501
- tests/test_downloadermiddleware_cookies.py E501
- tests/test_downloadermiddleware_defaultheaders.py E501
- tests/test_downloadermiddleware_downloadtimeout.py E501
- tests/test_downloadermiddleware_httpcache.py E501
- tests/test_downloadermiddleware_httpcompression.py E501
- tests/test_downloadermiddleware_decompression.py E501
- tests/test_downloadermiddleware_httpproxy.py E501
- tests/test_downloadermiddleware_redirect.py E501
- tests/test_downloadermiddleware_retry.py E501
- tests/test_downloadermiddleware_robotstxt.py E501
- tests/test_downloadermiddleware_stats.py E501
- tests/test_dupefilters.py E501
- tests/test_engine.py E501
- tests/test_exporters.py E501
- tests/test_feedexport.py E501
- tests/test_http_cookies.py E501
- tests/test_http_headers.py E501
- tests/test_http_request.py E402 E501
- tests/test_http_response.py E501
- tests/test_item.py E501
- tests/test_link.py E501
- tests/test_linkextractors.py E501
- tests/test_loader.py E501 E741
- tests/test_logformatter.py E501
- tests/test_mail.py E501
- tests/test_middleware.py E501
- tests/test_pipeline_crawl.py E501
- tests/test_pipeline_files.py E501
- tests/test_pipeline_images.py E501
- tests/test_pipeline_media.py E501
- tests/test_proxy_connect.py E501
- tests/test_request_cb_kwargs.py E501
- tests/test_responsetypes.py E501
- tests/test_robotstxt_interface.py E501 E501
- tests/test_scheduler.py E501
- tests/test_selector.py E501
- tests/test_spider.py E501
- tests/test_spidermiddleware.py E501
- tests/test_spidermiddleware_httperror.py E501
- tests/test_spidermiddleware_offsite.py E501
- tests/test_spidermiddleware_output_chain.py E501
- tests/test_spidermiddleware_referer.py E501
- tests/test_squeues.py E501
- tests/test_utils_asyncio.py E501
- tests/test_utils_conf.py E501
- tests/test_utils_curl.py E501
- tests/test_utils_datatypes.py E402 E501
- tests/test_utils_defer.py E501
- tests/test_utils_deprecate.py E501
- tests/test_utils_http.py E501
- tests/test_utils_iterators.py E501
- tests/test_utils_python.py E501
- tests/test_utils_reqser.py E501
- tests/test_utils_request.py E501
- tests/test_utils_response.py E501
- tests/test_utils_sitemap.py E501
- tests/test_utils_url.py E501 E501
- tests/test_webclient.py E501 E402
- tests/test_cmdline/__init__.py E501
- tests/test_settings/__init__.py E501
- tests/test_spiderloader/__init__.py E501
- tests/test_utils_misc/__init__.py E501
++ scrapy/utils/url.py F403 F405
++ tests/test_loader.py E741
++ tests/test_webclient.py E402
diff --cc tests/test_dupefilters.py
index ea0e664be,41a8d16bc..95a4fca0d
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@@ -177,10 -177,13 +177,14 @@@ class RFPDupeFilterTest(unittest.TestCa
dupefilter.log(r2, spider)
assert crawler.stats.get_value('dupefilter/filtered') == 2
- l.check_present(('scrapy.dupefilters', 'DEBUG',
- ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
- ' - no more duplicates will be shown'
- ' (see DUPEFILTER_DEBUG to show all duplicates)')))
+ log.check_present(
+ (
+ 'scrapy.dupefilters',
+ 'DEBUG',
- 'Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)'
++ 'Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more'
++ ' duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)'
+ )
+ )
dupefilter.close('finished')
@@@ -204,11 -206,19 +207,20 @@@
dupefilter.log(r2, spider)
assert crawler.stats.get_value('dupefilter/filtered') == 2
- l.check_present(('scrapy.dupefilters', 'DEBUG',
- ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
- ' (referer: None)')))
- l.check_present(('scrapy.dupefilters', 'DEBUG',
- ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
- ' (referer: http://scrapytest.org/INDEX.html)')))
+ log.check_present(
+ (
+ 'scrapy.dupefilters',
+ 'DEBUG',
+ 'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'
+ )
+ )
+ log.check_present(
+ (
+ 'scrapy.dupefilters',
+ 'DEBUG',
- 'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: http://scrapytest.org/INDEX.html)'
++ 'Filtered duplicate request: <GET http://scrapytest.org/index.html>'
++ ' (referer: http://scrapytest.org/INDEX.html)'
+ )
+ )
dupefilter.close('finished')
diff --cc tests/test_feedexport.py
index b41bf9794,cbc81bc35..8eeb29b6d
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@@ -715,16 -715,12 +715,15 @@@ class FeedExportTest(unittest.TestCase)
@defer.inlineCallbacks
def test_export_encoding(self):
items = [dict({'foo': u'Test\xd6'})]
- header = ['foo']
formats = {
- 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
- 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
- 'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
- 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
+ 'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
+ 'jsonlines': '{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
+ 'xml': (
+ '<?xml version="1.0" encoding="utf-8"?>\n'
+ '<items><item><foo>Test\xd6</foo></item></items>'
+ ).encode('utf-8'),
+ 'csv': 'foo\r\nTest\xd6\r\n'.encode('utf-8'),
}
for fmt, expected in formats.items():
diff --cc tests/test_linkextractors.py
index ed5f2560a,2a2650480..8d4538eed
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@@ -421,10 -419,8 +422,10 @@@ class Base
[
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
- Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='',
- nofollow=True),
- Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='',
- nofollow=False),
- Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
- Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
++ Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
++ fragment='', nofollow=True),
++ Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
++ fragment='', nofollow=False),
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
]
)
@@@ -437,10 -433,8 +438,10 @@@
[
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
- Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='',
- nofollow=True),
- Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='',
- nofollow=False),
- Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
- Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
++ Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
++ fragment='', nofollow=True),
++ Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
++ fragment='', nofollow=False),
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
]
)