Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' into flake8-max-line-length
Eugenio Lacuesta01d73dd3f017
diff --cc pytest.iniindex c267573b1,de0bccbf1..47f34e8f1--- a/pytest.ini+++ b/pytest.ini@@@ -28,137 -27,225 +28,135 @@@ flake8-ignorescrapy/core/downloader/handlers/http.py F401scrapy/http/__init__.py F401# Issues pending a review:- # extras- extras/qps-bench-server.py E501- extras/qpsclient.py E501 E501# scrapy/commands- scrapy/commands/__init__.py E128 E501- scrapy/commands/check.py E501- scrapy/commands/crawl.py E501- scrapy/commands/edit.py E501- scrapy/commands/fetch.py E401 E501 E128 E731- scrapy/commands/genspider.py E128 E501- scrapy/commands/parse.py E128 E501 E731- scrapy/commands/runspider.py E501+ scrapy/commands/__init__.py E128+ scrapy/commands/fetch.py E401 E128 E731+ scrapy/commands/genspider.py E128+ scrapy/commands/parse.py E128 E731scrapy/commands/settings.py E128- scrapy/commands/shell.py E128 E501- scrapy/commands/startproject.py E127 E501 E128- scrapy/commands/version.py E501 E128+ scrapy/commands/shell.py E128+ scrapy/commands/startproject.py E127 E128+ scrapy/commands/version.py E128# scrapy/contracts- scrapy/contracts/__init__.py E501 W504+ scrapy/contracts/__init__.py W504scrapy/contracts/default.py E128# scrapy/core- scrapy/core/engine.py E501 E128 E127- scrapy/core/scheduler.py E501- scrapy/core/scraper.py E501 E128 W504- scrapy/core/spidermw.py E501 E731 E126- scrapy/core/downloader/__init__.py E501- scrapy/core/downloader/contextfactory.py E501 E128 E126- scrapy/core/downloader/middleware.py E501- scrapy/core/downloader/tls.py E501- scrapy/core/downloader/webclient.py E731 E501 E128 E126- scrapy/core/downloader/handlers/__init__.py E501- scrapy/core/downloader/handlers/ftp.py E501 E128 E127- scrapy/core/downloader/handlers/http10.py E501- scrapy/core/downloader/handlers/http11.py E501- scrapy/core/downloader/handlers/s3.py E501 E128 E126+ scrapy/core/engine.py E128 E127+ scrapy/core/scraper.py E128 W504+ scrapy/core/spidermw.py E731 E126+ scrapy/core/downloader/contextfactory.py E128 E126- scrapy/core/downloader/tls.py E241+ scrapy/core/downloader/webclient.py E731 E128 E126+ scrapy/core/downloader/handlers/ftp.py E128 E127+ scrapy/core/downloader/handlers/s3.py E128 E126# scrapy/downloadermiddlewares- scrapy/downloadermiddlewares/ajaxcrawl.py E501- scrapy/downloadermiddlewares/decompression.py E501- scrapy/downloadermiddlewares/defaultheaders.py E501- scrapy/downloadermiddlewares/httpcache.py E501 E126- scrapy/downloadermiddlewares/httpcompression.py E501 E128- scrapy/downloadermiddlewares/httpproxy.py E501- scrapy/downloadermiddlewares/redirect.py E501 W504- scrapy/downloadermiddlewares/retry.py E501 E126- scrapy/downloadermiddlewares/robotstxt.py E501- scrapy/downloadermiddlewares/stats.py E501+ scrapy/downloadermiddlewares/httpcache.py E126+ scrapy/downloadermiddlewares/httpcompression.py E128+ scrapy/downloadermiddlewares/redirect.py W504+ scrapy/downloadermiddlewares/retry.py E126# scrapy/extensions- scrapy/extensions/closespider.py E501 E128 E123- scrapy/extensions/corestats.py E501- scrapy/extensions/feedexport.py E128 E501- scrapy/extensions/httpcache.py E128 E501- scrapy/extensions/memdebug.py E501- scrapy/extensions/spiderstate.py E501- scrapy/extensions/telnet.py E501 W504- scrapy/extensions/throttle.py E501+ scrapy/extensions/closespider.py E128 E123+ scrapy/extensions/feedexport.py E128+ scrapy/extensions/httpcache.py E128+ scrapy/extensions/telnet.py W504# scrapy/http- scrapy/http/common.py E501- scrapy/http/cookies.py E501- scrapy/http/request/__init__.py E501- scrapy/http/request/form.py E501 E123- scrapy/http/request/json_request.py E501- scrapy/http/response/__init__.py E501 E128- scrapy/http/response/text.py E501 E128 E124+ scrapy/http/request/form.py E123+ scrapy/http/response/__init__.py E128+ scrapy/http/response/text.py E128 E124# scrapy/linkextractors- scrapy/linkextractors/__init__.py E731 E501 E402 W504- scrapy/linkextractors/lxmlhtml.py E501 E731+ scrapy/linkextractors/__init__.py E731 E402 W504+ scrapy/linkextractors/lxmlhtml.py E731# scrapy/loader- scrapy/loader/__init__.py E501 E128- scrapy/loader/processors.py E501+ scrapy/loader/__init__.py E128# scrapy/pipelines- scrapy/pipelines/files.py E116 E266- scrapy/pipelines/images.py E265- scrapy/pipelines/media.py E125 E266- scrapy/pipelines/__init__.py E501- scrapy/pipelines/files.py E116 E501- scrapy/pipelines/images.py E501- scrapy/pipelines/media.py E125 E501++++ scrapy/pipelines/files.py E116++ scrapy/pipelines/media.py E125# scrapy/selectorscrapy/selector/__init__.py F403- scrapy/selector/unified.py E501 E111+ scrapy/selector/unified.py E111# scrapy/settings- scrapy/settings/__init__.py E501- scrapy/settings/default_settings.py E501 E114 E116- scrapy/settings/deprecated.py E501+ scrapy/settings/default_settings.py E114 E116# scrapy/spidermiddlewares- scrapy/spidermiddlewares/httperror.py E501- scrapy/spidermiddlewares/offsite.py E501- scrapy/spidermiddlewares/referer.py E501 E129 W504- scrapy/spidermiddlewares/urllength.py E501+ scrapy/spidermiddlewares/referer.py E129 W504# scrapy/spiders- scrapy/spiders/__init__.py E501 E402- scrapy/spiders/crawl.py E501- scrapy/spiders/feed.py E501- scrapy/spiders/sitemap.py E501+ scrapy/spiders/__init__.py E402# scrapy/utils- scrapy/utils/asyncio.py E501- scrapy/utils/benchserver.py E501- scrapy/utils/conf.py E402 E501- scrapy/utils/datatypes.py E501- scrapy/utils/decorators.py E501- scrapy/utils/defer.py E501 E128- scrapy/utils/deprecate.py E128 E501 E127- scrapy/utils/gz.py E501 W504+ scrapy/utils/conf.py E402+ scrapy/utils/defer.py E128+ scrapy/utils/deprecate.py E128 E127+ scrapy/utils/gz.py W504scrapy/utils/http.py F403- scrapy/utils/httpobj.py E501- scrapy/utils/iterators.py E501- scrapy/utils/log.py E128 E501+ scrapy/utils/log.py E128scrapy/utils/markup.py F403- scrapy/utils/misc.py E501scrapy/utils/multipart.py F403- scrapy/utils/project.py E501- scrapy/utils/python.py E501- scrapy/utils/reactor.py E501- scrapy/utils/reqser.py E501- scrapy/utils/request.py E127 E501- scrapy/utils/response.py E501 E128- scrapy/utils/signal.py E501 E128- scrapy/utils/sitemap.py E501- scrapy/utils/spider.py E501- scrapy/utils/ssl.py E501- scrapy/utils/test.py E501- scrapy/utils/url.py E501 F403 E128 F405+ scrapy/utils/request.py E127+ scrapy/utils/response.py E128+ scrapy/utils/signal.py E128+ scrapy/utils/url.py F403 E128 F405# scrapy- scrapy/__init__.py E402 E501- scrapy/cmdline.py E501- scrapy/crawler.py E501- scrapy/dupefilters.py E501- scrapy/exceptions.py E501- scrapy/exporters.py E501- scrapy/interfaces.py E501- scrapy/item.py E501 E128- scrapy/link.py E501- scrapy/logformatter.py E501- scrapy/mail.py E402 E128 E501- scrapy/middleware.py E128 E501- scrapy/pqueues.py E501- scrapy/resolver.py E501- scrapy/responsetypes.py E128 E501- scrapy/robotstxt.py E501- scrapy/shell.py E501- scrapy/signalmanager.py E501- scrapy/spiderloader.py F841 E501 E126+ scrapy/__init__.py E402- scrapy/dupefilters.py E202+ scrapy/item.py E128+ scrapy/mail.py E402 E128+ scrapy/middleware.py E128+ scrapy/responsetypes.py E128+ scrapy/spiderloader.py F841 E126scrapy/squeues.py E128- scrapy/statscollectors.py E501# tests- tests/__init__.py E402 E501- tests/mockserver.py E401 E501 E126 E123+ tests/__init__.py E402+ tests/mockserver.py E401 E126 E123tests/pipelines.py F841- tests/spiders.py E501 E127- tests/test_closespider.py E501 E127- tests/test_command_fetch.py E501- tests/test_command_parse.py E501 E128- tests/test_command_shell.py E501 E128- tests/test_commands.py E128 E501- tests/test_contracts.py E501 E128- tests/test_crawl.py E501 E741- tests/test_crawler.py F841 E501- tests/test_dependencies.py F841 E501- tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123- tests/test_downloadermiddleware.py E501- tests/test_downloadermiddleware_ajaxcrawlable.py E501- tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E126+ tests/spiders.py E127+ tests/test_closespider.py E127+ tests/test_command_parse.py E128+ tests/test_command_shell.py E128+ tests/test_commands.py E128+ tests/test_contracts.py E128- tests/test_crawl.py E741 E265++ tests/test_crawl.py E741+ tests/test_crawler.py F841+ tests/test_dependencies.py F841- tests/test_downloader_handlers.py E124 E127 E128 E265 E126 E123- tests/test_downloadermiddleware_cookies.py E731 E741 E128 E265 E126++ tests/test_downloader_handlers.py E124 E127 E128 E126 E123++ tests/test_downloadermiddleware_cookies.py E731 E741 E128 E126tests/test_downloadermiddleware_decompression.py E127- tests/test_downloadermiddleware_defaultheaders.py E501- tests/test_downloadermiddleware_downloadtimeout.py E501- tests/test_downloadermiddleware_httpcache.py E501- tests/test_downloadermiddleware_httpcompression.py E501 E126 E123- tests/test_downloadermiddleware_httpproxy.py E501 E128- tests/test_downloadermiddleware_redirect.py E501 E128 E127- tests/test_downloadermiddleware_retry.py E501 E128 E126- tests/test_downloadermiddleware_robotstxt.py E501- tests/test_downloadermiddleware_stats.py E501- tests/test_dupefilters.py E501 E741 E128 E124- tests/test_engine.py E401 E501 E128- tests/test_exporters.py E501 E731 E128 E124+ tests/test_downloadermiddleware_httpcompression.py E126 E123+ tests/test_downloadermiddleware_httpproxy.py E128+ tests/test_downloadermiddleware_redirect.py E128 E127+ tests/test_downloadermiddleware_retry.py E128 E126+ tests/test_dupefilters.py E741 E128 E124+ tests/test_engine.py E401 E128+ tests/test_exporters.py E731 E128 E124tests/test_extension_telnet.py F841- tests/test_feedexport.py F841 E241- tests/test_feedexport.py E501 F841- tests/test_http_cookies.py E501- tests/test_http_headers.py E501- tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123- tests/test_http_response.py E501 E128++ tests/test_feedexport.py F841+ tests/test_http_request.py E402 E127 E128 E128 E126 E123- tests/test_http_response.py E128 E265++ tests/test_http_response.py E128tests/test_item.py E128 F841- tests/test_link.py E501- tests/test_linkextractors.py E501 E128 E124- tests/test_loader.py E501 E731 E741 E128 E117- tests/test_logformatter.py E128 E501 E122- tests/test_mail.py E128 E501- tests/test_middleware.py E501 E128- tests/test_pipeline_crawl.py E501 E128 E126- tests/test_pipeline_files.py E501- tests/test_pipeline_images.py F841 E501- tests/test_pipeline_media.py E501 E741 E731 E128- tests/test_proxy_connect.py E501 E741- tests/test_request_cb_kwargs.py E501- tests/test_responsetypes.py E501- tests/test_robotstxt_interface.py E501 E501- tests/test_scheduler.py E501 E126 E123- tests/test_selector.py E501 E127- tests/test_spider.py E501- tests/test_spidermiddleware.py E501- tests/test_spidermiddleware_httperror.py E128 E501 E127 E121- tests/test_spidermiddleware_offsite.py E501 E128 E111- tests/test_spidermiddleware_output_chain.py E501- tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121- tests/test_squeues.py E501 E741- tests/test_utils_asyncio.py E501- tests/test_utils_conf.py E501 E128- tests/test_utils_curl.py E501- tests/test_utils_datatypes.py E402 E501- tests/test_utils_defer.py E501 F841- tests/test_utils_deprecate.py F841 E501- tests/test_utils_http.py E501 E128 W504- tests/test_utils_iterators.py E501 E128 E129+ tests/test_linkextractors.py E128 E124- tests/test_loader.py E731 E741 E128 E117 E241++ tests/test_loader.py E731 E741 E128 E117+ tests/test_logformatter.py E128 E122+ tests/test_mail.py E128+ tests/test_middleware.py E128+ tests/test_pipeline_crawl.py E128 E126+ tests/test_pipeline_images.py F841+ tests/test_pipeline_media.py E741 E731 E128+ tests/test_proxy_connect.py E741+ tests/test_scheduler.py E126 E123+ tests/test_selector.py E127+ tests/test_spidermiddleware_httperror.py E128 E127 E121+ tests/test_spidermiddleware_offsite.py E128 E111- tests/test_spidermiddleware_referer.py F841 E125 E201 E124 E241 E121++ tests/test_spidermiddleware_referer.py F841 E125 E124 E121+ tests/test_squeues.py E741+ tests/test_utils_conf.py E128+ tests/test_utils_datatypes.py E402+ tests/test_utils_defer.py F841+ tests/test_utils_deprecate.py F841+ tests/test_utils_http.py E128 W504- tests/test_utils_iterators.py E128 E129 E241++ tests/test_utils_iterators.py E128 E129tests/test_utils_log.py E741- tests/test_utils_python.py E501 E731- tests/test_utils_reqser.py E501 E128- tests/test_utils_request.py E501 E128- tests/test_utils_response.py E501+ tests/test_utils_python.py E731+ tests/test_utils_reqser.py E128+ tests/test_utils_request.py E128tests/test_utils_signal.py E741 F841 E731- tests/test_utils_sitemap.py E128 E501 E124- tests/test_utils_url.py E501 E127 E125 E501 E126 E123- tests/test_webclient.py E501 E128 E122 E402 E123 E126- tests/test_cmdline/__init__.py E501- tests/test_settings/__init__.py E501 E128- tests/test_spiderloader/__init__.py E128 E501- tests/test_utils_misc/__init__.py E501+ tests/test_utils_sitemap.py E128 E124- tests/test_utils_url.py E127 E125 E241 E126 E123- tests/test_webclient.py E128 E122 E402 E241 E123 E126++ tests/test_utils_url.py E127 E125 E126 E123++ tests/test_webclient.py E128 E122 E402 E123 E126+ tests/test_settings/__init__.py E128+ tests/test_spiderloader/__init__.py E128diff --cc tests/test_spidermiddleware_referer.pyindex 3974f7884,742adc64f..6291d1db8--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@@ -119,36 -119,28 +119,36 @@@ class MixinSameOrigin('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),- ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),+ (+ 'http://example.com:8888/page.html',+ 'http://example.com:8888/not-page.html',+ b'http://example.com:8888/page.html',+ ),# Different host: do NOT send referrer- ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None),- ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None),- ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None),+ ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None),+ ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None),+ ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None),# Different port: do NOT send referrer- ('https://example.com:444/page.html', 'https://example.com/not-page.html', None),- ('http://example.com:81/page.html', 'http://example.com/not-page.html', None),- ('http://example.com/page.html', 'http://example.com:81/not-page.html', None),+ ('https://example.com:444/page.html', 'https://example.com/not-page.html', None),+ ('http://example.com:81/page.html', 'http://example.com/not-page.html', None),+ ('http://example.com/page.html', 'http://example.com:81/not-page.html', None),# Different protocols: do NOT send refferer- ('https://example.com/page.html', 'http://example.com/not-page.html', None),- ('https://example.com/page.html', 'http://not.example.com/', None),- ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),- ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None),- ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),+ ('https://example.com/page.html', 'http://example.com/not-page.html', None),+ ('https://example.com/page.html', 'http://not.example.com/', None),+ ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),+ ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None),+ ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),# test for user/password stripping- ('https://user:[email protected]/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),('https://user:[email protected]/page.html', 'http://example.com/not-page.html', None),+ (+ 'https://user:[email protected]/page.html',+ 'https://example.com/not-page.html',+ b'https://example.com/page.html',+ ),]@@@ -192,42 -184,30 +192,42 @@@ class MixinOriginWhenCrossOrigin('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),- ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),+ (+ 'http://example.com:8888/page.html',+ 'http://example.com:8888/not-page.html',+ b'http://example.com:8888/page.html',+ ),# Different host: send origin as referrer- ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),- ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),- ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),+ ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),+ ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),+ ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),# exact match required- ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),+ ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),# Different port: send origin as referrer- ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),- ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),+ ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),+ ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),# Different protocols: send origin as referrer- ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'),- ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'),- ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),- ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),- ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),+ ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'),+ ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'),+ ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),+ ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),+ ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),# test for user/password stripping- ('https://user:[email protected]/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),+ (+ 'https://user:[email protected]/page.html',+ 'https://example5.com/not-page.html',+ b'https://example5.com/page.html',+ ),# TLS to non-TLS downgrade: send origin- ('https://user:[email protected]/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'),+ (+ 'https://user:[email protected]/page.html',+ 'http://example5.com/not-page.html',+ b'https://example5.com/',+ ),]@@@ -239,86 -219,70 +239,86 @@@ class MixinStrictOriginWhenCrossOrigin('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),- ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),+ (+ 'http://example.com:8888/page.html',+ 'http://example.com:8888/not-page.html',+ b'http://example.com:8888/page.html',+ ),# Different host: send origin as referrer- ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),- ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),- ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),+ ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),+ ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),+ ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),# exact match required- ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),+ ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),# Different port: send origin as referrer- ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),- ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),+ ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),+ ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),# downgrade- ('https://example4.com/page.html', 'http://example4.com/not-page.html', None),- ('https://example4.com/page.html', 'http://not.example4.com/', None),+ ('https://example4.com/page.html', 'http://example4.com/not-page.html', None),+ ('https://example4.com/page.html', 'http://not.example4.com/', None),# non-TLS to non-TLS- ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),+ ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),# upgrade- ('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'),- ('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'),+ ('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'),+ ('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'),# Different protocols: send origin as referrer- ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),- ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),+ ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),+ ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),# test for user/password stripping- ('https://user:[email protected]/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),+ (+ 'https://user:[email protected]/page.html',+ 'https://example5.com/not-page.html',+ b'https://example5.com/page.html',+ ),# TLS to non-TLS downgrade: send nothing- ('https://user:[email protected]/page.html', 'http://example5.com/not-page.html', None),+ ('https://user:[email protected]/page.html', 'http://example5.com/not-page.html', None),]class MixinUnsafeUrl:scenarii = [# TLS to TLS: send referrer- ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'),- ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'),- ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),- ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),- ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'),- ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'),+ ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'),+ ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'),+ ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),+ ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),+ ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'),+ ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'),# TLS to non-TLS: send referrer (yes, it's unsafe)- ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'),- ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'),- ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'),+ ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'),+ ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'),+ ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'),# non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)- ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'),- ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'),- ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'),- ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'),- ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'),- ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'),- ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'),- ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'),+ ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'),+ ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'),+ ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'),+ ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'),+ ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'),+ ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'),+ ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'),+ ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'),# test for user/password stripping- ('http://user:[email protected]/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'),- ('https://user:[email protected]/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'),+ (+ 'http://user:[email protected]/page.html',+ 'https://not.example4.com/',+ b'http://example4.com/page.html',+ ),+ (+ 'https://user:[email protected]/page.html',+ 'http://scrapy.org/',+ b'https://example4.com/page.html',+ ),]