Scrapy, a fast high-level web crawling & scraping framework for Python.
Let framework handle only HTTP redirects by default for fetch and shell commands
Paul Tremberth778bed07bf77
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.pyindex a157b19f8..6fe6d73b9 100644--- a/scrapy/commands/fetch.py+++ b/scrapy/commands/fetch.py@@ -5,6 +5,7 @@ from w3lib.url import is_urlfrom scrapy.commands import ScrapyCommandfrom scrapy.http import Requestfrom scrapy.exceptions import UsageError+from scrapy.utils.datatypes import SequenceExcludefrom scrapy.utils.spider import spidercls_for_request, DefaultSpiderclass Command(ScrapyCommand):@@ -27,8 +28,8 @@ class Command(ScrapyCommand):help="use this spider")parser.add_option("--headers", dest="headers", action="store_true", \help="print response HTTP headers instead of body")- parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \- default=False, help="do not handle status codes like redirects and print response as-is")+ parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \+ default=False, help="do not handle HTTP 3xx status codes and print response as-is")def _print_headers(self, headers, prefix):for key, values in headers.items():@@ -52,7 +53,11 @@ class Command(ScrapyCommand):raise UsageError()cb = lambda x: self._print_response(x, opts)request = Request(args[0], callback=cb, dont_filter=True)- if opts.no_status_aware:+ # by default, let the framework handle redirects,+ # i.e. command handles all codes expect 3xx+ if not opts.no_redirect:+ request.meta['handle_httpstatus_list'] = SequenceExclude(six.moves.range(300, 400))+ else:request.meta['handle_httpstatus_all'] = Truespidercls = DefaultSpiderdiff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.pyindex bc0203d89..40a58d94a 100644--- a/scrapy/commands/shell.py+++ b/scrapy/commands/shell.py@@ -36,8 +36,8 @@ class Command(ScrapyCommand):help="evaluate the code in the shell, print the result and exit")parser.add_option("--spider", dest="spider",help="use this spider")- parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \- default=False, help="do not transparently handle status codes like redirects")+ parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \+ default=False, help="do not handle HTTP 3xx status codes and print response as-is")def update_vars(self, vars):"""You can use this function to update the Scrapy objects that will be@@ -70,7 +70,7 @@ class Command(ScrapyCommand):self._start_crawler_thread()shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)- shell.start(url=url, handle_statuses=opts.no_status_aware)+ shell.start(url=url, redirect=not opts.no_redirect)def _start_crawler_thread(self):t = Thread(target=self.crawler_process.start,diff --git a/scrapy/shell.py b/scrapy/shell.pyindex 966003f17..6c78722be 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -7,6 +7,7 @@ from __future__ import print_functionimport osimport signal+from six.moves import rangeimport warningsfrom twisted.internet import reactor, threads, defer@@ -20,6 +21,7 @@ from scrapy.item import BaseItemfrom scrapy.settings import Settingsfrom scrapy.spiders import Spiderfrom scrapy.utils.console import start_python_console+from scrapy.utils.datatypes import SequenceExcludefrom scrapy.utils.misc import load_objectfrom scrapy.utils.response import open_in_browserfrom scrapy.utils.conf import get_config@@ -40,11 +42,11 @@ class Shell(object):self.code = codeself.vars = {}- def start(self, url=None, request=None, response=None, spider=None, handle_statuses=True):+ def start(self, url=None, request=None, response=None, spider=None, redirect=True):# disable accidental Ctrl-C key press from shutting down the enginesignal.signal(signal.SIGINT, signal.SIG_IGN)if url:- self.fetch(url, spider, handle_statuses=handle_statuses)+ self.fetch(url, spider, redirect=redirect)elif request:self.fetch(request, spider)elif response:@@ -98,13 +100,15 @@ class Shell(object):self.spider = spiderreturn spider- def fetch(self, request_or_url, spider=None, handle_statuses=False, **kwargs):+ def fetch(self, request_or_url, spider=None, redirect=True, **kwargs):if isinstance(request_or_url, Request):request = request_or_urlelse:url = any_to_uri(request_or_url)request = Request(url, dont_filter=True, **kwargs)- if handle_statuses:+ if redirect:+ request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400))+ else:request.meta['handle_httpstatus_all'] = Trueresponse = Nonetry:diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex d04b43176..e516185bd 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -304,3 +304,13 @@ class LocalCache(OrderedDict):while len(self) >= self.limit:self.popitem(last=False)super(LocalCache, self).__setitem__(key, value)+++class SequenceExclude(object):+ """Object to test if an item is NOT within some sequence."""++ def __init__(self, seq):+ self.seq = seq++ def __contains__(self, item):+ return item not in self.seqdiff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.pyindex 45d03a129..3fa3ed930 100644--- a/tests/test_command_fetch.py+++ b/tests/test_command_fetch.py@@ -21,7 +21,7 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):@defer.inlineCallbacksdef test_redirect_disabled(self):- _, out, err = yield self.execute(['--no-status-aware', self.url('/redirect-no-meta-refresh')])+ _, out, err = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh')])err = err.strip()self.assertIn(b'downloader/response_status_count/302', err, err)self.assertNotIn(b'downloader/response_status_count/200', err, err)