Scrapy, a fast high-level web crawling & scraping framework for Python.
Handle redirects transparently by default in shell and fetch
Paul Tremberth35b655d2f84d
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.pyindex f09a873c1..a157b19f8 100644--- a/scrapy/commands/fetch.py+++ b/scrapy/commands/fetch.py@@ -27,6 +27,8 @@ class Command(ScrapyCommand):help="use this spider")parser.add_option("--headers", dest="headers", action="store_true", \help="print response HTTP headers instead of body")+ parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \+ default=False, help="do not handle status codes like redirects and print response as-is")def _print_headers(self, headers, prefix):for key, values in headers.items():@@ -50,7 +52,8 @@ class Command(ScrapyCommand):raise UsageError()cb = lambda x: self._print_response(x, opts)request = Request(args[0], callback=cb, dont_filter=True)- request.meta['handle_httpstatus_all'] = True+ if opts.no_status_aware:+ request.meta['handle_httpstatus_all'] = Truespidercls = DefaultSpiderspider_loader = self.crawler_process.spider_loaderdiff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.pyindex 7be7f7256..bc0203d89 100644--- a/scrapy/commands/shell.py+++ b/scrapy/commands/shell.py@@ -36,6 +36,8 @@ class Command(ScrapyCommand):help="evaluate the code in the shell, print the result and exit")parser.add_option("--spider", dest="spider",help="use this spider")+ parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \+ default=False, help="do not transparently handle status codes like redirects")def update_vars(self, vars):"""You can use this function to update the Scrapy objects that will be@@ -68,7 +70,7 @@ class Command(ScrapyCommand):self._start_crawler_thread()shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)- shell.start(url=url)+ shell.start(url=url, handle_statuses=opts.no_status_aware)def _start_crawler_thread(self):t = Thread(target=self.crawler_process.start,diff --git a/scrapy/shell.py b/scrapy/shell.pyindex 183ee1f70..966003f17 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -40,11 +40,11 @@ class Shell(object):self.code = codeself.vars = {}- def start(self, url=None, request=None, response=None, spider=None):+ def start(self, url=None, request=None, response=None, spider=None, handle_statuses=True):# disable accidental Ctrl-C key press from shutting down the enginesignal.signal(signal.SIGINT, signal.SIG_IGN)if url:- self.fetch(url, spider)+ self.fetch(url, spider, handle_statuses=handle_statuses)elif request:self.fetch(request, spider)elif response:@@ -98,14 +98,14 @@ class Shell(object):self.spider = spiderreturn spider- def fetch(self, request_or_url, spider=None):+ def fetch(self, request_or_url, spider=None, handle_statuses=False, **kwargs):if isinstance(request_or_url, Request):request = request_or_url- url = request.urlelse:url = any_to_uri(request_or_url)- request = Request(url, dont_filter=True)- request.meta['handle_httpstatus_all'] = True+ request = Request(url, dont_filter=True, **kwargs)+ if handle_statuses:+ request.meta['handle_httpstatus_all'] = Trueresponse = Nonetry:response, spider = threads.blockingCallFromThread(