Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Handle redirects transparently by default in shell and fetch

Paul Tremberth35b655d2f84d
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index f09a873c1..a157b19f8 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -27,6 +27,8 @@ class Command(ScrapyCommand):
help="use this spider")
parser.add_option("--headers", dest="headers", action="store_true", \
help="print response HTTP headers instead of body")
+ parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \
+ default=False, help="do not handle status codes like redirects and print response as-is")
def _print_headers(self, headers, prefix):
for key, values in headers.items():
@@ -50,7 +52,8 @@ class Command(ScrapyCommand):
raise UsageError()
cb = lambda x: self._print_response(x, opts)
request = Request(args[0], callback=cb, dont_filter=True)
- request.meta['handle_httpstatus_all'] = True
+ if opts.no_status_aware:
+ request.meta['handle_httpstatus_all'] = True
spidercls = DefaultSpider
spider_loader = self.crawler_process.spider_loader
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 7be7f7256..bc0203d89 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -36,6 +36,8 @@ class Command(ScrapyCommand):
help="evaluate the code in the shell, print the result and exit")
parser.add_option("--spider", dest="spider",
help="use this spider")
+ parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \
+ default=False, help="do not transparently handle status codes like redirects")
def update_vars(self, vars):
"""You can use this function to update the Scrapy objects that will be
@@ -68,7 +70,7 @@ class Command(ScrapyCommand):
self._start_crawler_thread()
shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
- shell.start(url=url)
+ shell.start(url=url, handle_statuses=opts.no_status_aware)
def _start_crawler_thread(self):
t = Thread(target=self.crawler_process.start,
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 183ee1f70..966003f17 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -40,11 +40,11 @@ class Shell(object):
self.code = code
self.vars = {}
- def start(self, url=None, request=None, response=None, spider=None):
+ def start(self, url=None, request=None, response=None, spider=None, handle_statuses=True):
# disable accidental Ctrl-C key press from shutting down the engine
signal.signal(signal.SIGINT, signal.SIG_IGN)
if url:
- self.fetch(url, spider)
+ self.fetch(url, spider, handle_statuses=handle_statuses)
elif request:
self.fetch(request, spider)
elif response:
@@ -98,14 +98,14 @@ class Shell(object):
self.spider = spider
return spider
- def fetch(self, request_or_url, spider=None):
+ def fetch(self, request_or_url, spider=None, handle_statuses=False, **kwargs):
if isinstance(request_or_url, Request):
request = request_or_url
- url = request.url
else:
url = any_to_uri(request_or_url)
- request = Request(url, dont_filter=True)
- request.meta['handle_httpstatus_all'] = True
+ request = Request(url, dont_filter=True, **kwargs)
+ if handle_statuses:
+ request.meta['handle_httpstatus_all'] = True
response = None
try:
response, spider = threads.blockingCallFromThread(