Scrapy, a fast high-level web crawling & scraping framework for Python.
TST remove temporary files and folders
Mikhail Korobovb9928558b293
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 6333efceb..f9980f7b2 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -1,6 +1,7 @@import osimport siximport contextlib+import shutiltry:from unittest import mockexcept ImportError:@@ -84,11 +85,13 @@ class FileTestCase(unittest.TestCase):def setUp(self):self.tmpname = self.mktemp()- fd = open(self.tmpname + '^', 'w')- fd.write('0123456789')- fd.close()+ with open(self.tmpname + '^', 'w') as f:+ f.write('0123456789')self.download_request = FileDownloadHandler(Settings()).download_request+ def tearDown(self):+ os.unlink(self.tmpname + '^')+def test_download(self):def _test(response):self.assertEquals(response.url, request.url)@@ -134,10 +137,10 @@ class HttpTestCase(unittest.TestCase):certfile = 'keys/cert.pem'def setUp(self):- name = self.mktemp()- os.mkdir(name)- FilePath(name).child("file").setContent(b"0123456789")- r = static.File(name)+ self.tmpname = self.mktemp()+ os.mkdir(self.tmpname)+ FilePath(self.tmpname).child("file").setContent(b"0123456789")+ r = static.File(self.tmpname)r.putChild(b"redirect", util.Redirect(b"/file"))r.putChild(b"wait", ForeverTakingResource())r.putChild(b"hang-after-headers", ForeverTakingResource(write=True))@@ -165,6 +168,7 @@ class HttpTestCase(unittest.TestCase):yield self.port.stopListening()if hasattr(self.download_handler, 'close'):yield self.download_handler.close()+ shutil.rmtree(self.tmpname)def getURL(self, path):return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path)@@ -709,6 +713,9 @@ class FTPTestCase(unittest.TestCase):self.download_handler = FTPDownloadHandler(Settings())self.addCleanup(self.port.stopListening)+ def tearDown(self):+ shutil.rmtree(self.directory)+def _add_test_callbacks(self, deferred, callback=None, errback=None):def _clean(data):self.download_handler.client.transport.loseConnection()diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex e93d2bafb..2d137edf4 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -57,8 +57,11 @@ class FileFeedStorageTest(unittest.TestCase):file.write(b"content")yield storage.store(file)self.assertTrue(os.path.exists(path))- with open(path, 'rb') as fp:- self.assertEqual(fp.read(), b"content")+ try:+ with open(path, 'rb') as fp:+ self.assertEqual(fp.read(), b"content")+ finally:+ os.unlink(path)class FTPFeedStorageTest(unittest.TestCase):@@ -79,12 +82,15 @@ class FTPFeedStorageTest(unittest.TestCase):file.write(b"content")yield storage.store(file)self.assertTrue(os.path.exists(path))- with open(path, 'rb') as fp:- self.assertEqual(fp.read(), b"content")- # again, to check s3 objects are overwritten- yield storage.store(BytesIO(b"new content"))- with open(path, 'rb') as fp:- self.assertEqual(fp.read(), b"new content")+ try:+ with open(path, 'rb') as fp:+ self.assertEqual(fp.read(), b"content")+ # again, to check s3 objects are overwritten+ yield storage.store(BytesIO(b"new content"))+ with open(path, 'rb') as fp:+ self.assertEqual(fp.read(), b"new content")+ finally:+ os.unlink(path)class BlockingFeedStorageTest(unittest.TestCase):diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.pyindex d1d6debec..383fadfeb 100644--- a/tests/test_spiderstate.py+++ b/tests/test_spiderstate.py@@ -1,5 +1,6 @@import osfrom datetime import datetime+import shutilfrom twisted.trial import unittestfrom scrapy.extensions.spiderstate import SpiderState@@ -13,20 +14,23 @@ class SpiderStateTest(unittest.TestCase):def test_store_load(self):jobdir = self.mktemp()os.mkdir(jobdir)- spider = Spider(name='default')- dt = datetime.now()-- ss = SpiderState(jobdir)- ss.spider_opened(spider)- spider.state['one'] = 1- spider.state['dt'] = dt- ss.spider_closed(spider)-- spider2 = Spider(name='default')- ss2 = SpiderState(jobdir)- ss2.spider_opened(spider2)- self.assertEqual(spider.state, {'one': 1, 'dt': dt})- ss2.spider_closed(spider2)+ try:+ spider = Spider(name='default')+ dt = datetime.now()++ ss = SpiderState(jobdir)+ ss.spider_opened(spider)+ spider.state['one'] = 1+ spider.state['dt'] = dt+ ss.spider_closed(spider)++ spider2 = Spider(name='default')+ ss2 = SpiderState(jobdir)+ ss2.spider_opened(spider2)+ self.assertEqual(spider.state, {'one': 1, 'dt': dt})+ ss2.spider_closed(spider2)+ finally:+ shutil.rmtree(jobdir)def test_state_attribute(self):# state attribute must be present if jobdir is not set, to provide adiff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex 9b5beda4c..af531c81f 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -4,7 +4,7 @@ Tests borrowed from the twisted.web.client tests."""import osimport six-from six.moves.urllib.parse import urlparse+import shutilfrom twisted.trial import unittestfrom twisted.web import server, static, util, resource@@ -12,6 +12,7 @@ from twisted.internet import reactor, deferfrom twisted.test.proto_helpers import StringTransportfrom twisted.python.filepath import FilePathfrom twisted.protocols.policies import WrappingFactory+from twisted.internet.defer import inlineCallbacksfrom scrapy.core.downloader import webclient as clientfrom scrapy.http import Request, Headers@@ -229,10 +230,10 @@ class WebClientTestCase(unittest.TestCase):return reactor.listenTCP(0, site, interface="127.0.0.1")def setUp(self):- name = self.mktemp()- os.mkdir(name)- FilePath(name).child("file").setContent(b"0123456789")- r = static.File(name)+ self.tmpname = self.mktemp()+ os.mkdir(self.tmpname)+ FilePath(self.tmpname).child("file").setContent(b"0123456789")+ r = static.File(self.tmpname)r.putChild(b"redirect", util.Redirect(b"/file"))r.putChild(b"wait", ForeverTakingResource())r.putChild(b"error", ErrorResource())@@ -246,8 +247,10 @@ class WebClientTestCase(unittest.TestCase):self.port = self._listen(self.wrapper)self.portno = self.port.getHost().port+ @inlineCallbacksdef tearDown(self):- return self.port.stopListening()+ yield self.port.stopListening()+ shutil.rmtree(self.tmpname)def getURL(self, path):return "http://127.0.0.1:%d/%s" % (self.portno, path)@@ -266,7 +269,6 @@ class WebClientTestCase(unittest.TestCase):getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, to_bytes("www.example.com"))])-def test_getPage(self):"""L{client.getPage} returns a L{Deferred} which is called back with@@ -276,7 +278,6 @@ class WebClientTestCase(unittest.TestCase):d.addCallback(self.assertEquals, b"0123456789")return d-def test_getPageHead(self):"""L{client.getPage} returns a L{Deferred} which is called back with@@ -289,7 +290,6 @@ class WebClientTestCase(unittest.TestCase):_getPage("head").addCallback(self.assertEqual, b""),_getPage("HEAD").addCallback(self.assertEqual, b"")])-def test_timeoutNotTriggering(self):"""When a non-zero timeout is passed to L{getPage} and the page is@@ -301,7 +301,6 @@ class WebClientTestCase(unittest.TestCase):self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno))return d-def test_timeoutTriggering(self):"""When a non-zero timeout is passed to L{getPage} and that many