Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

TST remove temporary files and folders

Mikhail Korobovb9928558b293
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 6333efceb..f9980f7b2 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,6 +1,7 @@
import os
import six
import contextlib
+import shutil
try:
from unittest import mock
except ImportError:
@@ -84,11 +85,13 @@ class FileTestCase(unittest.TestCase):
def setUp(self):
self.tmpname = self.mktemp()
- fd = open(self.tmpname + '^', 'w')
- fd.write('0123456789')
- fd.close()
+ with open(self.tmpname + '^', 'w') as f:
+ f.write('0123456789')
self.download_request = FileDownloadHandler(Settings()).download_request
+ def tearDown(self):
+ os.unlink(self.tmpname + '^')
+
def test_download(self):
def _test(response):
self.assertEquals(response.url, request.url)
@@ -134,10 +137,10 @@ class HttpTestCase(unittest.TestCase):
certfile = 'keys/cert.pem'
def setUp(self):
- name = self.mktemp()
- os.mkdir(name)
- FilePath(name).child("file").setContent(b"0123456789")
- r = static.File(name)
+ self.tmpname = self.mktemp()
+ os.mkdir(self.tmpname)
+ FilePath(self.tmpname).child("file").setContent(b"0123456789")
+ r = static.File(self.tmpname)
r.putChild(b"redirect", util.Redirect(b"/file"))
r.putChild(b"wait", ForeverTakingResource())
r.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
@@ -165,6 +168,7 @@ class HttpTestCase(unittest.TestCase):
yield self.port.stopListening()
if hasattr(self.download_handler, 'close'):
yield self.download_handler.close()
+ shutil.rmtree(self.tmpname)
def getURL(self, path):
return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path)
@@ -709,6 +713,9 @@ class FTPTestCase(unittest.TestCase):
self.download_handler = FTPDownloadHandler(Settings())
self.addCleanup(self.port.stopListening)
+ def tearDown(self):
+ shutil.rmtree(self.directory)
+
def _add_test_callbacks(self, deferred, callback=None, errback=None):
def _clean(data):
self.download_handler.client.transport.loseConnection()
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e93d2bafb..2d137edf4 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -57,8 +57,11 @@ class FileFeedStorageTest(unittest.TestCase):
file.write(b"content")
yield storage.store(file)
self.assertTrue(os.path.exists(path))
- with open(path, 'rb') as fp:
- self.assertEqual(fp.read(), b"content")
+ try:
+ with open(path, 'rb') as fp:
+ self.assertEqual(fp.read(), b"content")
+ finally:
+ os.unlink(path)
class FTPFeedStorageTest(unittest.TestCase):
@@ -79,12 +82,15 @@ class FTPFeedStorageTest(unittest.TestCase):
file.write(b"content")
yield storage.store(file)
self.assertTrue(os.path.exists(path))
- with open(path, 'rb') as fp:
- self.assertEqual(fp.read(), b"content")
- # again, to check s3 objects are overwritten
- yield storage.store(BytesIO(b"new content"))
- with open(path, 'rb') as fp:
- self.assertEqual(fp.read(), b"new content")
+ try:
+ with open(path, 'rb') as fp:
+ self.assertEqual(fp.read(), b"content")
+ # again, to check s3 objects are overwritten
+ yield storage.store(BytesIO(b"new content"))
+ with open(path, 'rb') as fp:
+ self.assertEqual(fp.read(), b"new content")
+ finally:
+ os.unlink(path)
class BlockingFeedStorageTest(unittest.TestCase):
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index d1d6debec..383fadfeb 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -1,5 +1,6 @@
import os
from datetime import datetime
+import shutil
from twisted.trial import unittest
from scrapy.extensions.spiderstate import SpiderState
@@ -13,20 +14,23 @@ class SpiderStateTest(unittest.TestCase):
def test_store_load(self):
jobdir = self.mktemp()
os.mkdir(jobdir)
- spider = Spider(name='default')
- dt = datetime.now()
-
- ss = SpiderState(jobdir)
- ss.spider_opened(spider)
- spider.state['one'] = 1
- spider.state['dt'] = dt
- ss.spider_closed(spider)
-
- spider2 = Spider(name='default')
- ss2 = SpiderState(jobdir)
- ss2.spider_opened(spider2)
- self.assertEqual(spider.state, {'one': 1, 'dt': dt})
- ss2.spider_closed(spider2)
+ try:
+ spider = Spider(name='default')
+ dt = datetime.now()
+
+ ss = SpiderState(jobdir)
+ ss.spider_opened(spider)
+ spider.state['one'] = 1
+ spider.state['dt'] = dt
+ ss.spider_closed(spider)
+
+ spider2 = Spider(name='default')
+ ss2 = SpiderState(jobdir)
+ ss2.spider_opened(spider2)
+ self.assertEqual(spider.state, {'one': 1, 'dt': dt})
+ ss2.spider_closed(spider2)
+ finally:
+ shutil.rmtree(jobdir)
def test_state_attribute(self):
# state attribute must be present if jobdir is not set, to provide a
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 9b5beda4c..af531c81f 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -4,7 +4,7 @@ Tests borrowed from the twisted.web.client tests.
"""
import os
import six
-from six.moves.urllib.parse import urlparse
+import shutil
from twisted.trial import unittest
from twisted.web import server, static, util, resource
@@ -12,6 +12,7 @@ from twisted.internet import reactor, defer
from twisted.test.proto_helpers import StringTransport
from twisted.python.filepath import FilePath
from twisted.protocols.policies import WrappingFactory
+from twisted.internet.defer import inlineCallbacks
from scrapy.core.downloader import webclient as client
from scrapy.http import Request, Headers
@@ -229,10 +230,10 @@ class WebClientTestCase(unittest.TestCase):
return reactor.listenTCP(0, site, interface="127.0.0.1")
def setUp(self):
- name = self.mktemp()
- os.mkdir(name)
- FilePath(name).child("file").setContent(b"0123456789")
- r = static.File(name)
+ self.tmpname = self.mktemp()
+ os.mkdir(self.tmpname)
+ FilePath(self.tmpname).child("file").setContent(b"0123456789")
+ r = static.File(self.tmpname)
r.putChild(b"redirect", util.Redirect(b"/file"))
r.putChild(b"wait", ForeverTakingResource())
r.putChild(b"error", ErrorResource())
@@ -246,8 +247,10 @@ class WebClientTestCase(unittest.TestCase):
self.port = self._listen(self.wrapper)
self.portno = self.port.getHost().port
+ @inlineCallbacks
def tearDown(self):
- return self.port.stopListening()
+ yield self.port.stopListening()
+ shutil.rmtree(self.tmpname)
def getURL(self, path):
return "http://127.0.0.1:%d/%s" % (self.portno, path)
@@ -266,7 +269,6 @@ class WebClientTestCase(unittest.TestCase):
getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(
self.assertEquals, to_bytes("www.example.com"))])
-
def test_getPage(self):
"""
L{client.getPage} returns a L{Deferred} which is called back with
@@ -276,7 +278,6 @@ class WebClientTestCase(unittest.TestCase):
d.addCallback(self.assertEquals, b"0123456789")
return d
-
def test_getPageHead(self):
"""
L{client.getPage} returns a L{Deferred} which is called back with
@@ -289,7 +290,6 @@ class WebClientTestCase(unittest.TestCase):
_getPage("head").addCallback(self.assertEqual, b""),
_getPage("HEAD").addCallback(self.assertEqual, b"")])
-
def test_timeoutNotTriggering(self):
"""
When a non-zero timeout is passed to L{getPage} and the page is
@@ -301,7 +301,6 @@ class WebClientTestCase(unittest.TestCase):
self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno))
return d
-
def test_timeoutTriggering(self):
"""
When a non-zero timeout is passed to L{getPage} and that many