Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Move garbage_collect to scrapy.utils.python

Konstantin Lopuhin19ca986aa134
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index b546d030e..dc6b59fe0 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -4,7 +4,6 @@ import optparse
import cProfile
import inspect
import pkg_resources
-import gc
import scrapy
from scrapy.crawler import CrawlerProcess
@@ -12,6 +11,7 @@ from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
+from scrapy.utils.python import garbage_collect
from scrapy.settings.deprecated import check_deprecated_settings
def _iter_command_classes(module_name):
@@ -171,4 +171,4 @@ if __name__ == '__main__':
finally:
# Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
# on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
- gc.collect()
+ garbage_collect()
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 4c500abf4..d28d71bd3 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -1,6 +1,7 @@
"""
This module contains essential stuff that should've come with Python itself ;)
"""
+import gc
import os
import re
import inspect
@@ -8,6 +9,8 @@ import weakref
import errno
import six
from functools import partial, wraps
+import sys
+import time
from scrapy.utils.decorators import deprecated
@@ -355,3 +358,20 @@ def global_object_name(obj):
'scrapy.http.request.Request'
"""
return "%s.%s" % (obj.__module__, obj.__name__)
+
+
+if sys.platform.startswith('java'):
+ def garbage_collect():
+ # Some JVM GCs will execute finalizers in a different thread, meaning
+ # we need to wait for that to complete before we go on looking for the
+ # effects of that.
+ gc.collect()
+ time.sleep(0.1)
+elif hasattr(sys, "pypy_version_info"):
+ def garbage_collect():
+ # Collecting weakreferences can take two collections on PyPy.
+ gc.collect()
+ gc.collect()
+else:
+ def garbage_collect():
+ gc.collect()