Scrapy, a fast high-level web crawling & scraping framework for Python.
Move garbage_collect to scrapy.utils.python
Konstantin Lopuhin19ca986aa134
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.pyindex b546d030e..dc6b59fe0 100644--- a/scrapy/cmdline.py+++ b/scrapy/cmdline.py@@ -4,7 +4,6 @@ import optparseimport cProfileimport inspectimport pkg_resources-import gcimport scrapyfrom scrapy.crawler import CrawlerProcess@@ -12,6 +11,7 @@ from scrapy.commands import ScrapyCommandfrom scrapy.exceptions import UsageErrorfrom scrapy.utils.misc import walk_modulesfrom scrapy.utils.project import inside_project, get_project_settings+from scrapy.utils.python import garbage_collectfrom scrapy.settings.deprecated import check_deprecated_settingsdef _iter_command_classes(module_name):@@ -171,4 +171,4 @@ if __name__ == '__main__':finally:# Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()# on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies- gc.collect()+ garbage_collect()diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 4c500abf4..d28d71bd3 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -1,6 +1,7 @@"""This module contains essential stuff that should've come with Python itself ;)"""+import gcimport osimport reimport inspect@@ -8,6 +9,8 @@ import weakrefimport errnoimport sixfrom functools import partial, wraps+import sys+import timefrom scrapy.utils.decorators import deprecated@@ -355,3 +358,20 @@ def global_object_name(obj):'scrapy.http.request.Request'"""return "%s.%s" % (obj.__module__, obj.__name__)+++if sys.platform.startswith('java'):+ def garbage_collect():+ # Some JVM GCs will execute finalizers in a different thread, meaning+ # we need to wait for that to complete before we go on looking for the+ # effects of that.+ gc.collect()+ time.sleep(0.1)+elif hasattr(sys, "pypy_version_info"):+ def garbage_collect():+ # Collecting weakreferences can take two collections on PyPy.+ gc.collect()+ gc.collect()+else:+ def garbage_collect():+ gc.collect()