Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

function to get unique file queues for any type of base queue

Vostretsov Nikitaa25cf5c82f99
diff --git a/scrapy/core/queues.py b/scrapy/core/queues.py
new file mode 100644
index 000000000..96d582fc7
--- /dev/null
+++ b/scrapy/core/queues.py
@@ -0,0 +1,15 @@
+import uuid
+import os.path
+
+
+def unique_files_queue(queue_class):
+
+ class UniqueFilesQueue(queue_class):
+ def __init__(self, path):
+ path = path + "-" + uuid.uuid4().hex
+ while os.path.exists(path):
+ path = path + "-" + uuid.uuid4().hex
+
+ super().__init__(path)
+
+ return UniqueFilesQueue