🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(storage): possible threading issue with recursion + optimizations
Karim shoair7f11b6f59b8a
diff --git a/scrapling/core/storage_adaptors.py b/scrapling/core/storage_adaptors.pyindex 9d2e5c2..5821707 100644--- a/scrapling/core/storage_adaptors.py+++ b/scrapling/core/storage_adaptors.py@@ -1,14 +1,15 @@-import sqlite3-import threading+from sqlite3 import connect as db_connect+from threading import RLockfrom abc import ABC, abstractmethodfrom hashlib import sha256+from functools import lru_cache-import orjson-from lxml import html+from lxml.html import HtmlElement+from orjson import dumps, loadsfrom tldextract import extract as tld+from scrapling.core.utils import _StorageTools, logfrom scrapling.core._types import Dict, Optional, Union-from scrapling.core.utils import _StorageTools, log, lru_cacheclass StorageSystemMixin(ABC):@@ -35,7 +36,7 @@ class StorageSystemMixin(ABC):return default_value@abstractmethod- def save(self, element: html.HtmlElement, identifier: str) -> None:+ def save(self, element: HtmlElement, identifier: str) -> None:"""Saves the element's unique properties to the storage for retrieval and relocation later:param element: The element itself which we want to save to storage.@@ -81,11 +82,10 @@ class SQLiteStorageSystem(StorageSystemMixin):"""super().__init__(url)self.storage_file = storage_file- # We use a threading.Lock to ensure thread-safety instead of relying on thread-local storage.- self.lock = threading.Lock()+ self.lock = RLock() # Better than Lock for reentrancy# >SQLite default mode in the earlier version is 1 not 2 (1=thread-safe 2=serialized)# `check_same_thread=False` to allow it to be used across different threads.- self.connection = sqlite3.connect(self.storage_file, check_same_thread=False)+ self.connection = db_connect(self.storage_file, check_same_thread=False)# WAL (Write-Ahead Logging) allows for better concurrency.self.connection.execute("PRAGMA journal_mode=WAL")self.cursor = self.connection.cursor()@@ -106,7 +106,7 @@ class SQLiteStorageSystem(StorageSystemMixin):""")self.connection.commit()- def save(self, element: html.HtmlElement, identifier: str):+ def save(self, element: HtmlElement, identifier: str):"""Saves the elements unique properties to the storage for retrieval and relocation later:param element: The element itself which we want to save to storage.@@ -121,7 +121,7 @@ class SQLiteStorageSystem(StorageSystemMixin):INSERT OR REPLACE INTO storage (url, identifier, element_data)VALUES (?, ?, ?)""",- (url, identifier, orjson.dumps(element_data)),+ (url, identifier, dumps(element_data)),)self.cursor.fetchall()self.connection.commit()@@ -141,7 +141,7 @@ class SQLiteStorageSystem(StorageSystemMixin):)result = self.cursor.fetchone()if result:- return orjson.loads(result[0])+ return loads(result[0])return Nonedef close(self):