Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: decouple database from manager (#1291)
NTFSvolumedf87f59dff48
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 31ee0b25c..8417ff4f5 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -21,11 +21,11 @@ from cyberdrop_dl import constantsfrom cyberdrop_dl.clients.scraper_client import ScraperClientfrom cyberdrop_dl.data_structures.mediaprops import Resolutionfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, copy_signature+from cyberdrop_dl.database import get_db_pathfrom cyberdrop_dl.downloader.downloader import Downloaderfrom cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.scraper import filtersfrom cyberdrop_dl.utils import css, m3u8-from cyberdrop_dl.utils.database.tables.history_table import get_db_pathfrom cyberdrop_dl.utils.dates import TimeStamp, parse_human_date, to_timestampfrom cyberdrop_dl.utils.logger import log, log_debugfrom cyberdrop_dl.utils.strings import safe_formatdiff --git a/cyberdrop_dl/managers/db_manager.py b/cyberdrop_dl/database/__init__.pysimilarity index 57%rename from cyberdrop_dl/managers/db_manager.pyrename to cyberdrop_dl/database/__init__.pyindex 0250d9fe2..02b874af4 100644--- a/cyberdrop_dl/managers/db_manager.py+++ b/cyberdrop_dl/database/__init__.py@@ -1,39 +1,29 @@from __future__ import annotations-from dataclasses import fieldfrom typing import TYPE_CHECKINGimport aiosqlite-from cyberdrop_dl.utils.database.tables.hash_table import HashTable-from cyberdrop_dl.utils.database.tables.history_table import HistoryTable-from cyberdrop_dl.utils.database.tables.temp_referer_table import TempRefererTable+from .tables import HashTable, HistoryTable, TempRefererTable+from .tables.history import get_db_pathif TYPE_CHECKING:from pathlib import Path- from cyberdrop_dl.managers.manager import Manager--class DBManager:- def __init__(self, manager: Manager, db_path: Path) -> None:- self.manager = manager- self._db_conn: aiosqlite.Connection = field(init=False)+class Database:+ def __init__(self, db_path: Path, ignore_history: bool) -> None:+ self._db_conn: aiosqlite.Connectionself._db_path: Path = db_path-- self.ignore_history: bool = False-- self.history_table: HistoryTable = field(init=False)- self.hash_table: HashTable = field(init=False)- self.temp_referer_table: TempRefererTable = field(init=False)+ self.ignore_history = ignore_history+ self.history_table: HistoryTable+ self.hash_table: HashTable+ self.temp_referer_table: TempRefererTableasync def startup(self) -> None:"""Startup process for the DBManager."""self._db_conn = await aiosqlite.connect(self._db_path)self._db_conn.row_factory = aiosqlite.Row-- self.ignore_history = self.manager.config_manager.settings_data.runtime_options.ignore_history-self.history_table = HistoryTable(self)self.hash_table = HashTable(self)self.temp_referer_table = TempRefererTable(self)@@ -42,12 +32,6 @@ class DBManager:await self.history_table.startup()await self.hash_table.startup()await self.temp_referer_table.startup()- await self.run_fixes()-- async def run_fixes(self):- if not self.manager.cache_manager.get("fixed_empty_download_filenames"):- await self.history_table.delete_invalid_rows()- self.manager.cache_manager.save("fixed_empty_download_filenames", True)async def close(self) -> None:"""Close the DBManager."""@@ -70,3 +54,6 @@ class DBManager:if free_space and free_space[0] <= 1024:await self._db_conn.executescript(pre_allocate_script)await self._db_conn.commit()+++__all__ = ["Database", "get_db_path"]diff --git a/cyberdrop_dl/database/tables/__init__.py b/cyberdrop_dl/database/tables/__init__.pynew file mode 100644index 000000000..32cd78ef0--- /dev/null+++ b/cyberdrop_dl/database/tables/__init__.py@@ -0,0 +1,5 @@+from .hash import HashTable+from .history import HistoryTable+from .temp_referer import TempRefererTable++__all__ = ["HashTable", "HistoryTable", "TempRefererTable"]diff --git a/cyberdrop_dl/utils/database/table_definitions.py b/cyberdrop_dl/database/tables/definitions.pysimilarity index 100%rename from cyberdrop_dl/utils/database/table_definitions.pyrename to cyberdrop_dl/database/tables/definitions.pydiff --git a/cyberdrop_dl/utils/database/tables/hash_table.py b/cyberdrop_dl/database/tables/hash.pysimilarity index 97%rename from cyberdrop_dl/utils/database/tables/hash_table.pyrename to cyberdrop_dl/database/tables/hash.pyindex d4c1baab1..efcda3d1b 100644--- a/cyberdrop_dl/utils/database/tables/hash_table.py+++ b/cyberdrop_dl/database/tables/hash.py@@ -3,18 +3,19 @@ from __future__ import annotationsfrom pathlib import Pathfrom typing import TYPE_CHECKING, cast-from cyberdrop_dl.utils.database.table_definitions import create_files, create_hashfrom cyberdrop_dl.utils.logger import log+from .definitions import create_files, create_hash+if TYPE_CHECKING:import aiosqlitefrom yarl import URL- from cyberdrop_dl.managers.db_manager import DBManager+ from cyberdrop_dl.database import Databaseclass HashTable:- def __init__(self, database: DBManager) -> None:+ def __init__(self, database: Database) -> None:self._database = database@propertydiff --git a/cyberdrop_dl/utils/database/tables/history_table.py b/cyberdrop_dl/database/tables/history.pysimilarity index 98%rename from cyberdrop_dl/utils/database/tables/history_table.pyrename to cyberdrop_dl/database/tables/history.pyindex 9646354ba..4cf63e853 100644--- a/cyberdrop_dl/utils/database/tables/history_table.py+++ b/cyberdrop_dl/database/tables/history.py@@ -3,9 +3,10 @@ from __future__ import annotationsfrom sqlite3 import IntegrityError, Rowfrom typing import TYPE_CHECKING, cast-from cyberdrop_dl.utils.database.table_definitions import create_fixed_history, create_historyfrom cyberdrop_dl.utils.utilities import log+from .definitions import create_fixed_history, create_history+if TYPE_CHECKING:import datetime@@ -14,7 +15,7 @@ if TYPE_CHECKING:from cyberdrop_dl.crawlers import Crawlerfrom cyberdrop_dl.data_structures.url_objects import MediaItem- from cyberdrop_dl.managers.db_manager import DBManager+ from cyberdrop_dl.database import Databasedef get_db_path(url: URL, domain: str = "") -> str:@@ -36,7 +37,7 @@ def get_db_path(url: URL, domain: str = "") -> str:class HistoryTable:- def __init__(self, database: DBManager) -> None:+ def __init__(self, database: Database) -> None:self._database = database@propertydiff --git a/cyberdrop_dl/utils/database/tables/temp_referer_table.py b/cyberdrop_dl/database/tables/temp_referer.pysimilarity index 93%rename from cyberdrop_dl/utils/database/tables/temp_referer_table.pyrename to cyberdrop_dl/database/tables/temp_referer.pyindex 54ed09f5d..c7b69adc7 100644--- a/cyberdrop_dl/utils/database/tables/temp_referer_table.py+++ b/cyberdrop_dl/database/tables/temp_referer.py@@ -2,18 +2,18 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING-from cyberdrop_dl.utils.database.table_definitions import create_temp_referer+from .definitions import create_temp_refererif TYPE_CHECKING:import aiosqlitefrom yarl import URLfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL- from cyberdrop_dl.managers.db_manager import DBManager+ from cyberdrop_dl.database import Databaseclass TempRefererTable:- def __init__(self, database: DBManager) -> None:+ def __init__(self, database: Database) -> None:self._database = database@propertydiff --git a/cyberdrop_dl/utils/transfer.py b/cyberdrop_dl/database/transfer.pysimilarity index 97%rename from cyberdrop_dl/utils/transfer.pyrename to cyberdrop_dl/database/transfer.pyindex 087401a67..a5b3b1046 100644--- a/cyberdrop_dl/utils/transfer.py+++ b/cyberdrop_dl/database/transfer.py@@ -9,7 +9,7 @@ from shutil import copy2import arrowfrom rich.console import Console-from cyberdrop_dl.utils.database.table_definitions import create_files, create_temp_hash+from .tables.definitions import create_files, create_temp_hashconsole = Console()diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex e01bb22b7..24d891741 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -16,6 +16,7 @@ from aiohttp import ClientConnectorError, ClientError, ClientResponseErrorfrom cyberdrop_dl.constants import CustomHTTPStatusfrom cyberdrop_dl.data_structures.url_objects import HlsSegment, MediaItem+from cyberdrop_dl.database import get_db_pathfrom cyberdrop_dl.exceptions import (DownloadError,DurationError,@@ -25,7 +26,6 @@ from cyberdrop_dl.exceptions import (TooManyCrawlerErrors,)from cyberdrop_dl.utils import aio, ffmpeg-from cyberdrop_dl.utils.database.tables.history_table import get_db_pathfrom cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_size_or_none, parse_urldiff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex f28e96ae1..31013deb0 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -9,10 +9,11 @@ from typing import TYPE_CHECKING, Any, NamedTuple, TypeVarfrom pydantic import BaseModelfrom cyberdrop_dl import __version__, constants+from cyberdrop_dl.database import Database+from cyberdrop_dl.database.transfer import transfer_v5_db_to_v6from cyberdrop_dl.managers.cache_manager import CacheManagerfrom cyberdrop_dl.managers.client_manager import ClientManagerfrom cyberdrop_dl.managers.config_manager import ConfigManager-from cyberdrop_dl.managers.db_manager import DBManagerfrom cyberdrop_dl.managers.hash_manager import HashManagerfrom cyberdrop_dl.managers.live_manager import LiveManagerfrom cyberdrop_dl.managers.log_manager import LogManager@@ -22,7 +23,6 @@ from cyberdrop_dl.managers.storage_manager import StorageManagerfrom cyberdrop_dl.utils import ffmpegfrom cyberdrop_dl.utils.args import ParsedArgs, parse_argsfrom cyberdrop_dl.utils.logger import QueuedLogger, log-from cyberdrop_dl.utils.transfer import transfer_v5_db_to_v6from cyberdrop_dl.utils.utilities import close_if_defined, get_system_informationif TYPE_CHECKING:@@ -45,7 +45,7 @@ class Manager:self.hash_manager: HashManager = field(init=False)self.log_manager: LogManager = field(init=False)- self.db_manager: DBManager = field(init=False)+ self.db_manager: Database = field(init=False)self.client_manager: ClientManager = field(init=False)self.storage_manager: StorageManager = field(init=False)@@ -150,8 +150,11 @@ class Manager:constants.MAX_NAME_LENGTHS["FOLDER"] = self.config_manager.global_settings_data.general.max_folder_name_lengthasync def async_db_hash_startup(self) -> None:- if not isinstance(self.db_manager, DBManager):- self.db_manager = DBManager(self, self.path_manager.history_db)+ if not isinstance(self.db_manager, Database):+ self.db_manager = Database(+ self.path_manager.history_db,+ self.config.runtime_options.ignore_history,+ )await self.db_manager.startup()transfer_v5_db_to_v6(self.path_manager.history_db)if not isinstance(self.hash_manager, HashManager):diff --git a/cyberdrop_dl/utils/database/__init__.py b/cyberdrop_dl/utils/database/__init__.pydeleted file mode 100644index e69de29bb..000000000diff --git a/cyberdrop_dl/utils/database/tables/__init__.py b/cyberdrop_dl/utils/database/tables/__init__.pydeleted file mode 100644index e69de29bb..000000000