Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: decouple database from manager (#1291)

NTFSvolumedf87f59dff48
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 31ee0b25c..8417ff4f5 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -21,11 +21,11 @@ from cyberdrop_dl import constants
from cyberdrop_dl.clients.scraper_client import ScraperClient
from cyberdrop_dl.data_structures.mediaprops import Resolution
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, copy_signature
+from cyberdrop_dl.database import get_db_path
from cyberdrop_dl.downloader.downloader import Downloader
from cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeError
from cyberdrop_dl.scraper import filters
from cyberdrop_dl.utils import css, m3u8
-from cyberdrop_dl.utils.database.tables.history_table import get_db_path
from cyberdrop_dl.utils.dates import TimeStamp, parse_human_date, to_timestamp
from cyberdrop_dl.utils.logger import log, log_debug
from cyberdrop_dl.utils.strings import safe_format
diff --git a/cyberdrop_dl/managers/db_manager.py b/cyberdrop_dl/database/__init__.py
similarity index 57%
rename from cyberdrop_dl/managers/db_manager.py
rename to cyberdrop_dl/database/__init__.py
index 0250d9fe2..02b874af4 100644
--- a/cyberdrop_dl/managers/db_manager.py
+++ b/cyberdrop_dl/database/__init__.py
@@ -1,39 +1,29 @@
from __future__ import annotations
-from dataclasses import field
from typing import TYPE_CHECKING
import aiosqlite
-from cyberdrop_dl.utils.database.tables.hash_table import HashTable
-from cyberdrop_dl.utils.database.tables.history_table import HistoryTable
-from cyberdrop_dl.utils.database.tables.temp_referer_table import TempRefererTable
+from .tables import HashTable, HistoryTable, TempRefererTable
+from .tables.history import get_db_path
if TYPE_CHECKING:
from pathlib import Path
- from cyberdrop_dl.managers.manager import Manager
-
-class DBManager:
- def __init__(self, manager: Manager, db_path: Path) -> None:
- self.manager = manager
- self._db_conn: aiosqlite.Connection = field(init=False)
+class Database:
+ def __init__(self, db_path: Path, ignore_history: bool) -> None:
+ self._db_conn: aiosqlite.Connection
self._db_path: Path = db_path
-
- self.ignore_history: bool = False
-
- self.history_table: HistoryTable = field(init=False)
- self.hash_table: HashTable = field(init=False)
- self.temp_referer_table: TempRefererTable = field(init=False)
+ self.ignore_history = ignore_history
+ self.history_table: HistoryTable
+ self.hash_table: HashTable
+ self.temp_referer_table: TempRefererTable
async def startup(self) -> None:
"""Startup process for the DBManager."""
self._db_conn = await aiosqlite.connect(self._db_path)
self._db_conn.row_factory = aiosqlite.Row
-
- self.ignore_history = self.manager.config_manager.settings_data.runtime_options.ignore_history
-
self.history_table = HistoryTable(self)
self.hash_table = HashTable(self)
self.temp_referer_table = TempRefererTable(self)
@@ -42,12 +32,6 @@ class DBManager:
await self.history_table.startup()
await self.hash_table.startup()
await self.temp_referer_table.startup()
- await self.run_fixes()
-
- async def run_fixes(self):
- if not self.manager.cache_manager.get("fixed_empty_download_filenames"):
- await self.history_table.delete_invalid_rows()
- self.manager.cache_manager.save("fixed_empty_download_filenames", True)
async def close(self) -> None:
"""Close the DBManager."""
@@ -70,3 +54,6 @@ class DBManager:
if free_space and free_space[0] <= 1024:
await self._db_conn.executescript(pre_allocate_script)
await self._db_conn.commit()
+
+
+__all__ = ["Database", "get_db_path"]
diff --git a/cyberdrop_dl/database/tables/__init__.py b/cyberdrop_dl/database/tables/__init__.py
new file mode 100644
index 000000000..32cd78ef0
--- /dev/null
+++ b/cyberdrop_dl/database/tables/__init__.py
@@ -0,0 +1,5 @@
+from .hash import HashTable
+from .history import HistoryTable
+from .temp_referer import TempRefererTable
+
+__all__ = ["HashTable", "HistoryTable", "TempRefererTable"]
diff --git a/cyberdrop_dl/utils/database/table_definitions.py b/cyberdrop_dl/database/tables/definitions.py
similarity index 100%
rename from cyberdrop_dl/utils/database/table_definitions.py
rename to cyberdrop_dl/database/tables/definitions.py
diff --git a/cyberdrop_dl/utils/database/tables/hash_table.py b/cyberdrop_dl/database/tables/hash.py
similarity index 97%
rename from cyberdrop_dl/utils/database/tables/hash_table.py
rename to cyberdrop_dl/database/tables/hash.py
index d4c1baab1..efcda3d1b 100644
--- a/cyberdrop_dl/utils/database/tables/hash_table.py
+++ b/cyberdrop_dl/database/tables/hash.py
@@ -3,18 +3,19 @@ from __future__ import annotations
from pathlib import Path
from typing import TYPE_CHECKING, cast
-from cyberdrop_dl.utils.database.table_definitions import create_files, create_hash
from cyberdrop_dl.utils.logger import log
+from .definitions import create_files, create_hash
+
if TYPE_CHECKING:
import aiosqlite
from yarl import URL
- from cyberdrop_dl.managers.db_manager import DBManager
+ from cyberdrop_dl.database import Database
class HashTable:
- def __init__(self, database: DBManager) -> None:
+ def __init__(self, database: Database) -> None:
self._database = database
@property
diff --git a/cyberdrop_dl/utils/database/tables/history_table.py b/cyberdrop_dl/database/tables/history.py
similarity index 98%
rename from cyberdrop_dl/utils/database/tables/history_table.py
rename to cyberdrop_dl/database/tables/history.py
index 9646354ba..4cf63e853 100644
--- a/cyberdrop_dl/utils/database/tables/history_table.py
+++ b/cyberdrop_dl/database/tables/history.py
@@ -3,9 +3,10 @@ from __future__ import annotations
from sqlite3 import IntegrityError, Row
from typing import TYPE_CHECKING, cast
-from cyberdrop_dl.utils.database.table_definitions import create_fixed_history, create_history
from cyberdrop_dl.utils.utilities import log
+from .definitions import create_fixed_history, create_history
+
if TYPE_CHECKING:
import datetime
@@ -14,7 +15,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.crawlers import Crawler
from cyberdrop_dl.data_structures.url_objects import MediaItem
- from cyberdrop_dl.managers.db_manager import DBManager
+ from cyberdrop_dl.database import Database
def get_db_path(url: URL, domain: str = "") -> str:
@@ -36,7 +37,7 @@ def get_db_path(url: URL, domain: str = "") -> str:
class HistoryTable:
- def __init__(self, database: DBManager) -> None:
+ def __init__(self, database: Database) -> None:
self._database = database
@property
diff --git a/cyberdrop_dl/utils/database/tables/temp_referer_table.py b/cyberdrop_dl/database/tables/temp_referer.py
similarity index 93%
rename from cyberdrop_dl/utils/database/tables/temp_referer_table.py
rename to cyberdrop_dl/database/tables/temp_referer.py
index 54ed09f5d..c7b69adc7 100644
--- a/cyberdrop_dl/utils/database/tables/temp_referer_table.py
+++ b/cyberdrop_dl/database/tables/temp_referer.py
@@ -2,18 +2,18 @@ from __future__ import annotations
from typing import TYPE_CHECKING
-from cyberdrop_dl.utils.database.table_definitions import create_temp_referer
+from .definitions import create_temp_referer
if TYPE_CHECKING:
import aiosqlite
from yarl import URL
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
- from cyberdrop_dl.managers.db_manager import DBManager
+ from cyberdrop_dl.database import Database
class TempRefererTable:
- def __init__(self, database: DBManager) -> None:
+ def __init__(self, database: Database) -> None:
self._database = database
@property
diff --git a/cyberdrop_dl/utils/transfer.py b/cyberdrop_dl/database/transfer.py
similarity index 97%
rename from cyberdrop_dl/utils/transfer.py
rename to cyberdrop_dl/database/transfer.py
index 087401a67..a5b3b1046 100644
--- a/cyberdrop_dl/utils/transfer.py
+++ b/cyberdrop_dl/database/transfer.py
@@ -9,7 +9,7 @@ from shutil import copy2
import arrow
from rich.console import Console
-from cyberdrop_dl.utils.database.table_definitions import create_files, create_temp_hash
+from .tables.definitions import create_files, create_temp_hash
console = Console()
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index e01bb22b7..24d891741 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -16,6 +16,7 @@ from aiohttp import ClientConnectorError, ClientError, ClientResponseError
from cyberdrop_dl.constants import CustomHTTPStatus
from cyberdrop_dl.data_structures.url_objects import HlsSegment, MediaItem
+from cyberdrop_dl.database import get_db_path
from cyberdrop_dl.exceptions import (
DownloadError,
DurationError,
@@ -25,7 +26,6 @@ from cyberdrop_dl.exceptions import (
TooManyCrawlerErrors,
)
from cyberdrop_dl.utils import aio, ffmpeg
-from cyberdrop_dl.utils.database.tables.history_table import get_db_path
from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_size_or_none, parse_url
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index f28e96ae1..31013deb0 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -9,10 +9,11 @@ from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar
from pydantic import BaseModel
from cyberdrop_dl import __version__, constants
+from cyberdrop_dl.database import Database
+from cyberdrop_dl.database.transfer import transfer_v5_db_to_v6
from cyberdrop_dl.managers.cache_manager import CacheManager
from cyberdrop_dl.managers.client_manager import ClientManager
from cyberdrop_dl.managers.config_manager import ConfigManager
-from cyberdrop_dl.managers.db_manager import DBManager
from cyberdrop_dl.managers.hash_manager import HashManager
from cyberdrop_dl.managers.live_manager import LiveManager
from cyberdrop_dl.managers.log_manager import LogManager
@@ -22,7 +23,6 @@ from cyberdrop_dl.managers.storage_manager import StorageManager
from cyberdrop_dl.utils import ffmpeg
from cyberdrop_dl.utils.args import ParsedArgs, parse_args
from cyberdrop_dl.utils.logger import QueuedLogger, log
-from cyberdrop_dl.utils.transfer import transfer_v5_db_to_v6
from cyberdrop_dl.utils.utilities import close_if_defined, get_system_information
if TYPE_CHECKING:
@@ -45,7 +45,7 @@ class Manager:
self.hash_manager: HashManager = field(init=False)
self.log_manager: LogManager = field(init=False)
- self.db_manager: DBManager = field(init=False)
+ self.db_manager: Database = field(init=False)
self.client_manager: ClientManager = field(init=False)
self.storage_manager: StorageManager = field(init=False)
@@ -150,8 +150,11 @@ class Manager:
constants.MAX_NAME_LENGTHS["FOLDER"] = self.config_manager.global_settings_data.general.max_folder_name_length
async def async_db_hash_startup(self) -> None:
- if not isinstance(self.db_manager, DBManager):
- self.db_manager = DBManager(self, self.path_manager.history_db)
+ if not isinstance(self.db_manager, Database):
+ self.db_manager = Database(
+ self.path_manager.history_db,
+ self.config.runtime_options.ignore_history,
+ )
await self.db_manager.startup()
transfer_v5_db_to_v6(self.path_manager.history_db)
if not isinstance(self.hash_manager, HashManager):
diff --git a/cyberdrop_dl/utils/database/__init__.py b/cyberdrop_dl/utils/database/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/cyberdrop_dl/utils/database/tables/__init__.py b/cyberdrop_dl/utils/database/tables/__init__.py
deleted file mode 100644
index e69de29bb..000000000