Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: decouple table from db obj
NTFSvolume7aa8038585e0
diff --git a/cyberdrop_dl/database/_db.py b/cyberdrop_dl/database/_db.pyindex 1d5fc883d..ddbced11c 100644--- a/cyberdrop_dl/database/_db.py+++ b/cyberdrop_dl/database/_db.py@@ -43,11 +43,6 @@ class Database:_conn: aiosqlite.Connection = dataclasses.field(init=False)_is_new: bool = dataclasses.field(init=False)- def __post_init__(self) -> None:- self.history = HistoryTable(self)- self.hash = HashTable(self)- self.schema = SchemaTable(self)-@propertydef conn(self) -> aiosqlite.Connection:return self._conn@@ -55,6 +50,9 @@ class Database:async def _connect(self) -> None:self._is_new = not await aio.get_size(self.path)self._conn = await _connect(self.path)+ self.history = HistoryTable(self._conn, self.ignore_history)+ self.hash = HashTable(self._conn, self.ignore_history)+ self.schema = SchemaTable(self._conn, self.ignore_history)@contextlib.asynccontextmanagerasync def connect(self) -> AsyncGenerator[Self]:diff --git a/cyberdrop_dl/database/hash.py b/cyberdrop_dl/database/hash.pyindex 626f012c0..f0da1c112 100644--- a/cyberdrop_dl/database/hash.py+++ b/cyberdrop_dl/database/hash.py@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)@dataclasses.dataclass(slots=True)-class HashTable(table.Table):+class HashTable(table.Table, name="hash"):cwd: Path = dataclasses.field(init=False, default_factory=lambda: Path.cwd().expanduser().resolve())async def create(self) -> None:@@ -72,15 +72,15 @@ class HashTable(table.Table):"""try:- cursor = await self.db_conn.execute(query, (hash_value, size, hash_type))- return cast("list[aiosqlite.Row]", await cursor.fetchall())+ rows = await self.db_conn.execute_fetchall(query, (hash_value, size, hash_type))+ return cast("list[aiosqlite.Row]", rows)except Exception:logger.exception("Error retrieving folder and filename")return []async def check_hash_exists(self, hash_type: str, hash_value: str) -> bool:- if self._database.ignore_history:+ if self.ignore_history:return Falsequery = "SELECT 1 FROM hash WHERE hash.hash_type = ? AND hash.hash = ? LIMIT 1"diff --git a/cyberdrop_dl/database/history.py b/cyberdrop_dl/database/history.pyindex 9b854ab5a..57c3d6cb7 100644--- a/cyberdrop_dl/database/history.py+++ b/cyberdrop_dl/database/history.py@@ -23,7 +23,7 @@ _FETCH_MANY_SIZE: int = 1000logger = logging.getLogger(__name__)-class HistoryTable(table.Table):+class HistoryTable(table.Table, name="media"):async def create(self) -> None:await self.db_conn.execute(CREATE_HISTORY)await self.db_conn.executescript(CREATE_MEDIA_INDEX)@@ -35,7 +35,7 @@ class HistoryTable(table.Table):async def check_complete(self, domain: str, db_path: str) -> tuple[str, bool]:"""Checks whether an individual file has completed given its domain and url path."""- if self._database.ignore_history:+ if self.ignore_history:return "", Falsequery = "SELECT referer, completed FROM media WHERE domain = ? and url_path = ? LIMIT 1"@@ -51,7 +51,7 @@ class HistoryTable(table.Table):async def check_album(self, domain: str, album_id: str) -> dict[str, bool]:"""Checks whether an album has completed given its domain and album id."""- if self._database.ignore_history:+ if self.ignore_history:return {}query = "SELECT url_path, completed FROM media WHERE domain = ? and album_id = ?"@@ -67,7 +67,7 @@ class HistoryTable(table.Table):async def check_complete_by_referer(self, domain: str | None, referer: URL) -> bool:"""Checks whether an individual file has completed given its domain and url path."""- if self._database.ignore_history:+ if self.ignore_history:return Falseif domain is None:diff --git a/cyberdrop_dl/database/schema.py b/cyberdrop_dl/database/schema.pyindex 18c13d151..126fbbc29 100644--- a/cyberdrop_dl/database/schema.py+++ b/cyberdrop_dl/database/schema.py@@ -38,12 +38,12 @@ logger = logging.getLogger(__name__)@dataclasses.dataclass(slots=True)-class SchemaTable(table.Table):+class SchemaTable(table.Table, name="schema_version"):up_to_date: bool = Falseversion: Version | None = Noneasync def get_version(self) -> Version | None:- if not await table.exists(self.db_conn, "schema_version"):+ if not await self.exists():return Nonequery = "SELECT version FROM schema_version ORDER BY ROWID DESC LIMIT 1;"cursor = await self.db_conn.execute(query)@@ -78,8 +78,7 @@ class SchemaTable(table.Table):async def dump(db_conn: aiosqlite.Connection) -> str:query = "SELECT sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY name"- cursor = await db_conn.execute(query)- rows = await cursor.fetchall()+ rows = await db_conn.execute_fetchall(query)index_queries: list[str] = []table_queries: list[str] = []diff --git a/cyberdrop_dl/database/table.py b/cyberdrop_dl/database/table.pyindex 028c4582f..4ff37ffab 100644--- a/cyberdrop_dl/database/table.py+++ b/cyberdrop_dl/database/table.py@@ -2,27 +2,26 @@ from __future__ import annotationsimport dataclassesfrom abc import ABC, abstractmethod-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, ClassVarif TYPE_CHECKING:import aiosqlite- from cyberdrop_dl.database import Database-@dataclasses.dataclass(slots=True)class Table(ABC):- _database: Database = dataclasses.field(repr=False)+ NAME: ClassVar[str]+ db_conn: aiosqlite.Connection+ ignore_history: bool = False- @property- def db_conn(self) -> aiosqlite.Connection:- return self._database.conn+ def __init_subclass__(cls, name: str | None = None) -> None:+ if name:+ cls.NAME = name@abstractmethodasync def create(self) -> None: ...--async def exists(db_conn: aiosqlite.Connection, table: str) -> bool:- query = "SELECT 1 FROM sqlite_master WHERE type='table' AND name= ? ;"- cursor = await db_conn.execute(query, (table,))- return await cursor.fetchone() is not None+ async def exists(self) -> bool:+ query = "SELECT 1 FROM sqlite_master WHERE type='table' AND name= ? LIMIT 1;"+ cursor = await self.db_conn.execute(query, (self.NAME,))+ return await cursor.fetchone() is not None