Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: decouple table from db obj

NTFSvolume7aa8038585e0
diff --git a/cyberdrop_dl/database/_db.py b/cyberdrop_dl/database/_db.py
index 1d5fc883d..ddbced11c 100644
--- a/cyberdrop_dl/database/_db.py
+++ b/cyberdrop_dl/database/_db.py
@@ -43,11 +43,6 @@ class Database:
_conn: aiosqlite.Connection = dataclasses.field(init=False)
_is_new: bool = dataclasses.field(init=False)
- def __post_init__(self) -> None:
- self.history = HistoryTable(self)
- self.hash = HashTable(self)
- self.schema = SchemaTable(self)
-
@property
def conn(self) -> aiosqlite.Connection:
return self._conn
@@ -55,6 +50,9 @@ class Database:
async def _connect(self) -> None:
self._is_new = not await aio.get_size(self.path)
self._conn = await _connect(self.path)
+ self.history = HistoryTable(self._conn, self.ignore_history)
+ self.hash = HashTable(self._conn, self.ignore_history)
+ self.schema = SchemaTable(self._conn, self.ignore_history)
@contextlib.asynccontextmanager
async def connect(self) -> AsyncGenerator[Self]:
diff --git a/cyberdrop_dl/database/hash.py b/cyberdrop_dl/database/hash.py
index 626f012c0..f0da1c112 100644
--- a/cyberdrop_dl/database/hash.py
+++ b/cyberdrop_dl/database/hash.py
@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
@dataclasses.dataclass(slots=True)
-class HashTable(table.Table):
+class HashTable(table.Table, name="hash"):
cwd: Path = dataclasses.field(init=False, default_factory=lambda: Path.cwd().expanduser().resolve())
async def create(self) -> None:
@@ -72,15 +72,15 @@ class HashTable(table.Table):
"""
try:
- cursor = await self.db_conn.execute(query, (hash_value, size, hash_type))
- return cast("list[aiosqlite.Row]", await cursor.fetchall())
+ rows = await self.db_conn.execute_fetchall(query, (hash_value, size, hash_type))
+ return cast("list[aiosqlite.Row]", rows)
except Exception:
logger.exception("Error retrieving folder and filename")
return []
async def check_hash_exists(self, hash_type: str, hash_value: str) -> bool:
- if self._database.ignore_history:
+ if self.ignore_history:
return False
query = "SELECT 1 FROM hash WHERE hash.hash_type = ? AND hash.hash = ? LIMIT 1"
diff --git a/cyberdrop_dl/database/history.py b/cyberdrop_dl/database/history.py
index 9b854ab5a..57c3d6cb7 100644
--- a/cyberdrop_dl/database/history.py
+++ b/cyberdrop_dl/database/history.py
@@ -23,7 +23,7 @@ _FETCH_MANY_SIZE: int = 1000
logger = logging.getLogger(__name__)
-class HistoryTable(table.Table):
+class HistoryTable(table.Table, name="media"):
async def create(self) -> None:
await self.db_conn.execute(CREATE_HISTORY)
await self.db_conn.executescript(CREATE_MEDIA_INDEX)
@@ -35,7 +35,7 @@ class HistoryTable(table.Table):
async def check_complete(self, domain: str, db_path: str) -> tuple[str, bool]:
"""Checks whether an individual file has completed given its domain and url path."""
- if self._database.ignore_history:
+ if self.ignore_history:
return "", False
query = "SELECT referer, completed FROM media WHERE domain = ? and url_path = ? LIMIT 1"
@@ -51,7 +51,7 @@ class HistoryTable(table.Table):
async def check_album(self, domain: str, album_id: str) -> dict[str, bool]:
"""Checks whether an album has completed given its domain and album id."""
- if self._database.ignore_history:
+ if self.ignore_history:
return {}
query = "SELECT url_path, completed FROM media WHERE domain = ? and album_id = ?"
@@ -67,7 +67,7 @@ class HistoryTable(table.Table):
async def check_complete_by_referer(self, domain: str | None, referer: URL) -> bool:
"""Checks whether an individual file has completed given its domain and url path."""
- if self._database.ignore_history:
+ if self.ignore_history:
return False
if domain is None:
diff --git a/cyberdrop_dl/database/schema.py b/cyberdrop_dl/database/schema.py
index 18c13d151..126fbbc29 100644
--- a/cyberdrop_dl/database/schema.py
+++ b/cyberdrop_dl/database/schema.py
@@ -38,12 +38,12 @@ logger = logging.getLogger(__name__)
@dataclasses.dataclass(slots=True)
-class SchemaTable(table.Table):
+class SchemaTable(table.Table, name="schema_version"):
up_to_date: bool = False
version: Version | None = None
async def get_version(self) -> Version | None:
- if not await table.exists(self.db_conn, "schema_version"):
+ if not await self.exists():
return None
query = "SELECT version FROM schema_version ORDER BY ROWID DESC LIMIT 1;"
cursor = await self.db_conn.execute(query)
@@ -78,8 +78,7 @@ class SchemaTable(table.Table):
async def dump(db_conn: aiosqlite.Connection) -> str:
query = "SELECT sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY name"
- cursor = await db_conn.execute(query)
- rows = await cursor.fetchall()
+ rows = await db_conn.execute_fetchall(query)
index_queries: list[str] = []
table_queries: list[str] = []
diff --git a/cyberdrop_dl/database/table.py b/cyberdrop_dl/database/table.py
index 028c4582f..4ff37ffab 100644
--- a/cyberdrop_dl/database/table.py
+++ b/cyberdrop_dl/database/table.py
@@ -2,27 +2,26 @@ from __future__ import annotations
import dataclasses
from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, ClassVar
if TYPE_CHECKING:
import aiosqlite
- from cyberdrop_dl.database import Database
-
@dataclasses.dataclass(slots=True)
class Table(ABC):
- _database: Database = dataclasses.field(repr=False)
+ NAME: ClassVar[str]
+ db_conn: aiosqlite.Connection
+ ignore_history: bool = False
- @property
- def db_conn(self) -> aiosqlite.Connection:
- return self._database.conn
+ def __init_subclass__(cls, name: str | None = None) -> None:
+ if name:
+ cls.NAME = name
@abstractmethod
async def create(self) -> None: ...
-
-async def exists(db_conn: aiosqlite.Connection, table: str) -> bool:
- query = "SELECT 1 FROM sqlite_master WHERE type='table' AND name= ? ;"
- cursor = await db_conn.execute(query, (table,))
- return await cursor.fetchone() is not None
+ async def exists(self) -> bool:
+ query = "SELECT 1 FROM sqlite_master WHERE type='table' AND name= ? LIMIT 1;"
+ cursor = await self.db_conn.execute(query, (self.NAME,))
+ return await cursor.fetchone() is not None