Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: no circular imports on db
NTFSvolume8717735d48dd
diff --git a/cyberdrop_dl/database/__init__.py b/cyberdrop_dl/database/__init__.pyindex 351913096..f905762f5 100644--- a/cyberdrop_dl/database/__init__.py+++ b/cyberdrop_dl/database/__init__.py@@ -1,3 +1,5 @@-from ._db import Database, connect, pre_allocate_100mb+from __future__ import annotations-__all__ = ["Database", "connect", "pre_allocate_100mb"]+from ._db import Database++__all__ = ["Database"]diff --git a/cyberdrop_dl/database/_db.py b/cyberdrop_dl/database/_db.pyindex ddbced11c..494b43584 100644--- a/cyberdrop_dl/database/_db.py+++ b/cyberdrop_dl/database/_db.py@@ -4,31 +4,18 @@ import contextlibimport dataclassesfrom typing import TYPE_CHECKING, Self-import aiosqlite-from cyberdrop_dl import aio-from cyberdrop_dl.database.hash import HashTable-from cyberdrop_dl.database.history import HistoryTable-from cyberdrop_dl.database.schema import SchemaTable++from .common import pre_allocate_100mb, raw_connect+from .hash import HashTable+from .history import HistoryTable+from .schema import SchemaTableif TYPE_CHECKING:from collections.abc import AsyncGeneratorfrom pathlib import Path--async def _connect(path: Path) -> aiosqlite.Connection:- db_conn = await aiosqlite.connect(path, timeout=20)- db_conn.row_factory = aiosqlite.Row- return db_conn---async def connect(path: Path) -> AsyncGenerator[aiosqlite.Connection]:- db_conn = await _connect(path)- try:- yield db_conn- finally:- await db_conn.close()+ import aiosqlite@dataclasses.dataclass(slots=True)@@ -40,19 +27,15 @@ class Database:hash: HashTable = dataclasses.field(init=False)schema: SchemaTable = dataclasses.field(init=False)- _conn: aiosqlite.Connection = dataclasses.field(init=False)- _is_new: bool = dataclasses.field(init=False)-- @property- def conn(self) -> aiosqlite.Connection:- return self._conn+ conn: aiosqlite.Connection = dataclasses.field(init=False)+ is_new: bool = dataclasses.field(init=False)async def _connect(self) -> None:- self._is_new = not await aio.get_size(self.path)- self._conn = await _connect(self.path)- self.history = HistoryTable(self._conn, self.ignore_history)- self.hash = HashTable(self._conn, self.ignore_history)- self.schema = SchemaTable(self._conn, self.ignore_history)+ self.is_new = not await aio.get_size(self.path)+ self.conn = await raw_connect(self.path)+ self.history = HistoryTable(self.conn, self.ignore_history)+ self.hash = HashTable(self.conn, self.ignore_history)+ self.schema = SchemaTable(self.conn, self.ignore_history)@contextlib.asynccontextmanagerasync def connect(self) -> AsyncGenerator[Self]:@@ -64,19 +47,19 @@ class Database:async def _create_tables(self) -> None:await self.schema.create()- if not self._is_new:+ if not self.is_new:self.schema.check_version()await pre_allocate_100mb(self.conn)await self.history.create()await self.hash.create()- if self._is_new:+ if self.is_new:await self.schema.update()async def create_tables(self) -> None:try:await self._create_tables()except Exception:- if self._is_new:+ if self.is_new:await self.conn.close()try:await aio.unlink(self.path, missing_ok=True)@@ -84,7 +67,7 @@ class Database:passraiseelse:- if not (self._is_new or self.schema.up_to_date):+ if not (self.is_new or self.schema.up_to_date):await self.history.apply_updates()await self.schema.update()@@ -95,22 +78,3 @@ class Database:async def __aexit__(self, *_: object) -> None:await self.conn.close()---async def pre_allocate_100mb(db_conn: aiosqlite.Connection) -> None:- """Pre-allocate 100MB of space to the SQL file just in case the user runs out of disk space."""-- cursor = await db_conn.execute("PRAGMA freelist_count;")- free_space = await cursor.fetchone()- assert free_space is not None-- if free_space[0] > 1024:- return-- pre_allocate_script = (- "CREATE TABLE IF NOT EXISTS t(x);"- "INSERT INTO t VALUES(zeroblob(100*1024*1024));" # 100 MB- "DROP TABLE t;"- )- _ = await db_conn.executescript(pre_allocate_script)- await db_conn.commit()diff --git a/cyberdrop_dl/database/common.py b/cyberdrop_dl/database/common.pynew file mode 100644index 000000000..341dbc378--- /dev/null+++ b/cyberdrop_dl/database/common.py@@ -0,0 +1,65 @@+from __future__ import annotations++import contextlib+import dataclasses+from abc import ABC, abstractmethod+from typing import TYPE_CHECKING, ClassVar++import aiosqlite++if TYPE_CHECKING:+ from collections.abc import AsyncGenerator+ from pathlib import Path++[email protected](slots=True)+class Table(ABC):+ NAME: ClassVar[str]+ db_conn: aiosqlite.Connection+ ignore_history: bool = False++ def __init_subclass__(cls, name: str | None = None) -> None:+ if name:+ cls.NAME = name++ @abstractmethod+ async def create(self) -> None: ...++ async def exists(self) -> bool:+ query = "SELECT 1 FROM sqlite_master WHERE type='table' AND name= ? LIMIT 1;"+ cursor = await self.db_conn.execute(query, (self.NAME,))+ return await cursor.fetchone() is not None+++async def raw_connect(path: Path) -> aiosqlite.Connection:+ db_conn = await aiosqlite.connect(path, timeout=20)+ db_conn.row_factory = aiosqlite.Row+ return db_conn+++async def connect(path: Path) -> AsyncGenerator[aiosqlite.Connection]:+ db_conn = await raw_connect(path)+ try:+ yield db_conn+ finally:+ await db_conn.close()+++async def pre_allocate_100mb(db_conn: aiosqlite.Connection) -> None:+ """Pre-allocate 100MB of space to the SQL file just in case the user runs out of disk space."""++ cursor = await db_conn.execute("PRAGMA freelist_count;")+ free_space = await cursor.fetchone()+ assert free_space is not None++ if free_space[0] > 1024:+ return++ pre_allocate_script = (+ "CREATE TABLE IF NOT EXISTS t(x);"+ "INSERT INTO t VALUES(zeroblob(100*1024*1024));" # 100 MB+ "DROP TABLE t;"+ )+ _ = await db_conn.executescript(pre_allocate_script)+ await db_conn.commit()diff --git a/cyberdrop_dl/database/definitions.py b/cyberdrop_dl/database/definitions.pyindex 5dc3526e4..90a33325d 100644--- a/cyberdrop_dl/database/definitions.py+++ b/cyberdrop_dl/database/definitions.py@@ -1,3 +1,5 @@+from __future__ import annotations+CREATE_HISTORY = """CREATE TABLE IF NOT EXISTS media (domain TEXT,url_path TEXT,diff --git a/cyberdrop_dl/database/hash.py b/cyberdrop_dl/database/hash.pyindex f0da1c112..8e8fb5d0b 100644--- a/cyberdrop_dl/database/hash.py+++ b/cyberdrop_dl/database/hash.py@@ -5,8 +5,8 @@ import loggingfrom pathlib import Pathfrom typing import TYPE_CHECKING, cast-from cyberdrop_dl.database import table-from cyberdrop_dl.database.definitions import CREATE_FILES, CREATE_HASH, CREATE_HASH_INDEX+from .common import Table+from .definitions import CREATE_FILES, CREATE_HASH, CREATE_HASH_INDEXif TYPE_CHECKING:import aiosqlite@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)@dataclasses.dataclass(slots=True)-class HashTable(table.Table, name="hash"):+class HashTable(Table, name="hash"):cwd: Path = dataclasses.field(init=False, default_factory=lambda: Path.cwd().expanduser().resolve())async def create(self) -> None:diff --git a/cyberdrop_dl/database/history.py b/cyberdrop_dl/database/history.pyindex 57c3d6cb7..b4f2cd111 100644--- a/cyberdrop_dl/database/history.py+++ b/cyberdrop_dl/database/history.py@@ -6,8 +6,8 @@ import timefrom sqlite3 import IntegrityError, Rowfrom typing import TYPE_CHECKING, Any, cast-from cyberdrop_dl.database import table-from cyberdrop_dl.database.definitions import CREATE_HISTORY, CREATE_MEDIA_INDEX+from .common import Table+from .definitions import CREATE_HISTORY, CREATE_MEDIA_INDEXif TYPE_CHECKING:from collections.abc import AsyncGenerator, Callable, Generator@@ -23,7 +23,7 @@ _FETCH_MANY_SIZE: int = 1000logger = logging.getLogger(__name__)-class HistoryTable(table.Table, name="media"):+class HistoryTable(Table, name="media"):async def create(self) -> None:await self.db_conn.execute(CREATE_HISTORY)await self.db_conn.executescript(CREATE_MEDIA_INDEX)diff --git a/cyberdrop_dl/database/schema.py b/cyberdrop_dl/database/schema.pyindex 126fbbc29..5e535655f 100644--- a/cyberdrop_dl/database/schema.py+++ b/cyberdrop_dl/database/schema.py@@ -6,10 +6,11 @@ from typing import TYPE_CHECKING, NamedTuple, finalimport aiosqlite-from cyberdrop_dl.database import table-from cyberdrop_dl.database.definitions import CREATE_SCHEMAfrom cyberdrop_dl.exceptions import DatabaseError+from .common import Table+from .definitions import CREATE_SCHEMA+if TYPE_CHECKING:import aiosqlite@@ -38,7 +39,7 @@ logger = logging.getLogger(__name__)@dataclasses.dataclass(slots=True)-class SchemaTable(table.Table, name="schema_version"):+class SchemaTable(Table, name="schema_version"):up_to_date: bool = Falseversion: Version | None = Nonediff --git a/cyberdrop_dl/database/table.py b/cyberdrop_dl/database/table.pydeleted file mode 100644index 4ff37ffab..000000000--- a/cyberdrop_dl/database/table.py+++ /dev/null@@ -1,27 +0,0 @@-from __future__ import annotations--import dataclasses-from abc import ABC, abstractmethod-from typing import TYPE_CHECKING, ClassVar--if TYPE_CHECKING:- import aiosqlite--[email protected](slots=True)-class Table(ABC):- NAME: ClassVar[str]- db_conn: aiosqlite.Connection- ignore_history: bool = False-- def __init_subclass__(cls, name: str | None = None) -> None:- if name:- cls.NAME = name-- @abstractmethod- async def create(self) -> None: ...-- async def exists(self) -> bool:- query = "SELECT 1 FROM sqlite_master WHERE type='table' AND name= ? LIMIT 1;"- cursor = await self.db_conn.execute(query, (self.NAME,))- return await cursor.fetchone() is not Nonediff --git a/cyberdrop_dl/database/transfer.py b/cyberdrop_dl/database/transfer.pyindex 55809d83f..101c0d0b5 100644--- a/cyberdrop_dl/database/transfer.py+++ b/cyberdrop_dl/database/transfer.py@@ -8,6 +8,7 @@ from pathlib import Pathfrom typing import LiteralStringfrom cyberdrop_dl import database+from cyberdrop_dl.database import commonfrom cyberdrop_dl.database.history import apply_fixesfrom cyberdrop_dl.database.schema import CURRENT_VERSION, Versionfrom cyberdrop_dl.logs import setup_console_logging@@ -195,7 +196,7 @@ def _create_new_database(path: Path) -> None:def _apply_fixes(db_path: Path) -> None:async def fix() -> None:- async with database.connect(db_path) as conn:+ async with common.connect(db_path) as conn:await apply_fixes(conn)asyncio.run(fix())diff --git a/cyberdrop_dl/url_objects.py b/cyberdrop_dl/url_objects.pyindex d88a70370..dc4d2299e 100644--- a/cyberdrop_dl/url_objects.py+++ b/cyberdrop_dl/url_objects.py@@ -18,7 +18,6 @@ if TYPE_CHECKING:from collections.abc import Generator, Iterable, Sequencefrom cyberdrop_dl import signature- from cyberdrop_dl.manager import Managerclass AbsoluteHttpURL(yarl.URL):@signature.copy(yarl.URL.__new__)@@ -266,7 +265,7 @@ class ScrapeItem:self.folders.append(folder)- def _set_type(self, scrape_item_type: ScrapeItemType | None, _: Manager | None = None) -> None:+ def _set_type(self, scrape_item_type: ScrapeItemType | None) -> None:self.type = scrape_item_typeself._children_count = self._children_limit = 0if self.type is None:diff --git a/tests/test_database.py b/tests/test_database.pyindex 5a0293ad1..7b0ce64ba 100644--- a/tests/test_database.py+++ b/tests/test_database.py@@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, castimport pytest-from cyberdrop_dl import aio, database, scrape_mapper-from cyberdrop_dl.database import Database, schema+from cyberdrop_dl import aio, scrape_mapper+from cyberdrop_dl.database import Database, common, schemafrom cyberdrop_dl.exceptions import DatabaseErrorfrom cyberdrop_dl.scrape_mapper import _create_item_from_rowfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem@@ -129,7 +129,7 @@ async def test_database_creation(tmp_cwd: Path) -> None:async with db:pass- assert db._is_new+ assert db.is_newsize = await aio.get_size(db_file)assert sizeassert db.schema.up_to_date@@ -137,12 +137,12 @@ async def test_database_creation(tmp_cwd: Path) -> None:async def test_pre_allocation(tmp_cwd: Path) -> None:db_file = tmp_cwd / "test_db.db"- async with database.connect(db_file) as db:+ async with common.connect(db_file) as db:size = await aio.get_size(db_file)assert size == 0- async with database.connect(db_file) as db:- await database.pre_allocate_100mb(db)+ async with common.connect(db_file) as db:+ await common.pre_allocate_100mb(db)size = await aio.get_size(db_file)assert size@@ -154,12 +154,12 @@ async def test_database_version_check(tmp_cwd: Path) -> None:db_file.touch()async with Database(db_file).connect() as db:await db._create_tables()- assert db._is_new+ assert db.is_newawait db.conn.execute("DROP TABLE 'schema_version'")await db.conn.commit()async with Database(db_file).connect() as db:- assert not db._is_new+ assert not db.is_newassert not db.schema.up_to_dateawait db.schema.create()assert db.schema.version is None