Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: no circular imports on db

NTFSvolume8717735d48dd
diff --git a/cyberdrop_dl/database/__init__.py b/cyberdrop_dl/database/__init__.py
index 351913096..f905762f5 100644
--- a/cyberdrop_dl/database/__init__.py
+++ b/cyberdrop_dl/database/__init__.py
@@ -1,3 +1,5 @@
-from ._db import Database, connect, pre_allocate_100mb
+from __future__ import annotations
-__all__ = ["Database", "connect", "pre_allocate_100mb"]
+from ._db import Database
+
+__all__ = ["Database"]
diff --git a/cyberdrop_dl/database/_db.py b/cyberdrop_dl/database/_db.py
index ddbced11c..494b43584 100644
--- a/cyberdrop_dl/database/_db.py
+++ b/cyberdrop_dl/database/_db.py
@@ -4,31 +4,18 @@ import contextlib
import dataclasses
from typing import TYPE_CHECKING, Self
-import aiosqlite
-
from cyberdrop_dl import aio
-from cyberdrop_dl.database.hash import HashTable
-from cyberdrop_dl.database.history import HistoryTable
-from cyberdrop_dl.database.schema import SchemaTable
+
+from .common import pre_allocate_100mb, raw_connect
+from .hash import HashTable
+from .history import HistoryTable
+from .schema import SchemaTable
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
from pathlib import Path
-
-async def _connect(path: Path) -> aiosqlite.Connection:
- db_conn = await aiosqlite.connect(path, timeout=20)
- db_conn.row_factory = aiosqlite.Row
- return db_conn
-
-
-async def connect(path: Path) -> AsyncGenerator[aiosqlite.Connection]:
- db_conn = await _connect(path)
- try:
- yield db_conn
- finally:
- await db_conn.close()
+ import aiosqlite
@dataclasses.dataclass(slots=True)
@@ -40,19 +27,15 @@ class Database:
hash: HashTable = dataclasses.field(init=False)
schema: SchemaTable = dataclasses.field(init=False)
- _conn: aiosqlite.Connection = dataclasses.field(init=False)
- _is_new: bool = dataclasses.field(init=False)
-
- @property
- def conn(self) -> aiosqlite.Connection:
- return self._conn
+ conn: aiosqlite.Connection = dataclasses.field(init=False)
+ is_new: bool = dataclasses.field(init=False)
async def _connect(self) -> None:
- self._is_new = not await aio.get_size(self.path)
- self._conn = await _connect(self.path)
- self.history = HistoryTable(self._conn, self.ignore_history)
- self.hash = HashTable(self._conn, self.ignore_history)
- self.schema = SchemaTable(self._conn, self.ignore_history)
+ self.is_new = not await aio.get_size(self.path)
+ self.conn = await raw_connect(self.path)
+ self.history = HistoryTable(self.conn, self.ignore_history)
+ self.hash = HashTable(self.conn, self.ignore_history)
+ self.schema = SchemaTable(self.conn, self.ignore_history)
@contextlib.asynccontextmanager
async def connect(self) -> AsyncGenerator[Self]:
@@ -64,19 +47,19 @@ class Database:
async def _create_tables(self) -> None:
await self.schema.create()
- if not self._is_new:
+ if not self.is_new:
self.schema.check_version()
await pre_allocate_100mb(self.conn)
await self.history.create()
await self.hash.create()
- if self._is_new:
+ if self.is_new:
await self.schema.update()
async def create_tables(self) -> None:
try:
await self._create_tables()
except Exception:
- if self._is_new:
+ if self.is_new:
await self.conn.close()
try:
await aio.unlink(self.path, missing_ok=True)
@@ -84,7 +67,7 @@ class Database:
pass
raise
else:
- if not (self._is_new or self.schema.up_to_date):
+ if not (self.is_new or self.schema.up_to_date):
await self.history.apply_updates()
await self.schema.update()
@@ -95,22 +78,3 @@ class Database:
async def __aexit__(self, *_: object) -> None:
await self.conn.close()
-
-
-async def pre_allocate_100mb(db_conn: aiosqlite.Connection) -> None:
- """Pre-allocate 100MB of space to the SQL file just in case the user runs out of disk space."""
-
- cursor = await db_conn.execute("PRAGMA freelist_count;")
- free_space = await cursor.fetchone()
- assert free_space is not None
-
- if free_space[0] > 1024:
- return
-
- pre_allocate_script = (
- "CREATE TABLE IF NOT EXISTS t(x);"
- "INSERT INTO t VALUES(zeroblob(100*1024*1024));" # 100 MB
- "DROP TABLE t;"
- )
- _ = await db_conn.executescript(pre_allocate_script)
- await db_conn.commit()
diff --git a/cyberdrop_dl/database/common.py b/cyberdrop_dl/database/common.py
new file mode 100644
index 000000000..341dbc378
--- /dev/null
+++ b/cyberdrop_dl/database/common.py
@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import contextlib
+import dataclasses
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, ClassVar
+
+import aiosqlite
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator
+ from pathlib import Path
+
+
[email protected](slots=True)
+class Table(ABC):
+ NAME: ClassVar[str]
+ db_conn: aiosqlite.Connection
+ ignore_history: bool = False
+
+ def __init_subclass__(cls, name: str | None = None) -> None:
+ if name:
+ cls.NAME = name
+
+ @abstractmethod
+ async def create(self) -> None: ...
+
+ async def exists(self) -> bool:
+ query = "SELECT 1 FROM sqlite_master WHERE type='table' AND name= ? LIMIT 1;"
+ cursor = await self.db_conn.execute(query, (self.NAME,))
+ return await cursor.fetchone() is not None
+
+
+async def raw_connect(path: Path) -> aiosqlite.Connection:
+ db_conn = await aiosqlite.connect(path, timeout=20)
+ db_conn.row_factory = aiosqlite.Row
+ return db_conn
+
+
+async def connect(path: Path) -> AsyncGenerator[aiosqlite.Connection]:
+ db_conn = await raw_connect(path)
+ try:
+ yield db_conn
+ finally:
+ await db_conn.close()
+
+
+async def pre_allocate_100mb(db_conn: aiosqlite.Connection) -> None:
+ """Pre-allocate 100MB of space to the SQL file just in case the user runs out of disk space."""
+
+ cursor = await db_conn.execute("PRAGMA freelist_count;")
+ free_space = await cursor.fetchone()
+ assert free_space is not None
+
+ if free_space[0] > 1024:
+ return
+
+ pre_allocate_script = (
+ "CREATE TABLE IF NOT EXISTS t(x);"
+ "INSERT INTO t VALUES(zeroblob(100*1024*1024));" # 100 MB
+ "DROP TABLE t;"
+ )
+ _ = await db_conn.executescript(pre_allocate_script)
+ await db_conn.commit()
diff --git a/cyberdrop_dl/database/definitions.py b/cyberdrop_dl/database/definitions.py
index 5dc3526e4..90a33325d 100644
--- a/cyberdrop_dl/database/definitions.py
+++ b/cyberdrop_dl/database/definitions.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
CREATE_HISTORY = """CREATE TABLE IF NOT EXISTS media (
domain TEXT,
url_path TEXT,
diff --git a/cyberdrop_dl/database/hash.py b/cyberdrop_dl/database/hash.py
index f0da1c112..8e8fb5d0b 100644
--- a/cyberdrop_dl/database/hash.py
+++ b/cyberdrop_dl/database/hash.py
@@ -5,8 +5,8 @@ import logging
from pathlib import Path
from typing import TYPE_CHECKING, cast
-from cyberdrop_dl.database import table
-from cyberdrop_dl.database.definitions import CREATE_FILES, CREATE_HASH, CREATE_HASH_INDEX
+from .common import Table
+from .definitions import CREATE_FILES, CREATE_HASH, CREATE_HASH_INDEX
if TYPE_CHECKING:
import aiosqlite
@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
@dataclasses.dataclass(slots=True)
-class HashTable(table.Table, name="hash"):
+class HashTable(Table, name="hash"):
cwd: Path = dataclasses.field(init=False, default_factory=lambda: Path.cwd().expanduser().resolve())
async def create(self) -> None:
diff --git a/cyberdrop_dl/database/history.py b/cyberdrop_dl/database/history.py
index 57c3d6cb7..b4f2cd111 100644
--- a/cyberdrop_dl/database/history.py
+++ b/cyberdrop_dl/database/history.py
@@ -6,8 +6,8 @@ import time
from sqlite3 import IntegrityError, Row
from typing import TYPE_CHECKING, Any, cast
-from cyberdrop_dl.database import table
-from cyberdrop_dl.database.definitions import CREATE_HISTORY, CREATE_MEDIA_INDEX
+from .common import Table
+from .definitions import CREATE_HISTORY, CREATE_MEDIA_INDEX
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable, Generator
@@ -23,7 +23,7 @@ _FETCH_MANY_SIZE: int = 1000
logger = logging.getLogger(__name__)
-class HistoryTable(table.Table, name="media"):
+class HistoryTable(Table, name="media"):
async def create(self) -> None:
await self.db_conn.execute(CREATE_HISTORY)
await self.db_conn.executescript(CREATE_MEDIA_INDEX)
diff --git a/cyberdrop_dl/database/schema.py b/cyberdrop_dl/database/schema.py
index 126fbbc29..5e535655f 100644
--- a/cyberdrop_dl/database/schema.py
+++ b/cyberdrop_dl/database/schema.py
@@ -6,10 +6,11 @@ from typing import TYPE_CHECKING, NamedTuple, final
import aiosqlite
-from cyberdrop_dl.database import table
-from cyberdrop_dl.database.definitions import CREATE_SCHEMA
from cyberdrop_dl.exceptions import DatabaseError
+from .common import Table
+from .definitions import CREATE_SCHEMA
+
if TYPE_CHECKING:
import aiosqlite
@@ -38,7 +39,7 @@ logger = logging.getLogger(__name__)
@dataclasses.dataclass(slots=True)
-class SchemaTable(table.Table, name="schema_version"):
+class SchemaTable(Table, name="schema_version"):
up_to_date: bool = False
version: Version | None = None
diff --git a/cyberdrop_dl/database/table.py b/cyberdrop_dl/database/table.py
deleted file mode 100644
index 4ff37ffab..000000000
--- a/cyberdrop_dl/database/table.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from __future__ import annotations
-
-import dataclasses
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, ClassVar
-
-if TYPE_CHECKING:
- import aiosqlite
-
-
[email protected](slots=True)
-class Table(ABC):
- NAME: ClassVar[str]
- db_conn: aiosqlite.Connection
- ignore_history: bool = False
-
- def __init_subclass__(cls, name: str | None = None) -> None:
- if name:
- cls.NAME = name
-
- @abstractmethod
- async def create(self) -> None: ...
-
- async def exists(self) -> bool:
- query = "SELECT 1 FROM sqlite_master WHERE type='table' AND name= ? LIMIT 1;"
- cursor = await self.db_conn.execute(query, (self.NAME,))
- return await cursor.fetchone() is not None
diff --git a/cyberdrop_dl/database/transfer.py b/cyberdrop_dl/database/transfer.py
index 55809d83f..101c0d0b5 100644
--- a/cyberdrop_dl/database/transfer.py
+++ b/cyberdrop_dl/database/transfer.py
@@ -8,6 +8,7 @@ from pathlib import Path
from typing import LiteralString
from cyberdrop_dl import database
+from cyberdrop_dl.database import common
from cyberdrop_dl.database.history import apply_fixes
from cyberdrop_dl.database.schema import CURRENT_VERSION, Version
from cyberdrop_dl.logs import setup_console_logging
@@ -195,7 +196,7 @@ def _create_new_database(path: Path) -> None:
def _apply_fixes(db_path: Path) -> None:
async def fix() -> None:
- async with database.connect(db_path) as conn:
+ async with common.connect(db_path) as conn:
await apply_fixes(conn)
asyncio.run(fix())
diff --git a/cyberdrop_dl/url_objects.py b/cyberdrop_dl/url_objects.py
index d88a70370..dc4d2299e 100644
--- a/cyberdrop_dl/url_objects.py
+++ b/cyberdrop_dl/url_objects.py
@@ -18,7 +18,6 @@ if TYPE_CHECKING:
from collections.abc import Generator, Iterable, Sequence
from cyberdrop_dl import signature
- from cyberdrop_dl.manager import Manager
class AbsoluteHttpURL(yarl.URL):
@signature.copy(yarl.URL.__new__)
@@ -266,7 +265,7 @@ class ScrapeItem:
self.folders.append(folder)
- def _set_type(self, scrape_item_type: ScrapeItemType | None, _: Manager | None = None) -> None:
+ def _set_type(self, scrape_item_type: ScrapeItemType | None) -> None:
self.type = scrape_item_type
self._children_count = self._children_limit = 0
if self.type is None:
diff --git a/tests/test_database.py b/tests/test_database.py
index 5a0293ad1..7b0ce64ba 100644
--- a/tests/test_database.py
+++ b/tests/test_database.py
@@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, cast
import pytest
-from cyberdrop_dl import aio, database, scrape_mapper
-from cyberdrop_dl.database import Database, schema
+from cyberdrop_dl import aio, scrape_mapper
+from cyberdrop_dl.database import Database, common, schema
from cyberdrop_dl.exceptions import DatabaseError
from cyberdrop_dl.scrape_mapper import _create_item_from_row
from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem
@@ -129,7 +129,7 @@ async def test_database_creation(tmp_cwd: Path) -> None:
async with db:
pass
- assert db._is_new
+ assert db.is_new
size = await aio.get_size(db_file)
assert size
assert db.schema.up_to_date
@@ -137,12 +137,12 @@ async def test_database_creation(tmp_cwd: Path) -> None:
async def test_pre_allocation(tmp_cwd: Path) -> None:
db_file = tmp_cwd / "test_db.db"
- async with database.connect(db_file) as db:
+ async with common.connect(db_file) as db:
size = await aio.get_size(db_file)
assert size == 0
- async with database.connect(db_file) as db:
- await database.pre_allocate_100mb(db)
+ async with common.connect(db_file) as db:
+ await common.pre_allocate_100mb(db)
size = await aio.get_size(db_file)
assert size
@@ -154,12 +154,12 @@ async def test_database_version_check(tmp_cwd: Path) -> None:
db_file.touch()
async with Database(db_file).connect() as db:
await db._create_tables()
- assert db._is_new
+ assert db.is_new
await db.conn.execute("DROP TABLE 'schema_version'")
await db.conn.commit()
async with Database(db_file).connect() as db:
- assert not db._is_new
+ assert not db.is_new
assert not db.schema.up_to_date
await db.schema.create()
assert db.schema.version is None