Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: save the database path as an attribute of `MediaItem` (#1354)

NTFSvolume6cccd25f1f36
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 6613b6df3..01f167bd4 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -20,7 +20,6 @@ from cyberdrop_dl import constants
from cyberdrop_dl.clients.scraper_client import ScraperClient
from cyberdrop_dl.data_structures.mediaprops import Resolution
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, copy_signature
-from cyberdrop_dl.database import get_db_path
from cyberdrop_dl.downloader.downloader import Downloader
from cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeError
from cyberdrop_dl.scraper import filters
@@ -376,7 +375,7 @@ class Crawler(ABC):
assert is_absolute_http_url(debrid_link)
download_folder = get_download_path(self.manager, scrape_item, self.FOLDER_DOMAIN)
media_item = MediaItem.from_item(
- scrape_item, url, download_folder, filename, original_filename, debrid_link, ext=ext
+ scrape_item, url, self.DOMAIN, download_folder, filename, original_filename, debrid_link, ext=ext
)
self.create_task(self.handle_media_item(media_item, m3u8))
@@ -497,7 +496,7 @@ class Crawler(ABC):
"""Checks whether an album has completed given its domain and album id."""
if not album_results:
return False
- url_path = get_db_path(url, self.DOMAIN)
+ url_path = MediaItem.create_db_path(url, self.DOMAIN)
if url_path in album_results and album_results[url_path] != 0:
log(f"Skipping {url} as it has already been downloaded")
self.manager.progress_manager.download_progress.add_previously_completed()
diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.py
index 852d7a0a9..a3499ba25 100644
--- a/cyberdrop_dl/data_structures/url_objects.py
+++ b/cyberdrop_dl/data_structures/url_objects.py
@@ -152,6 +152,7 @@ class HlsSegment(NamedTuple):
@dataclass(unsafe_hash=True, slots=True, kw_only=True)
class MediaItem:
url: AbsoluteHttpURL
+ domain: str
referer: AbsoluteHttpURL
download_folder: Path
filename: str
@@ -177,13 +178,34 @@ class MediaItem:
downloaded: bool = field(default=False, compare=False)
parent_media_item: MediaItem | None = field(default=None, compare=False)
+ db_path: str = field(init=False, repr=False)
_task_id: TaskID | None = field(default=None, compare=False)
+ def __post_init__(self) -> None:
+ self.db_path = self.create_db_path(self.url, self.domain)
+
+ @staticmethod
+ def create_db_path(url: yarl.URL, domain: str) -> str:
+ """Gets the URL path to be put into the DB and checked from the DB."""
+
+ if domain:
+ if "e-hentai" in domain:
+ return url.path.split("keystamp")[0][:-1]
+
+ if "mediafire" in domain:
+ return url.name
+
+ if "mega.nz" in domain:
+ return url.path_qs if not (frag := url.fragment) else f"{url.path_qs}#{frag}"
+
+ return url.path
+
@staticmethod
def from_item(
origin: ScrapeItem | MediaItem,
url: AbsoluteHttpURL,
/,
+ domain: str,
download_folder: Path,
filename: str,
original_filename: str | None = None,
@@ -195,6 +217,7 @@ class MediaItem:
) -> MediaItem:
return MediaItem(
url=url,
+ domain=domain,
download_folder=download_folder,
filename=filename,
debrid_link=debrid_link,
diff --git a/cyberdrop_dl/database/__init__.py b/cyberdrop_dl/database/__init__.py
index d880ddca9..dcceb80a8 100644
--- a/cyberdrop_dl/database/__init__.py
+++ b/cyberdrop_dl/database/__init__.py
@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING
import aiosqlite
from .tables import HashTable, HistoryTable, SchemaVersionTable, TempRefererTable
-from .tables.history import get_db_path
if TYPE_CHECKING:
from pathlib import Path
@@ -58,4 +57,4 @@ class Database:
await self._db_conn.commit()
-__all__ = ["Database", "get_db_path"]
+__all__ = ["Database"]
diff --git a/cyberdrop_dl/database/tables/history.py b/cyberdrop_dl/database/tables/history.py
index 0135a6ee0..837c97fd7 100644
--- a/cyberdrop_dl/database/tables/history.py
+++ b/cyberdrop_dl/database/tables/history.py
@@ -3,6 +3,7 @@ from __future__ import annotations
from sqlite3 import IntegrityError, Row
from typing import TYPE_CHECKING, cast
+from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.utils.utilities import log
from .definitions import create_fixed_history, create_history
@@ -15,29 +16,10 @@ if TYPE_CHECKING:
from yarl import URL
from cyberdrop_dl.crawlers import Crawler
- from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.database import Database
-_FETCH_MANY_SIZE = 1000
-
-
-def get_db_path(url: URL, domain: str = "") -> str:
- """Gets the URL path to be put into the DB and checked from the DB."""
-
- # TODO: domain SHOULD be mandatory, not optional. Make it mandatory and update any place that uses it
-
- if domain:
- if "e-hentai" in domain:
- return url.path.split("keystamp")[0][:-1]
-
- if "mediafire" in domain:
- return url.name
-
- if "mega.nz" in domain:
- return url.path_qs if not (frag := url.fragment) else f"{url.path_qs}#{frag}"
-
- return url.path
+_FETCH_MANY_SIZE: int = 1000
class HistoryTable:
@@ -97,7 +79,7 @@ class HistoryTable:
if self._database.ignore_history:
return False
- url_path = get_db_path(url, domain)
+ url_path = MediaItem.create_db_path(url, domain)
async def select_referer_and_completed() -> tuple[str, bool]:
query = "SELECT referer, completed FROM media WHERE domain = ? and url_path = ?"
@@ -132,9 +114,8 @@ class HistoryTable:
async def set_album_id(self, domain: str, media_item: MediaItem) -> None:
"""Sets an album_id in the database."""
- url_path = get_db_path(media_item.url, str(media_item.referer))
query = "UPDATE media SET album_id = ? WHERE domain = ? and url_path = ?"
- await self.db_conn.execute(query, (media_item.album_id, domain, url_path))
+ await self.db_conn.execute(query, (media_item.album_id, domain, media_item.db_path))
await self.db_conn.commit()
async def check_complete_by_referer(self, domain: str | None, referer: URL) -> bool:
@@ -162,7 +143,7 @@ class HistoryTable:
async def insert_incompleted(self, domain: str, media_item: MediaItem) -> None:
"""Inserts an uncompleted file into the database."""
- url_path = get_db_path(media_item.url, str(media_item.referer))
+ url_path = media_item.db_path
download_filename = media_item.download_filename or ""
cursor = await self.db_conn.cursor()
query = "UPDATE media SET domain = ?, album_id = ? WHERE domain = 'no_crawler' and url_path = ? and referer = ?"
@@ -199,7 +180,7 @@ class HistoryTable:
async def mark_complete(self, domain: str, media_item: MediaItem) -> None:
"""Mark a download as completed in the database."""
- url_path = get_db_path(media_item.url, str(media_item.referer))
+ url_path = media_item.db_path
query = "UPDATE media SET completed = 1, completed_at = CURRENT_TIMESTAMP WHERE domain = ? and url_path = ?"
await self.db_conn.execute(query, (domain, url_path))
await self.db_conn.commit()
@@ -207,7 +188,7 @@ class HistoryTable:
async def add_filesize(self, domain: str, media_item: MediaItem) -> None:
"""Adds the file size to the db."""
- url_path = get_db_path(media_item.url, str(media_item.referer))
+ url_path = media_item.db_path
file_size = media_item.complete_file.stat().st_size
query = """UPDATE media SET file_size=? WHERE domain = ? and url_path = ?"""
await self.db_conn.execute(query, (file_size, domain, url_path))
@@ -216,7 +197,7 @@ class HistoryTable:
async def add_duration(self, domain: str, media_item: MediaItem) -> None:
"""Adds the duration to the db."""
- url_path = get_db_path(media_item.url, str(media_item.referer))
+ url_path = media_item.db_path
query = "UPDATE media SET duration=? WHERE domain = ? and url_path = ?"
await self.db_conn.execute(query, (media_item.duration, domain, url_path))
await self.db_conn.commit()
@@ -226,7 +207,7 @@ class HistoryTable:
if media_item.is_segment:
return
- url_path = get_db_path(media_item.url, str(media_item.referer))
+ url_path = media_item.db_path
query = "SELECT duration FROM media WHERE domain = ? and url_path = ?"
cursor = await self.db_conn.execute(query, (domain, url_path))
if row := await cursor.fetchone():
@@ -234,7 +215,7 @@ class HistoryTable:
async def add_download_filename(self, domain: str, media_item: MediaItem) -> None:
"""Add the download_filename to the db."""
- url_path = get_db_path(media_item.url, str(media_item.referer))
+ url_path = media_item.db_path
query = "UPDATE media SET download_filename=? WHERE domain = ? and url_path = ? and download_filename = ''"
await self.db_conn.execute(query, (media_item.download_filename, domain, url_path))
await self.db_conn.commit()
@@ -253,7 +234,7 @@ class HistoryTable:
if media_item.is_segment:
return media_item.filename
- url_path = get_db_path(media_item.url, str(media_item.referer))
+ url_path = media_item.db_path
query = "SELECT download_filename FROM media WHERE domain = ? and url_path = ?"
cursor = await self.db_conn.execute(query, (domain, url_path))
if row := await cursor.fetchone():
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index 24d891741..6755a20eb 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -16,7 +16,6 @@ from aiohttp import ClientConnectorError, ClientError, ClientResponseError
from cyberdrop_dl.constants import CustomHTTPStatus
from cyberdrop_dl.data_structures.url_objects import HlsSegment, MediaItem
-from cyberdrop_dl.database import get_db_path
from cyberdrop_dl.exceptions import (
DownloadError,
DurationError,
@@ -167,7 +166,7 @@ class Downloader:
async with self._semaphore:
await self.manager.states.RUNNING.wait()
self.waiting_items -= 1
- self.processed_items.add(get_db_path(media_item.url, self.domain))
+ self.processed_items.add(media_item.db_path)
self.update_queued_files(increase_total=False)
async with self.manager.client_manager.global_download_slots:
return await self.start_download(media_item)
@@ -274,6 +273,7 @@ class Downloader:
seg_media_item = MediaItem.from_item(
media_item,
segment.url,
+ domain=media_item.domain,
download_folder=download_folder,
filename=segment.name,
ext=media_item.ext,
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index 0a308deb7..99524d6d3 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -238,12 +238,13 @@ class ScrapeMapper:
scrape_item.add_to_parent_title("Loose Files")
scrape_item.part_of_album = True
- download_folder = get_download_path(self.manager, scrape_item, "no_crawler")
+ domain = "no_crawler"
+ download_folder = get_download_path(self.manager, scrape_item, domain)
try:
filename, _ = get_filename_and_ext(scrape_item.url.name)
except NoExtensionError:
filename, _ = get_filename_and_ext(scrape_item.url.name, forum=True)
- media_item = MediaItem.from_item(scrape_item, scrape_item.url, download_folder, filename)
+ media_item = MediaItem.from_item(scrape_item, scrape_item.url, domain, download_folder, filename)
self.manager.task_group.create_task(self.no_crawler_downloader.run(media_item))
return
diff --git a/tests/test_database.py b/tests/test_database.py
index e535230bf..665bd64dd 100644
--- a/tests/test_database.py
+++ b/tests/test_database.py
@@ -1,12 +1,17 @@
+from __future__ import annotations
+
from datetime import datetime
from pathlib import Path
-from typing import cast
+from typing import TYPE_CHECKING, cast
-import aiosqlite
import pytest
-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem
from cyberdrop_dl.scraper.scrape_mapper import _create_item_from_row
+from cyberdrop_dl.utils.utilities import parse_url
+
+if TYPE_CHECKING:
+ import aiosqlite
_MOCK_ROW = {
"referer": "https://drive.google.com/file/d/1F0YBsnQRvrMbK0p9UlnyLu88kqQ0j_F6/edit",
@@ -81,3 +86,28 @@ def test_invalid_date_format(row) -> None:
with pytest.raises(ValueError):
_create_item_from_row(row)
+
+
+ "url, expected",
+ [
+ (
+ "https://megacloud.blog/embed-2/v3/e-1/TZb4gRkOQ642?k=1&autoPlay=1&oa=0&asi=1",
+ "/embed-2/v3/e-1/TZb4gRkOQ642",
+ ),
+ ("https://www.mediafire.com/file/ctppmpm7giofsgv/ADOFAI.vpk", "ADOFAI.vpk"),
+ ("https://mega.nz/#!Ue5VRSIQ!kC2E4a4JwfWWCWYNJovGFHlbz8F", "/#!Ue5VRSIQ!kC2E4a4JwfWWCWYNJovGFHlbz8F"),
+ (
+ "https://mega.nz/folder/oZZxyBrY#oU4jASLPpJVvqGHJIMRcgQ/file/IYZABDGY",
+ "/folder/oZZxyBrY#oU4jASLPpJVvqGHJIMRcgQ/file/IYZABDGY",
+ ),
+ (
+ "https://c.bunkr-cache.se/HwdRnHMUiWOQevCg/1df93418-5063-4e1b-851e-9470cb8fc5c6.mp4",
+ "/HwdRnHMUiWOQevCg/1df93418-5063-4e1b-851e-9470cb8fc5c6.mp4",
+ ),
+ ],
+)
+def test_create_db_path(url: str, expected: str) -> None:
+ url_ = parse_url(url)
+ path = MediaItem.create_db_path(url_, url_.host)
+ assert path == expected