Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: save the database path as an attribute of `MediaItem` (#1354)
NTFSvolume6cccd25f1f36
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 6613b6df3..01f167bd4 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -20,7 +20,6 @@ from cyberdrop_dl import constantsfrom cyberdrop_dl.clients.scraper_client import ScraperClientfrom cyberdrop_dl.data_structures.mediaprops import Resolutionfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, copy_signature-from cyberdrop_dl.database import get_db_pathfrom cyberdrop_dl.downloader.downloader import Downloaderfrom cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.scraper import filters@@ -376,7 +375,7 @@ class Crawler(ABC):assert is_absolute_http_url(debrid_link)download_folder = get_download_path(self.manager, scrape_item, self.FOLDER_DOMAIN)media_item = MediaItem.from_item(- scrape_item, url, download_folder, filename, original_filename, debrid_link, ext=ext+ scrape_item, url, self.DOMAIN, download_folder, filename, original_filename, debrid_link, ext=ext)self.create_task(self.handle_media_item(media_item, m3u8))@@ -497,7 +496,7 @@ class Crawler(ABC):"""Checks whether an album has completed given its domain and album id."""if not album_results:return False- url_path = get_db_path(url, self.DOMAIN)+ url_path = MediaItem.create_db_path(url, self.DOMAIN)if url_path in album_results and album_results[url_path] != 0:log(f"Skipping {url} as it has already been downloaded")self.manager.progress_manager.download_progress.add_previously_completed()diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.pyindex 852d7a0a9..a3499ba25 100644--- a/cyberdrop_dl/data_structures/url_objects.py+++ b/cyberdrop_dl/data_structures/url_objects.py@@ -152,6 +152,7 @@ class HlsSegment(NamedTuple):@dataclass(unsafe_hash=True, slots=True, kw_only=True)class MediaItem:url: AbsoluteHttpURL+ domain: strreferer: AbsoluteHttpURLdownload_folder: Pathfilename: str@@ -177,13 +178,34 @@ class MediaItem:downloaded: bool = field(default=False, compare=False)parent_media_item: MediaItem | None = field(default=None, compare=False)+ db_path: str = field(init=False, repr=False)_task_id: TaskID | None = field(default=None, compare=False)+ def __post_init__(self) -> None:+ self.db_path = self.create_db_path(self.url, self.domain)++ @staticmethod+ def create_db_path(url: yarl.URL, domain: str) -> str:+ """Gets the URL path to be put into the DB and checked from the DB."""++ if domain:+ if "e-hentai" in domain:+ return url.path.split("keystamp")[0][:-1]++ if "mediafire" in domain:+ return url.name++ if "mega.nz" in domain:+ return url.path_qs if not (frag := url.fragment) else f"{url.path_qs}#{frag}"++ return url.path+@staticmethoddef from_item(origin: ScrapeItem | MediaItem,url: AbsoluteHttpURL,/,+ domain: str,download_folder: Path,filename: str,original_filename: str | None = None,@@ -195,6 +217,7 @@ class MediaItem:) -> MediaItem:return MediaItem(url=url,+ domain=domain,download_folder=download_folder,filename=filename,debrid_link=debrid_link,diff --git a/cyberdrop_dl/database/__init__.py b/cyberdrop_dl/database/__init__.pyindex d880ddca9..dcceb80a8 100644--- a/cyberdrop_dl/database/__init__.py+++ b/cyberdrop_dl/database/__init__.py@@ -5,7 +5,6 @@ from typing import TYPE_CHECKINGimport aiosqlitefrom .tables import HashTable, HistoryTable, SchemaVersionTable, TempRefererTable-from .tables.history import get_db_pathif TYPE_CHECKING:from pathlib import Path@@ -58,4 +57,4 @@ class Database:await self._db_conn.commit()-__all__ = ["Database", "get_db_path"]+__all__ = ["Database"]diff --git a/cyberdrop_dl/database/tables/history.py b/cyberdrop_dl/database/tables/history.pyindex 0135a6ee0..837c97fd7 100644--- a/cyberdrop_dl/database/tables/history.py+++ b/cyberdrop_dl/database/tables/history.py@@ -3,6 +3,7 @@ from __future__ import annotationsfrom sqlite3 import IntegrityError, Rowfrom typing import TYPE_CHECKING, cast+from cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.utils.utilities import logfrom .definitions import create_fixed_history, create_history@@ -15,29 +16,10 @@ if TYPE_CHECKING:from yarl import URLfrom cyberdrop_dl.crawlers import Crawler- from cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.database import Database-_FETCH_MANY_SIZE = 1000---def get_db_path(url: URL, domain: str = "") -> str:- """Gets the URL path to be put into the DB and checked from the DB."""-- # TODO: domain SHOULD be mandatory, not optional. Make it mandatory and update any place that uses it-- if domain:- if "e-hentai" in domain:- return url.path.split("keystamp")[0][:-1]-- if "mediafire" in domain:- return url.name-- if "mega.nz" in domain:- return url.path_qs if not (frag := url.fragment) else f"{url.path_qs}#{frag}"-- return url.path+_FETCH_MANY_SIZE: int = 1000class HistoryTable:@@ -97,7 +79,7 @@ class HistoryTable:if self._database.ignore_history:return False- url_path = get_db_path(url, domain)+ url_path = MediaItem.create_db_path(url, domain)async def select_referer_and_completed() -> tuple[str, bool]:query = "SELECT referer, completed FROM media WHERE domain = ? and url_path = ?"@@ -132,9 +114,8 @@ class HistoryTable:async def set_album_id(self, domain: str, media_item: MediaItem) -> None:"""Sets an album_id in the database."""- url_path = get_db_path(media_item.url, str(media_item.referer))query = "UPDATE media SET album_id = ? WHERE domain = ? and url_path = ?"- await self.db_conn.execute(query, (media_item.album_id, domain, url_path))+ await self.db_conn.execute(query, (media_item.album_id, domain, media_item.db_path))await self.db_conn.commit()async def check_complete_by_referer(self, domain: str | None, referer: URL) -> bool:@@ -162,7 +143,7 @@ class HistoryTable:async def insert_incompleted(self, domain: str, media_item: MediaItem) -> None:"""Inserts an uncompleted file into the database."""- url_path = get_db_path(media_item.url, str(media_item.referer))+ url_path = media_item.db_pathdownload_filename = media_item.download_filename or ""cursor = await self.db_conn.cursor()query = "UPDATE media SET domain = ?, album_id = ? WHERE domain = 'no_crawler' and url_path = ? and referer = ?"@@ -199,7 +180,7 @@ class HistoryTable:async def mark_complete(self, domain: str, media_item: MediaItem) -> None:"""Mark a download as completed in the database."""- url_path = get_db_path(media_item.url, str(media_item.referer))+ url_path = media_item.db_pathquery = "UPDATE media SET completed = 1, completed_at = CURRENT_TIMESTAMP WHERE domain = ? and url_path = ?"await self.db_conn.execute(query, (domain, url_path))await self.db_conn.commit()@@ -207,7 +188,7 @@ class HistoryTable:async def add_filesize(self, domain: str, media_item: MediaItem) -> None:"""Adds the file size to the db."""- url_path = get_db_path(media_item.url, str(media_item.referer))+ url_path = media_item.db_pathfile_size = media_item.complete_file.stat().st_sizequery = """UPDATE media SET file_size=? WHERE domain = ? and url_path = ?"""await self.db_conn.execute(query, (file_size, domain, url_path))@@ -216,7 +197,7 @@ class HistoryTable:async def add_duration(self, domain: str, media_item: MediaItem) -> None:"""Adds the duration to the db."""- url_path = get_db_path(media_item.url, str(media_item.referer))+ url_path = media_item.db_pathquery = "UPDATE media SET duration=? WHERE domain = ? and url_path = ?"await self.db_conn.execute(query, (media_item.duration, domain, url_path))await self.db_conn.commit()@@ -226,7 +207,7 @@ class HistoryTable:if media_item.is_segment:return- url_path = get_db_path(media_item.url, str(media_item.referer))+ url_path = media_item.db_pathquery = "SELECT duration FROM media WHERE domain = ? and url_path = ?"cursor = await self.db_conn.execute(query, (domain, url_path))if row := await cursor.fetchone():@@ -234,7 +215,7 @@ class HistoryTable:async def add_download_filename(self, domain: str, media_item: MediaItem) -> None:"""Add the download_filename to the db."""- url_path = get_db_path(media_item.url, str(media_item.referer))+ url_path = media_item.db_pathquery = "UPDATE media SET download_filename=? WHERE domain = ? and url_path = ? and download_filename = ''"await self.db_conn.execute(query, (media_item.download_filename, domain, url_path))await self.db_conn.commit()@@ -253,7 +234,7 @@ class HistoryTable:if media_item.is_segment:return media_item.filename- url_path = get_db_path(media_item.url, str(media_item.referer))+ url_path = media_item.db_pathquery = "SELECT download_filename FROM media WHERE domain = ? and url_path = ?"cursor = await self.db_conn.execute(query, (domain, url_path))if row := await cursor.fetchone():diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 24d891741..6755a20eb 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -16,7 +16,6 @@ from aiohttp import ClientConnectorError, ClientError, ClientResponseErrorfrom cyberdrop_dl.constants import CustomHTTPStatusfrom cyberdrop_dl.data_structures.url_objects import HlsSegment, MediaItem-from cyberdrop_dl.database import get_db_pathfrom cyberdrop_dl.exceptions import (DownloadError,DurationError,@@ -167,7 +166,7 @@ class Downloader:async with self._semaphore:await self.manager.states.RUNNING.wait()self.waiting_items -= 1- self.processed_items.add(get_db_path(media_item.url, self.domain))+ self.processed_items.add(media_item.db_path)self.update_queued_files(increase_total=False)async with self.manager.client_manager.global_download_slots:return await self.start_download(media_item)@@ -274,6 +273,7 @@ class Downloader:seg_media_item = MediaItem.from_item(media_item,segment.url,+ domain=media_item.domain,download_folder=download_folder,filename=segment.name,ext=media_item.ext,diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex 0a308deb7..99524d6d3 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -238,12 +238,13 @@ class ScrapeMapper:scrape_item.add_to_parent_title("Loose Files")scrape_item.part_of_album = True- download_folder = get_download_path(self.manager, scrape_item, "no_crawler")+ domain = "no_crawler"+ download_folder = get_download_path(self.manager, scrape_item, domain)try:filename, _ = get_filename_and_ext(scrape_item.url.name)except NoExtensionError:filename, _ = get_filename_and_ext(scrape_item.url.name, forum=True)- media_item = MediaItem.from_item(scrape_item, scrape_item.url, download_folder, filename)+ media_item = MediaItem.from_item(scrape_item, scrape_item.url, domain, download_folder, filename)self.manager.task_group.create_task(self.no_crawler_downloader.run(media_item))returndiff --git a/tests/test_database.py b/tests/test_database.pyindex e535230bf..665bd64dd 100644--- a/tests/test_database.py+++ b/tests/test_database.py@@ -1,12 +1,17 @@+from __future__ import annotations+from datetime import datetimefrom pathlib import Path-from typing import cast+from typing import TYPE_CHECKING, cast-import aiosqliteimport pytest-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItemfrom cyberdrop_dl.scraper.scrape_mapper import _create_item_from_row+from cyberdrop_dl.utils.utilities import parse_url++if TYPE_CHECKING:+ import aiosqlite_MOCK_ROW = {"referer": "https://drive.google.com/file/d/1F0YBsnQRvrMbK0p9UlnyLu88kqQ0j_F6/edit",@@ -81,3 +86,28 @@ def test_invalid_date_format(row) -> None:with pytest.raises(ValueError):_create_item_from_row(row)+++ "url, expected",+ [+ (+ "https://megacloud.blog/embed-2/v3/e-1/TZb4gRkOQ642?k=1&autoPlay=1&oa=0&asi=1",+ "/embed-2/v3/e-1/TZb4gRkOQ642",+ ),+ ("https://www.mediafire.com/file/ctppmpm7giofsgv/ADOFAI.vpk", "ADOFAI.vpk"),+ ("https://mega.nz/#!Ue5VRSIQ!kC2E4a4JwfWWCWYNJovGFHlbz8F", "/#!Ue5VRSIQ!kC2E4a4JwfWWCWYNJovGFHlbz8F"),+ (+ "https://mega.nz/folder/oZZxyBrY#oU4jASLPpJVvqGHJIMRcgQ/file/IYZABDGY",+ "/folder/oZZxyBrY#oU4jASLPpJVvqGHJIMRcgQ/file/IYZABDGY",+ ),+ (+ "https://c.bunkr-cache.se/HwdRnHMUiWOQevCg/1df93418-5063-4e1b-851e-9470cb8fc5c6.mp4",+ "/HwdRnHMUiWOQevCg/1df93418-5063-4e1b-851e-9470cb8fc5c6.mp4",+ ),+ ],+)+def test_create_db_path(url: str, expected: str) -> None:+ url_ = parse_url(url)+ path = MediaItem.create_db_path(url_, url_.host)+ assert path == expected