Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: add `check_complete_by_hash` method (#1363)
NTFSvolumef694a9c74053
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 01f167bd4..622de2c30 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -467,6 +467,19 @@ class Crawler(ABC):return Truereturn False+ @final+ async def check_complete_by_hash(+ self: Crawler, scrape_item: ScrapeItem | URL, hash_type: str, hash_value: str+ ) -> bool:+ """Returns `True` if at least 1 file with this hash is recorded on the database"""+ downloaded = await self.manager.db_manager.hash_table.check_hash_exists(hash_type, hash_value)+ if downloaded:+ url = scrape_item if isinstance(scrape_item, URL) else scrape_item.url+ log(f"Skipping {url} as its hash ({hash_type}:{hash_value}) has already been downloaded", 10)+ self.manager.progress_manager.download_progress.add_previously_completed()+ return True+ return False+async def get_album_results(self, album_id: str) -> dict[str, int]:"""Checks whether an album has completed given its domain and album id."""return await self.manager.db_manager.history_table.check_album(self.DOMAIN, album_id)diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.pyindex 677a422d5..c3286d1ae 100644--- a/cyberdrop_dl/crawlers/kemono.py+++ b/cyberdrop_dl/crawlers/kemono.py@@ -6,6 +6,7 @@ import itertoolsimport refrom collections import defaultdictfrom datetime import datetime # noqa: TC003+from pathlib import Pathfrom typing import TYPE_CHECKING, Annotated, Any, ClassVar, Concatenate, Literal, NamedTuple, NotRequired, ParamSpecfrom pydantic import AliasChoices, BeforeValidator, Field@@ -337,12 +338,17 @@ class KemonoBaseCrawler(Crawler, is_abc=True):async def handle_direct_link(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:scrape_item.url = _thumbnail_to_src(scrape_item.url)link = _thumbnail_to_src(url or scrape_item.url)+ hash_value = Path(link.name).stem+ if await self.check_complete_by_hash(link, "sha256", hash_value):+ return+try:filename, ext = self.get_filename_and_ext(link.query.get("f") or link.name)except NoExtensionError:# Some patreon URLs have another URL as the filename:# ex: https://kemono.su/data/7a...27ad7e40bd.jpg?f=https://www.patreon.com/media-u/Z0F..00672794_filename, ext = self.get_filename_and_ext(link.name)+await self.handle_file(link, scrape_item, link.name, ext, custom_filename=filename)# ~~~~~~~~ INTERNAL METHODS, not expected to be overriden, but could be ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~diff --git a/cyberdrop_dl/database/tables/hash.py b/cyberdrop_dl/database/tables/hash.pyindex efcda3d1b..76808b4a8 100644--- a/cyberdrop_dl/database/tables/hash.py+++ b/cyberdrop_dl/database/tables/hash.py@@ -87,6 +87,15 @@ class HashTable:log(f"Error retrieving folder and filename: {e}", 40, exc_info=e)return []+ async def check_hash_exists(self, hash_type: str, hash_value: str) -> bool:+ if self._database.ignore_history:+ return False++ query = "SELECT 1 FROM hash WHERE hash.hash_type = ? AND hash.hash = ? LIMIT 1"+ cursor = await self.db_conn.execute(query, (hash_type, hash_value))+ result = await cursor.fetchone()+ return result is not None+async def insert_or_update_hash_db(self, hash_value: str, hash_type: str, file: Path | str, original_filename: str | None, referer: URL | None) -> bool: