Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add `check_complete_by_hash` method (#1363)

NTFSvolumef694a9c74053
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 01f167bd4..622de2c30 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -467,6 +467,19 @@ class Crawler(ABC):
return True
return False
+ @final
+ async def check_complete_by_hash(
+ self: Crawler, scrape_item: ScrapeItem | URL, hash_type: str, hash_value: str
+ ) -> bool:
+ """Returns `True` if at least 1 file with this hash is recorded on the database"""
+ downloaded = await self.manager.db_manager.hash_table.check_hash_exists(hash_type, hash_value)
+ if downloaded:
+ url = scrape_item if isinstance(scrape_item, URL) else scrape_item.url
+ log(f"Skipping {url} as its hash ({hash_type}:{hash_value}) has already been downloaded", 10)
+ self.manager.progress_manager.download_progress.add_previously_completed()
+ return True
+ return False
+
async def get_album_results(self, album_id: str) -> dict[str, int]:
"""Checks whether an album has completed given its domain and album id."""
return await self.manager.db_manager.history_table.check_album(self.DOMAIN, album_id)
diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.py
index 677a422d5..c3286d1ae 100644
--- a/cyberdrop_dl/crawlers/kemono.py
+++ b/cyberdrop_dl/crawlers/kemono.py
@@ -6,6 +6,7 @@ import itertools
import re
from collections import defaultdict
from datetime import datetime # noqa: TC003
+from pathlib import Path
from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Concatenate, Literal, NamedTuple, NotRequired, ParamSpec
from pydantic import AliasChoices, BeforeValidator, Field
@@ -337,12 +338,17 @@ class KemonoBaseCrawler(Crawler, is_abc=True):
async def handle_direct_link(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:
scrape_item.url = _thumbnail_to_src(scrape_item.url)
link = _thumbnail_to_src(url or scrape_item.url)
+ hash_value = Path(link.name).stem
+ if await self.check_complete_by_hash(link, "sha256", hash_value):
+ return
+
try:
filename, ext = self.get_filename_and_ext(link.query.get("f") or link.name)
except NoExtensionError:
# Some patreon URLs have another URL as the filename:
# ex: https://kemono.su/data/7a...27ad7e40bd.jpg?f=https://www.patreon.com/media-u/Z0F..00672794_
filename, ext = self.get_filename_and_ext(link.name)
+
await self.handle_file(link, scrape_item, link.name, ext, custom_filename=filename)
# ~~~~~~~~ INTERNAL METHODS, not expected to be overriden, but could be ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/cyberdrop_dl/database/tables/hash.py b/cyberdrop_dl/database/tables/hash.py
index efcda3d1b..76808b4a8 100644
--- a/cyberdrop_dl/database/tables/hash.py
+++ b/cyberdrop_dl/database/tables/hash.py
@@ -87,6 +87,15 @@ class HashTable:
log(f"Error retrieving folder and filename: {e}", 40, exc_info=e)
return []
+ async def check_hash_exists(self, hash_type: str, hash_value: str) -> bool:
+ if self._database.ignore_history:
+ return False
+
+ query = "SELECT 1 FROM hash WHERE hash.hash_type = ? AND hash.hash = ? LIMIT 1"
+ cursor = await self.db_conn.execute(query, (hash_type, hash_value))
+ result = await cursor.fetchone()
+ return result is not None
+
async def insert_or_update_hash_db(
self, hash_value: str, hash_type: str, file: Path | str, original_filename: str | None, referer: URL | None
) -> bool: