Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: warn if hash does not have the right size
NTFSvolume61e9fc427155
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 761b4932a..dda10cc3c 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -610,12 +610,24 @@ class Crawler(HTTPMixin, HLSMixin, ABC):@finalasync def check_complete_by_hash(- self: Crawler, url: AbsoluteHttpURL, hash_type: Literal["md5", "sha256"], hash_value: str+ self: Crawler, url: AbsoluteHttpURL, hash_algo: Literal["md5", "sha256"], checksum: str) -> bool:"""Returns `True` if at least 1 file with this hash is recorded on the database"""- downloaded = await self.database.hash.check_hash_exists(hash_type, hash_value)++ expected_len = 32 if hash_algo == "md5" else 64+ if len(checksum) != expected_len:+ self.log.warning(+ "Hash %s with %s characters does not match the expected size for a %s hex digest (%s characters), ignoring...",+ checksum,+ len(checksum),+ hash_algo,+ expected_len,+ )+ return False++ downloaded = await self.database.hash.check_hash_exists(hash_algo, checksum)if downloaded:- logger.info(f"Skipping {url} as its hash ({hash_type}:{hash_value}) has already been downloaded")+ logger.info(f"Skipping {url} as its hash ({hash_algo}:{checksum}) has already been downloaded")self.tui.files.stats.prev_completed += 1return downloaded