Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: take fragment into account for some crawlers
NTFSvolume22166b647e31
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex fd9b90575..e8615a008 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -22,6 +22,13 @@ All notable changes to this project will be documented here. For more details, vThe format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).+## [10.3.0] - UNRELEASED++### Fixed++- Some URLs being wrongly skipped as duplicates when using any of the `retry` options if they came from the same collection/folder/album.+ This affected sites where multiple files may share identical URL path and query, with the fragment being what uniquely identifies each file (Mega.nz, Koofr, Transfer.it)+## [10.2.1] - 2026-07-21### Changeddiff --git a/cyberdrop_dl/crawlers/bandcamp.py b/cyberdrop_dl/crawlers/bandcamp.pyindex 58845a484..16ce8314e 100644--- a/cyberdrop_dl/crawlers/bandcamp.py+++ b/cyberdrop_dl/crawlers/bandcamp.py@@ -26,7 +26,8 @@ class Format:name: str-class BandcampCrawler(Crawler, db_path="path_qs_frag"):[email protected]_path_builder("path_qs_frag")+class BandcampCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Album": "/album/<slug>","Song": "/track/<slug>",diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex fb0e36b76..7539e6396 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -79,13 +79,20 @@ class _PlaceHolderConfigInclude:_include = _PlaceHolderConfigInclude()-_DB_PATH_BUILDERS: MappingProxyType[str, Callable[[AbsoluteHttpURL], str]] = MappingProxyType(+type URLHasher = Callable[[AbsoluteHttpURL], str]+++def _path_qs_frag(url: AbsoluteHttpURL) -> str:+ return f"{url.path_qs}#{frag}" if (frag := url.fragment) else url.path_qs+++_DB_PATH_BUILDERS: MappingProxyType[str, URLHasher] = MappingProxyType({"url": str,"name": lambda url: url.name,"path": lambda url: url.path,"path_qs": lambda url: url.path_qs,- "path_qs_frag": lambda url: f"{url.path_qs}#{frag}" if (frag := url.fragment) else url.path_qs,+ "path_qs_frag": _path_qs_frag,"path_frag": lambda url: f"{url.path}#{frag}" if (frag := url.fragment) else url.path,})@@ -151,6 +158,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):_DOWNLOAD_SLOTS: ClassVar[int | None] = None_SCRAPE_SLOTS: ClassVar[int] = 20_USE_DOWNLOAD_SERVERS_LOCKS: ClassVar[bool] = False+ _IGNORE_FRAGMENT: ClassVar[bool] = Truedisabled: bool = False@@ -163,6 +171,17 @@ class Crawler(HTTPMixin, HLSMixin, ABC):)return f"<{type(self).__name__}({', '.join(fields)})>"+ @final+ @staticmethod+ def db_path_builder(key: Literal["url", "name", "path", "path_qs", "path_qs_frag", "path_frag"]):++ def apply[T: Crawler](c: type[T]) -> type[T]:+ c.__db_path__ = staticmethod(_DB_PATH_BUILDERS[key])+ c._IGNORE_FRAGMENT = "frag" not in key # pyright: ignore[reportConstantRedefinition]+ return c++ return apply+@staticmethoddef __db_path__(url: AbsoluteHttpURL, /) -> str:return url.path@@ -222,13 +241,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):This method its called once and only if the crawler is actually going to be scrape something"""def __init_subclass__(- cls,- *,- is_abc: bool = False,- is_generic: bool = False,- is_debug: bool = False,- db_path: Literal["url", "name", "path", "path_qs", "path_qs_frag", "path_frag"] | None = None,- **kwargs: Any,+ cls, *, is_abc: bool = False, is_generic: bool = False, is_debug: bool = False, **kwargs: Any) -> None:assert cls.__name__.endswith("Crawler"), f"{cls.__name__} does not end with 'Crawler'"assert cls.__name__ not in Registry.names@@ -246,9 +259,6 @@ class Crawler(HTTPMixin, HLSMixin, ABC):add_to_registry = bool(not is_debug or (is_debug and env.ENABLE_DEBUG_CRAWLERS))- if db_path:- cls.__db_path__ = staticmethod(_DB_PATH_BUILDERS[db_path])-if cls.IS_GENERIC:cls.SCRAPE_MAPPER_KEYS = ()cls.INFO: CrawlerInfo = CrawlerInfo.generic(cls.NAME, cls.SUPPORTED_PATHS)@@ -385,11 +395,12 @@ class Crawler(HTTPMixin, HLSMixin, ABC):with scrape_item.track_changes:scrape_item.url = url = self.transform_url(scrape_item.url)- if url.path_qs in self._scraped_items:+ lookup = url.path_qs if self._IGNORE_FRAGMENT else _path_qs_frag(url)+ if lookup in self._scraped_items:logger.info(f"Skipping {url} as it has already been scraped")return- self._scraped_items.add(url.path_qs)+ self._scraped_items.add(lookup)if not self.ALLOW_EMPTY_PATH and url.path == "/":self.raise_exc(scrape_item, ScrapeError.unsupported())diff --git a/cyberdrop_dl/crawlers/koofr.py b/cyberdrop_dl/crawlers/koofr.pyindex dad9d37a4..f69de68a2 100644--- a/cyberdrop_dl/crawlers/koofr.py+++ b/cyberdrop_dl/crawlers/koofr.py@@ -24,7 +24,8 @@ class Node:hash: str = "" # md5-class KooFrCrawler(Crawler, db_path="path_qs_frag"):[email protected]_path_builder("path_qs_frag")+class KooFrCrawler(Crawler):SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "koofr.net", "koofr.eu", _SHORT_LINK_CDN.hostSUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Public Share": (diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.pyindex 3f89150b3..1d73f76b7 100644--- a/cyberdrop_dl/crawlers/mediafire.py+++ b/cyberdrop_dl/crawlers/mediafire.py@@ -50,7 +50,8 @@ _API_ERRORS_OVERRIDES: dict[int, int] = {}-class MediaFireCrawler(Crawler, db_path="name"):[email protected]_path_builder("name")+class MediaFireCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"File": ("/file/<quick_key>",diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.pyindex 754710723..db16e86cd 100644--- a/cyberdrop_dl/crawlers/mega_nz.py+++ b/cyberdrop_dl/crawlers/mega_nz.py@@ -29,7 +29,8 @@ if TYPE_CHECKING:@HTTPConfig.default_headers(user_agent=CDL_USER_AGENT)-class MegaNzCrawler(Crawler, db_path="path_qs_frag"):[email protected]_path_builder("path_qs_frag")+class MegaNzCrawler(Crawler):SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "mega.io", "mega.nz"SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"File": (diff --git a/cyberdrop_dl/crawlers/transfer_it.py b/cyberdrop_dl/crawlers/transfer_it.pyindex 535893f9a..160b9df1d 100644--- a/cyberdrop_dl/crawlers/transfer_it.py+++ b/cyberdrop_dl/crawlers/transfer_it.py@@ -19,7 +19,8 @@ if TYPE_CHECKING:@HTTPConfig.default_headers(user_agent=CDL_USER_AGENT)-class TransferItCrawler(Crawler, db_path="path_qs_frag"):[email protected]_path_builder("path_qs_frag")+class TransferItCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Transfer": "/t/<transfer_id>"}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://transfer.it")DOMAIN: ClassVar[str] = "transfer.it"diff --git a/cyberdrop_dl/crawlers/vidara.py b/cyberdrop_dl/crawlers/vidara.pyindex a7e603cf7..7a4343e50 100644--- a/cyberdrop_dl/crawlers/vidara.py+++ b/cyberdrop_dl/crawlers/vidara.py@@ -11,7 +11,8 @@ if TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem-class VidaraCrawler(Crawler, db_path="path_qs_frag"):[email protected]_path_builder("path_qs_frag")+class VidaraCrawler(Crawler):SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = ("xca.cymru","vidara.to",diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.pyindex ed237c389..36b74b24a 100644--- a/cyberdrop_dl/downloader/http.py+++ b/cyberdrop_dl/downloader/http.py@@ -252,7 +252,7 @@ class Downloader:yieldasync def run(self, media_item: MediaItem) -> bool:- if media_item.url.path in self._processed_items and not self.config.ignore_history:+ if media_item.db_path in self._processed_items and not self.config.ignore_history:return Falseasync with self.__download_context(media_item):@@ -260,7 +260,7 @@ class Downloader:@error_handling_wrapperasync def download_hls(self, media_item: MediaItem, m3u8_group: Rendition) -> None:- if media_item.url.path in self._processed_items and not self.config.ignore_history:+ if media_item.db_path in self._processed_items and not self.config.ignore_history:returnassert ffmpeg.is_installed()