Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: take fragment into account for some crawlers

NTFSvolume22166b647e31
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fd9b90575..e8615a008 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,13 @@ All notable changes to this project will be documented here. For more details, v
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [10.3.0] - UNRELEASED
+
+### Fixed
+
+- Some URLs being wrongly skipped as duplicates when using any of the `retry` options if they came from the same collection/folder/album.
+ This affected sites where multiple files may share identical URL path and query, with the fragment being what uniquely identifies each file (Mega.nz, Koofr, Transfer.it)
+
## [10.2.1] - 2026-07-21
### Changed
diff --git a/cyberdrop_dl/crawlers/bandcamp.py b/cyberdrop_dl/crawlers/bandcamp.py
index 58845a484..16ce8314e 100644
--- a/cyberdrop_dl/crawlers/bandcamp.py
+++ b/cyberdrop_dl/crawlers/bandcamp.py
@@ -26,7 +26,8 @@ class Format:
name: str
-class BandcampCrawler(Crawler, db_path="path_qs_frag"):
[email protected]_path_builder("path_qs_frag")
+class BandcampCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Album": "/album/<slug>",
"Song": "/track/<slug>",
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index fb0e36b76..7539e6396 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -79,13 +79,20 @@ class _PlaceHolderConfigInclude:
_include = _PlaceHolderConfigInclude()
-_DB_PATH_BUILDERS: MappingProxyType[str, Callable[[AbsoluteHttpURL], str]] = MappingProxyType(
+type URLHasher = Callable[[AbsoluteHttpURL], str]
+
+
+def _path_qs_frag(url: AbsoluteHttpURL) -> str:
+ return f"{url.path_qs}#{frag}" if (frag := url.fragment) else url.path_qs
+
+
+_DB_PATH_BUILDERS: MappingProxyType[str, URLHasher] = MappingProxyType(
{
"url": str,
"name": lambda url: url.name,
"path": lambda url: url.path,
"path_qs": lambda url: url.path_qs,
- "path_qs_frag": lambda url: f"{url.path_qs}#{frag}" if (frag := url.fragment) else url.path_qs,
+ "path_qs_frag": _path_qs_frag,
"path_frag": lambda url: f"{url.path}#{frag}" if (frag := url.fragment) else url.path,
}
)
@@ -151,6 +158,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
_DOWNLOAD_SLOTS: ClassVar[int | None] = None
_SCRAPE_SLOTS: ClassVar[int] = 20
_USE_DOWNLOAD_SERVERS_LOCKS: ClassVar[bool] = False
+ _IGNORE_FRAGMENT: ClassVar[bool] = True
disabled: bool = False
@@ -163,6 +171,17 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
)
return f"<{type(self).__name__}({', '.join(fields)})>"
+ @final
+ @staticmethod
+ def db_path_builder(key: Literal["url", "name", "path", "path_qs", "path_qs_frag", "path_frag"]):
+
+ def apply[T: Crawler](c: type[T]) -> type[T]:
+ c.__db_path__ = staticmethod(_DB_PATH_BUILDERS[key])
+ c._IGNORE_FRAGMENT = "frag" not in key # pyright: ignore[reportConstantRedefinition]
+ return c
+
+ return apply
+
@staticmethod
def __db_path__(url: AbsoluteHttpURL, /) -> str:
return url.path
@@ -222,13 +241,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
This method its called once and only if the crawler is actually going to be scrape something"""
def __init_subclass__(
- cls,
- *,
- is_abc: bool = False,
- is_generic: bool = False,
- is_debug: bool = False,
- db_path: Literal["url", "name", "path", "path_qs", "path_qs_frag", "path_frag"] | None = None,
- **kwargs: Any,
+ cls, *, is_abc: bool = False, is_generic: bool = False, is_debug: bool = False, **kwargs: Any
) -> None:
assert cls.__name__.endswith("Crawler"), f"{cls.__name__} does not end with 'Crawler'"
assert cls.__name__ not in Registry.names
@@ -246,9 +259,6 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
add_to_registry = bool(not is_debug or (is_debug and env.ENABLE_DEBUG_CRAWLERS))
- if db_path:
- cls.__db_path__ = staticmethod(_DB_PATH_BUILDERS[db_path])
-
if cls.IS_GENERIC:
cls.SCRAPE_MAPPER_KEYS = ()
cls.INFO: CrawlerInfo = CrawlerInfo.generic(cls.NAME, cls.SUPPORTED_PATHS)
@@ -385,11 +395,12 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
with scrape_item.track_changes:
scrape_item.url = url = self.transform_url(scrape_item.url)
- if url.path_qs in self._scraped_items:
+ lookup = url.path_qs if self._IGNORE_FRAGMENT else _path_qs_frag(url)
+ if lookup in self._scraped_items:
logger.info(f"Skipping {url} as it has already been scraped")
return
- self._scraped_items.add(url.path_qs)
+ self._scraped_items.add(lookup)
if not self.ALLOW_EMPTY_PATH and url.path == "/":
self.raise_exc(scrape_item, ScrapeError.unsupported())
diff --git a/cyberdrop_dl/crawlers/koofr.py b/cyberdrop_dl/crawlers/koofr.py
index dad9d37a4..f69de68a2 100644
--- a/cyberdrop_dl/crawlers/koofr.py
+++ b/cyberdrop_dl/crawlers/koofr.py
@@ -24,7 +24,8 @@ class Node:
hash: str = "" # md5
-class KooFrCrawler(Crawler, db_path="path_qs_frag"):
[email protected]_path_builder("path_qs_frag")
+class KooFrCrawler(Crawler):
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "koofr.net", "koofr.eu", _SHORT_LINK_CDN.host
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Public Share": (
diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.py
index 3f89150b3..1d73f76b7 100644
--- a/cyberdrop_dl/crawlers/mediafire.py
+++ b/cyberdrop_dl/crawlers/mediafire.py
@@ -50,7 +50,8 @@ _API_ERRORS_OVERRIDES: dict[int, int] = {
}
-class MediaFireCrawler(Crawler, db_path="name"):
[email protected]_path_builder("name")
+class MediaFireCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"File": (
"/file/<quick_key>",
diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.py
index 754710723..db16e86cd 100644
--- a/cyberdrop_dl/crawlers/mega_nz.py
+++ b/cyberdrop_dl/crawlers/mega_nz.py
@@ -29,7 +29,8 @@ if TYPE_CHECKING:
@HTTPConfig.default_headers(user_agent=CDL_USER_AGENT)
-class MegaNzCrawler(Crawler, db_path="path_qs_frag"):
[email protected]_path_builder("path_qs_frag")
+class MegaNzCrawler(Crawler):
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "mega.io", "mega.nz"
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"File": (
diff --git a/cyberdrop_dl/crawlers/transfer_it.py b/cyberdrop_dl/crawlers/transfer_it.py
index 535893f9a..160b9df1d 100644
--- a/cyberdrop_dl/crawlers/transfer_it.py
+++ b/cyberdrop_dl/crawlers/transfer_it.py
@@ -19,7 +19,8 @@ if TYPE_CHECKING:
@HTTPConfig.default_headers(user_agent=CDL_USER_AGENT)
-class TransferItCrawler(Crawler, db_path="path_qs_frag"):
[email protected]_path_builder("path_qs_frag")
+class TransferItCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Transfer": "/t/<transfer_id>"}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://transfer.it")
DOMAIN: ClassVar[str] = "transfer.it"
diff --git a/cyberdrop_dl/crawlers/vidara.py b/cyberdrop_dl/crawlers/vidara.py
index a7e603cf7..7a4343e50 100644
--- a/cyberdrop_dl/crawlers/vidara.py
+++ b/cyberdrop_dl/crawlers/vidara.py
@@ -11,7 +11,8 @@ if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
-class VidaraCrawler(Crawler, db_path="path_qs_frag"):
[email protected]_path_builder("path_qs_frag")
+class VidaraCrawler(Crawler):
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (
"xca.cymru",
"vidara.to",
diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.py
index ed237c389..36b74b24a 100644
--- a/cyberdrop_dl/downloader/http.py
+++ b/cyberdrop_dl/downloader/http.py
@@ -252,7 +252,7 @@ class Downloader:
yield
async def run(self, media_item: MediaItem) -> bool:
- if media_item.url.path in self._processed_items and not self.config.ignore_history:
+ if media_item.db_path in self._processed_items and not self.config.ignore_history:
return False
async with self.__download_context(media_item):
@@ -260,7 +260,7 @@ class Downloader:
@error_handling_wrapper
async def download_hls(self, media_item: MediaItem, m3u8_group: Rendition) -> None:
- if media_item.url.path in self._processed_items and not self.config.ignore_history:
+ if media_item.db_path in self._processed_items and not self.config.ignore_history:
return
assert ffmpeg.is_installed()