Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: domain exclude match
NTFSvolume4489d3082b36
diff --git a/cyberdrop_dl/crawlers/twitter/crawler.py b/cyberdrop_dl/crawlers/twitter/crawler.pyindex bf11ab7d6..e30c24f5e 100644--- a/cyberdrop_dl/crawlers/twitter/crawler.py+++ b/cyberdrop_dl/crawlers/twitter/crawler.py@@ -28,6 +28,11 @@ class TwitterShortURLCrawler(Crawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://t.co")DOMAIN: ClassVar[str] = "t.co"+ @classmethod+ @override+ def check_host_match(cls, host: str) -> bool:+ return super().check_host_match(host) and host == "t.co"+async def fetch(self, scrape_item: ScrapeItem) -> None:if scrape_item.url.host != self.PRIMARY_URL.host:raise ValueError@@ -115,7 +120,7 @@ class TwitterCrawler(Crawler):@classmethod@overridedef check_host_match(cls, host: str) -> bool:- return host == "x.com" or not host.endswith("x.com")+ return super().check_host_match(host) and (host == "x.com" or not host.endswith("x.com"))@property@overridediff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.pyindex fb9e86796..f55bfdcbb 100644--- a/cyberdrop_dl/scrape_mapper.py+++ b/cyberdrop_dl/scrape_mapper.py@@ -467,7 +467,7 @@ def _best_match[T: Crawler | type[Crawler]](crawlers: dict[str, T], domain: str)if found := crawlers.get(domain):return found- matches = (host for host, cls in crawlers.items() if host in domain and cls.check_host_match(host))+ matches = (host for host, cls in crawlers.items() if host in domain and cls.check_host_match(domain))try:best_match = max(matches, key=len)