Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: domain exclude match

NTFSvolume4489d3082b36
diff --git a/cyberdrop_dl/crawlers/twitter/crawler.py b/cyberdrop_dl/crawlers/twitter/crawler.py
index bf11ab7d6..e30c24f5e 100644
--- a/cyberdrop_dl/crawlers/twitter/crawler.py
+++ b/cyberdrop_dl/crawlers/twitter/crawler.py
@@ -28,6 +28,11 @@ class TwitterShortURLCrawler(Crawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://t.co")
DOMAIN: ClassVar[str] = "t.co"
+ @classmethod
+ @override
+ def check_host_match(cls, host: str) -> bool:
+ return super().check_host_match(host) and host == "t.co"
+
async def fetch(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.host != self.PRIMARY_URL.host:
raise ValueError
@@ -115,7 +120,7 @@ class TwitterCrawler(Crawler):
@classmethod
@override
def check_host_match(cls, host: str) -> bool:
- return host == "x.com" or not host.endswith("x.com")
+ return super().check_host_match(host) and (host == "x.com" or not host.endswith("x.com"))
@property
@override
diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.py
index fb9e86796..f55bfdcbb 100644
--- a/cyberdrop_dl/scrape_mapper.py
+++ b/cyberdrop_dl/scrape_mapper.py
@@ -467,7 +467,7 @@ def _best_match[T: Crawler | type[Crawler]](crawlers: dict[str, T], domain: str)
if found := crawlers.get(domain):
return found
- matches = (host for host, cls in crawlers.items() if host in domain and cls.check_host_match(host))
+ matches = (host for host, cls in crawlers.items() if host in domain and cls.check_host_match(domain))
try:
best_match = max(matches, key=len)