Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: named scrape tasks
NTFSvolume3cf68fd5010e
diff --git a/cyberdrop_dl/aio.py b/cyberdrop_dl/aio.pyindex f0404b680..35da46546 100644--- a/cyberdrop_dl/aio.py+++ b/cyberdrop_dl/aio.py@@ -62,6 +62,8 @@ class EagerTaskGroup(asyncio.TaskGroup):return await cororun = lazy()+ run.__name__ = coro.__name__+ run.__repr__ = coro.__repr__else:run = corodiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 9ca35d76d..7714e75e6 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -448,7 +448,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):lookup = url.path_qs if self.__url_config__.ignore_fragment else _path_qs_frag(url)if lookup in self._scraped_items:- logger.info(f"Skipping {url} as it has already been scrapped")+ logger.info("Skipping %s as it has already been scrapped", url)returnself._scraped_items.add(lookup)diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.pyindex f55bfdcbb..c83ab5727 100644--- a/cyberdrop_dl/scrape_mapper.py+++ b/cyberdrop_dl/scrape_mapper.py@@ -304,12 +304,12 @@ class ScrapeMapper:await crawler.__async_init__()if crawler.__url_config__.trim:scrape_item.url = remove_trailing_slash(scrape_item.url)- self.task_mngr.scrape.create_task(crawler.run(scrape_item))+ self.task_mngr.scrape.create_task(crawler.run(scrape_item), name=str(scrape_item.url))returnif not self._real_debrid.disabled and self._real_debrid.api.is_supported(scrape_item.url):logger.info(f"Using RealDebrid for unsupported URL: {scrape_item.url}")- self.task_mngr.scrape.create_task(self._real_debrid.run(scrape_item))+ self.task_mngr.scrape.create_task(self._real_debrid.run(scrape_item), name=str(scrape_item.url))returntry: