Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: simplify
NTFSvolumecace67ddd39a
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex ed9d34fa9..5015ea48d 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -45,7 +45,6 @@ P = ParamSpec("P")R = TypeVar("R")T = TypeVar("T")_T_co = TypeVar("_T_co", covariant=True)-_CRAWLER_DOMAIN: Any = object()if TYPE_CHECKING:@@ -349,18 +348,16 @@ class Crawler(ABC):return False+ @finalasync def check_complete_from_referer(- self, scrape_item: ScrapeItem | URL, domain: str | None = _CRAWLER_DOMAIN+ self: Crawler, scrape_item: ScrapeItem | URL, any_crawler: bool = False) -> bool:"""Checks if the scrape item has already been scraped.- by default, it only checks database entries from this crawler. Pass a custom value in domain to check referers from other crawlers-- if domain is `None`, checks database entries for all crawlers and returns `True` if at least 1 of them has marked it as completed+ if `any_crawler` is `True`, checks database entries for all crawlers and returns `True` if at least 1 of them has marked it as completed"""url = scrape_item if isinstance(scrape_item, URL) else scrape_item.url- if domain is _CRAWLER_DOMAIN:- domain = self.DOMAIN+ domain = None if any_crawler else self.DOMAINdownloaded = await self.manager.db_manager.history_table.check_complete_by_referer(domain, url)if downloaded:log(f"Skipping {url} as it has already been downloaded", 10)