Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: expose ORIGIN as public crawler API (#1941)
NTFSvolume794fdf3294b6
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.pyindex bce70210e..fb97969cb 100644--- a/cyberdrop_dl/clients/http.py+++ b/cyberdrop_dl/clients/http.py@@ -6,7 +6,6 @@ import dataclassesimport loggingimport timeimport warnings-from abc import ABC, abstractmethodfrom contextvars import ContextVarfrom http import HTTPStatusfrom typing import TYPE_CHECKING, Any, Literal, Protocol, Self, cast, final@@ -339,10 +338,10 @@ async def _check_json(response: AbstractResponse[Any]) -> None:return-class HTTPMixin(ABC):- @abstractmethod+class HTTPMixin:@signature.copy(HTTPClient.request)- def request(self, *args: Any, **kwargs: Any) -> contextlib._AsyncGeneratorContextManager[AbstractResponse[Any]]: ... # pyright: ignore[reportPrivateUsage]+ def request(self, *args: Any, **kwargs: Any) -> contextlib._AsyncGeneratorContextManager[AbstractResponse[Any]]:+ raise NotImplementedError@signature.copy(request)async def request_json(self, *args: Any, **kwargs: Any) -> Any:diff --git a/cyberdrop_dl/crawlers/archive_org.py b/cyberdrop_dl/crawlers/archive_org.pyindex b1d455632..a61453095 100644--- a/cyberdrop_dl/crawlers/archive_org.py+++ b/cyberdrop_dl/crawlers/archive_org.py@@ -85,7 +85,7 @@ class ArchiveOrgCrawler(Crawler):class ArchiveOrgAPI(API):async def metadata(self, identifier: str) -> dict[str, Any]:- return await self._request(self.crawler.PRIMARY_URL / "metadata" / identifier)+ return await self._request(self.PRIMARY_URL / "metadata" / identifier)async def item(self, identifier: str) -> Item:metadata = await self.metadata(identifier)diff --git a/cyberdrop_dl/crawlers/clyp_it.py b/cyberdrop_dl/crawlers/clyp_it.pyindex 54371c8b6..bb6fc2da4 100644--- a/cyberdrop_dl/crawlers/clyp_it.py+++ b/cyberdrop_dl/crawlers/clyp_it.py@@ -91,7 +91,7 @@ class ClypItAPI(API):async def wav(self, audio_id: str, token: str | None) -> AbsoluteHttpURL | None:if env.CLYPIT_PREFER_MP3:return None- url = self.crawler.PRIMARY_URL / audio_id+ url = self.PRIMARY_URL / audio_idif token:url = url.with_query(token=token)text = await self.request_text(url)@@ -100,7 +100,7 @@ class ClypItAPI(API):except ValueError:return Noneelse:- return self.crawler.parse_url(src)+ return self.parse_url(src)def user_uploads(self, user_id: str) -> AsyncGenerator[map[Audio]]:api_url = self.ENTRYPOINT / "User" / user_id / "Uploads"@@ -114,7 +114,7 @@ class ClypItAPI(API):next_page: str | None = resp.get("Paging").get("Next")if not next_page:break- url = self.crawler.parse_url(next_page)+ url = self.parse_url(next_page)@dataclasses.dataclass(slots=True)diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex d56fa812a..b4679d02f 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -60,6 +60,7 @@ SupportedDomains = OneOrTuple[str]RateLimit = tuple[float, float]SKIP_DOWNLOAD: ContextVar[bool] = ContextVar("SKIP_DOWNLOAD", default=False)ALLOW_NO_EXT: ContextVar[bool] = ContextVar("ALLOW_NO_EXT", default=False)+ORIGIN: ContextVar[AbsoluteHttpURL] = ContextVar("ORIGIN")_HASH_PREFIXES = "md5:", "sha1:", "sha256:", "xxh128:"@@ -375,6 +376,11 @@ class Crawler(HTTPMixin, HLSMixin, ABC):def deep_scrape(self) -> bool:return self.manager.config.deep_scrape+ @final+ @property+ def origin(self) -> AbsoluteHttpURL:+ return ORIGIN.get()+@propertydef separate_posts(self) -> bool:return self.manager.config.settings.download_options.separate_posts@@ -438,6 +444,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):def new_task_id(self, url: AbsoluteHttpURL):"""Creates a new task_id (shows the URL in the UI and logs)"""self.log.info(f"Scraping {url}")+ _ = ORIGIN.set(url.origin())return self.manager.scrape_mapper.tui.scrape.new(url)@final@@ -954,30 +961,27 @@ _CrawlerT = TypeVar("_CrawlerT", bound=Crawler)_CrawlerT_generic = TypeVar("_CrawlerT_generic", bound=Crawler, default=Crawler)-class API(HTTPMixin, Generic[_CrawlerT_generic]):- crawler: _CrawlerT_generic+class API(HTTPMixin, ABC, Generic[_CrawlerT_generic]):+ _crawler: _CrawlerT_generic@finaldef __init__(self, crawler: _CrawlerT_generic) -> None:- self.crawler = crawler+ self._crawler = crawler+ self.PRIMARY_URL: Final = crawler.PRIMARY_URL+ self.parse_url: Final = crawler.parse_url+ self.request: Final = crawler.request+ self.config: Final = crawler.manager.configself.__post_init__()- def __post_init__(self): ...+ def __post_init__(self) -> None: ...def __repr__(self) -> str:- return f"{type(self).__name__}(crawler={self.crawler.__name__!r})"-- @signature.copy(Crawler.request)- def request(self, *args: Any, **kwargs: Any) -> contextlib._AsyncGeneratorContextManager[AbstractResponse[Any]]: # pyright: ignore[reportPrivateUsage]- return self.crawler.request(*args, **kwargs)-- @property- def parse_url(self):- return self.crawler.parse_url+ return f"{type(self).__name__}(crawler={self._crawler.__name__!r})"+ @final@property- def PRIMARY_URL(self) -> AbsoluteHttpURL: # noqa: N802- return self.crawler.PRIMARY_URL+ def origin(self) -> AbsoluteHttpURL:+ return ORIGIN.get()def _make_scrape_mapper_keys(cls: type[Crawler] | Crawler) -> tuple[str, ...]:diff --git a/cyberdrop_dl/crawlers/pixeldrain.py b/cyberdrop_dl/crawlers/pixeldrain.pyindex ca06385b1..551674875 100644--- a/cyberdrop_dl/crawlers/pixeldrain.py+++ b/cyberdrop_dl/crawlers/pixeldrain.py@@ -3,7 +3,6 @@from __future__ import annotationsimport dataclasses-from contextvars import ContextVarfrom typing import TYPE_CHECKING, Any, ClassVar, Literal, finalfrom typing_extensions import override@@ -19,7 +18,6 @@ if TYPE_CHECKING:_PRIMARY_URL = AbsoluteHttpURL("https://pixeldrain.com")-_CURRENT_ORIGIN: ContextVar[AbsoluteHttpURL] = ContextVar("_CURRENT_ORIGIN")@final@@ -146,8 +144,6 @@ class PixelDrainCrawler(Crawler):@overrideasync def fetch(self, scrape_item: ScrapeItem) -> None:- _CURRENT_ORIGIN.set(scrape_item.url.origin())-match scrape_item.url.parts[1:]:case ["u", file_id]:return await self.file(scrape_item, file_id)@@ -194,7 +190,7 @@ class PixelDrainCrawler(Crawler):if self.check_album_results(_build_download_url(file), results):continue- url = _CURRENT_ORIGIN.get() / "u" / file.id+ url = self.origin / "u" / file.idnew_scrape_item = scrape_item.create_child(url)self.create_task(self._file_task(new_scrape_item, file))scrape_item.add_children()@@ -229,7 +225,7 @@ class PixelDrainCrawler(Crawler):if node.name == ".search_index.gz":continue- url = _CURRENT_ORIGIN.get() / "d" / node.path.removeprefix("/")+ url = self.origin / "d" / node.path.removeprefix("/")new_scrape_item = scrape_item.create_child(url)if node.type == "file":@@ -262,7 +258,7 @@ class PixelDrainCrawler(Crawler):if "text/plain" in file.mime_type:return await self.text(scrape_item, file)- src = _build_download_url(file).with_host(_CURRENT_ORIGIN.get().host)+ src = _build_download_url(file).with_host(self.origin.host)if await self.check_complete_by_hash(src, "sha256", file.hash_sha256):return None@@ -291,7 +287,7 @@ class PixelDrainCrawler(Crawler):class PixelDrainAPI(API):def __post_init__(self) -> None:self.headers: dict[str, str] = {}- if api_key := self.crawler.manager.config.auth.pixeldrain.api_key:+ if api_key := self.config.auth.pixeldrain.api_key:self.headers["Authorization"] = basic_auth("Cyberdrop-DL", api_key)@property@@ -299,21 +295,21 @@ class PixelDrainAPI(API):return bool(self.headers)async def file(self, file_id: str) -> File:- api_url = _CURRENT_ORIGIN.get() / "api/file" / file_id / "info"+ api_url = self.origin / "api/file" / file_id / "info"resp = await self._request(api_url)return type_adapter(File).validate_json(resp)async def text(self, file_id: str) -> str:- api_url = _CURRENT_ORIGIN.get() / "api/file" / file_id+ api_url = self.origin / "api/file" / file_idreturn await self._request(api_url)async def folder(self, list_id: str) -> Folder:- api_url = _CURRENT_ORIGIN.get() / "api/list" / list_id+ api_url = self.origin / "api/list" / list_idresp = await self._request(api_url)return type_adapter(Folder).validate_json(resp)async def filesystem(self, path: str) -> FileSystem:- api_url = (_CURRENT_ORIGIN.get() / "api/filesystem" / path.removeprefix("/")).with_query("stat")+ api_url = (self.origin / "api/filesystem" / path.removeprefix("/")).with_query("stat")resp = await self._request(api_url)return type_adapter(FileSystem).validate_json(resp)