Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: expose ORIGIN as public crawler API (#1941)

NTFSvolume794fdf3294b6
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.py
index bce70210e..fb97969cb 100644
--- a/cyberdrop_dl/clients/http.py
+++ b/cyberdrop_dl/clients/http.py
@@ -6,7 +6,6 @@ import dataclasses
import logging
import time
import warnings
-from abc import ABC, abstractmethod
from contextvars import ContextVar
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, Literal, Protocol, Self, cast, final
@@ -339,10 +338,10 @@ async def _check_json(response: AbstractResponse[Any]) -> None:
return
-class HTTPMixin(ABC):
- @abstractmethod
+class HTTPMixin:
@signature.copy(HTTPClient.request)
- def request(self, *args: Any, **kwargs: Any) -> contextlib._AsyncGeneratorContextManager[AbstractResponse[Any]]: ... # pyright: ignore[reportPrivateUsage]
+ def request(self, *args: Any, **kwargs: Any) -> contextlib._AsyncGeneratorContextManager[AbstractResponse[Any]]:
+ raise NotImplementedError
@signature.copy(request)
async def request_json(self, *args: Any, **kwargs: Any) -> Any:
diff --git a/cyberdrop_dl/crawlers/archive_org.py b/cyberdrop_dl/crawlers/archive_org.py
index b1d455632..a61453095 100644
--- a/cyberdrop_dl/crawlers/archive_org.py
+++ b/cyberdrop_dl/crawlers/archive_org.py
@@ -85,7 +85,7 @@ class ArchiveOrgCrawler(Crawler):
class ArchiveOrgAPI(API):
async def metadata(self, identifier: str) -> dict[str, Any]:
- return await self._request(self.crawler.PRIMARY_URL / "metadata" / identifier)
+ return await self._request(self.PRIMARY_URL / "metadata" / identifier)
async def item(self, identifier: str) -> Item:
metadata = await self.metadata(identifier)
diff --git a/cyberdrop_dl/crawlers/clyp_it.py b/cyberdrop_dl/crawlers/clyp_it.py
index 54371c8b6..bb6fc2da4 100644
--- a/cyberdrop_dl/crawlers/clyp_it.py
+++ b/cyberdrop_dl/crawlers/clyp_it.py
@@ -91,7 +91,7 @@ class ClypItAPI(API):
async def wav(self, audio_id: str, token: str | None) -> AbsoluteHttpURL | None:
if env.CLYPIT_PREFER_MP3:
return None
- url = self.crawler.PRIMARY_URL / audio_id
+ url = self.PRIMARY_URL / audio_id
if token:
url = url.with_query(token=token)
text = await self.request_text(url)
@@ -100,7 +100,7 @@ class ClypItAPI(API):
except ValueError:
return None
else:
- return self.crawler.parse_url(src)
+ return self.parse_url(src)
def user_uploads(self, user_id: str) -> AsyncGenerator[map[Audio]]:
api_url = self.ENTRYPOINT / "User" / user_id / "Uploads"
@@ -114,7 +114,7 @@ class ClypItAPI(API):
next_page: str | None = resp.get("Paging").get("Next")
if not next_page:
break
- url = self.crawler.parse_url(next_page)
+ url = self.parse_url(next_page)
@dataclasses.dataclass(slots=True)
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index d56fa812a..b4679d02f 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -60,6 +60,7 @@ SupportedDomains = OneOrTuple[str]
RateLimit = tuple[float, float]
SKIP_DOWNLOAD: ContextVar[bool] = ContextVar("SKIP_DOWNLOAD", default=False)
ALLOW_NO_EXT: ContextVar[bool] = ContextVar("ALLOW_NO_EXT", default=False)
+ORIGIN: ContextVar[AbsoluteHttpURL] = ContextVar("ORIGIN")
_HASH_PREFIXES = "md5:", "sha1:", "sha256:", "xxh128:"
@@ -375,6 +376,11 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
def deep_scrape(self) -> bool:
return self.manager.config.deep_scrape
+ @final
+ @property
+ def origin(self) -> AbsoluteHttpURL:
+ return ORIGIN.get()
+
@property
def separate_posts(self) -> bool:
return self.manager.config.settings.download_options.separate_posts
@@ -438,6 +444,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
def new_task_id(self, url: AbsoluteHttpURL):
"""Creates a new task_id (shows the URL in the UI and logs)"""
self.log.info(f"Scraping {url}")
+ _ = ORIGIN.set(url.origin())
return self.manager.scrape_mapper.tui.scrape.new(url)
@final
@@ -954,30 +961,27 @@ _CrawlerT = TypeVar("_CrawlerT", bound=Crawler)
_CrawlerT_generic = TypeVar("_CrawlerT_generic", bound=Crawler, default=Crawler)
-class API(HTTPMixin, Generic[_CrawlerT_generic]):
- crawler: _CrawlerT_generic
+class API(HTTPMixin, ABC, Generic[_CrawlerT_generic]):
+ _crawler: _CrawlerT_generic
@final
def __init__(self, crawler: _CrawlerT_generic) -> None:
- self.crawler = crawler
+ self._crawler = crawler
+ self.PRIMARY_URL: Final = crawler.PRIMARY_URL
+ self.parse_url: Final = crawler.parse_url
+ self.request: Final = crawler.request
+ self.config: Final = crawler.manager.config
self.__post_init__()
- def __post_init__(self): ...
+ def __post_init__(self) -> None: ...
def __repr__(self) -> str:
- return f"{type(self).__name__}(crawler={self.crawler.__name__!r})"
-
- @signature.copy(Crawler.request)
- def request(self, *args: Any, **kwargs: Any) -> contextlib._AsyncGeneratorContextManager[AbstractResponse[Any]]: # pyright: ignore[reportPrivateUsage]
- return self.crawler.request(*args, **kwargs)
-
- @property
- def parse_url(self):
- return self.crawler.parse_url
+ return f"{type(self).__name__}(crawler={self._crawler.__name__!r})"
+ @final
@property
- def PRIMARY_URL(self) -> AbsoluteHttpURL: # noqa: N802
- return self.crawler.PRIMARY_URL
+ def origin(self) -> AbsoluteHttpURL:
+ return ORIGIN.get()
def _make_scrape_mapper_keys(cls: type[Crawler] | Crawler) -> tuple[str, ...]:
diff --git a/cyberdrop_dl/crawlers/pixeldrain.py b/cyberdrop_dl/crawlers/pixeldrain.py
index ca06385b1..551674875 100644
--- a/cyberdrop_dl/crawlers/pixeldrain.py
+++ b/cyberdrop_dl/crawlers/pixeldrain.py
@@ -3,7 +3,6 @@
from __future__ import annotations
import dataclasses
-from contextvars import ContextVar
from typing import TYPE_CHECKING, Any, ClassVar, Literal, final
from typing_extensions import override
@@ -19,7 +18,6 @@ if TYPE_CHECKING:
_PRIMARY_URL = AbsoluteHttpURL("https://pixeldrain.com")
-_CURRENT_ORIGIN: ContextVar[AbsoluteHttpURL] = ContextVar("_CURRENT_ORIGIN")
@final
@@ -146,8 +144,6 @@ class PixelDrainCrawler(Crawler):
@override
async def fetch(self, scrape_item: ScrapeItem) -> None:
- _CURRENT_ORIGIN.set(scrape_item.url.origin())
-
match scrape_item.url.parts[1:]:
case ["u", file_id]:
return await self.file(scrape_item, file_id)
@@ -194,7 +190,7 @@ class PixelDrainCrawler(Crawler):
if self.check_album_results(_build_download_url(file), results):
continue
- url = _CURRENT_ORIGIN.get() / "u" / file.id
+ url = self.origin / "u" / file.id
new_scrape_item = scrape_item.create_child(url)
self.create_task(self._file_task(new_scrape_item, file))
scrape_item.add_children()
@@ -229,7 +225,7 @@ class PixelDrainCrawler(Crawler):
if node.name == ".search_index.gz":
continue
- url = _CURRENT_ORIGIN.get() / "d" / node.path.removeprefix("/")
+ url = self.origin / "d" / node.path.removeprefix("/")
new_scrape_item = scrape_item.create_child(url)
if node.type == "file":
@@ -262,7 +258,7 @@ class PixelDrainCrawler(Crawler):
if "text/plain" in file.mime_type:
return await self.text(scrape_item, file)
- src = _build_download_url(file).with_host(_CURRENT_ORIGIN.get().host)
+ src = _build_download_url(file).with_host(self.origin.host)
if await self.check_complete_by_hash(src, "sha256", file.hash_sha256):
return None
@@ -291,7 +287,7 @@ class PixelDrainCrawler(Crawler):
class PixelDrainAPI(API):
def __post_init__(self) -> None:
self.headers: dict[str, str] = {}
- if api_key := self.crawler.manager.config.auth.pixeldrain.api_key:
+ if api_key := self.config.auth.pixeldrain.api_key:
self.headers["Authorization"] = basic_auth("Cyberdrop-DL", api_key)
@property
@@ -299,21 +295,21 @@ class PixelDrainAPI(API):
return bool(self.headers)
async def file(self, file_id: str) -> File:
- api_url = _CURRENT_ORIGIN.get() / "api/file" / file_id / "info"
+ api_url = self.origin / "api/file" / file_id / "info"
resp = await self._request(api_url)
return type_adapter(File).validate_json(resp)
async def text(self, file_id: str) -> str:
- api_url = _CURRENT_ORIGIN.get() / "api/file" / file_id
+ api_url = self.origin / "api/file" / file_id
return await self._request(api_url)
async def folder(self, list_id: str) -> Folder:
- api_url = _CURRENT_ORIGIN.get() / "api/list" / list_id
+ api_url = self.origin / "api/list" / list_id
resp = await self._request(api_url)
return type_adapter(Folder).validate_json(resp)
async def filesystem(self, path: str) -> FileSystem:
- api_url = (_CURRENT_ORIGIN.get() / "api/filesystem" / path.removeprefix("/")).with_query("stat")
+ api_url = (self.origin / "api/filesystem" / path.removeprefix("/")).with_query("stat")
resp = await self._request(api_url)
return type_adapter(FileSystem).validate_json(resp)