Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: switch to API (OmegaScans) (#1850)
NTFSvolume9159669ae986
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex dce5c08e8..c29f98423 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Final, Literal, Pafrom typing_extensions import deprecated-from cyberdrop_dl import aio, env+from cyberdrop_dl import aio, env, signaturefrom cyberdrop_dl.clients.http import HTTPClient, HTTPClientProxyfrom cyberdrop_dl.crawlers._hls import HLSParserfrom cyberdrop_dl.downloader.http import Downloader@@ -164,6 +164,9 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):_USE_DOWNLOAD_SERVERS_LOCKS: ClassVar[bool] = Falsedisabled: bool = False+ def __repr__(self) -> str:+ return f"<{type(self).__name__}(domain={self.DOMAIN!r}, primary_url={self.PRIMARY_URL!r}, disabled={self.disabled!r}, _ready={self._ready!r})>"+@staticmethoddef __db_path__(url: AbsoluteHttpURL, /) -> str:return url.path@@ -894,6 +897,19 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):)[email protected](slots=True)+class CrawlerAPI:+ crawler: Crawler++ @signature.copy(HTTPClient.request)+ async def request_text(self, *args, **kwargs) -> str:+ return await self.crawler.request_text(*args, **kwargs)++ @signature.copy(request_text)+ async def request_json(self, *args, **kwargs) -> Any:+ return await self.crawler.request_json(*args, **kwargs)++def _make_scrape_mapper_keys(cls: type[Crawler] | Crawler) -> tuple[str, ...]:hosts = cls.SUPPORTED_DOMAINS or cls.DOMAINif isinstance(hosts, str):diff --git a/cyberdrop_dl/crawlers/omegascans.py b/cyberdrop_dl/crawlers/omegascans.pyindex 20a06f850..7e96dfeed 100644--- a/cyberdrop_dl/crawlers/omegascans.py+++ b/cyberdrop_dl/crawlers/omegascans.py@@ -2,39 +2,25 @@ from __future__ import annotationsimport dataclassesimport itertools-from typing import TYPE_CHECKING, Any, ClassVar+from typing import TYPE_CHECKING, Any, ClassVar, override-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths-from cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.crawlers.crawler import Crawler, CrawlerAPI, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, next_js+from cyberdrop_dl.utils import error_handling_wrapper, parse_urlif TYPE_CHECKING:- from bs4 import BeautifulSoup+ from collections.abc import AsyncGeneratorfrom cyberdrop_dl.url_objects import ScrapeItem-_API_ENTRYPOINT = AbsoluteHttpURL("https://api.omegascans.org/chapter/query")---class Selector:- SERIES_ID = "script:-soup-contains('series_id')"- IMAGE = "#content img.h-auto.object-contain"--[email protected](slots=True)[email protected](slots=True, kw_only=True)class Chapter:name: strslug: strcreated_at: str--[email protected](slots=True)-class Series:- title: str- thumbnail: str- slug: str+ series_title: str+ images: tuple[AbsoluteHttpURL, ...]class OmegaScansCrawler(Crawler):@@ -49,63 +35,60 @@ class OmegaScansCrawler(Crawler):async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:- case ["series", _]:- return await self.series(scrape_item)- case ["series", _, _]:- return await self.chapter(scrape_item)+ case ["series", series_slug]:+ return await self.series(scrape_item, series_slug)+ case ["series", series_slug, chapter_slug]:+ return await self.chapter(scrape_item, series_slug, chapter_slug)case ["file", *_]:await self.direct_file(scrape_item)case _:raise ValueError- @error_handling_wrapper- async def series(self, scrape_item: ScrapeItem) -> None:- soup = await self.request_soup(scrape_item.url)- js_script = css.select_text(soup, Selector.SERIES_ID)- series_id = js_script.split('series_id\\":')[1].split(",")[0]- scrape_item.setup_as_album("", album_id=series_id)- # TODO: Add title- # title: str = ""- api_url = _API_ENTRYPOINT.with_query(series_id=series_id, perPage=30)+ @override+ async def __async_post_init__(self) -> None:+ self.api: OmegaScansAPI = OmegaScansAPI(self)- for page in itertools.count(1):- json_resp: dict[str, Any] = await self.request_json(api_url.update_query(page=page))- for chapter in json_resp["data"]:- chapter_url = scrape_item.url / chapter["chapter_slug"]+ @error_handling_wrapper+ async def series(self, scrape_item: ScrapeItem, series_slug: str) -> None:+ scrape_item.setup_as_album("", album_id=series_slug)+ async for chapter_slugs in self.api.series_chapters(series_slug):+ for slug in chapter_slugs:+ chapter_url = self.PRIMARY_URL / "series" / series_slug / slugnew_scrape_item = scrape_item.create_child(chapter_url)self.create_task(self.run(new_scrape_item))scrape_item.add_children()- if json_resp["meta"]["current_page"] == json_resp["meta"]["last_page"]:- break-@error_handling_wrapper- async def chapter(self, scrape_item: ScrapeItem) -> None:- soup = await self.request_soup(scrape_item.url)-- if "This chapter is premium" in soup.get_text():- raise ScrapeError(401, "This chapter is premium")-- series, chapter = _extract_info(soup)- scrape_item.setup_as_album(self.create_title(series.title))+ async def chapter(self, scrape_item: ScrapeItem, series_slug: str, chapter_slug: str) -> None:+ chapter = await self.api.chapter(series_slug, chapter_slug)+ scrape_item.setup_as_album(self.create_title(chapter.series_title))scrape_item.add_to_parent_title(chapter.name)-scrape_item.uploaded_at = self.parse_iso_date(chapter.created_at)- for _, link in self.iter_tags(soup, Selector.IMAGE, "src"):- self.create_task(self.direct_file(scrape_item, link))+ for img in chapter.images:+ self.create_task(self.direct_file(scrape_item, img))scrape_item.add_children()-def _extract_info(soup: BeautifulSoup) -> tuple[Series, Chapter]:- data = next_js.extract(soup)- chapter = next_js.find(data, "id", "series_id", "chapter_title", "created_at")- series = next_js.find(data, "id", "series_slug", "title")- return Series(- title=series["title"],- slug=series["series_slug"],- thumbnail=series["thumbnail"],- ), Chapter(- name=chapter["chapter_name"],- slug=chapter["chapter_slug"],- created_at=chapter["created_at"],- )+class OmegaScansAPI(CrawlerAPI):+ ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://api.omegascans.org")++ async def chapter(self, series_slug: str, chapter_slug: str) -> Chapter:+ api_url = self.ENTRYPOINT / "chapter" / series_slug / chapter_slug+ chapter = (await self.request_json(api_url))["chapter"]+ return Chapter(+ name=chapter["chapter_name"],+ slug=chapter["chapter_slug"],+ created_at=chapter["created_at"],+ images=tuple(map(parse_url, chapter["chapter_data"]["images"])),+ series_title=chapter["series"]["title"],+ )++ async def series_chapters(self, series_slug: str) -> AsyncGenerator[tuple[str,]]:+ api_url = self.ENTRYPOINT / "series" / series_slug+ series_id: int = (await self.request_json(api_url))["id"]+ api_url = (self.ENTRYPOINT / "chapter/query").with_query(series_id=series_id, perPage=10_000)+ for page in itertools.count(1):+ resp: dict[str, Any] = await self.request_json(api_url.update_query(page=page))+ yield tuple(data["chapter_slug"] for data in resp["data"])+ if resp["meta"]["current_page"] == resp["meta"]["last_page"]:+ break