Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: switch to API (OmegaScans) (#1850)

NTFSvolume9159669ae986
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index dce5c08e8..c29f98423 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Final, Literal, Pa
from typing_extensions import deprecated
-from cyberdrop_dl import aio, env
+from cyberdrop_dl import aio, env, signature
from cyberdrop_dl.clients.http import HTTPClient, HTTPClientProxy
from cyberdrop_dl.crawlers._hls import HLSParser
from cyberdrop_dl.downloader.http import Downloader
@@ -164,6 +164,9 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
_USE_DOWNLOAD_SERVERS_LOCKS: ClassVar[bool] = False
disabled: bool = False
+ def __repr__(self) -> str:
+ return f"<{type(self).__name__}(domain={self.DOMAIN!r}, primary_url={self.PRIMARY_URL!r}, disabled={self.disabled!r}, _ready={self._ready!r})>"
+
@staticmethod
def __db_path__(url: AbsoluteHttpURL, /) -> str:
return url.path
@@ -894,6 +897,19 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
)
[email protected](slots=True)
+class CrawlerAPI:
+ crawler: Crawler
+
+ @signature.copy(HTTPClient.request)
+ async def request_text(self, *args, **kwargs) -> str:
+ return await self.crawler.request_text(*args, **kwargs)
+
+ @signature.copy(request_text)
+ async def request_json(self, *args, **kwargs) -> Any:
+ return await self.crawler.request_json(*args, **kwargs)
+
+
def _make_scrape_mapper_keys(cls: type[Crawler] | Crawler) -> tuple[str, ...]:
hosts = cls.SUPPORTED_DOMAINS or cls.DOMAIN
if isinstance(hosts, str):
diff --git a/cyberdrop_dl/crawlers/omegascans.py b/cyberdrop_dl/crawlers/omegascans.py
index 20a06f850..7e96dfeed 100644
--- a/cyberdrop_dl/crawlers/omegascans.py
+++ b/cyberdrop_dl/crawlers/omegascans.py
@@ -2,39 +2,25 @@ from __future__ import annotations
import dataclasses
import itertools
-from typing import TYPE_CHECKING, Any, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar, override
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
-from cyberdrop_dl.exceptions import ScrapeError
+from cyberdrop_dl.crawlers.crawler import Crawler, CrawlerAPI, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, next_js
+from cyberdrop_dl.utils import error_handling_wrapper, parse_url
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
+ from collections.abc import AsyncGenerator
from cyberdrop_dl.url_objects import ScrapeItem
-_API_ENTRYPOINT = AbsoluteHttpURL("https://api.omegascans.org/chapter/query")
-
-
-class Selector:
- SERIES_ID = "script:-soup-contains('series_id')"
- IMAGE = "#content img.h-auto.object-contain"
-
-
[email protected](slots=True)
[email protected](slots=True, kw_only=True)
class Chapter:
name: str
slug: str
created_at: str
-
-
[email protected](slots=True)
-class Series:
- title: str
- thumbnail: str
- slug: str
+ series_title: str
+ images: tuple[AbsoluteHttpURL, ...]
class OmegaScansCrawler(Crawler):
@@ -49,63 +35,60 @@ class OmegaScansCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
- case ["series", _]:
- return await self.series(scrape_item)
- case ["series", _, _]:
- return await self.chapter(scrape_item)
+ case ["series", series_slug]:
+ return await self.series(scrape_item, series_slug)
+ case ["series", series_slug, chapter_slug]:
+ return await self.chapter(scrape_item, series_slug, chapter_slug)
case ["file", *_]:
await self.direct_file(scrape_item)
case _:
raise ValueError
- @error_handling_wrapper
- async def series(self, scrape_item: ScrapeItem) -> None:
- soup = await self.request_soup(scrape_item.url)
- js_script = css.select_text(soup, Selector.SERIES_ID)
- series_id = js_script.split('series_id\\":')[1].split(",")[0]
- scrape_item.setup_as_album("", album_id=series_id)
- # TODO: Add title
- # title: str = ""
- api_url = _API_ENTRYPOINT.with_query(series_id=series_id, perPage=30)
+ @override
+ async def __async_post_init__(self) -> None:
+ self.api: OmegaScansAPI = OmegaScansAPI(self)
- for page in itertools.count(1):
- json_resp: dict[str, Any] = await self.request_json(api_url.update_query(page=page))
- for chapter in json_resp["data"]:
- chapter_url = scrape_item.url / chapter["chapter_slug"]
+ @error_handling_wrapper
+ async def series(self, scrape_item: ScrapeItem, series_slug: str) -> None:
+ scrape_item.setup_as_album("", album_id=series_slug)
+ async for chapter_slugs in self.api.series_chapters(series_slug):
+ for slug in chapter_slugs:
+ chapter_url = self.PRIMARY_URL / "series" / series_slug / slug
new_scrape_item = scrape_item.create_child(chapter_url)
self.create_task(self.run(new_scrape_item))
scrape_item.add_children()
- if json_resp["meta"]["current_page"] == json_resp["meta"]["last_page"]:
- break
-
@error_handling_wrapper
- async def chapter(self, scrape_item: ScrapeItem) -> None:
- soup = await self.request_soup(scrape_item.url)
-
- if "This chapter is premium" in soup.get_text():
- raise ScrapeError(401, "This chapter is premium")
-
- series, chapter = _extract_info(soup)
- scrape_item.setup_as_album(self.create_title(series.title))
+ async def chapter(self, scrape_item: ScrapeItem, series_slug: str, chapter_slug: str) -> None:
+ chapter = await self.api.chapter(series_slug, chapter_slug)
+ scrape_item.setup_as_album(self.create_title(chapter.series_title))
scrape_item.add_to_parent_title(chapter.name)
-
scrape_item.uploaded_at = self.parse_iso_date(chapter.created_at)
- for _, link in self.iter_tags(soup, Selector.IMAGE, "src"):
- self.create_task(self.direct_file(scrape_item, link))
+ for img in chapter.images:
+ self.create_task(self.direct_file(scrape_item, img))
scrape_item.add_children()
-def _extract_info(soup: BeautifulSoup) -> tuple[Series, Chapter]:
- data = next_js.extract(soup)
- chapter = next_js.find(data, "id", "series_id", "chapter_title", "created_at")
- series = next_js.find(data, "id", "series_slug", "title")
- return Series(
- title=series["title"],
- slug=series["series_slug"],
- thumbnail=series["thumbnail"],
- ), Chapter(
- name=chapter["chapter_name"],
- slug=chapter["chapter_slug"],
- created_at=chapter["created_at"],
- )
+class OmegaScansAPI(CrawlerAPI):
+ ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://api.omegascans.org")
+
+ async def chapter(self, series_slug: str, chapter_slug: str) -> Chapter:
+ api_url = self.ENTRYPOINT / "chapter" / series_slug / chapter_slug
+ chapter = (await self.request_json(api_url))["chapter"]
+ return Chapter(
+ name=chapter["chapter_name"],
+ slug=chapter["chapter_slug"],
+ created_at=chapter["created_at"],
+ images=tuple(map(parse_url, chapter["chapter_data"]["images"])),
+ series_title=chapter["series"]["title"],
+ )
+
+ async def series_chapters(self, series_slug: str) -> AsyncGenerator[tuple[str,]]:
+ api_url = self.ENTRYPOINT / "series" / series_slug
+ series_id: int = (await self.request_json(api_url))["id"]
+ api_url = (self.ENTRYPOINT / "chapter/query").with_query(series_id=series_id, perPage=10_000)
+ for page in itertools.count(1):
+ resp: dict[str, Any] = await self.request_json(api_url.update_query(page=page))
+ yield tuple(data["chapter_slug"] for data in resp["data"])
+ if resp["meta"]["current_page"] == resp["meta"]["last_page"]:
+ break