Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: albums (imgbb) (#1376)
NTFSvolume943683edbdfb
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex 3f1bf8f2e..e14c11190 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -1,7 +1,7 @@from __future__ import annotationsimport base64-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, ClassVar, finalfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, copy_signature@@ -44,7 +44,7 @@ class CheveretoCrawler(Crawler, is_generic=True):"/image/<id>","/image/<name>.<id>",),- "Profiles": "/<user_name>",+ "Profile": "/<user_name>","Video": ("/video/<id>","/video/<name>.<id>",@@ -81,6 +81,20 @@ class CheveretoCrawler(Crawler, is_generic=True):case _:raise ValueError+ @final+ @staticmethod+ def _thumbnail_to_src(url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ new_name = url.name+ for trash in (".md.", ".th.", ".fr."):+ new_name = new_name.replace(trash, ".")+ return url.with_name(new_name)++ @classmethod+ def _match_img(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:+ match url.parts[1:]:+ case ["img" | "image" as part, image_slug, *_]:+ return url.origin() / part / _id(image_slug)+@copy_signature(Crawler.request_soup)async def request_soup(self, url: AbsoluteHttpURL, *args, impersonate: bool = False, **kwargs) -> BeautifulSoup:# chevereto redirects are URL encoded and aiohttp always reencodes them by default, leading to an infinite redirect loop, so we use cURL@@ -150,13 +164,13 @@ class CheveretoCrawler(Crawler, is_generic=True):self, scrape_item: ScrapeItem, soup: BeautifulSoup, results: dict[str, int] | None = None) -> None:for thumb, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ITEM):- if image_url := _match_img(new_scrape_item.url):+ if image_url := self._match_img(new_scrape_item.url):new_scrape_item.url = image_urlif thumb:# for images, we can download the file from the thumbnail, skipping an additional request per img# cons: we won't get the upload date- source = _thumbnail_to_src(thumb)+ source = self._thumbnail_to_src(thumb)if results and self.check_album_results(source, results):continue@@ -183,7 +197,7 @@ class CheveretoCrawler(Crawler, is_generic=True):async def direct_file(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None, assume_ext: str | None = None) -> None:- link = _thumbnail_to_src(url or scrape_item.url)+ link = self._thumbnail_to_src(url or scrape_item.url)await super().direct_file(scrape_item, link, assume_ext)@error_handling_wrapper@@ -217,19 +231,6 @@ def _id(slug: str) -> str:return slug.rsplit(".")[-1]-def _match_img(url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:- match url.parts[1:]:- case ["img" | "image" as part, image_slug, *_]:- return url.origin() / part / _id(image_slug)---def _thumbnail_to_src(url: AbsoluteHttpURL) -> AbsoluteHttpURL:- new_name = url.name- for trash in (".md.", ".th.", ".fr."):- new_name = new_name.replace(trash, ".")- return url.with_name(new_name)--def _sort_by_new(url: AbsoluteHttpURL) -> AbsoluteHttpURL:init_page = int(url.query.get("page") or 1)if url.name:diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.pyindex b28bfba0b..1412948f2 100644--- a/cyberdrop_dl/crawlers/imgbb.py+++ b/cyberdrop_dl/crawlers/imgbb.py@@ -1,91 +1,48 @@from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, ClassVar, Final-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths+from cyberdrop_dl.crawlers._chevereto import CheveretoCrawlerfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css-from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:+ from cyberdrop_dl.crawlers.crawler import SupportedDomains, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import ScrapeItem-PRIMARY_URL = AbsoluteHttpURL("https://ibb.co")-IMAGE_PAGE_SELECTOR = "a[class*=image-container]"-ALBUM_PAGE_SELECTOR = "a[class='image-container --media']"-FIRST_PAGE_SELECTOR = "a[id=list-most-recent-link]"+IMAGES_CDN: Final = "i.ibb.co"-IMAGE_SELECTOR = "div[id=image-viewer-container] img"-DATE_SELECTOR = "p[class*=description-meta] span"-ALBUM_TITLE_SELECTOR = "a[data-text=album-name]"-MAIN_HOST = "ibb.co"---class ImgBBCrawler(Crawler):- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Album": "/album/...", "Image": "/..."}+class ImgBBCrawler(CheveretoCrawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Album": "/album/<album_id>",+ "Image": "/<image_id>",+ "Profile": "<user_name>.imgbb.co/",+ }SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "ibb.co", "imgbb.co"- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL- NEXT_PAGE_SELECTOR: ClassVar[str] = "a[data-pagination=next]"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://ibb.co")DOMAIN: ClassVar[str] = "imgbb"FOLDER_DOMAIN: ClassVar[str] = "ImgBB"+ SKIP_PRE_CHECK: ClassVar[bool] = Trueasync def fetch(self, scrape_item: ScrapeItem) -> None:- if is_cdn(scrape_item.url):- image_id = scrape_item.url.parts[1]- scrape_item.url = PRIMARY_URL / image_id-- scrape_item.url = scrape_item.url.with_host(MAIN_HOST)- if "album" in scrape_item.url.parts:- return await self.album(scrape_item)- await self.image(scrape_item)-- @error_handling_wrapper- async def album(self, scrape_item: ScrapeItem) -> None:- title: str = ""- first_page = None-- async for soup in self.web_pager(scrape_item.url / "sub"):- if not title:- album_id = scrape_item.url.parts[2]- title_portion = css.select_one_get_text(soup, ALBUM_TITLE_SELECTOR)- title = self.create_title(title_portion, album_id)- scrape_item.setup_as_album(title, album_id=album_id)- first_page_str: str = css.select_one_get_attr(soup, FIRST_PAGE_SELECTOR, "href")- first_page = self.parse_url(first_page_str)-- for _, sub_album in self.iter_children(scrape_item, soup, ALBUM_PAGE_SELECTOR):- self.create_task(self.run(sub_album))-- assert first_page- async for soup in self.web_pager(first_page):- for _, image in self.iter_children(scrape_item, soup, IMAGE_PAGE_SELECTOR):- self.create_task(self.run(image))-- @error_handling_wrapper- async def image(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):- return-- soup = await self.request_soup(scrape_item.url)-- link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")- link = self.parse_url(link_str)- date_str: str = css.select_one_get_attr(soup, DATE_SELECTOR, "title")- scrape_item.possible_datetime = self.parse_iso_date(date_str)-- filename, ext = self.get_filename_and_ext(link.name)- await self.handle_file(link, scrape_item, filename, ext)-- @error_handling_wrapper- async def handle_direct_link(self, scrape_item: ScrapeItem) -> None:- """Handles a direct link."""- scrape_item.url = scrape_item.url.with_name(scrape_item.url.name.replace(".md.", ".").replace(".th.", "."))- filename, ext = self.get_filename_and_ext(scrape_item.url.name)- await self.handle_file(scrape_item.url, scrape_item, filename, ext)---async def is_cdn(url: AbsoluteHttpURL) -> bool:- """Determines if the url is a direct link or not."""- return url.host == "i.ibb.co"+ username, _, rest = scrape_item.url.host.partition(".imgbb.")+ if username and rest:+ return await self.profile(scrape_item)++ if scrape_item.url.host == IMAGES_CDN:+ return await self.direct_file(scrape_item)++ match scrape_item.url.parts[1:]:+ case ["album", album_id]:+ return await self.album(scrape_item, album_id)+ case [_]:+ return await self.media(scrape_item)+ case _:+ raise ValueError++ @classmethod+ def _match_img(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:+ match url.parts[1:]:+ case [_]:+ return urldiff --git a/tests/crawlers/test_cases/imgbb.py b/tests/crawlers/test_cases/imgbb.pynew file mode 100644index 000000000..1ffda7c10--- /dev/null+++ b/tests/crawlers/test_cases/imgbb.py@@ -0,0 +1,39 @@+DOMAIN = "imgbb"+TEST_CASES = [+ (+ "https://ibb.co/FbtMCg43",+ [+ {+ "url": "https://i.ibb.co/DDnyT5fW/Long-toes-are-the-best.jpg",+ "filename": "Long-toes-are-the-best.jpg",+ "referer": "https://ibb.co/FbtMCg43",+ "album_id": None,+ "datetime": 1740984052,+ }+ ],+ ),+ (+ "https://i.ibb.co/DDnyT5fW/Long-toes-are-the-best.jpg",+ [+ {+ "url": "https://i.ibb.co/DDnyT5fW/Long-toes-are-the-best.jpg",+ "filename": "Long-toes-are-the-best.jpg",+ "referer": "https://i.ibb.co/DDnyT5fW/Long-toes-are-the-best.jpg",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ (+ "https://ibb.co/album/yhdTNv",+ [+ {+ "url": "re:.jpg",+ "album_id": "yhdTNv",+ "download_filename": r"re:ALEXA_2024/25 \(ImgBB\)",+ "datetime": None,+ }+ ],+ 1537,+ ),+]