Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: albums (imgbb) (#1376)

NTFSvolume943683edbdfb
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index 3f1bf8f2e..e14c11190 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import base64
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, ClassVar, final
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, copy_signature
@@ -44,7 +44,7 @@ class CheveretoCrawler(Crawler, is_generic=True):
"/image/<id>",
"/image/<name>.<id>",
),
- "Profiles": "/<user_name>",
+ "Profile": "/<user_name>",
"Video": (
"/video/<id>",
"/video/<name>.<id>",
@@ -81,6 +81,20 @@ class CheveretoCrawler(Crawler, is_generic=True):
case _:
raise ValueError
+ @final
+ @staticmethod
+ def _thumbnail_to_src(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ new_name = url.name
+ for trash in (".md.", ".th.", ".fr."):
+ new_name = new_name.replace(trash, ".")
+ return url.with_name(new_name)
+
+ @classmethod
+ def _match_img(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
+ match url.parts[1:]:
+ case ["img" | "image" as part, image_slug, *_]:
+ return url.origin() / part / _id(image_slug)
+
@copy_signature(Crawler.request_soup)
async def request_soup(self, url: AbsoluteHttpURL, *args, impersonate: bool = False, **kwargs) -> BeautifulSoup:
# chevereto redirects are URL encoded and aiohttp always reencodes them by default, leading to an infinite redirect loop, so we use cURL
@@ -150,13 +164,13 @@ class CheveretoCrawler(Crawler, is_generic=True):
self, scrape_item: ScrapeItem, soup: BeautifulSoup, results: dict[str, int] | None = None
) -> None:
for thumb, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ITEM):
- if image_url := _match_img(new_scrape_item.url):
+ if image_url := self._match_img(new_scrape_item.url):
new_scrape_item.url = image_url
if thumb:
# for images, we can download the file from the thumbnail, skipping an additional request per img
# cons: we won't get the upload date
- source = _thumbnail_to_src(thumb)
+ source = self._thumbnail_to_src(thumb)
if results and self.check_album_results(source, results):
continue
@@ -183,7 +197,7 @@ class CheveretoCrawler(Crawler, is_generic=True):
async def direct_file(
self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None, assume_ext: str | None = None
) -> None:
- link = _thumbnail_to_src(url or scrape_item.url)
+ link = self._thumbnail_to_src(url or scrape_item.url)
await super().direct_file(scrape_item, link, assume_ext)
@error_handling_wrapper
@@ -217,19 +231,6 @@ def _id(slug: str) -> str:
return slug.rsplit(".")[-1]
-def _match_img(url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
- match url.parts[1:]:
- case ["img" | "image" as part, image_slug, *_]:
- return url.origin() / part / _id(image_slug)
-
-
-def _thumbnail_to_src(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
- new_name = url.name
- for trash in (".md.", ".th.", ".fr."):
- new_name = new_name.replace(trash, ".")
- return url.with_name(new_name)
-
-
def _sort_by_new(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
init_page = int(url.query.get("page") or 1)
if url.name:
diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.py
index b28bfba0b..1412948f2 100644
--- a/cyberdrop_dl/crawlers/imgbb.py
+++ b/cyberdrop_dl/crawlers/imgbb.py
@@ -1,91 +1,48 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, ClassVar, Final
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
+from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css
-from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
+ from cyberdrop_dl.crawlers.crawler import SupportedDomains, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
-PRIMARY_URL = AbsoluteHttpURL("https://ibb.co")
-IMAGE_PAGE_SELECTOR = "a[class*=image-container]"
-ALBUM_PAGE_SELECTOR = "a[class='image-container --media']"
-FIRST_PAGE_SELECTOR = "a[id=list-most-recent-link]"
+IMAGES_CDN: Final = "i.ibb.co"
-IMAGE_SELECTOR = "div[id=image-viewer-container] img"
-DATE_SELECTOR = "p[class*=description-meta] span"
-ALBUM_TITLE_SELECTOR = "a[data-text=album-name]"
-MAIN_HOST = "ibb.co"
-
-
-class ImgBBCrawler(Crawler):
- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Album": "/album/...", "Image": "/..."}
+class ImgBBCrawler(CheveretoCrawler):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Album": "/album/<album_id>",
+ "Image": "/<image_id>",
+ "Profile": "<user_name>.imgbb.co/",
+ }
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "ibb.co", "imgbb.co"
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
- NEXT_PAGE_SELECTOR: ClassVar[str] = "a[data-pagination=next]"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://ibb.co")
DOMAIN: ClassVar[str] = "imgbb"
FOLDER_DOMAIN: ClassVar[str] = "ImgBB"
+ SKIP_PRE_CHECK: ClassVar[bool] = True
async def fetch(self, scrape_item: ScrapeItem) -> None:
- if is_cdn(scrape_item.url):
- image_id = scrape_item.url.parts[1]
- scrape_item.url = PRIMARY_URL / image_id
-
- scrape_item.url = scrape_item.url.with_host(MAIN_HOST)
- if "album" in scrape_item.url.parts:
- return await self.album(scrape_item)
- await self.image(scrape_item)
-
- @error_handling_wrapper
- async def album(self, scrape_item: ScrapeItem) -> None:
- title: str = ""
- first_page = None
-
- async for soup in self.web_pager(scrape_item.url / "sub"):
- if not title:
- album_id = scrape_item.url.parts[2]
- title_portion = css.select_one_get_text(soup, ALBUM_TITLE_SELECTOR)
- title = self.create_title(title_portion, album_id)
- scrape_item.setup_as_album(title, album_id=album_id)
- first_page_str: str = css.select_one_get_attr(soup, FIRST_PAGE_SELECTOR, "href")
- first_page = self.parse_url(first_page_str)
-
- for _, sub_album in self.iter_children(scrape_item, soup, ALBUM_PAGE_SELECTOR):
- self.create_task(self.run(sub_album))
-
- assert first_page
- async for soup in self.web_pager(first_page):
- for _, image in self.iter_children(scrape_item, soup, IMAGE_PAGE_SELECTOR):
- self.create_task(self.run(image))
-
- @error_handling_wrapper
- async def image(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
- return
-
- soup = await self.request_soup(scrape_item.url)
-
- link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")
- link = self.parse_url(link_str)
- date_str: str = css.select_one_get_attr(soup, DATE_SELECTOR, "title")
- scrape_item.possible_datetime = self.parse_iso_date(date_str)
-
- filename, ext = self.get_filename_and_ext(link.name)
- await self.handle_file(link, scrape_item, filename, ext)
-
- @error_handling_wrapper
- async def handle_direct_link(self, scrape_item: ScrapeItem) -> None:
- """Handles a direct link."""
- scrape_item.url = scrape_item.url.with_name(scrape_item.url.name.replace(".md.", ".").replace(".th.", "."))
- filename, ext = self.get_filename_and_ext(scrape_item.url.name)
- await self.handle_file(scrape_item.url, scrape_item, filename, ext)
-
-
-async def is_cdn(url: AbsoluteHttpURL) -> bool:
- """Determines if the url is a direct link or not."""
- return url.host == "i.ibb.co"
+ username, _, rest = scrape_item.url.host.partition(".imgbb.")
+ if username and rest:
+ return await self.profile(scrape_item)
+
+ if scrape_item.url.host == IMAGES_CDN:
+ return await self.direct_file(scrape_item)
+
+ match scrape_item.url.parts[1:]:
+ case ["album", album_id]:
+ return await self.album(scrape_item, album_id)
+ case [_]:
+ return await self.media(scrape_item)
+ case _:
+ raise ValueError
+
+ @classmethod
+ def _match_img(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
+ match url.parts[1:]:
+ case [_]:
+ return url
diff --git a/tests/crawlers/test_cases/imgbb.py b/tests/crawlers/test_cases/imgbb.py
new file mode 100644
index 000000000..1ffda7c10
--- /dev/null
+++ b/tests/crawlers/test_cases/imgbb.py
@@ -0,0 +1,39 @@
+DOMAIN = "imgbb"
+TEST_CASES = [
+ (
+ "https://ibb.co/FbtMCg43",
+ [
+ {
+ "url": "https://i.ibb.co/DDnyT5fW/Long-toes-are-the-best.jpg",
+ "filename": "Long-toes-are-the-best.jpg",
+ "referer": "https://ibb.co/FbtMCg43",
+ "album_id": None,
+ "datetime": 1740984052,
+ }
+ ],
+ ),
+ (
+ "https://i.ibb.co/DDnyT5fW/Long-toes-are-the-best.jpg",
+ [
+ {
+ "url": "https://i.ibb.co/DDnyT5fW/Long-toes-are-the-best.jpg",
+ "filename": "Long-toes-are-the-best.jpg",
+ "referer": "https://i.ibb.co/DDnyT5fW/Long-toes-are-the-best.jpg",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+ (
+ "https://ibb.co/album/yhdTNv",
+ [
+ {
+ "url": "re:.jpg",
+ "album_id": "yhdTNv",
+ "download_filename": r"re:ALEXA_2024/25 \(ImgBB\)",
+ "datetime": None,
+ }
+ ],
+ 1537,
+ ),
+]