Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: remove redundant methods (crawler) (#94)

NTFSvolumef1dd719c3746
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.py
index 8ad74e06e..88e0780fc 100644
--- a/cyberdrop_dl/clients/__init__.py
+++ b/cyberdrop_dl/clients/__init__.py
@@ -21,7 +21,6 @@ if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Mapping
from bs4 import BeautifulSoup
- from curl_cffi.requests.impersonate import BrowserTypeLiteral
from curl_cffi.requests.session import HttpMethod
from cyberdrop_dl.clients import flaresolverr
@@ -78,7 +77,7 @@ class HTTPClient:
/,
method: HttpMethod = "GET",
headers: Mapping[str, str] | None = None,
- impersonate: BrowserTypeLiteral | bool | None = None,
+ impersonate: str | bool | None = None,
data: Any = None,
json: Any = None,
cache_disabled: bool = False,
@@ -223,7 +222,7 @@ def _write_resp_to_disk(
class HTTPClientProxy:
DOMAIN: ClassVar[str]
- _IMPERSONATE: ClassVar[BrowserTypeLiteral | bool | None] = None
+ _IMPERSONATE: ClassVar[str | bool | None] = None
client: HTTPClient # pyright: ignore[reportUninitializedInstanceVariable]
@classmethod
@@ -249,7 +248,7 @@ class HTTPClientProxy:
@signature.copy(HTTPClient.request)
@contextlib.asynccontextmanager
async def request(
- self, *args, impersonate: BrowserTypeLiteral | bool | None = None, **kwargs
+ self, *args, impersonate: str | bool | None = None, **kwargs
) -> AsyncGenerator[AbstractResponse[Any]]:
if impersonate is None:
impersonate = self._IMPERSONATE
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index e69de29bb..cc6f0b5f9 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING, TypeVar
+
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils.utilities import is_absolute_http_url, remove_trailing_slash
+
+if TYPE_CHECKING:
+ from collections.abc import Iterable
+
+ from cyberdrop_dl.crawlers.crawler import Crawler
+
+ _CrawlerT = TypeVar("_CrawlerT", bound=Crawler)
+
+
+def create_crawlers(
+ urls: Iterable[str] | Iterable[AbsoluteHttpURL], base_crawler: type[_CrawlerT]
+) -> set[type[_CrawlerT]]:
+ """Creates new subclasses of the base crawler from the urls"""
+ return {_create_subclass(url, base_crawler) for url in urls}
+
+
+def _create_subclass(url: AbsoluteHttpURL | str, base_class: type[_CrawlerT]) -> type[_CrawlerT]:
+ url = AbsoluteHttpURL(url)
+ assert is_absolute_http_url(url)
+ primary_url = remove_trailing_slash(url)
+ domain = primary_url.host.removeprefix("www.")
+ class_name = _make_crawler_name(domain)
+ class_attributes = {
+ "PRIMARY_URL": primary_url,
+ "DOMAIN": domain,
+ "SUPPORTED_DOMAINS": (),
+ "FOLDER_DOMAIN": "",
+ }
+ return type(class_name, (base_class,), class_attributes) # type: ignore # pyright: ignore[reportReturnType]
+
+
+def _make_crawler_name(input_string: str) -> str:
+ clean_string = re.sub(r"[^a-zA-Z0-9]+", " ", input_string).strip()
+ cap_name = clean_string.title().replace(" ", "")
+ assert cap_name and cap_name.isalnum(), (
+ f"Can not generate a valid class name from {input_string}. Needs to be defined as a concrete class"
+ )
+ if cap_name[0].isdigit():
+ cap_name = "_" + cap_name
+ return f"{cap_name}Crawler"
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index b466aca6e..0abfb40b7 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -10,7 +10,7 @@ from cyberdrop_dl.utils import css, json, open_graph
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Generator
+ from collections.abc import Generator, Mapping
from bs4 import BeautifulSoup
@@ -109,16 +109,10 @@ class CheveretoCrawler(Crawler, is_generic=True):
return url
- async def web_pager(
- self, url: AbsoluteHttpURL, next_page_selector: str | None = None, *, cffi: bool = False, **kwargs: Any
- ) -> AsyncGenerator[BeautifulSoup]:
- async for soup in super().web_pager(_sort_by_new(url), next_page_selector, cffi=cffi, trim=False, **kwargs):
- yield soup
-
@error_handling_wrapper
async def profile(self, scrape_item: ScrapeItem, *, albums: bool = False) -> None:
title: str = ""
- async for soup in self.web_pager(scrape_item.url):
+ async for soup in self.web_pager(_sort_by_new(scrape_item.url), trim=False):
if not title:
title = self.create_title(open_graph.title(soup))
scrape_item.setup_as_profile(title)
@@ -133,12 +127,12 @@ class CheveretoCrawler(Crawler, is_generic=True):
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem, album_id: str) -> None:
- results: dict[str, int] = {}
+ results: dict[str, bool] = {}
title: str = ""
scrape_item.url = await self._get_final_album_url(scrape_item.url)
- async for soup in self.web_pager(scrape_item.url):
+ async for soup in self.web_pager(_sort_by_new(scrape_item.url), trim=False):
if not title:
if soup.select_one("form"):
await self._unlock_pw_protected_album(scrape_item, soup)
@@ -151,12 +145,15 @@ class CheveretoCrawler(Crawler, is_generic=True):
self._iter_album_files(scrape_item, soup, results)
- async for soup in self.web_pager(scrape_item.url / "sub"):
+ async for soup in self.web_pager(_sort_by_new(scrape_item.url / "sub"), trim=False):
for _, sub_album in self.iter_children(scrape_item, soup, Selector.ITEM):
self.create_task(self.run(sub_album))
def _iter_album_files(
- self, scrape_item: ScrapeItem, soup: BeautifulSoup, results: dict[str, int] | None = None
+ self,
+ scrape_item: ScrapeItem,
+ soup: BeautifulSoup,
+ results: Mapping[str, bool] | None = None,
) -> None:
results = results or {}
for web_url, src_url in self._get_album_files(soup):
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.py
index dbfaba338..ebdddcda9 100644
--- a/cyberdrop_dl/crawlers/_forum.py
+++ b/cyberdrop_dl/crawlers/_forum.py
@@ -535,7 +535,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
yield get_text_between(css.attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")
async def thread_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:
- async for soup in self._web_pager(scrape_item.url, self.get_next_page):
+ async for soup in self.web_pager(scrape_item.url, self.get_next_page):
yield soup
def get_next_page(self, soup: BeautifulSoup) -> str | None:
diff --git a/cyberdrop_dl/crawlers/_one_manager.py b/cyberdrop_dl/crawlers/_one_manager.py
index c4ba28c30..30f441332 100644
--- a/cyberdrop_dl/crawlers/_one_manager.py
+++ b/cyberdrop_dl/crawlers/_one_manager.py
@@ -36,7 +36,7 @@ _SELECTORS = Selectors()
class OneManagerCrawler(Crawler, is_abc=True):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Any path": "/..."}
- SKIP_PRE_CHECK = True
+ ALLOW_EMPTY_PATH: ClassVar[bool] = True
async def fetch(self, scrape_item: ScrapeItem) -> None:
scrape_item.url = scrape_item.url.with_query(None)
diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.py
index a084b100f..0b29a8e51 100644
--- a/cyberdrop_dl/crawlers/ashemaletube.py
+++ b/cyberdrop_dl/crawlers/ashemaletube.py
@@ -104,14 +104,14 @@ class AShemaleTubeCrawler(Crawler):
@error_handling_wrapper
async def gallery(self, scrape_item: ScrapeItem) -> None:
- async for soup in self.web_pager(scrape_item.url, cffi=True):
+ async for soup in self.web_pager(scrape_item.url, impersonate=True):
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.GALLERY_ALBUM):
self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
album_title = ""
- async for soup in self.web_pager(scrape_item.url, cffi=True):
+ async for soup in self.web_pager(scrape_item.url, impersonate=True):
if not album_title:
album_title = self.create_collection_title(soup, CollectionType.ALBUM)
scrape_item.setup_as_album(album_title)
@@ -122,7 +122,7 @@ class AShemaleTubeCrawler(Crawler):
@error_handling_wrapper
async def collection(self, scrape_item: ScrapeItem, collection_type: CollectionType) -> None:
collection_title = ""
- async for soup in self.web_pager(scrape_item.url, cffi=True):
+ async for soup in self.web_pager(scrape_item.url, impersonate=True):
if not collection_title:
collection_title = self.create_collection_title(soup, collection_type)
if collection_type == CollectionType.MODEL:
diff --git a/cyberdrop_dl/crawlers/bunkrr_albums_io.py b/cyberdrop_dl/crawlers/bunkrr_albums_io.py
index a2ba8f3b4..5ca8beafc 100644
--- a/cyberdrop_dl/crawlers/bunkrr_albums_io.py
+++ b/cyberdrop_dl/crawlers/bunkrr_albums_io.py
@@ -11,12 +11,12 @@ if TYPE_CHECKING:
class BunkrAlbumsIOCrawler(Crawler):
- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Search": "/s?search=<query>"}
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Search": "/?search=<query>"}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://bunkr-albums.io/")
DOMAIN: ClassVar[str] = "bunkr-albums.io"
FOLDER_DOMAIN: ClassVar[str] = "Bunkr-Albums.io"
NEXT_PAGE_SELECTOR: ClassVar[str] = "nav a:-soup-contains(Next)"
- SKIP_PRE_CHECK: ClassVar[bool] = True
+ ALLOW_EMPTY_PATH: ClassVar[bool] = True
async def fetch(self, scrape_item: ScrapeItem) -> None:
if query := scrape_item.url.query.get("search"):
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 45a7fdd40..7a07d6e0a 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -5,7 +5,6 @@ import contextlib
import dataclasses
import datetime
import importlib
-import inspect
import logging
import pkgutil
import re
@@ -14,19 +13,8 @@ from abc import ABC, abstractmethod
from collections import Counter
from functools import wraps
from pathlib import Path
-from typing import (
- TYPE_CHECKING,
- Any,
- ClassVar,
- Concatenate,
- Literal,
- NamedTuple,
- ParamSpec,
- Self,
- TypeAlias,
- TypeVar,
- final,
-)
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Final, Literal, ParamSpec, Self, TypeAlias, TypeVar, final
from aiolimiter import AsyncLimiter
from typing_extensions import deprecated
@@ -50,18 +38,17 @@ from cyberdrop_dl.utils.utilities import (
is_blob_or_svg,
parse_url,
remove_file_id,
- remove_trailing_slash,
sanitize_filename,
truncate_str,
)
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Callable, Coroutine, Generator, Iterable, Mapping
- from http.cookies import BaseCookie
+ from collections.abc import AsyncGenerator, AsyncIterator, Callable, Coroutine, Generator, Iterable, Mapping
from types import ModuleType
import yarl
from bs4 import BeautifulSoup, Tag
+ from curl_cffi.requests.impersonate import BrowserTypeLiteral
from rich.progress import TaskID
from cyberdrop_dl.managers.manager import Manager
@@ -71,7 +58,7 @@ logger = logging.getLogger(__name__)
_P = ParamSpec("_P")
_R = TypeVar("_R")
_T = TypeVar("_T")
-_T_co = TypeVar("_T_co", covariant=True)
+
OneOrTuple: TypeAlias = _T | tuple[_T, ...]
SupportedPaths: TypeAlias = dict[str, OneOrTuple[str]]
SupportedDomains: TypeAlias = OneOrTuple[str]
@@ -92,21 +79,24 @@ class _PlaceHolderConfigInclude:
_include = _PlaceHolderConfigInclude()
-_DB_PATH_BUILDERS: dict[str, Callable[[AbsoluteHttpURL], str]] = {
- "url": lambda url: str(url),
- "name": lambda url: url.name,
- "path": lambda url: url.path,
- "path_qs": lambda url: url.path_qs,
- "path_qs_frag": lambda url: f"{url.path_qs}#{frag}" if (frag := url.fragment) else url.path_qs,
- "path_frag": lambda url: f"{url.path}#{frag}" if (frag := url.fragment) else url.path,
-}
+_DB_PATH_BUILDERS: MappingProxyType[str, Callable[[AbsoluteHttpURL], str]] = MappingProxyType(
+ {
+ "url": lambda url: str(url),
+ "name": lambda url: url.name,
+ "path": lambda url: url.path,
+ "path_qs": lambda url: url.path_qs,
+ "path_qs_frag": lambda url: f"{url.path_qs}#{frag}" if (frag := url.fragment) else url.path_qs,
+ "path_frag": lambda url: f"{url.path}#{frag}" if (frag := url.fragment) else url.path,
+ }
+)
def _url(item: ScrapeItem | AbsoluteHttpURL) -> AbsoluteHttpURL:
return item if isinstance(item, AbsoluteHttpURL) else item.url
-class CrawlerInfo(NamedTuple):
[email protected](slots=True, frozen=True, order=True)
+class CrawlerInfo:
site: str
primary_url: AbsoluteHttpURL
supported_domains: tuple[str, ...]
@@ -152,7 +142,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {id} - {title}"
UPDATE_UNSUPPORTED: ClassVar[bool] = False
- SKIP_PRE_CHECK: ClassVar[bool] = False
+ ALLOW_EMPTY_PATH: ClassVar[bool] = False
NEXT_PAGE_SELECTOR: ClassVar[str] = ""
DEFAULT_TRIM_URLS: ClassVar[bool] = True
@@ -177,20 +167,13 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
self._ready: bool = False
self.disabled: bool = False
self.logged_in: bool = False
- self.scraped_items: set[str] = set()
- self.RATE_LIMIT = AsyncLimiter(*self._RATE_LIMIT)
+ self._scraped_items: set[str] = set()
self.log = log
self.log_debug = log_debug
self._semaphore: asyncio.Semaphore = asyncio.Semaphore(20)
self.__post_init__()
- @property
- def waiting_items(self) -> int:
- if self._semaphore._waiters is None:
- return 0
- return len(self._semaphore._waiters)
-
def __post_init__(self) -> None:
"""Override in subclasses to add custom init logic
@@ -198,17 +181,16 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
@final
async def __async_init__(self) -> None:
- """Starts the crawler."""
async with self._startup_lock:
if self._ready:
return
self.client = self.manager.client_manager.scraper_client
- self.manager.client_manager.rate_limits[self.DOMAIN] = self.RATE_LIMIT
+ self.manager.client_manager.rate_limits[self.DOMAIN] = AsyncLimiter(*self._RATE_LIMIT)
if self._DOWNLOAD_SLOTS:
self.manager.client_manager.download_slots[self.DOMAIN] = self._DOWNLOAD_SLOTS
if self._USE_DOWNLOAD_SERVERS_LOCKS:
self.manager.client_manager.download_client.server_locked_domains.add(self.DOMAIN)
- self.downloader = self._init_downloader()
+ self.__init_downloader__()
await self.__async_post_init__()
self._ready = True
@@ -282,6 +264,10 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
)
Registry.concrete.add(cls)
+ def __init_downloader__(self) -> None:
+ self.downloader = dl = Downloader(self.manager, self.DOMAIN)
+ dl.startup()
+
@final
@staticmethod
def _assert_fields_overrides(subclass: type[Crawler], *fields: str) -> None:
@@ -289,24 +275,34 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
assert getattr(subclass, field_name, None), f"Subclass {subclass.__name__} must override: {field_name}"
@final
- def create_task(self, coro: Coroutine[Any, Any, _T_co]) -> asyncio.Task[_T_co]:
- return self.manager.task_group.create_task(coro)
+ def create_task(self, coro: Coroutine[Any, Any, _T]) -> None:
+ _ = self.manager.task_group.create_task(coro)
@abstractmethod
- async def fetch(self, scrape_item: ScrapeItem) -> None: ...
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ """Here goes the main logic to parse URL paths.
+
+ This method MUST NOT raise any exceptions other that ValueError to indiated that the path is not supported"""
+ @final
@property
- def allow_no_extension(self) -> bool:
- return not self.manager.config_manager.settings_data.ignore_options.exclude_files_with_no_extension
+ def waiting_items(self) -> int:
+ if self._semaphore._waiters is None:
+ return 0
+ return len(self._semaphore._waiters)
+ @final
@property
def deep_scrape(self) -> bool:
return self.manager.config_manager.deep_scrape
- def _init_downloader(self) -> Downloader:
- self.downloader = dl = Downloader(self.manager, self.DOMAIN)
- dl.startup()
- return dl
+ @property
+ def allow_no_extension(self) -> bool:
+ return not self.manager.config.ignore_options.exclude_files_with_no_extension
+
+ @property
+ def separate_posts(self) -> bool:
+ return self.manager.config.download_options.separate_posts
@final
@contextlib.contextmanager
@@ -318,13 +314,10 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
self.disabled = True
raise
- catch_errors = final(error_handling_context)
+ catch_errors: Final = error_handling_context
@final
async def run(self, scrape_item: ScrapeItem) -> None:
- """Runs the crawler loop."""
- if not scrape_item.url.host:
- return
if self.disabled:
return
@@ -332,17 +325,21 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
with scrape_item.track_changes():
scrape_item.url = url = self.transform_url(scrape_item.url)
- if url.path_qs in self.scraped_items:
- return logger.info(f"Skipping {url} as it has already been scraped")
+ if url.path_qs in self._scraped_items:
+ logger.info(f"Skipping {url} as it has already been scraped")
+ return
- self.scraped_items.add(url.path_qs)
- async with self._fetch_context(scrape_item):
- self.pre_check_scrape_item(scrape_item)
- await self.fetch(scrape_item)
+ self._scraped_items.add(url.path_qs)
- def pre_check_scrape_item(self, scrape_item: ScrapeItem) -> None:
- if not self.SKIP_PRE_CHECK and scrape_item.url.path == "/":
- raise ValueError
+ with self.new_task_id(scrape_item.url):
+ try:
+ if not self.ALLOW_EMPTY_PATH and scrape_item.url.path == "/":
+ raise ValueError
+ await self.fetch(scrape_item)
+ except ValueError:
+ self.raise_exc(scrape_item, ScrapeError("Unknown URL path"))
+ except MaxChildrenError as e:
+ self.raise_exc(scrape_item, e)
@classmethod
def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
@@ -354,19 +351,6 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
return url.with_host(new_host)
return url
- @final
- @contextlib.asynccontextmanager
- async def _fetch_context(self, scrape_item: ScrapeItem) -> AsyncGenerator[TaskID]:
- with self.new_task_id(scrape_item.url) as task_id:
- try:
- yield task_id
- except ValueError:
- self.raise_exc(scrape_item, ScrapeError("Unknown URL path"))
- except MaxChildrenError as e:
- self.raise_exc(scrape_item, e)
- finally:
- pass
-
@error_handling_wrapper
def raise_exc(self, scrape_item: ScrapeItem, exc: type[Exception] | Exception | str) -> None:
if isinstance(exc, str):
@@ -470,22 +454,21 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
data = [media_item.__json__()]
await self.manager.logs.write_jsonl(data)
+ @final
async def check_complete(self, url: AbsoluteHttpURL, referer: AbsoluteHttpURL) -> bool:
"""Checks if this URL has been download before.
This method is called automatically on a created media item,
but Crawler code can use it to skip unnecessary requests"""
db_path = self.__db_path__(url)
- check_complete = await self.manager.db_manager.history_table.check_complete(self.DOMAIN, url, referer, db_path)
- if check_complete:
+ was_completed = await self.manager.db_manager.history_table.check_complete(self.DOMAIN, url, referer, db_path)
+ if was_completed:
logger.info(f"Skipping {url} as it has already been downloaded")
self.manager.progress_manager.download_progress.add_previously_completed()
- return check_complete
+ return was_completed
async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.Rendition | None = None) -> None:
-
- check_complete = await self.check_complete(media_item.url, media_item.referer)
- if check_complete:
+ if await self.check_complete(media_item.url, media_item.referer):
if media_item.album_id:
await self.manager.db_manager.history_table.set_album_id(self.DOMAIN, media_item)
return
@@ -499,18 +482,17 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
@final
async def check_skip_by_config(self, media_item: MediaItem) -> bool:
media_host = media_item.url.host
+ ignore_options = self.manager.config.ignore_options
- if (hosts := self.manager.config.ignore_options.skip_hosts) and any(host in media_host for host in hosts):
+ if (hosts := ignore_options.skip_hosts) and any(host in media_host for host in hosts):
logger.info(f"Download skipped {media_item.url} due to skip_hosts config")
return True
- if (hosts := self.manager.config.ignore_options.only_hosts) and not any(host in media_host for host in hosts):
+ if (hosts := ignore_options.only_hosts) and not any(host in media_host for host in hosts):
logger.info(f"Download skipped {media_item.url} due to only_hosts config")
return True
- if (regex := self.manager.config.ignore_options.filename_regex_filter) and re.search(
- regex, media_item.filename
- ):
+ if (regex := ignore_options.filename_regex_filter) and re.search(regex, media_item.filename):
logger.info(f"Download skipped {media_item.url} due to filename regex filter config")
return True
@@ -530,8 +512,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
if downloaded:
logger.info(f"Skipping {url} as it has already been downloaded")
self.manager.progress_manager.download_progress.add_previously_completed()
- return True
- return False
+ return downloaded
@final
async def check_complete_by_hash(
@@ -545,20 +526,26 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
self.manager.progress_manager.download_progress.add_previously_completed()
return downloaded
- async def get_album_results(self, album_id: str) -> dict[str, int]:
+ @final
+ async def get_album_results(self, album_id: str) -> dict[str, bool]:
"""Checks whether an album has completed given its domain and album id."""
return await self.manager.db_manager.history_table.check_album(self.DOMAIN, album_id)
+ @final
def handle_external_links(self, scrape_item: ScrapeItem, reset: bool = True) -> None:
"""Maps external links to the scraper class."""
if reset:
scrape_item.reset()
self.create_task(self.manager.scrape_mapper.filter_and_send_to_crawler(scrape_item))
- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
+ @final
def get_filename_and_ext(
- self, filename: str, forum: bool = False, assume_ext: str | None = ".mp4", *, mime_type: str | None = None
+ self,
+ filename: str,
+ forum: bool = False,
+ assume_ext: str | None = ".mp4",
+ *,
+ mime_type: str | None = None,
) -> tuple[str, str]:
"""Wrapper around `utils.get_filename_and_ext`.
Calls it as is.
@@ -572,12 +559,13 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
raise
@final
- def check_album_results(self, url: AbsoluteHttpURL, album_results: dict[str, Any]) -> bool:
+ def check_album_results(self, url: AbsoluteHttpURL, album_results: Mapping[str, bool]) -> bool:
"""Checks whether an album has completed given its domain and album id."""
if not album_results:
return False
+
url_path = self.__db_path__(url)
- if url_path in album_results and album_results[url_path] != 0:
+ if album_results.get(url_path) is True:
logger.info(f"Skipping {url} as it has already been downloaded")
self.manager.progress_manager.download_progress.add_previously_completed()
return True
@@ -586,10 +574,8 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
@final
def create_title(self, title: str, album_id: str | None = None, thread_id: int | None = None) -> str:
"""Creates the title for the scrape item."""
- if not title:
- title = "Untitled"
+ title = (title or "Untitled").strip()
- title = title.strip()
if album_id and self.manager.config.download_options.include_album_id_in_folder_name:
title = f"{title} {album_id}"
@@ -600,16 +586,9 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
title = f"{title} ({self.FOLDER_DOMAIN})"
# Remove double spaces
- while True:
- title = title.replace(" ", " ")
- if " " not in title:
- break
+ title = " ".join(title.split(""))
return title
- @property
- def separate_posts(self) -> bool:
- return self.manager.config.download_options.separate_posts
-
@final
def create_separate_post_title(
self,
@@ -631,14 +610,24 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
@classmethod
def parse_url(
- cls, link_str: yarl.URL | str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool | None = None
+ cls,
+ url: yarl.URL | str,
+ /,
+ relative_to: AbsoluteHttpURL | None = None,
+ *,
+ trim: bool | None = None,
) -> AbsoluteHttpURL:
"""Wrapper around `utils.parse_url` to use `self.PRIMARY_URL` as base"""
base = relative_to or cls.PRIMARY_URL
assert is_absolute_http_url(base)
if trim is None:
trim = cls.DEFAULT_TRIM_URLS
- return parse_url(link_str, base, trim=trim)
+ return parse_url(url, base, trim=trim)
+
+ @final
+ def get_cookie_value(self, name: str) -> str | None:
+ if morsel := self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL).get(name):
+ return morsel.value
@final
def update_cookies(self, cookies: dict[str, Any], url: yarl.URL | None = None) -> None:
@@ -657,7 +646,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
/,
attribute: str = "href",
*,
- results: dict[str, int] | None = None,
+ results: Mapping[str, bool] | None = None,
) -> Generator[tuple[AbsoluteHttpURL | None, AbsoluteHttpURL]]:
"""Generates tuples with an URL from the `src` value of the first image tag (AKA the thumbnail) and an URL from the value of `attribute`
@@ -674,11 +663,13 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
link = self.parse_url(link_str)
if self.check_album_results(link, album_results):
continue
- if t_tag := tag.select_one("img"):
- thumb_str: str | None = css.attr_or_none(t_tag, "src")
+ try:
+ thumb_str: str | None = css.select(tag, "img", "src")
+ except css.SelectorError:
+ thumb = None
else:
- thumb_str = None
- thumb = self.parse_url(thumb_str) if thumb_str and not is_blob_or_svg(thumb_str) else None
+ thumb = None if is_blob_or_svg(thumb_str) else self.parse_url(thumb_str)
+
yield thumb, link
@final
@@ -690,92 +681,51 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
/,
attribute: str = "href",
*,
- results: dict[str, int] | None = None,
+ results: Mapping[str, bool] | None = None,
**kwargs: Any,
) -> Generator[tuple[AbsoluteHttpURL | None, ScrapeItem]]:
"""Generates tuples with an URL from the `src` value of the first image tag (AKA the thumbnail) and a new scrape item from the value of `attribute`
:param results: must be the output of `self.get_album_results`.
- If provided, it will be used as a filter, to only yield items that has not been downloaded before
- :param **kwargs: Will be forwarded to `scrape.item.create_child`"""
+ If provided, it will be used as a filter, to only yield items that has not been downloaded before"""
for thumb, link in self.iter_tags(soup, selector, attribute, results=results):
new_scrape_item = scrape_item.create_child(link, **kwargs)
yield thumb, new_scrape_item
scrape_item.add_children()
- @final
- async def crawl_children(
- self,
- scrape_item: ScrapeItem,
- selector: str,
- /,
- attribute: str = "href",
- *,
- results: dict[str, int] | None = None,
- next_page_selector: str | None = None,
- coro_factory: Callable[[ScrapeItem], Coroutine[Any, Any, Any]] | None = None,
- ) -> None:
- """Crawls children URLs and schedules scrape tasks for them.
-
- Uses `self.web_pager` to iterate through pages and extracts child links based on `selector` and `attribute`. For
- each extracted URL, a new `ScrapeItem` is created, and a task
- (by default, `self.run`) is scheduled in the global task group to process it
- """
-
- if coro_factory is None:
- coro_factory = self.run
-
- async for soup in self.web_pager(scrape_item.url, next_page_selector):
- for _, new_scrape_item in self.iter_children(scrape_item, soup, selector, attribute, results=results):
- self.create_task(coro_factory(new_scrape_item))
-
async def web_pager(
- self, url: AbsoluteHttpURL, next_page_selector: str | None = None, *, cffi: bool = False, **kwargs: Any
- ) -> AsyncGenerator[BeautifulSoup]:
- """Generator of website pages.
-
- :param next_page_selector: If `None`, `self.next_page_selector` will be used
- :param cffi: If `True`, uses `curl_cffi` to get the soup for each page. Otherwise, `aiohttp` will be used
- :param **kwargs: Will be forwarded to `self.parse_url` to parse each new page"""
-
- async for soup in self._web_pager(url, next_page_selector, cffi=cffi, **kwargs):
- yield soup
-
- async def _web_pager(
self,
url: AbsoluteHttpURL,
- selector: Callable[[BeautifulSoup], str | None] | str | None = None,
+ selector: Callable[[BeautifulSoup], yarl.URL | str | None] | str | None = None,
*,
- cffi: bool = False,
- **kwargs: Any,
- ) -> AsyncGenerator[BeautifulSoup]:
- """Generator of website pages.
-
- :param next_page_selector: If `None`, `self.next_page_selector` will be used
- :param cffi: If `True`, uses `curl_cffi` to get the soup for each page. Otherwise, `aiohttp` will be used
- :param **kwargs: Will be forwarded to `self.parse_url` to parse each new page"""
+ impersonate: BrowserTypeLiteral | bool | None = False,
+ relative_to: AbsoluteHttpURL | None = None,
+ trim: bool | None = None,
+ ) -> AsyncIterator[BeautifulSoup]:
+ """Generator of website pages"""
- kwargs.setdefault("relative_to", url.origin())
+ relative_to = relative_to or url.origin()
page_url = url
if callable(selector):
get_next_page = selector
+
else:
selector = selector or self.NEXT_PAGE_SELECTOR
assert selector, f"No selector was provided and {self.DOMAIN} does define a next_page_selector"
- def get_next_page(soup: BeautifulSoup, /) -> str | None:
+ def get_next_page(soup: BeautifulSoup, /) -> yarl.URL | str | None:
try:
return css.select(soup, selector, "href")
except css.SelectorError:
return
while True:
- soup = await self.request_soup(page_url, impersonate=cffi or None)
+ soup = await self.request_soup(page_url, impersonate=impersonate or None)
yield soup
page_url_str = get_next_page(soup)
if not page_url_str:
break
- page_url = self.parse_url(page_url_str, **kwargs)
+ page_url = self.parse_url(page_url_str, relative_to=relative_to, trim=trim)
@error_handling_wrapper
async def direct_file(
@@ -795,7 +745,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
@final
@classmethod
- def parse_date(cls, date_or_datetime: str, format: str) -> dates.TimeStamp | None:
+ def parse_date(cls, date_or_datetime: str, /, format: str) -> dates.TimeStamp | None:
return dates.to_timestamp(dates.parse_format(date_or_datetime, format))
@final
@@ -852,6 +802,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
) -> m3u8.M3U8:
return await self._request_m3u8(url, headers, media_type)
+ @final
def create_custom_filename(
self,
name: str,
@@ -895,23 +846,6 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
logger.warning(msg)
return filename
- @final
- def get_cookies(self, partial_match_domain: bool = False) -> Iterable[tuple[str, BaseCookie[str]]]:
- if partial_match_domain:
- yield from self.client.client_manager.filter_cookies_by_word_in_domain(self.DOMAIN)
- else:
- yield str(self.PRIMARY_URL.host), self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL)
-
- @final
- def get_cookie_value(self, cookie_name: str, partial_match_domain: bool = False) -> str | None:
- def get_morsels_by_name():
- for _, cookie in self.get_cookies(partial_match_domain):
- if morsel := cookie.get(cookie_name):
- yield morsel
-
- if newest := max(get_morsels_by_name(), key=lambda x: int(x["max-age"] or 0), default=None):
- return newest.value
-
@final
def handle_subs(self, scrape_item: ScrapeItem, video_filename: str, subtitles: Iterable[ISO639Subtitle]) -> None:
counter = Counter()
@@ -962,44 +896,6 @@ def _make_custom_filename(stem: str, ext: str, extra_info: list[str], only_trunc
return f"{truncated_stem}{ext}", invalid_extra_info_chars
-class Site(NamedTuple):
- PRIMARY_URL: AbsoluteHttpURL
- DOMAIN: str
- SUPPORTED_DOMAINS: SupportedDomains = ()
- FOLDER_DOMAIN: str = ""
-
-
-_CrawlerT = TypeVar("_CrawlerT", bound=Crawler)
-
-
-def create_crawlers(
- urls: Iterable[str] | Iterable[AbsoluteHttpURL], base_crawler: type[_CrawlerT]
-) -> set[type[_CrawlerT]]:
- """Creates new subclasses of the base crawler from the urls"""
- return {_create_subclass(url, base_crawler) for url in urls}
-
-
-def _create_subclass(url: AbsoluteHttpURL | str, base_class: type[_CrawlerT]) -> type[_CrawlerT]:
- url = AbsoluteHttpURL(url)
- assert is_absolute_http_url(url)
- primary_url = remove_trailing_slash(url)
- domain = primary_url.host.removeprefix("www.")
- class_name = _make_crawler_name(domain)
- class_attributes = Site(primary_url, domain)._asdict()
- return type(class_name, (base_class,), class_attributes) # type: ignore # pyright: ignore[reportReturnType]
-
-
-def _make_crawler_name(input_string: str) -> str:
- clean_string = re.sub(r"[^a-zA-Z0-9]+", " ", input_string).strip()
- cap_name = clean_string.title().replace(" ", "")
- assert cap_name and cap_name.isalnum(), (
- f"Can not generate a valid class name from {input_string}. Needs to be defined as a concrete class"
- )
- if cap_name[0].isdigit():
- cap_name = "_" + cap_name
- return f"{cap_name}Crawler"
-
-
def _validate_supported_paths(cls: type[Crawler]) -> None:
for path_name, paths in cls.SUPPORTED_PATHS.items():
assert path_name, f"{cls.__name__}, Invalid path: {path_name}"
@@ -1008,9 +904,11 @@ def _validate_supported_paths(cls: type[Crawler]) -> None:
assert paths, f"{cls.__name__} has not paths for {path_name}"
if path_name.startswith("*"): # note
- return
+ continue
+
if isinstance(paths, str):
paths = (paths,)
+
for path in paths:
assert "`" not in path, f"{cls.__name__}, Invalid path {path_name}: {path}"
@@ -1034,18 +932,17 @@ def _sort_supported_paths(supported_paths: SupportedPaths) -> dict[str, OneOrTup
return dict(sorted(path_pairs, key=lambda x: x[0].casefold()))
+_CrawlerT = TypeVar("_CrawlerT", bound=Crawler)
+
+
def auto_task_id(
- func: Callable[Concatenate[_CrawlerT, ScrapeItem, _P], _R | Coroutine[None, None, _R]],
+ func: Callable[Concatenate[_CrawlerT, ScrapeItem, _P], Coroutine[None, None, _R]],
) -> Callable[Concatenate[_CrawlerT, ScrapeItem, _P], Coroutine[None, None, _R]]:
"""Autocreate a new `task_id` from the scrape_item of the method"""
@wraps(func)
async def wrapper(self: _CrawlerT, scrape_item: ScrapeItem, *args: _P.args, **kwargs: _P.kwargs) -> _R:
-
with self.new_task_id(scrape_item.url):
- result = func(self, scrape_item, *args, **kwargs)
- if inspect.isawaitable(result):
- return await result
- return result
+ return await func(self, scrape_item, *args, **kwargs)
return wrapper
diff --git a/cyberdrop_dl/crawlers/discourse/__init__.py b/cyberdrop_dl/crawlers/discourse/__init__.py
index b515c4c5a..71657ee03 100644
--- a/cyberdrop_dl/crawlers/discourse/__init__.py
+++ b/cyberdrop_dl/crawlers/discourse/__init__.py
@@ -2,7 +2,7 @@
# We can create subclasses dynamically by their URL
-from cyberdrop_dl.crawlers.crawler import create_crawlers
+from cyberdrop_dl.crawlers import create_crawlers
from ._discourse import DiscourseCrawler
diff --git a/cyberdrop_dl/crawlers/efukt.py b/cyberdrop_dl/crawlers/efukt.py
index e3cc6583a..a8fb6a0b9 100644
--- a/cyberdrop_dl/crawlers/efukt.py
+++ b/cyberdrop_dl/crawlers/efukt.py
@@ -40,8 +40,8 @@ class EfuktCrawler(Crawler):
DOMAIN: ClassVar[str] = "efukt.com"
FOLDER_DOMAIN: ClassVar[str] = "eFukt"
NEXT_PAGE_SELECTOR = _SELECTORS.NEXT_PAGE
- SKIP_PRE_CHECK = True
- DEFAULT_POST_TITLE_FORMAT = "{date:%Y-%m-%d} {title}"
+ ALLOW_EMPTY_PATH: ClassVar[bool] = True
+ DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date:%Y-%m-%d} {title}"
async def fetch(self, scrape_item: ScrapeItem) -> None:
if is_series(scrape_item.url) or is_homepage(scrape_item.url):
diff --git a/cyberdrop_dl/crawlers/erome.py b/cyberdrop_dl/crawlers/erome.py
index 144cbda6d..0f73f2598 100644
--- a/cyberdrop_dl/crawlers/erome.py
+++ b/cyberdrop_dl/crawlers/erome.py
@@ -44,13 +44,18 @@ class EromeCrawler(Crawler):
async def profile(self, scrape_item: ScrapeItem, name: str) -> None:
title = self.create_title(name)
scrape_item.setup_as_profile(title)
- await self.crawl_children(scrape_item, Selector.ALBUM)
+
+ async for soup in self.web_pager(scrape_item.url):
+ for _, new_item in self.iter_children(scrape_item, soup, Selector.ALBUM):
+ self.create_task(self.run(new_item))
@error_handling_wrapper
async def search(self, scrape_item: ScrapeItem, query: str) -> None:
title = self.create_title(f"{query} [search]")
scrape_item.setup_as_album(title)
- await self.crawl_children(scrape_item, Selector.ALBUM)
+ async for soup in self.web_pager(scrape_item.url):
+ for _, new_item in self.iter_children(scrape_item, soup, Selector.ALBUM):
+ self.create_task(self.run(new_item))
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem, album_id: str) -> None:
diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.py
index 80f224c33..2550d905e 100644
--- a/cyberdrop_dl/crawlers/imgbb.py
+++ b/cyberdrop_dl/crawlers/imgbb.py
@@ -23,7 +23,7 @@ class ImgBBCrawler(CheveretoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://ibb.co")
DOMAIN: ClassVar[str] = "imgbb"
FOLDER_DOMAIN: ClassVar[str] = "ImgBB"
- SKIP_PRE_CHECK: ClassVar[bool] = True
+ ALLOW_EMPTY_PATH: ClassVar[bool] = True
async def fetch(self, scrape_item: ScrapeItem) -> None:
username, _, rest = scrape_item.url.host.partition(".imgbb.")
diff --git a/cyberdrop_dl/crawlers/luxuretv.py b/cyberdrop_dl/crawlers/luxuretv.py
index 58c2d1c01..8b6f47aaa 100644
--- a/cyberdrop_dl/crawlers/luxuretv.py
+++ b/cyberdrop_dl/crawlers/luxuretv.py
@@ -52,7 +52,7 @@ class LuxureTVCrawler(Crawler):
if url.name and not url.name.endswith(".html"):
url = url / ""
- async for soup in self.web_pager(url, cffi=True):
+ async for soup in self.web_pager(url, impersonate=True):
for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.VIDEOS_THUMBS):
self.create_task(self.run(new_scrape_item))
diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.py
index e3342234d..3f06c8723 100644
--- a/cyberdrop_dl/crawlers/mediafire.py
+++ b/cyberdrop_dl/crawlers/mediafire.py
@@ -61,7 +61,7 @@ class MediaFireCrawler(Crawler, db_path="name"):
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URL
DOMAIN: ClassVar[str] = "mediafire"
- SKIP_PRE_CHECK: ClassVar[bool] = True
+ ALLOW_EMPTY_PATH: ClassVar[bool] = True
def __post_init__(self) -> None:
self.api = MediaFireAPI(self)
diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.py
index 3a683a92b..ab6ee0b70 100644
--- a/cyberdrop_dl/crawlers/mega_nz.py
+++ b/cyberdrop_dl/crawlers/mega_nz.py
@@ -13,6 +13,7 @@ from mega.api import MegaAPI
from mega.core import MegaCore
from mega.crypto import b64_to_a32
from mega.data_structures import Crypto
+from typing_extensions import override
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, auto_task_id
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
@@ -42,7 +43,7 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):
"**NOTE**": "Downloads can not be resumed. Partial downloads will always be deleted and new downloads will start over",
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://mega.nz")
- SKIP_PRE_CHECK: ClassVar[bool] = True
+ ALLOW_EMPTY_PATH: ClassVar[bool] = True
DOMAIN: ClassVar[str] = "mega.nz"
FOLDER_DOMAIN: ClassVar[str] = "MegaNz"
OLD_DOMAINS: ClassVar[tuple[str, ...]] = ("mega.co.nz",)
@@ -58,11 +59,11 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):
def password(self) -> str | None:
return self.manager.auth_config.meganz.password or None
- def _init_downloader(self) -> MegaDownloader:
+ @override
+ def __init_downloader__(self) -> None:
self.core = MegaCore(MegaAPI(self.manager.client_manager._session))
self.downloader = dl = MegaDownloader(self.manager, self.DOMAIN) # type: ignore[reportIncompatibleVariableOverride]
dl.startup()
- return dl
async def __async_post_init__(self) -> None:
await self.login(self.PRIMARY_URL)
diff --git a/cyberdrop_dl/crawlers/missav.py b/cyberdrop_dl/crawlers/missav.py
index 2f0e8a311..43a395d87 100644
--- a/cyberdrop_dl/crawlers/missav.py
+++ b/cyberdrop_dl/crawlers/missav.py
@@ -49,7 +49,7 @@ class MissAVCrawler(Crawler):
title = self.create_title(f"{name} [{collection_type}]")
scrape_item.setup_as_album(title)
- async for soup in self.web_pager(scrape_item.url.update_query(page=1), cffi=True):
+ async for soup in self.web_pager(scrape_item.url.update_query(page=1), impersonate=True):
for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ITEM):
self.create_task(self.run(new_scrape_item))
diff --git a/cyberdrop_dl/crawlers/nhentai.py b/cyberdrop_dl/crawlers/nhentai.py
index d478ea1d1..56581655b 100644
--- a/cyberdrop_dl/crawlers/nhentai.py
+++ b/cyberdrop_dl/crawlers/nhentai.py
@@ -53,7 +53,7 @@ class NHentaiCrawler(Crawler):
@error_handling_wrapper
async def collection(self, scrape_item: ScrapeItem, collection_type: str) -> None:
title: str = ""
- async for soup in self.web_pager(scrape_item.url, cffi=True):
+ async for soup in self.web_pager(scrape_item.url, impersonate=True):
if not title:
if collection_type == "favorites":
title_tag = css.select(soup, Selector.FAVORITES_TITLE)
diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.py
index 01f280f8b..9cddc1cbb 100644
--- a/cyberdrop_dl/crawlers/onedrive.py
+++ b/cyberdrop_dl/crawlers/onedrive.py
@@ -97,7 +97,7 @@ class OneDriveCrawler(Crawler):
}
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = SHARE_LINK_HOST, "onedrive.live.com"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://onedrive.com/")
- SKIP_PRE_CHECK: ClassVar[bool] = True # URLs with not path could be valid
+ ALLOW_EMPTY_PATH: ClassVar[bool] = True
DOMAIN: ClassVar[str] = "onedrive"
FOLDER_DOMAIN: ClassVar[str] = "OneDrive"
diff --git a/cyberdrop_dl/crawlers/spankbang.py b/cyberdrop_dl/crawlers/spankbang.py
index 17d302fca..b7604ab70 100644
--- a/cyberdrop_dl/crawlers/spankbang.py
+++ b/cyberdrop_dl/crawlers/spankbang.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import dataclasses
-from typing import TYPE_CHECKING, Any, ClassVar
+from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPaths
from cyberdrop_dl.data_structures.mediaprops import Resolution
@@ -11,7 +11,7 @@ from cyberdrop_dl.utils import css, json
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Generator
+ from collections.abc import Generator
from bs4 import BeautifulSoup
@@ -93,12 +93,6 @@ class SpankBangCrawler(Crawler):
case _:
return url
- async def web_pager(
- self, url: AbsoluteHttpURL, next_page_selector: str | None = None, *, cffi: bool = False, **kwargs: Any
- ) -> AsyncGenerator[BeautifulSoup]:
- async for soup in super()._web_pager(url, next_page_selector, cffi=True, **kwargs):
- yield soup
-
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
# old referer logic. video_id may be canonical (unique per video) or relative (different on each playlist)
diff --git a/cyberdrop_dl/database/tables/history.py b/cyberdrop_dl/database/tables/history.py
index 4cc381bda..69e43fab0 100644
--- a/cyberdrop_dl/database/tables/history.py
+++ b/cyberdrop_dl/database/tables/history.py
@@ -119,7 +119,7 @@ class HistoryTable:
return completed
- async def check_album(self, domain: str, album_id: str) -> dict[str, int]:
+ async def check_album(self, domain: str, album_id: str) -> dict[str, bool]:
"""Checks whether an album has completed given its domain and album id."""
if self._database.ignore_history:
return {}
@@ -127,7 +127,7 @@ class HistoryTable:
query = "SELECT url_path, completed FROM media WHERE domain = ? and album_id = ?"
cursor = await self.db_conn.execute(query, (domain, album_id))
rows = await cursor.fetchall()
- return {row[0]: row[1] for row in rows}
+ return {row[0]: bool(row[1]) for row in rows}
async def set_album_id(self, domain: str, media_item: MediaItem) -> None:
"""Sets an album_id in the database."""
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index 740aa93c5..0fc9c34fa 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -13,8 +13,9 @@ from yarl import URL
from cyberdrop_dl.clients.jdownloader import JDownloader
from cyberdrop_dl.constants import REGEX_LINKS, BlockedDomains
+from cyberdrop_dl.crawlers import create_crawlers
from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler
-from cyberdrop_dl.crawlers.crawler import Crawler, create_crawlers
+from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.crawlers.discourse import DiscourseCrawler
from cyberdrop_dl.crawlers.http_direct import DirectHttpFile
from cyberdrop_dl.crawlers.realdebrid import RealDebridCrawler
@@ -122,7 +123,7 @@ class ScrapeMapper:
logger.exception("Failed to connect to jDownloader")
await self.start_real_debrid()
- self.direct_crawler._init_downloader()
+ self.direct_crawler.__init_downloader__()
async for item in self.get_input_items():
self.manager.task_group.create_task(self.send_to_crawler(item))
diff --git a/tests/test_scrape_mapper.py b/tests/test_scrape_mapper.py
index 7236b175d..65522000f 100644
--- a/tests/test_scrape_mapper.py
+++ b/tests/test_scrape_mapper.py
@@ -2,8 +2,8 @@ from __future__ import annotations
import pytest
+from cyberdrop_dl.crawlers import create_crawlers
from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler
-from cyberdrop_dl.crawlers.crawler import create_crawlers
from cyberdrop_dl.scraper import scrape_mapper