Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: remove redundant methods (crawler) (#94)
NTFSvolumef1dd719c3746
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.pyindex 8ad74e06e..88e0780fc 100644--- a/cyberdrop_dl/clients/__init__.py+++ b/cyberdrop_dl/clients/__init__.py@@ -21,7 +21,6 @@ if TYPE_CHECKING:from collections.abc import AsyncGenerator, Mappingfrom bs4 import BeautifulSoup- from curl_cffi.requests.impersonate import BrowserTypeLiteralfrom curl_cffi.requests.session import HttpMethodfrom cyberdrop_dl.clients import flaresolverr@@ -78,7 +77,7 @@ class HTTPClient:/,method: HttpMethod = "GET",headers: Mapping[str, str] | None = None,- impersonate: BrowserTypeLiteral | bool | None = None,+ impersonate: str | bool | None = None,data: Any = None,json: Any = None,cache_disabled: bool = False,@@ -223,7 +222,7 @@ def _write_resp_to_disk(class HTTPClientProxy:DOMAIN: ClassVar[str]- _IMPERSONATE: ClassVar[BrowserTypeLiteral | bool | None] = None+ _IMPERSONATE: ClassVar[str | bool | None] = Noneclient: HTTPClient # pyright: ignore[reportUninitializedInstanceVariable]@classmethod@@ -249,7 +248,7 @@ class HTTPClientProxy:@signature.copy(HTTPClient.request)@contextlib.asynccontextmanagerasync def request(- self, *args, impersonate: BrowserTypeLiteral | bool | None = None, **kwargs+ self, *args, impersonate: str | bool | None = None, **kwargs) -> AsyncGenerator[AbstractResponse[Any]]:if impersonate is None:impersonate = self._IMPERSONATEdiff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex e69de29bb..cc6f0b5f9 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -0,0 +1,47 @@+from __future__ import annotations++import re+from typing import TYPE_CHECKING, TypeVar++from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils.utilities import is_absolute_http_url, remove_trailing_slash++if TYPE_CHECKING:+ from collections.abc import Iterable++ from cyberdrop_dl.crawlers.crawler import Crawler++ _CrawlerT = TypeVar("_CrawlerT", bound=Crawler)+++def create_crawlers(+ urls: Iterable[str] | Iterable[AbsoluteHttpURL], base_crawler: type[_CrawlerT]+) -> set[type[_CrawlerT]]:+ """Creates new subclasses of the base crawler from the urls"""+ return {_create_subclass(url, base_crawler) for url in urls}+++def _create_subclass(url: AbsoluteHttpURL | str, base_class: type[_CrawlerT]) -> type[_CrawlerT]:+ url = AbsoluteHttpURL(url)+ assert is_absolute_http_url(url)+ primary_url = remove_trailing_slash(url)+ domain = primary_url.host.removeprefix("www.")+ class_name = _make_crawler_name(domain)+ class_attributes = {+ "PRIMARY_URL": primary_url,+ "DOMAIN": domain,+ "SUPPORTED_DOMAINS": (),+ "FOLDER_DOMAIN": "",+ }+ return type(class_name, (base_class,), class_attributes) # type: ignore # pyright: ignore[reportReturnType]+++def _make_crawler_name(input_string: str) -> str:+ clean_string = re.sub(r"[^a-zA-Z0-9]+", " ", input_string).strip()+ cap_name = clean_string.title().replace(" ", "")+ assert cap_name and cap_name.isalnum(), (+ f"Can not generate a valid class name from {input_string}. Needs to be defined as a concrete class"+ )+ if cap_name[0].isdigit():+ cap_name = "_" + cap_name+ return f"{cap_name}Crawler"diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex b466aca6e..0abfb40b7 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -10,7 +10,7 @@ from cyberdrop_dl.utils import css, json, open_graphfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from collections.abc import AsyncGenerator, Generator+ from collections.abc import Generator, Mappingfrom bs4 import BeautifulSoup@@ -109,16 +109,10 @@ class CheveretoCrawler(Crawler, is_generic=True):return url- async def web_pager(- self, url: AbsoluteHttpURL, next_page_selector: str | None = None, *, cffi: bool = False, **kwargs: Any- ) -> AsyncGenerator[BeautifulSoup]:- async for soup in super().web_pager(_sort_by_new(url), next_page_selector, cffi=cffi, trim=False, **kwargs):- yield soup-@error_handling_wrapperasync def profile(self, scrape_item: ScrapeItem, *, albums: bool = False) -> None:title: str = ""- async for soup in self.web_pager(scrape_item.url):+ async for soup in self.web_pager(_sort_by_new(scrape_item.url), trim=False):if not title:title = self.create_title(open_graph.title(soup))scrape_item.setup_as_profile(title)@@ -133,12 +127,12 @@ class CheveretoCrawler(Crawler, is_generic=True):@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem, album_id: str) -> None:- results: dict[str, int] = {}+ results: dict[str, bool] = {}title: str = ""scrape_item.url = await self._get_final_album_url(scrape_item.url)- async for soup in self.web_pager(scrape_item.url):+ async for soup in self.web_pager(_sort_by_new(scrape_item.url), trim=False):if not title:if soup.select_one("form"):await self._unlock_pw_protected_album(scrape_item, soup)@@ -151,12 +145,15 @@ class CheveretoCrawler(Crawler, is_generic=True):self._iter_album_files(scrape_item, soup, results)- async for soup in self.web_pager(scrape_item.url / "sub"):+ async for soup in self.web_pager(_sort_by_new(scrape_item.url / "sub"), trim=False):for _, sub_album in self.iter_children(scrape_item, soup, Selector.ITEM):self.create_task(self.run(sub_album))def _iter_album_files(- self, scrape_item: ScrapeItem, soup: BeautifulSoup, results: dict[str, int] | None = None+ self,+ scrape_item: ScrapeItem,+ soup: BeautifulSoup,+ results: Mapping[str, bool] | None = None,) -> None:results = results or {}for web_url, src_url in self._get_album_files(soup):diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.pyindex dbfaba338..ebdddcda9 100644--- a/cyberdrop_dl/crawlers/_forum.py+++ b/cyberdrop_dl/crawlers/_forum.py@@ -535,7 +535,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):yield get_text_between(css.attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")async def thread_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:- async for soup in self._web_pager(scrape_item.url, self.get_next_page):+ async for soup in self.web_pager(scrape_item.url, self.get_next_page):yield soupdef get_next_page(self, soup: BeautifulSoup) -> str | None:diff --git a/cyberdrop_dl/crawlers/_one_manager.py b/cyberdrop_dl/crawlers/_one_manager.pyindex c4ba28c30..30f441332 100644--- a/cyberdrop_dl/crawlers/_one_manager.py+++ b/cyberdrop_dl/crawlers/_one_manager.py@@ -36,7 +36,7 @@ _SELECTORS = Selectors()class OneManagerCrawler(Crawler, is_abc=True):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Any path": "/..."}- SKIP_PRE_CHECK = True+ ALLOW_EMPTY_PATH: ClassVar[bool] = Trueasync def fetch(self, scrape_item: ScrapeItem) -> None:scrape_item.url = scrape_item.url.with_query(None)diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.pyindex a084b100f..0b29a8e51 100644--- a/cyberdrop_dl/crawlers/ashemaletube.py+++ b/cyberdrop_dl/crawlers/ashemaletube.py@@ -104,14 +104,14 @@ class AShemaleTubeCrawler(Crawler):@error_handling_wrapperasync def gallery(self, scrape_item: ScrapeItem) -> None:- async for soup in self.web_pager(scrape_item.url, cffi=True):+ async for soup in self.web_pager(scrape_item.url, impersonate=True):for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.GALLERY_ALBUM):self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:album_title = ""- async for soup in self.web_pager(scrape_item.url, cffi=True):+ async for soup in self.web_pager(scrape_item.url, impersonate=True):if not album_title:album_title = self.create_collection_title(soup, CollectionType.ALBUM)scrape_item.setup_as_album(album_title)@@ -122,7 +122,7 @@ class AShemaleTubeCrawler(Crawler):@error_handling_wrapperasync def collection(self, scrape_item: ScrapeItem, collection_type: CollectionType) -> None:collection_title = ""- async for soup in self.web_pager(scrape_item.url, cffi=True):+ async for soup in self.web_pager(scrape_item.url, impersonate=True):if not collection_title:collection_title = self.create_collection_title(soup, collection_type)if collection_type == CollectionType.MODEL:diff --git a/cyberdrop_dl/crawlers/bunkrr_albums_io.py b/cyberdrop_dl/crawlers/bunkrr_albums_io.pyindex a2ba8f3b4..5ca8beafc 100644--- a/cyberdrop_dl/crawlers/bunkrr_albums_io.py+++ b/cyberdrop_dl/crawlers/bunkrr_albums_io.py@@ -11,12 +11,12 @@ if TYPE_CHECKING:class BunkrAlbumsIOCrawler(Crawler):- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Search": "/s?search=<query>"}+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Search": "/?search=<query>"}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://bunkr-albums.io/")DOMAIN: ClassVar[str] = "bunkr-albums.io"FOLDER_DOMAIN: ClassVar[str] = "Bunkr-Albums.io"NEXT_PAGE_SELECTOR: ClassVar[str] = "nav a:-soup-contains(Next)"- SKIP_PRE_CHECK: ClassVar[bool] = True+ ALLOW_EMPTY_PATH: ClassVar[bool] = Trueasync def fetch(self, scrape_item: ScrapeItem) -> None:if query := scrape_item.url.query.get("search"):diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 45a7fdd40..7a07d6e0a 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -5,7 +5,6 @@ import contextlibimport dataclassesimport datetimeimport importlib-import inspectimport loggingimport pkgutilimport re@@ -14,19 +13,8 @@ from abc import ABC, abstractmethodfrom collections import Counterfrom functools import wrapsfrom pathlib import Path-from typing import (- TYPE_CHECKING,- Any,- ClassVar,- Concatenate,- Literal,- NamedTuple,- ParamSpec,- Self,- TypeAlias,- TypeVar,- final,-)+from types import MappingProxyType+from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Final, Literal, ParamSpec, Self, TypeAlias, TypeVar, finalfrom aiolimiter import AsyncLimiterfrom typing_extensions import deprecated@@ -50,18 +38,17 @@ from cyberdrop_dl.utils.utilities import (is_blob_or_svg,parse_url,remove_file_id,- remove_trailing_slash,sanitize_filename,truncate_str,)if TYPE_CHECKING:- from collections.abc import AsyncGenerator, Callable, Coroutine, Generator, Iterable, Mapping- from http.cookies import BaseCookie+ from collections.abc import AsyncGenerator, AsyncIterator, Callable, Coroutine, Generator, Iterable, Mappingfrom types import ModuleTypeimport yarlfrom bs4 import BeautifulSoup, Tag+ from curl_cffi.requests.impersonate import BrowserTypeLiteralfrom rich.progress import TaskIDfrom cyberdrop_dl.managers.manager import Manager@@ -71,7 +58,7 @@ logger = logging.getLogger(__name__)_P = ParamSpec("_P")_R = TypeVar("_R")_T = TypeVar("_T")-_T_co = TypeVar("_T_co", covariant=True)+OneOrTuple: TypeAlias = _T | tuple[_T, ...]SupportedPaths: TypeAlias = dict[str, OneOrTuple[str]]SupportedDomains: TypeAlias = OneOrTuple[str]@@ -92,21 +79,24 @@ class _PlaceHolderConfigInclude:_include = _PlaceHolderConfigInclude()-_DB_PATH_BUILDERS: dict[str, Callable[[AbsoluteHttpURL], str]] = {- "url": lambda url: str(url),- "name": lambda url: url.name,- "path": lambda url: url.path,- "path_qs": lambda url: url.path_qs,- "path_qs_frag": lambda url: f"{url.path_qs}#{frag}" if (frag := url.fragment) else url.path_qs,- "path_frag": lambda url: f"{url.path}#{frag}" if (frag := url.fragment) else url.path,-}+_DB_PATH_BUILDERS: MappingProxyType[str, Callable[[AbsoluteHttpURL], str]] = MappingProxyType(+ {+ "url": lambda url: str(url),+ "name": lambda url: url.name,+ "path": lambda url: url.path,+ "path_qs": lambda url: url.path_qs,+ "path_qs_frag": lambda url: f"{url.path_qs}#{frag}" if (frag := url.fragment) else url.path_qs,+ "path_frag": lambda url: f"{url.path}#{frag}" if (frag := url.fragment) else url.path,+ }+)def _url(item: ScrapeItem | AbsoluteHttpURL) -> AbsoluteHttpURL:return item if isinstance(item, AbsoluteHttpURL) else item.url-class CrawlerInfo(NamedTuple):[email protected](slots=True, frozen=True, order=True)+class CrawlerInfo:site: strprimary_url: AbsoluteHttpURLsupported_domains: tuple[str, ...]@@ -152,7 +142,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {id} - {title}"UPDATE_UNSUPPORTED: ClassVar[bool] = False- SKIP_PRE_CHECK: ClassVar[bool] = False+ ALLOW_EMPTY_PATH: ClassVar[bool] = FalseNEXT_PAGE_SELECTOR: ClassVar[str] = ""DEFAULT_TRIM_URLS: ClassVar[bool] = True@@ -177,20 +167,13 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):self._ready: bool = Falseself.disabled: bool = Falseself.logged_in: bool = False- self.scraped_items: set[str] = set()- self.RATE_LIMIT = AsyncLimiter(*self._RATE_LIMIT)+ self._scraped_items: set[str] = set()self.log = logself.log_debug = log_debugself._semaphore: asyncio.Semaphore = asyncio.Semaphore(20)self.__post_init__()- @property- def waiting_items(self) -> int:- if self._semaphore._waiters is None:- return 0- return len(self._semaphore._waiters)-def __post_init__(self) -> None:"""Override in subclasses to add custom init logic@@ -198,17 +181,16 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):@finalasync def __async_init__(self) -> None:- """Starts the crawler."""async with self._startup_lock:if self._ready:returnself.client = self.manager.client_manager.scraper_client- self.manager.client_manager.rate_limits[self.DOMAIN] = self.RATE_LIMIT+ self.manager.client_manager.rate_limits[self.DOMAIN] = AsyncLimiter(*self._RATE_LIMIT)if self._DOWNLOAD_SLOTS:self.manager.client_manager.download_slots[self.DOMAIN] = self._DOWNLOAD_SLOTSif self._USE_DOWNLOAD_SERVERS_LOCKS:self.manager.client_manager.download_client.server_locked_domains.add(self.DOMAIN)- self.downloader = self._init_downloader()+ self.__init_downloader__()await self.__async_post_init__()self._ready = True@@ -282,6 +264,10 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):)Registry.concrete.add(cls)+ def __init_downloader__(self) -> None:+ self.downloader = dl = Downloader(self.manager, self.DOMAIN)+ dl.startup()+@final@staticmethoddef _assert_fields_overrides(subclass: type[Crawler], *fields: str) -> None:@@ -289,24 +275,34 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):assert getattr(subclass, field_name, None), f"Subclass {subclass.__name__} must override: {field_name}"@final- def create_task(self, coro: Coroutine[Any, Any, _T_co]) -> asyncio.Task[_T_co]:- return self.manager.task_group.create_task(coro)+ def create_task(self, coro: Coroutine[Any, Any, _T]) -> None:+ _ = self.manager.task_group.create_task(coro)@abstractmethod- async def fetch(self, scrape_item: ScrapeItem) -> None: ...+ async def fetch(self, scrape_item: ScrapeItem) -> None:+ """Here goes the main logic to parse URL paths.++ This method MUST NOT raise any exceptions other that ValueError to indiated that the path is not supported"""+ @final@property- def allow_no_extension(self) -> bool:- return not self.manager.config_manager.settings_data.ignore_options.exclude_files_with_no_extension+ def waiting_items(self) -> int:+ if self._semaphore._waiters is None:+ return 0+ return len(self._semaphore._waiters)+ @final@propertydef deep_scrape(self) -> bool:return self.manager.config_manager.deep_scrape- def _init_downloader(self) -> Downloader:- self.downloader = dl = Downloader(self.manager, self.DOMAIN)- dl.startup()- return dl+ @property+ def allow_no_extension(self) -> bool:+ return not self.manager.config.ignore_options.exclude_files_with_no_extension++ @property+ def separate_posts(self) -> bool:+ return self.manager.config.download_options.separate_posts@final@contextlib.contextmanager@@ -318,13 +314,10 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):self.disabled = Trueraise- catch_errors = final(error_handling_context)+ catch_errors: Final = error_handling_context@finalasync def run(self, scrape_item: ScrapeItem) -> None:- """Runs the crawler loop."""- if not scrape_item.url.host:- returnif self.disabled:return@@ -332,17 +325,21 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):with scrape_item.track_changes():scrape_item.url = url = self.transform_url(scrape_item.url)- if url.path_qs in self.scraped_items:- return logger.info(f"Skipping {url} as it has already been scraped")+ if url.path_qs in self._scraped_items:+ logger.info(f"Skipping {url} as it has already been scraped")+ return- self.scraped_items.add(url.path_qs)- async with self._fetch_context(scrape_item):- self.pre_check_scrape_item(scrape_item)- await self.fetch(scrape_item)+ self._scraped_items.add(url.path_qs)- def pre_check_scrape_item(self, scrape_item: ScrapeItem) -> None:- if not self.SKIP_PRE_CHECK and scrape_item.url.path == "/":- raise ValueError+ with self.new_task_id(scrape_item.url):+ try:+ if not self.ALLOW_EMPTY_PATH and scrape_item.url.path == "/":+ raise ValueError+ await self.fetch(scrape_item)+ except ValueError:+ self.raise_exc(scrape_item, ScrapeError("Unknown URL path"))+ except MaxChildrenError as e:+ self.raise_exc(scrape_item, e)@classmethoddef transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:@@ -354,19 +351,6 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):return url.with_host(new_host)return url- @final- @contextlib.asynccontextmanager- async def _fetch_context(self, scrape_item: ScrapeItem) -> AsyncGenerator[TaskID]:- with self.new_task_id(scrape_item.url) as task_id:- try:- yield task_id- except ValueError:- self.raise_exc(scrape_item, ScrapeError("Unknown URL path"))- except MaxChildrenError as e:- self.raise_exc(scrape_item, e)- finally:- pass-@error_handling_wrapperdef raise_exc(self, scrape_item: ScrapeItem, exc: type[Exception] | Exception | str) -> None:if isinstance(exc, str):@@ -470,22 +454,21 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):data = [media_item.__json__()]await self.manager.logs.write_jsonl(data)+ @finalasync def check_complete(self, url: AbsoluteHttpURL, referer: AbsoluteHttpURL) -> bool:"""Checks if this URL has been download before.This method is called automatically on a created media item,but Crawler code can use it to skip unnecessary requests"""db_path = self.__db_path__(url)- check_complete = await self.manager.db_manager.history_table.check_complete(self.DOMAIN, url, referer, db_path)- if check_complete:+ was_completed = await self.manager.db_manager.history_table.check_complete(self.DOMAIN, url, referer, db_path)+ if was_completed:logger.info(f"Skipping {url} as it has already been downloaded")self.manager.progress_manager.download_progress.add_previously_completed()- return check_complete+ return was_completedasync def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.Rendition | None = None) -> None:-- check_complete = await self.check_complete(media_item.url, media_item.referer)- if check_complete:+ if await self.check_complete(media_item.url, media_item.referer):if media_item.album_id:await self.manager.db_manager.history_table.set_album_id(self.DOMAIN, media_item)return@@ -499,18 +482,17 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):@finalasync def check_skip_by_config(self, media_item: MediaItem) -> bool:media_host = media_item.url.host+ ignore_options = self.manager.config.ignore_options- if (hosts := self.manager.config.ignore_options.skip_hosts) and any(host in media_host for host in hosts):+ if (hosts := ignore_options.skip_hosts) and any(host in media_host for host in hosts):logger.info(f"Download skipped {media_item.url} due to skip_hosts config")return True- if (hosts := self.manager.config.ignore_options.only_hosts) and not any(host in media_host for host in hosts):+ if (hosts := ignore_options.only_hosts) and not any(host in media_host for host in hosts):logger.info(f"Download skipped {media_item.url} due to only_hosts config")return True- if (regex := self.manager.config.ignore_options.filename_regex_filter) and re.search(- regex, media_item.filename- ):+ if (regex := ignore_options.filename_regex_filter) and re.search(regex, media_item.filename):logger.info(f"Download skipped {media_item.url} due to filename regex filter config")return True@@ -530,8 +512,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):if downloaded:logger.info(f"Skipping {url} as it has already been downloaded")self.manager.progress_manager.download_progress.add_previously_completed()- return True- return False+ return downloaded@finalasync def check_complete_by_hash(@@ -545,20 +526,26 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):self.manager.progress_manager.download_progress.add_previously_completed()return downloaded- async def get_album_results(self, album_id: str) -> dict[str, int]:+ @final+ async def get_album_results(self, album_id: str) -> dict[str, bool]:"""Checks whether an album has completed given its domain and album id."""return await self.manager.db_manager.history_table.check_album(self.DOMAIN, album_id)+ @finaldef handle_external_links(self, scrape_item: ScrapeItem, reset: bool = True) -> None:"""Maps external links to the scraper class."""if reset:scrape_item.reset()self.create_task(self.manager.scrape_mapper.filter_and_send_to_crawler(scrape_item))- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-+ @finaldef get_filename_and_ext(- self, filename: str, forum: bool = False, assume_ext: str | None = ".mp4", *, mime_type: str | None = None+ self,+ filename: str,+ forum: bool = False,+ assume_ext: str | None = ".mp4",+ *,+ mime_type: str | None = None,) -> tuple[str, str]:"""Wrapper around `utils.get_filename_and_ext`.Calls it as is.@@ -572,12 +559,13 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):raise@final- def check_album_results(self, url: AbsoluteHttpURL, album_results: dict[str, Any]) -> bool:+ def check_album_results(self, url: AbsoluteHttpURL, album_results: Mapping[str, bool]) -> bool:"""Checks whether an album has completed given its domain and album id."""if not album_results:return False+url_path = self.__db_path__(url)- if url_path in album_results and album_results[url_path] != 0:+ if album_results.get(url_path) is True:logger.info(f"Skipping {url} as it has already been downloaded")self.manager.progress_manager.download_progress.add_previously_completed()return True@@ -586,10 +574,8 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):@finaldef create_title(self, title: str, album_id: str | None = None, thread_id: int | None = None) -> str:"""Creates the title for the scrape item."""- if not title:- title = "Untitled"+ title = (title or "Untitled").strip()- title = title.strip()if album_id and self.manager.config.download_options.include_album_id_in_folder_name:title = f"{title} {album_id}"@@ -600,16 +586,9 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):title = f"{title} ({self.FOLDER_DOMAIN})"# Remove double spaces- while True:- title = title.replace(" ", " ")- if " " not in title:- break+ title = " ".join(title.split(""))return title- @property- def separate_posts(self) -> bool:- return self.manager.config.download_options.separate_posts-@finaldef create_separate_post_title(self,@@ -631,14 +610,24 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):@classmethoddef parse_url(- cls, link_str: yarl.URL | str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool | None = None+ cls,+ url: yarl.URL | str,+ /,+ relative_to: AbsoluteHttpURL | None = None,+ *,+ trim: bool | None = None,) -> AbsoluteHttpURL:"""Wrapper around `utils.parse_url` to use `self.PRIMARY_URL` as base"""base = relative_to or cls.PRIMARY_URLassert is_absolute_http_url(base)if trim is None:trim = cls.DEFAULT_TRIM_URLS- return parse_url(link_str, base, trim=trim)+ return parse_url(url, base, trim=trim)++ @final+ def get_cookie_value(self, name: str) -> str | None:+ if morsel := self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL).get(name):+ return morsel.value@finaldef update_cookies(self, cookies: dict[str, Any], url: yarl.URL | None = None) -> None:@@ -657,7 +646,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):/,attribute: str = "href",*,- results: dict[str, int] | None = None,+ results: Mapping[str, bool] | None = None,) -> Generator[tuple[AbsoluteHttpURL | None, AbsoluteHttpURL]]:"""Generates tuples with an URL from the `src` value of the first image tag (AKA the thumbnail) and an URL from the value of `attribute`@@ -674,11 +663,13 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):link = self.parse_url(link_str)if self.check_album_results(link, album_results):continue- if t_tag := tag.select_one("img"):- thumb_str: str | None = css.attr_or_none(t_tag, "src")+ try:+ thumb_str: str | None = css.select(tag, "img", "src")+ except css.SelectorError:+ thumb = Noneelse:- thumb_str = None- thumb = self.parse_url(thumb_str) if thumb_str and not is_blob_or_svg(thumb_str) else None+ thumb = None if is_blob_or_svg(thumb_str) else self.parse_url(thumb_str)+yield thumb, link@final@@ -690,92 +681,51 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):/,attribute: str = "href",*,- results: dict[str, int] | None = None,+ results: Mapping[str, bool] | None = None,**kwargs: Any,) -> Generator[tuple[AbsoluteHttpURL | None, ScrapeItem]]:"""Generates tuples with an URL from the `src` value of the first image tag (AKA the thumbnail) and a new scrape item from the value of `attribute`:param results: must be the output of `self.get_album_results`.- If provided, it will be used as a filter, to only yield items that has not been downloaded before- :param **kwargs: Will be forwarded to `scrape.item.create_child`"""+ If provided, it will be used as a filter, to only yield items that has not been downloaded before"""for thumb, link in self.iter_tags(soup, selector, attribute, results=results):new_scrape_item = scrape_item.create_child(link, **kwargs)yield thumb, new_scrape_itemscrape_item.add_children()- @final- async def crawl_children(- self,- scrape_item: ScrapeItem,- selector: str,- /,- attribute: str = "href",- *,- results: dict[str, int] | None = None,- next_page_selector: str | None = None,- coro_factory: Callable[[ScrapeItem], Coroutine[Any, Any, Any]] | None = None,- ) -> None:- """Crawls children URLs and schedules scrape tasks for them.-- Uses `self.web_pager` to iterate through pages and extracts child links based on `selector` and `attribute`. For- each extracted URL, a new `ScrapeItem` is created, and a task- (by default, `self.run`) is scheduled in the global task group to process it- """-- if coro_factory is None:- coro_factory = self.run-- async for soup in self.web_pager(scrape_item.url, next_page_selector):- for _, new_scrape_item in self.iter_children(scrape_item, soup, selector, attribute, results=results):- self.create_task(coro_factory(new_scrape_item))-async def web_pager(- self, url: AbsoluteHttpURL, next_page_selector: str | None = None, *, cffi: bool = False, **kwargs: Any- ) -> AsyncGenerator[BeautifulSoup]:- """Generator of website pages.-- :param next_page_selector: If `None`, `self.next_page_selector` will be used- :param cffi: If `True`, uses `curl_cffi` to get the soup for each page. Otherwise, `aiohttp` will be used- :param **kwargs: Will be forwarded to `self.parse_url` to parse each new page"""-- async for soup in self._web_pager(url, next_page_selector, cffi=cffi, **kwargs):- yield soup-- async def _web_pager(self,url: AbsoluteHttpURL,- selector: Callable[[BeautifulSoup], str | None] | str | None = None,+ selector: Callable[[BeautifulSoup], yarl.URL | str | None] | str | None = None,*,- cffi: bool = False,- **kwargs: Any,- ) -> AsyncGenerator[BeautifulSoup]:- """Generator of website pages.-- :param next_page_selector: If `None`, `self.next_page_selector` will be used- :param cffi: If `True`, uses `curl_cffi` to get the soup for each page. Otherwise, `aiohttp` will be used- :param **kwargs: Will be forwarded to `self.parse_url` to parse each new page"""+ impersonate: BrowserTypeLiteral | bool | None = False,+ relative_to: AbsoluteHttpURL | None = None,+ trim: bool | None = None,+ ) -> AsyncIterator[BeautifulSoup]:+ """Generator of website pages"""- kwargs.setdefault("relative_to", url.origin())+ relative_to = relative_to or url.origin()page_url = urlif callable(selector):get_next_page = selector+else:selector = selector or self.NEXT_PAGE_SELECTORassert selector, f"No selector was provided and {self.DOMAIN} does define a next_page_selector"- def get_next_page(soup: BeautifulSoup, /) -> str | None:+ def get_next_page(soup: BeautifulSoup, /) -> yarl.URL | str | None:try:return css.select(soup, selector, "href")except css.SelectorError:returnwhile True:- soup = await self.request_soup(page_url, impersonate=cffi or None)+ soup = await self.request_soup(page_url, impersonate=impersonate or None)yield souppage_url_str = get_next_page(soup)if not page_url_str:break- page_url = self.parse_url(page_url_str, **kwargs)+ page_url = self.parse_url(page_url_str, relative_to=relative_to, trim=trim)@error_handling_wrapperasync def direct_file(@@ -795,7 +745,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):@final@classmethod- def parse_date(cls, date_or_datetime: str, format: str) -> dates.TimeStamp | None:+ def parse_date(cls, date_or_datetime: str, /, format: str) -> dates.TimeStamp | None:return dates.to_timestamp(dates.parse_format(date_or_datetime, format))@final@@ -852,6 +802,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):) -> m3u8.M3U8:return await self._request_m3u8(url, headers, media_type)+ @finaldef create_custom_filename(self,name: str,@@ -895,23 +846,6 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):logger.warning(msg)return filename- @final- def get_cookies(self, partial_match_domain: bool = False) -> Iterable[tuple[str, BaseCookie[str]]]:- if partial_match_domain:- yield from self.client.client_manager.filter_cookies_by_word_in_domain(self.DOMAIN)- else:- yield str(self.PRIMARY_URL.host), self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL)-- @final- def get_cookie_value(self, cookie_name: str, partial_match_domain: bool = False) -> str | None:- def get_morsels_by_name():- for _, cookie in self.get_cookies(partial_match_domain):- if morsel := cookie.get(cookie_name):- yield morsel-- if newest := max(get_morsels_by_name(), key=lambda x: int(x["max-age"] or 0), default=None):- return newest.value-@finaldef handle_subs(self, scrape_item: ScrapeItem, video_filename: str, subtitles: Iterable[ISO639Subtitle]) -> None:counter = Counter()@@ -962,44 +896,6 @@ def _make_custom_filename(stem: str, ext: str, extra_info: list[str], only_truncreturn f"{truncated_stem}{ext}", invalid_extra_info_chars-class Site(NamedTuple):- PRIMARY_URL: AbsoluteHttpURL- DOMAIN: str- SUPPORTED_DOMAINS: SupportedDomains = ()- FOLDER_DOMAIN: str = ""---_CrawlerT = TypeVar("_CrawlerT", bound=Crawler)---def create_crawlers(- urls: Iterable[str] | Iterable[AbsoluteHttpURL], base_crawler: type[_CrawlerT]-) -> set[type[_CrawlerT]]:- """Creates new subclasses of the base crawler from the urls"""- return {_create_subclass(url, base_crawler) for url in urls}---def _create_subclass(url: AbsoluteHttpURL | str, base_class: type[_CrawlerT]) -> type[_CrawlerT]:- url = AbsoluteHttpURL(url)- assert is_absolute_http_url(url)- primary_url = remove_trailing_slash(url)- domain = primary_url.host.removeprefix("www.")- class_name = _make_crawler_name(domain)- class_attributes = Site(primary_url, domain)._asdict()- return type(class_name, (base_class,), class_attributes) # type: ignore # pyright: ignore[reportReturnType]---def _make_crawler_name(input_string: str) -> str:- clean_string = re.sub(r"[^a-zA-Z0-9]+", " ", input_string).strip()- cap_name = clean_string.title().replace(" ", "")- assert cap_name and cap_name.isalnum(), (- f"Can not generate a valid class name from {input_string}. Needs to be defined as a concrete class"- )- if cap_name[0].isdigit():- cap_name = "_" + cap_name- return f"{cap_name}Crawler"--def _validate_supported_paths(cls: type[Crawler]) -> None:for path_name, paths in cls.SUPPORTED_PATHS.items():assert path_name, f"{cls.__name__}, Invalid path: {path_name}"@@ -1008,9 +904,11 @@ def _validate_supported_paths(cls: type[Crawler]) -> None:assert paths, f"{cls.__name__} has not paths for {path_name}"if path_name.startswith("*"): # note- return+ continue+if isinstance(paths, str):paths = (paths,)+for path in paths:assert "`" not in path, f"{cls.__name__}, Invalid path {path_name}: {path}"@@ -1034,18 +932,17 @@ def _sort_supported_paths(supported_paths: SupportedPaths) -> dict[str, OneOrTupreturn dict(sorted(path_pairs, key=lambda x: x[0].casefold()))+_CrawlerT = TypeVar("_CrawlerT", bound=Crawler)++def auto_task_id(- func: Callable[Concatenate[_CrawlerT, ScrapeItem, _P], _R | Coroutine[None, None, _R]],+ func: Callable[Concatenate[_CrawlerT, ScrapeItem, _P], Coroutine[None, None, _R]],) -> Callable[Concatenate[_CrawlerT, ScrapeItem, _P], Coroutine[None, None, _R]]:"""Autocreate a new `task_id` from the scrape_item of the method"""@wraps(func)async def wrapper(self: _CrawlerT, scrape_item: ScrapeItem, *args: _P.args, **kwargs: _P.kwargs) -> _R:-with self.new_task_id(scrape_item.url):- result = func(self, scrape_item, *args, **kwargs)- if inspect.isawaitable(result):- return await result- return result+ return await func(self, scrape_item, *args, **kwargs)return wrapperdiff --git a/cyberdrop_dl/crawlers/discourse/__init__.py b/cyberdrop_dl/crawlers/discourse/__init__.pyindex b515c4c5a..71657ee03 100644--- a/cyberdrop_dl/crawlers/discourse/__init__.py+++ b/cyberdrop_dl/crawlers/discourse/__init__.py@@ -2,7 +2,7 @@# We can create subclasses dynamically by their URL-from cyberdrop_dl.crawlers.crawler import create_crawlers+from cyberdrop_dl.crawlers import create_crawlersfrom ._discourse import DiscourseCrawlerdiff --git a/cyberdrop_dl/crawlers/efukt.py b/cyberdrop_dl/crawlers/efukt.pyindex e3cc6583a..a8fb6a0b9 100644--- a/cyberdrop_dl/crawlers/efukt.py+++ b/cyberdrop_dl/crawlers/efukt.py@@ -40,8 +40,8 @@ class EfuktCrawler(Crawler):DOMAIN: ClassVar[str] = "efukt.com"FOLDER_DOMAIN: ClassVar[str] = "eFukt"NEXT_PAGE_SELECTOR = _SELECTORS.NEXT_PAGE- SKIP_PRE_CHECK = True- DEFAULT_POST_TITLE_FORMAT = "{date:%Y-%m-%d} {title}"+ ALLOW_EMPTY_PATH: ClassVar[bool] = True+ DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date:%Y-%m-%d} {title}"async def fetch(self, scrape_item: ScrapeItem) -> None:if is_series(scrape_item.url) or is_homepage(scrape_item.url):diff --git a/cyberdrop_dl/crawlers/erome.py b/cyberdrop_dl/crawlers/erome.pyindex 144cbda6d..0f73f2598 100644--- a/cyberdrop_dl/crawlers/erome.py+++ b/cyberdrop_dl/crawlers/erome.py@@ -44,13 +44,18 @@ class EromeCrawler(Crawler):async def profile(self, scrape_item: ScrapeItem, name: str) -> None:title = self.create_title(name)scrape_item.setup_as_profile(title)- await self.crawl_children(scrape_item, Selector.ALBUM)++ async for soup in self.web_pager(scrape_item.url):+ for _, new_item in self.iter_children(scrape_item, soup, Selector.ALBUM):+ self.create_task(self.run(new_item))@error_handling_wrapperasync def search(self, scrape_item: ScrapeItem, query: str) -> None:title = self.create_title(f"{query} [search]")scrape_item.setup_as_album(title)- await self.crawl_children(scrape_item, Selector.ALBUM)+ async for soup in self.web_pager(scrape_item.url):+ for _, new_item in self.iter_children(scrape_item, soup, Selector.ALBUM):+ self.create_task(self.run(new_item))@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem, album_id: str) -> None:diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.pyindex 80f224c33..2550d905e 100644--- a/cyberdrop_dl/crawlers/imgbb.py+++ b/cyberdrop_dl/crawlers/imgbb.py@@ -23,7 +23,7 @@ class ImgBBCrawler(CheveretoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://ibb.co")DOMAIN: ClassVar[str] = "imgbb"FOLDER_DOMAIN: ClassVar[str] = "ImgBB"- SKIP_PRE_CHECK: ClassVar[bool] = True+ ALLOW_EMPTY_PATH: ClassVar[bool] = Trueasync def fetch(self, scrape_item: ScrapeItem) -> None:username, _, rest = scrape_item.url.host.partition(".imgbb.")diff --git a/cyberdrop_dl/crawlers/luxuretv.py b/cyberdrop_dl/crawlers/luxuretv.pyindex 58c2d1c01..8b6f47aaa 100644--- a/cyberdrop_dl/crawlers/luxuretv.py+++ b/cyberdrop_dl/crawlers/luxuretv.py@@ -52,7 +52,7 @@ class LuxureTVCrawler(Crawler):if url.name and not url.name.endswith(".html"):url = url / ""- async for soup in self.web_pager(url, cffi=True):+ async for soup in self.web_pager(url, impersonate=True):for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.VIDEOS_THUMBS):self.create_task(self.run(new_scrape_item))diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.pyindex e3342234d..3f06c8723 100644--- a/cyberdrop_dl/crawlers/mediafire.py+++ b/cyberdrop_dl/crawlers/mediafire.py@@ -61,7 +61,7 @@ class MediaFireCrawler(Crawler, db_path="name"):}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URLDOMAIN: ClassVar[str] = "mediafire"- SKIP_PRE_CHECK: ClassVar[bool] = True+ ALLOW_EMPTY_PATH: ClassVar[bool] = Truedef __post_init__(self) -> None:self.api = MediaFireAPI(self)diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.pyindex 3a683a92b..ab6ee0b70 100644--- a/cyberdrop_dl/crawlers/mega_nz.py+++ b/cyberdrop_dl/crawlers/mega_nz.py@@ -13,6 +13,7 @@ from mega.api import MegaAPIfrom mega.core import MegaCorefrom mega.crypto import b64_to_a32from mega.data_structures import Crypto+from typing_extensions import overridefrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, auto_task_idfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL@@ -42,7 +43,7 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):"**NOTE**": "Downloads can not be resumed. Partial downloads will always be deleted and new downloads will start over",}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://mega.nz")- SKIP_PRE_CHECK: ClassVar[bool] = True+ ALLOW_EMPTY_PATH: ClassVar[bool] = TrueDOMAIN: ClassVar[str] = "mega.nz"FOLDER_DOMAIN: ClassVar[str] = "MegaNz"OLD_DOMAINS: ClassVar[tuple[str, ...]] = ("mega.co.nz",)@@ -58,11 +59,11 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):def password(self) -> str | None:return self.manager.auth_config.meganz.password or None- def _init_downloader(self) -> MegaDownloader:+ @override+ def __init_downloader__(self) -> None:self.core = MegaCore(MegaAPI(self.manager.client_manager._session))self.downloader = dl = MegaDownloader(self.manager, self.DOMAIN) # type: ignore[reportIncompatibleVariableOverride]dl.startup()- return dlasync def __async_post_init__(self) -> None:await self.login(self.PRIMARY_URL)diff --git a/cyberdrop_dl/crawlers/missav.py b/cyberdrop_dl/crawlers/missav.pyindex 2f0e8a311..43a395d87 100644--- a/cyberdrop_dl/crawlers/missav.py+++ b/cyberdrop_dl/crawlers/missav.py@@ -49,7 +49,7 @@ class MissAVCrawler(Crawler):title = self.create_title(f"{name} [{collection_type}]")scrape_item.setup_as_album(title)- async for soup in self.web_pager(scrape_item.url.update_query(page=1), cffi=True):+ async for soup in self.web_pager(scrape_item.url.update_query(page=1), impersonate=True):for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ITEM):self.create_task(self.run(new_scrape_item))diff --git a/cyberdrop_dl/crawlers/nhentai.py b/cyberdrop_dl/crawlers/nhentai.pyindex d478ea1d1..56581655b 100644--- a/cyberdrop_dl/crawlers/nhentai.py+++ b/cyberdrop_dl/crawlers/nhentai.py@@ -53,7 +53,7 @@ class NHentaiCrawler(Crawler):@error_handling_wrapperasync def collection(self, scrape_item: ScrapeItem, collection_type: str) -> None:title: str = ""- async for soup in self.web_pager(scrape_item.url, cffi=True):+ async for soup in self.web_pager(scrape_item.url, impersonate=True):if not title:if collection_type == "favorites":title_tag = css.select(soup, Selector.FAVORITES_TITLE)diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.pyindex 01f280f8b..9cddc1cbb 100644--- a/cyberdrop_dl/crawlers/onedrive.py+++ b/cyberdrop_dl/crawlers/onedrive.py@@ -97,7 +97,7 @@ class OneDriveCrawler(Crawler):}SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = SHARE_LINK_HOST, "onedrive.live.com"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://onedrive.com/")- SKIP_PRE_CHECK: ClassVar[bool] = True # URLs with not path could be valid+ ALLOW_EMPTY_PATH: ClassVar[bool] = TrueDOMAIN: ClassVar[str] = "onedrive"FOLDER_DOMAIN: ClassVar[str] = "OneDrive"diff --git a/cyberdrop_dl/crawlers/spankbang.py b/cyberdrop_dl/crawlers/spankbang.pyindex 17d302fca..b7604ab70 100644--- a/cyberdrop_dl/crawlers/spankbang.py+++ b/cyberdrop_dl/crawlers/spankbang.py@@ -1,7 +1,7 @@from __future__ import annotationsimport dataclasses-from typing import TYPE_CHECKING, Any, ClassVar+from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPathsfrom cyberdrop_dl.data_structures.mediaprops import Resolution@@ -11,7 +11,7 @@ from cyberdrop_dl.utils import css, jsonfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:- from collections.abc import AsyncGenerator, Generator+ from collections.abc import Generatorfrom bs4 import BeautifulSoup@@ -93,12 +93,6 @@ class SpankBangCrawler(Crawler):case _:return url- async def web_pager(- self, url: AbsoluteHttpURL, next_page_selector: str | None = None, *, cffi: bool = False, **kwargs: Any- ) -> AsyncGenerator[BeautifulSoup]:- async for soup in super()._web_pager(url, next_page_selector, cffi=True, **kwargs):- yield soup-@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:# old referer logic. video_id may be canonical (unique per video) or relative (different on each playlist)diff --git a/cyberdrop_dl/database/tables/history.py b/cyberdrop_dl/database/tables/history.pyindex 4cc381bda..69e43fab0 100644--- a/cyberdrop_dl/database/tables/history.py+++ b/cyberdrop_dl/database/tables/history.py@@ -119,7 +119,7 @@ class HistoryTable:return completed- async def check_album(self, domain: str, album_id: str) -> dict[str, int]:+ async def check_album(self, domain: str, album_id: str) -> dict[str, bool]:"""Checks whether an album has completed given its domain and album id."""if self._database.ignore_history:return {}@@ -127,7 +127,7 @@ class HistoryTable:query = "SELECT url_path, completed FROM media WHERE domain = ? and album_id = ?"cursor = await self.db_conn.execute(query, (domain, album_id))rows = await cursor.fetchall()- return {row[0]: row[1] for row in rows}+ return {row[0]: bool(row[1]) for row in rows}async def set_album_id(self, domain: str, media_item: MediaItem) -> None:"""Sets an album_id in the database."""diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex 740aa93c5..0fc9c34fa 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -13,8 +13,9 @@ from yarl import URLfrom cyberdrop_dl.clients.jdownloader import JDownloaderfrom cyberdrop_dl.constants import REGEX_LINKS, BlockedDomains+from cyberdrop_dl.crawlers import create_crawlersfrom cyberdrop_dl.crawlers._chevereto import CheveretoCrawler-from cyberdrop_dl.crawlers.crawler import Crawler, create_crawlers+from cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.crawlers.discourse import DiscourseCrawlerfrom cyberdrop_dl.crawlers.http_direct import DirectHttpFilefrom cyberdrop_dl.crawlers.realdebrid import RealDebridCrawler@@ -122,7 +123,7 @@ class ScrapeMapper:logger.exception("Failed to connect to jDownloader")await self.start_real_debrid()- self.direct_crawler._init_downloader()+ self.direct_crawler.__init_downloader__()async for item in self.get_input_items():self.manager.task_group.create_task(self.send_to_crawler(item))diff --git a/tests/test_scrape_mapper.py b/tests/test_scrape_mapper.pyindex 7236b175d..65522000f 100644--- a/tests/test_scrape_mapper.py+++ b/tests/test_scrape_mapper.py@@ -2,8 +2,8 @@ from __future__ import annotationsimport pytest+from cyberdrop_dl.crawlers import create_crawlersfrom cyberdrop_dl.crawlers._chevereto import CheveretoCrawler-from cyberdrop_dl.crawlers.crawler import create_crawlersfrom cyberdrop_dl.scraper import scrape_mapper