Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: remove mock manager (#81)
NTFSvolume4d2bf3a19c7c
diff --git a/cyberdrop_dl/managers/mock_manager.py b/cyberdrop_dl/managers/mock_manager.pydeleted file mode 100644index 3cf903507..000000000--- a/cyberdrop_dl/managers/mock_manager.py+++ /dev/null@@ -1,52 +0,0 @@-from __future__ import annotations--from typing import Any--MOCK_MANAGER = None---class MockCallable:- def __init__(self, return_obj: Any = None) -> None:- self.return_obj = return_obj-- def __getitem__(self, parameters: Any) -> object: ...- def __or__(self, other: Any) -> MockCallable: ...- def __ror__(self, other: Any) -> MockCallable: ...- def __call__(self, *args, **kwargs):- return self.return_obj---class Mock(Any):- def __init__(self, name: str, /) -> None:- self._nested_attrs: dict[str, Mock] = {}- self._mock_name = name-- def __call__(self, *args, **kwargs) -> None: ...-- def __getattribute__(self, name: str, /) -> Any:- if name == "manager" and MOCK_MANAGER is not None:- return MOCK_MANAGER- try:- return super().__getattribute__(name)- except AttributeError:- if name == "_nested_attrs":- raise # Avoid infinite recursion- return self._nested_attrs.get(name, Mock(name))---class MockCacheManager(Mock):- def __init__(self) -> None:- self.get = self.save = MockCallable()- super().__init__("cache_manager")---class MockManager(Mock):- def __init__(self):- global MOCK_MANAGER- assert MOCK_MANAGER is None, "A global MockManager already exists. Only 1 should be created"- super().__init__("manager")- self.cache_manager = MockCacheManager()- MOCK_MANAGER = self---MOCK_MANAGER = MockManager()diff --git a/cyberdrop_dl/models/types.py b/cyberdrop_dl/models/types.pyindex 07736a0f0..0a0178b14 100755--- a/cyberdrop_dl/models/types.py+++ b/cyberdrop_dl/models/types.py@@ -3,23 +3,24 @@ from __future__ import annotationsfrom pathlib import Pathfrom typing import Annotated, TypeVar-import yarlfrom pydantic import (AfterValidator,BeforeValidator,ByteSize,+ HttpUrl,NonNegativeInt,PlainSerializer,PlainValidator,StringConstraints,)+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.models.validators import (bytesize_to_str,change_path_suffix,falsy_as_list,falsy_as_none,- to_yarl_url_w_pydantyc_validation,+ to_yarl_url,)T = TypeVar("T")@@ -38,7 +39,7 @@ MainLogPath = Annotated[LogPath, AfterValidator(change_path_suffix(".log"))]# URL with pydantic.HttpUrl validation (must be absolute, must be http/https, detailed validation error).# In type hints it's a yarl.URL. After validation the result is parsed with `parse_url` so this is also a yarl.URL at runtime# Only use for config validation. To parse URLs internally while scraping, call `parse_url` directly-HttpURL = Annotated[yarl.URL, PlainValidator(to_yarl_url_w_pydantyc_validation), StrSerializer]+HttpURL = Annotated[AbsoluteHttpURL, PlainValidator(lambda x: to_yarl_url(HttpUrl(str(x)))), StrSerializer]# ~~~~~ Others ~~~~~~~ByteSizeSerilized = Annotated[ByteSize, PlainSerializer(bytesize_to_str, return_type=str)]diff --git a/cyberdrop_dl/models/validators.py b/cyberdrop_dl/models/validators.pyindex cede80af2..cf4b147ed 100755--- a/cyberdrop_dl/models/validators.py+++ b/cyberdrop_dl/models/validators.py@@ -15,13 +15,14 @@ from typing import (overload,)-import yarl-from pydantic import AnyUrl, ByteSize, HttpUrl, TypeAdapter+from pydantic import ByteSize, TypeAdapterif TYPE_CHECKING:from collections.abc import Callablefrom pathlib import Path+ import yarl+_DATE_PATTERN_REGEX = r"(\d+)\s*(second|seconds|minute|minutes|hour|hours|day|days|week|weeks|month|months|year|years)"_DATE_PATTERN = re.compile(_DATE_PATTERN_REGEX, re.IGNORECASE)@@ -39,22 +40,15 @@ def bytesize_to_str(value: _ConvertibleToInt) -> str:return value.human_readable(decimal=True)-def to_yarl_url(value: AnyUrl | str, *args, **kwargs) -> yarl.URL:+def to_yarl_url(value: object) -> yarl.URL:from cyberdrop_dl.utils.utilities import parse_urltry:- return parse_url(str(value), *args, **kwargs)+ return parse_url(str(value))except Exception as e:raise ValueError(str(e)) from e-def to_yarl_url_w_pydantyc_validation(value: str) -> yarl.URL:- if isinstance(value, yarl.URL):- value = str(value)- url = HttpUrl(value)- return to_yarl_url(url)--def to_bytesize(value: ByteSize | str | int) -> ByteSize:return _BYTE_SIZE_ADAPTER.validate_python(value)diff --git a/cyberdrop_dl/scraper/filters.py b/cyberdrop_dl/scraper/filters.pydeleted file mode 100644index 7c6dc8993..000000000--- a/cyberdrop_dl/scraper/filters.py+++ /dev/null@@ -1,46 +0,0 @@-from __future__ import annotations--import datetime-from typing import TYPE_CHECKING--from yarl import URL--from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL--if TYPE_CHECKING:- from collections.abc import Sequence-- from cyberdrop_dl.data_structures.url_objects import ScrapeItem---def is_valid_url(scrape_item: ScrapeItem) -> bool:- if not scrape_item.url:- return False- if not isinstance(scrape_item.url, URL):- try:- scrape_item.url = AbsoluteHttpURL(scrape_item.url)- except AttributeError:- return False- try:- if not scrape_item.url.host:- return False- except AttributeError:- return False-- return True---def is_outside_date_range(scrape_item: ScrapeItem, before: datetime.date | None, after: datetime.date | None) -> bool:- skip = False- item_date = scrape_item.completed_at or scrape_item.created_at- if not item_date:- return False- date = datetime.datetime.fromtimestamp(item_date).date()- if (after and date < after) or (before and date > before):- skip = True-- return skip---def is_in_domain_list(scrape_item: ScrapeItem, domain_list: Sequence[str]) -> bool:- return any(domain in scrape_item.url.host for domain in domain_list)diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex b5cff10d8..a18f35b92 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -2,9 +2,9 @@ from __future__ import annotationsimport asyncioimport contextlib+import datetimeimport loggingimport re-from datetime import date, datetimefrom pathlib import Pathfrom typing import TYPE_CHECKING, Literal, Self@@ -21,18 +21,17 @@ from cyberdrop_dl.crawlers.realdebrid import RealDebridCrawlerfrom cyberdrop_dl.crawlers.wordpress import WordPressHTMLCrawler, WordPressMediaCrawlerfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItemfrom cyberdrop_dl.exceptions import JDownloaderError, NoExtensionError-from cyberdrop_dl.scraper.filters import is_in_domain_list, is_outside_date_range, is_valid_url+from cyberdrop_dl.managers.manager import Managerfrom cyberdrop_dl.utils.logger import log_spacerfrom cyberdrop_dl.utils.utilities import get_download_path, remove_trailing_slashif TYPE_CHECKING:- from collections.abc import AsyncGenerator, Generator+ from collections.abc import AsyncGenerator, Generator, Sequenceimport aiosqlitefrom cyberdrop_dl.config.global_model import GenericCrawlerInstances, GlobalSettings- from cyberdrop_dl.crawlers import Crawler- from cyberdrop_dl.managers.manager import Manager+logger = logging.getLogger(__name__)existing_crawlers: dict[str, Crawler] = {}@@ -40,6 +39,22 @@ _seen_urls: set[AbsoluteHttpURL] = set()_crawlers_disabled_at_runtime: set[str] = set()+def is_outside_date_range(scrape_item: ScrapeItem, before: datetime.date | None, after: datetime.date | None) -> bool:+ skip = False+ item_date = scrape_item.completed_at or scrape_item.created_at+ if not item_date:+ return False+ date = datetime.datetime.fromtimestamp(item_date).date()+ if (after and date < after) or (before and date > before):+ skip = True++ return skip+++def is_in_domain_list(scrape_item: ScrapeItem, domain_list: Sequence[str]) -> bool:+ return any(domain in scrape_item.url.host for domain in domain_list)++class ScrapeMapper:"""This class maps links to their respective handlers, or JDownloader if they are unsupported."""@@ -186,8 +201,8 @@ class ScrapeMapper:async def load_all_links(self) -> AsyncGenerator[ScrapeItem]:"""Loads all links from database."""- after = self.manager.parsed_args.cli_only_args.completed_after or date.min- before = self.manager.parsed_args.cli_only_args.completed_before or datetime.now().date()+ after = self.manager.parsed_args.cli_only_args.completed_after or datetime.date.min+ before = self.manager.parsed_args.cli_only_args.completed_before or datetime.date.today()async for rows in self.manager.db_manager.history_table.get_all_items(after, before):for row in rows:yield _create_item_from_row(row)@@ -260,8 +275,6 @@ class ScrapeMapper:def filter_items(self, scrape_item: ScrapeItem) -> bool:"""Pre-filter scrape items base on URL."""- if not is_valid_url(scrape_item):- return Falseif scrape_item.url in _seen_urls:return False@@ -339,9 +352,9 @@ def _create_item_from_row(row: aiosqlite.Row) -> ScrapeItem:url = AbsoluteHttpURL(referer, encoded="%" in referer)item = ScrapeItem(url=url, retry_path=Path(row["download_path"]), part_of_album=True)if completed_at := row["completed_at"]:- item.completed_at = int(datetime.fromisoformat(completed_at).timestamp())+ item.completed_at = int(datetime.datetime.fromisoformat(completed_at).timestamp())if created_at := row["created_at"]:- item.created_at = int(datetime.fromisoformat(created_at).timestamp())+ item.created_at = int(datetime.datetime.fromisoformat(created_at).timestamp())return item@@ -353,9 +366,8 @@ def get_crawlers_mapping(manager: Manager | None = None, include_generics: boolIf manager is `None`, the `MOCK_MANAGER` will be used, which means the crawlers won't be able to actually run"""from cyberdrop_dl.crawlers import CRAWLERS- from cyberdrop_dl.managers.mock_manager import MOCK_MANAGER- manager_ = manager or MOCK_MANAGER+ manager_ = manager or Manager()global existing_crawlersif not existing_crawlers:for crawler in CRAWLERS: