Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: remove mock manager (#81)

NTFSvolume4d2bf3a19c7c
diff --git a/cyberdrop_dl/managers/mock_manager.py b/cyberdrop_dl/managers/mock_manager.py
deleted file mode 100644
index 3cf903507..000000000
--- a/cyberdrop_dl/managers/mock_manager.py
+++ /dev/null
@@ -1,52 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-MOCK_MANAGER = None
-
-
-class MockCallable:
- def __init__(self, return_obj: Any = None) -> None:
- self.return_obj = return_obj
-
- def __getitem__(self, parameters: Any) -> object: ...
- def __or__(self, other: Any) -> MockCallable: ...
- def __ror__(self, other: Any) -> MockCallable: ...
- def __call__(self, *args, **kwargs):
- return self.return_obj
-
-
-class Mock(Any):
- def __init__(self, name: str, /) -> None:
- self._nested_attrs: dict[str, Mock] = {}
- self._mock_name = name
-
- def __call__(self, *args, **kwargs) -> None: ...
-
- def __getattribute__(self, name: str, /) -> Any:
- if name == "manager" and MOCK_MANAGER is not None:
- return MOCK_MANAGER
- try:
- return super().__getattribute__(name)
- except AttributeError:
- if name == "_nested_attrs":
- raise # Avoid infinite recursion
- return self._nested_attrs.get(name, Mock(name))
-
-
-class MockCacheManager(Mock):
- def __init__(self) -> None:
- self.get = self.save = MockCallable()
- super().__init__("cache_manager")
-
-
-class MockManager(Mock):
- def __init__(self):
- global MOCK_MANAGER
- assert MOCK_MANAGER is None, "A global MockManager already exists. Only 1 should be created"
- super().__init__("manager")
- self.cache_manager = MockCacheManager()
- MOCK_MANAGER = self
-
-
-MOCK_MANAGER = MockManager()
diff --git a/cyberdrop_dl/models/types.py b/cyberdrop_dl/models/types.py
index 07736a0f0..0a0178b14 100755
--- a/cyberdrop_dl/models/types.py
+++ b/cyberdrop_dl/models/types.py
@@ -3,23 +3,24 @@ from __future__ import annotations
from pathlib import Path
from typing import Annotated, TypeVar
-import yarl
from pydantic import (
AfterValidator,
BeforeValidator,
ByteSize,
+ HttpUrl,
NonNegativeInt,
PlainSerializer,
PlainValidator,
StringConstraints,
)
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.models.validators import (
bytesize_to_str,
change_path_suffix,
falsy_as_list,
falsy_as_none,
- to_yarl_url_w_pydantyc_validation,
+ to_yarl_url,
)
T = TypeVar("T")
@@ -38,7 +39,7 @@ MainLogPath = Annotated[LogPath, AfterValidator(change_path_suffix(".log"))]
# URL with pydantic.HttpUrl validation (must be absolute, must be http/https, detailed validation error).
# In type hints it's a yarl.URL. After validation the result is parsed with `parse_url` so this is also a yarl.URL at runtime
# Only use for config validation. To parse URLs internally while scraping, call `parse_url` directly
-HttpURL = Annotated[yarl.URL, PlainValidator(to_yarl_url_w_pydantyc_validation), StrSerializer]
+HttpURL = Annotated[AbsoluteHttpURL, PlainValidator(lambda x: to_yarl_url(HttpUrl(str(x)))), StrSerializer]
# ~~~~~ Others ~~~~~~~
ByteSizeSerilized = Annotated[ByteSize, PlainSerializer(bytesize_to_str, return_type=str)]
diff --git a/cyberdrop_dl/models/validators.py b/cyberdrop_dl/models/validators.py
index cede80af2..cf4b147ed 100755
--- a/cyberdrop_dl/models/validators.py
+++ b/cyberdrop_dl/models/validators.py
@@ -15,13 +15,14 @@ from typing import (
overload,
)
-import yarl
-from pydantic import AnyUrl, ByteSize, HttpUrl, TypeAdapter
+from pydantic import ByteSize, TypeAdapter
if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path
+ import yarl
+
_DATE_PATTERN_REGEX = r"(\d+)\s*(second|seconds|minute|minutes|hour|hours|day|days|week|weeks|month|months|year|years)"
_DATE_PATTERN = re.compile(_DATE_PATTERN_REGEX, re.IGNORECASE)
@@ -39,22 +40,15 @@ def bytesize_to_str(value: _ConvertibleToInt) -> str:
return value.human_readable(decimal=True)
-def to_yarl_url(value: AnyUrl | str, *args, **kwargs) -> yarl.URL:
+def to_yarl_url(value: object) -> yarl.URL:
from cyberdrop_dl.utils.utilities import parse_url
try:
- return parse_url(str(value), *args, **kwargs)
+ return parse_url(str(value))
except Exception as e:
raise ValueError(str(e)) from e
-def to_yarl_url_w_pydantyc_validation(value: str) -> yarl.URL:
- if isinstance(value, yarl.URL):
- value = str(value)
- url = HttpUrl(value)
- return to_yarl_url(url)
-
-
def to_bytesize(value: ByteSize | str | int) -> ByteSize:
return _BYTE_SIZE_ADAPTER.validate_python(value)
diff --git a/cyberdrop_dl/scraper/filters.py b/cyberdrop_dl/scraper/filters.py
deleted file mode 100644
index 7c6dc8993..000000000
--- a/cyberdrop_dl/scraper/filters.py
+++ /dev/null
@@ -1,46 +0,0 @@
-from __future__ import annotations
-
-import datetime
-from typing import TYPE_CHECKING
-
-from yarl import URL
-
-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-
-if TYPE_CHECKING:
- from collections.abc import Sequence
-
- from cyberdrop_dl.data_structures.url_objects import ScrapeItem
-
-
-def is_valid_url(scrape_item: ScrapeItem) -> bool:
- if not scrape_item.url:
- return False
- if not isinstance(scrape_item.url, URL):
- try:
- scrape_item.url = AbsoluteHttpURL(scrape_item.url)
- except AttributeError:
- return False
- try:
- if not scrape_item.url.host:
- return False
- except AttributeError:
- return False
-
- return True
-
-
-def is_outside_date_range(scrape_item: ScrapeItem, before: datetime.date | None, after: datetime.date | None) -> bool:
- skip = False
- item_date = scrape_item.completed_at or scrape_item.created_at
- if not item_date:
- return False
- date = datetime.datetime.fromtimestamp(item_date).date()
- if (after and date < after) or (before and date > before):
- skip = True
-
- return skip
-
-
-def is_in_domain_list(scrape_item: ScrapeItem, domain_list: Sequence[str]) -> bool:
- return any(domain in scrape_item.url.host for domain in domain_list)
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index b5cff10d8..a18f35b92 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -2,9 +2,9 @@ from __future__ import annotations
import asyncio
import contextlib
+import datetime
import logging
import re
-from datetime import date, datetime
from pathlib import Path
from typing import TYPE_CHECKING, Literal, Self
@@ -21,18 +21,17 @@ from cyberdrop_dl.crawlers.realdebrid import RealDebridCrawler
from cyberdrop_dl.crawlers.wordpress import WordPressHTMLCrawler, WordPressMediaCrawler
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
from cyberdrop_dl.exceptions import JDownloaderError, NoExtensionError
-from cyberdrop_dl.scraper.filters import is_in_domain_list, is_outside_date_range, is_valid_url
+from cyberdrop_dl.managers.manager import Manager
from cyberdrop_dl.utils.logger import log_spacer
from cyberdrop_dl.utils.utilities import get_download_path, remove_trailing_slash
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Generator
+ from collections.abc import AsyncGenerator, Generator, Sequence
import aiosqlite
from cyberdrop_dl.config.global_model import GenericCrawlerInstances, GlobalSettings
- from cyberdrop_dl.crawlers import Crawler
- from cyberdrop_dl.managers.manager import Manager
+
logger = logging.getLogger(__name__)
existing_crawlers: dict[str, Crawler] = {}
@@ -40,6 +39,22 @@ _seen_urls: set[AbsoluteHttpURL] = set()
_crawlers_disabled_at_runtime: set[str] = set()
+def is_outside_date_range(scrape_item: ScrapeItem, before: datetime.date | None, after: datetime.date | None) -> bool:
+ skip = False
+ item_date = scrape_item.completed_at or scrape_item.created_at
+ if not item_date:
+ return False
+ date = datetime.datetime.fromtimestamp(item_date).date()
+ if (after and date < after) or (before and date > before):
+ skip = True
+
+ return skip
+
+
+def is_in_domain_list(scrape_item: ScrapeItem, domain_list: Sequence[str]) -> bool:
+ return any(domain in scrape_item.url.host for domain in domain_list)
+
+
class ScrapeMapper:
"""This class maps links to their respective handlers, or JDownloader if they are unsupported."""
@@ -186,8 +201,8 @@ class ScrapeMapper:
async def load_all_links(self) -> AsyncGenerator[ScrapeItem]:
"""Loads all links from database."""
- after = self.manager.parsed_args.cli_only_args.completed_after or date.min
- before = self.manager.parsed_args.cli_only_args.completed_before or datetime.now().date()
+ after = self.manager.parsed_args.cli_only_args.completed_after or datetime.date.min
+ before = self.manager.parsed_args.cli_only_args.completed_before or datetime.date.today()
async for rows in self.manager.db_manager.history_table.get_all_items(after, before):
for row in rows:
yield _create_item_from_row(row)
@@ -260,8 +275,6 @@ class ScrapeMapper:
def filter_items(self, scrape_item: ScrapeItem) -> bool:
"""Pre-filter scrape items base on URL."""
- if not is_valid_url(scrape_item):
- return False
if scrape_item.url in _seen_urls:
return False
@@ -339,9 +352,9 @@ def _create_item_from_row(row: aiosqlite.Row) -> ScrapeItem:
url = AbsoluteHttpURL(referer, encoded="%" in referer)
item = ScrapeItem(url=url, retry_path=Path(row["download_path"]), part_of_album=True)
if completed_at := row["completed_at"]:
- item.completed_at = int(datetime.fromisoformat(completed_at).timestamp())
+ item.completed_at = int(datetime.datetime.fromisoformat(completed_at).timestamp())
if created_at := row["created_at"]:
- item.created_at = int(datetime.fromisoformat(created_at).timestamp())
+ item.created_at = int(datetime.datetime.fromisoformat(created_at).timestamp())
return item
@@ -353,9 +366,8 @@ def get_crawlers_mapping(manager: Manager | None = None, include_generics: bool
If manager is `None`, the `MOCK_MANAGER` will be used, which means the crawlers won't be able to actually run"""
from cyberdrop_dl.crawlers import CRAWLERS
- from cyberdrop_dl.managers.mock_manager import MOCK_MANAGER
- manager_ = manager or MOCK_MANAGER
+ manager_ = manager or Manager()
global existing_crawlers
if not existing_crawlers:
for crawler in CRAWLERS: