Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: remove manager states (#93)

NTFSvolume84fe7c481e20
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.py
index 44dc5a6db..8ad74e06e 100644
--- a/cyberdrop_dl/clients/__init__.py
+++ b/cyberdrop_dl/clients/__init__.py
@@ -55,7 +55,6 @@ class HTTPClient:
with self.client_manager.request_context(domain):
domain_limiter = self.client_manager.get_rate_limiter(domain)
async with self.client_manager.global_rate_limiter, domain_limiter:
- await self.client_manager.manager.states.RUNNING.wait()
yield
def _prepare_headers(self, headers: Mapping[str, str] | None = None) -> CIMultiDict[str]:
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index 277bf880a..cbf3fc831 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -64,7 +64,6 @@ class DownloadClient:
@contextlib.asynccontextmanager
async def _track_errors(self, domain: str):
with self.client_manager.request_context(domain):
- await self.client_manager.manager.states.RUNNING.wait()
yield
def _get_download_headers(self, domain: str, referer: AbsoluteHttpURL) -> dict[str, str]:
@@ -207,7 +206,7 @@ class DownloadClient:
process_response: Callable[[aiohttp.ClientResponse | AbstractResponse], Coroutine[None, None, bool]],
) -> bool:
download_url = media_item.debrid_link or media_item.url
- await self.manager.states.RUNNING.wait()
+
fallback_url_generator = _fallback_generator(media_item)
fallback_count = 0
@@ -251,7 +250,6 @@ class DownloadClient:
async with aiofiles.open(media_item.partial_file, mode="ab") as f:
async for chunk in content.iter_chunked(self.client_manager.speed_limiter.chunk_size):
- await self.manager.states.RUNNING.wait()
await check_free_space()
chunk_size = len(chunk)
await self.client_manager.speed_limiter.acquire(chunk_size)
diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.py
index 712006c97..c1142de88 100644
--- a/cyberdrop_dl/clients/hash_client.py
+++ b/cyberdrop_dl/clients/hash_client.py
@@ -81,7 +81,7 @@ class HashClient:
return
if self.manager.config_manager.settings_data.dupe_cleanup_options.hashing != Hashing.IN_PLACE:
return
- await self.manager.states.RUNNING.wait()
+
try:
assert media_item.original_filename
hash = await self.update_db_and_retrive_hash(
diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.py
index 4d4cae1f7..5a94f5f99 100644
--- a/cyberdrop_dl/crawlers/bunkrr.py
+++ b/cyberdrop_dl/crawlers/bunkrr.py
@@ -264,7 +264,7 @@ class BunkrrCrawler(Crawler):
if self._known_good_host:
return await self.request_soup(url.with_host(self._known_good_host))
- async with self.startup_lock:
+ async with self._startup_lock:
if url.host not in known_bad_hosts:
if soup := await self._try_request_soup(url):
return soup
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index e22d11f79..45a7fdd40 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -78,8 +78,7 @@ SupportedDomains: TypeAlias = OneOrTuple[str]
RateLimit = tuple[float, float]
-HASH_PREFIXES = "md5:", "sha1:", "sha256:", "xxh128:"
-VALID_RESOLUTION_NAMES = "4K", "8K", "HQ", "Unknown"
+_HASH_PREFIXES = "md5:", "sha1:", "sha256:", "xxh128:"
@dataclasses.dataclass(slots=True, frozen=True)
@@ -171,21 +170,27 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
@final
def __init__(self, manager: Manager) -> None:
- self.manager = manager
+ self.manager: Manager = manager
self.downloader: Downloader = dataclasses.field(init=False)
self.client: HTTPClient = dataclasses.field(init=False)
- self.startup_lock = asyncio.Lock()
- self.ready: bool = False
+ self._startup_lock: asyncio.Lock = asyncio.Lock()
+ self._ready: bool = False
self.disabled: bool = False
self.logged_in: bool = False
self.scraped_items: set[str] = set()
self.RATE_LIMIT = AsyncLimiter(*self._RATE_LIMIT)
- self.waiting_items = 0
+
self.log = log
self.log_debug = log_debug
- self._semaphore = asyncio.Semaphore(20)
+ self._semaphore: asyncio.Semaphore = asyncio.Semaphore(20)
self.__post_init__()
+ @property
+ def waiting_items(self) -> int:
+ if self._semaphore._waiters is None:
+ return 0
+ return len(self._semaphore._waiters)
+
def __post_init__(self) -> None:
"""Override in subclasses to add custom init logic
@@ -194,8 +199,8 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
@final
async def __async_init__(self) -> None:
"""Starts the crawler."""
- async with self.startup_lock:
- if self.ready:
+ async with self._startup_lock:
+ if self._ready:
return
self.client = self.manager.client_manager.scraper_client
self.manager.client_manager.rate_limits[self.DOMAIN] = self.RATE_LIMIT
@@ -205,7 +210,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
self.manager.client_manager.download_client.server_locked_domains.add(self.DOMAIN)
self.downloader = self._init_downloader()
await self.__async_post_init__()
- self.ready = True
+ self._ready = True
async def __async_post_init__(self) -> None:
"""Perform additional setup that requires I/O
@@ -214,6 +219,11 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
This method its called once and only if the crawler is actually going to be scrape something"""
+ @final
+ async def ready(self) -> None:
+ if not self._ready:
+ await self.__async_init__()
+
def __init_subclass__(
cls,
is_abc: bool = False,
@@ -223,13 +233,15 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
) -> None:
super().__init_subclass__(**kwargs)
- msg = (
- f"Subclass {cls.__name__} must not override __init__/_async_init_ method,"
- "use __post_init__ for additional setup"
- "use _async_post_init_ for setup that requires manipulating cookies or any IO (database access, https requests, etc...)"
+ assert cls.__init__ is Crawler.__init__, (
+ f"Subclass {cls.__name__} must not override __init__,use __post_init__ for additional setup"
)
- assert cls.__init__ is Crawler.__init__, msg
- assert cls.__async_init__ is Crawler.__async_init__, msg
+
+ assert cls.__async_init__ is Crawler.__async_init__, (
+ f"Subclass {cls.__name__} must not override __async_init__,"
+ "use __async_post_init__ for setup that requires manipulating cookies or any IO (database access, HTTP requests, etc...)"
+ )
+
cls.NAME: str = cls.__name__.removesuffix("Crawler")
cls.IS_GENERIC: bool = is_generic
cls.SUPPORTED_PATHS = _sort_supported_paths(cls.SUPPORTED_PATHS) # pyright: ignore[reportConstantRedefinition]
@@ -316,10 +328,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
if self.disabled:
return
- self.waiting_items += 1
async with self._semaphore:
- await self.manager.states.RUNNING.wait()
- self.waiting_items -= 1
with scrape_item.track_changes():
scrape_item.url = url = self.transform_url(scrape_item.url)
@@ -474,7 +483,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
return check_complete
async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.Rendition | None = None) -> None:
- await self.manager.states.RUNNING.wait()
+
check_complete = await self.check_complete(media_item.url, media_item.referer)
if check_complete:
if media_item.album_id:
@@ -873,7 +882,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
yield res.name
if _include.hash and hash_string:
- assert any(hash_string.startswith(x) for x in HASH_PREFIXES), f"Invalid: {hash_string = }"
+ assert any(hash_string.startswith(x) for x in _HASH_PREFIXES), f"Invalid: {hash_string = }"
yield hash_string
filename, had_invalid_chars = _make_custom_filename(stem, ext, list(extra_info()), only_truncate_stem)
@@ -1032,7 +1041,7 @@ def auto_task_id(
@wraps(func)
async def wrapper(self: _CrawlerT, scrape_item: ScrapeItem, *args: _P.args, **kwargs: _P.kwargs) -> _R:
- await self.manager.states.RUNNING.wait()
+
with self.new_task_id(scrape_item.url):
result = func(self, scrape_item, *args, **kwargs)
if inspect.isawaitable(result):
diff --git a/cyberdrop_dl/crawlers/ehentai.py b/cyberdrop_dl/crawlers/ehentai.py
index 47b2fdce3..3c120231f 100644
--- a/cyberdrop_dl/crawlers/ehentai.py
+++ b/cyberdrop_dl/crawlers/ehentai.py
@@ -51,7 +51,7 @@ class EHentaiCrawler(Crawler):
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
- async with self.startup_lock:
+ async with self._startup_lock:
if not self._warnings_set:
await self.set_no_warnings(scrape_item)
diff --git a/cyberdrop_dl/crawlers/hitomi_la.py b/cyberdrop_dl/crawlers/hitomi_la.py
index cfc456c90..9eff8f3b5 100644
--- a/cyberdrop_dl/crawlers/hitomi_la.py
+++ b/cyberdrop_dl/crawlers/hitomi_la.py
@@ -166,7 +166,7 @@ class HitomiLaCrawler(Crawler):
return json.loads(js_text.split("=", 1)[-1])
async def get_servers(self) -> Servers:
- async with self.startup_lock:
+ async with self._startup_lock:
if self._servers is None or (datetime.now() - self._servers.fetch_datetime > SERVERS_EXPIRE_AFTER):
self._servers = await self._get_servers()
return self._servers
diff --git a/cyberdrop_dl/crawlers/send_now.py b/cyberdrop_dl/crawlers/send_now.py
index a8ac9b288..ace6272fc 100644
--- a/cyberdrop_dl/crawlers/send_now.py
+++ b/cyberdrop_dl/crawlers/send_now.py
@@ -51,7 +51,7 @@ class SendNowCrawler(Crawler):
async def _get_cookies(self, scrape_item: ScrapeItem) -> None:
if self.got_cookies:
return
- async with self.startup_lock:
+ async with self._startup_lock:
if self.got_cookies:
return
diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.py
index dd21cf945..3f82e4d38 100644
--- a/cyberdrop_dl/director.py
+++ b/cyberdrop_dl/director.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import asyncio
import contextlib
import logging
from datetime import datetime
@@ -91,20 +90,12 @@ async def _run_manager(manager: Manager) -> None:
async def _scheduler(manager: Manager) -> None:
for func in (_runtime, _post_runtime):
- if manager.states.SHUTTING_DOWN.is_set():
- return
-
- try:
- await func(manager)
- except asyncio.CancelledError:
- if not manager.states.SHUTTING_DOWN.is_set():
- raise
+ await func(manager)
async def _runtime(manager: Manager) -> None:
"""Main runtime loop for the program, this will run until all scraping and downloading is complete."""
- manager.states.RUNNING.set()
with manager.live_manager.get_main_live(stop=True):
async with ScrapeMapper.managed(manager) as scrape_mapper:
await scrape_mapper.run()
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index 2bd7839a1..9d38fc0e5 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -155,7 +155,7 @@ class Downloader:
@contextlib.asynccontextmanager
async def _download_context(self, media_item: MediaItem):
- await self.manager.states.RUNNING.wait()
+
media_item.attempts = 0
await self.client.mark_incomplete(media_item, self.domain)
if media_item.is_segment:
@@ -173,7 +173,6 @@ class Downloader:
)
async with server_limit, domain_limit, global_limit:
- await self.manager.states.RUNNING.wait()
self.processed_items.add(media_item.db_path)
self.update_queued_files(increase_total=False)
self.waiting_items -= 1
@@ -448,7 +447,6 @@ class Downloader:
if url_as_str in KNOWN_BAD_URLS:
raise DownloadError(KNOWN_BAD_URLS[url_as_str])
try:
- await self.manager.states.RUNNING.wait()
self.client.client_manager.check_domain_errors(self.domain)
media_item.attempts = media_item.attempts or 1
if not media_item.is_segment:
diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.py
index 828a4571c..ea3d398d8 100644
--- a/cyberdrop_dl/downloader/mega_nz.py
+++ b/cyberdrop_dl/downloader/mega_nz.py
@@ -41,7 +41,6 @@ class MegaDownloadClient(DownloadClient):
async with aiofiles.open(media_item.partial_file, mode="ab") as f:
for _, chunk_size in get_chunks(file_size):
- await self.manager.states.RUNNING.wait()
raw_chunk = await content.readexactly(chunk_size)
chunk = chunk_decryptor.read(raw_chunk)
await check_free_space()
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index 7e5e8d944..75b353609 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -7,7 +7,7 @@ import logging
from dataclasses import Field, field
from pathlib import Path
from time import perf_counter
-from typing import TYPE_CHECKING, Any, NamedTuple, Self, TypeVar
+from typing import TYPE_CHECKING, Any, Self, TypeVar
from pydantic import BaseModel
@@ -33,11 +33,6 @@ if TYPE_CHECKING:
from cyberdrop_dl.scraper.scrape_mapper import ScrapeMapper
-class AsyncioEvents(NamedTuple):
- SHUTTING_DOWN: asyncio.Event
- RUNNING: asyncio.Event
-
-
logger = logging.getLogger(__name__)
@@ -68,7 +63,7 @@ class Manager:
self.downloaded_data: int = 0
self.loggers: dict[str, QueuedLogger] = {}
self.args = args
- self.states: AsyncioEvents
+
self._appdata: AppData | None = None
self._completed_downloads: list[MediaItem] = []
@@ -127,7 +122,7 @@ class Manager:
async def async_startup(self) -> None:
"""Async startup process for the manager."""
- self.states = AsyncioEvents(asyncio.Event(), asyncio.Event())
+
self.args_logging()
if not isinstance(self.client_manager, ClientManager):
@@ -217,7 +212,6 @@ class Manager:
async def close(self) -> None:
"""Closes the manager."""
- self.states.RUNNING.clear()
await self.async_db_close()
diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.py
index 6cb867ddf..e15ae2d74 100644
--- a/cyberdrop_dl/managers/progress_manager.py
+++ b/cyberdrop_dl/managers/progress_manager.py
@@ -75,21 +75,6 @@ class ProgressManager:
finally:
self.status_message.update(self.status_message_task_id, visible=False)
- def pause_or_resume(self):
- if self.manager.states.RUNNING.is_set():
- self.pause()
- else:
- self.resume()
-
- def pause(self, msg: str = ""):
- self.manager.states.RUNNING.clear()
- suffix = f" [{msg}]" if msg else ""
- self.activity.update(self.activity_task_id, description=f"Paused{suffix}")
-
- def resume(self):
- self.manager.states.RUNNING.set()
- self.activity.update(self.activity_task_id, description="Running Cyberdrop-DL")
-
def startup(self) -> None:
"""Startup process for the progress manager."""
spinner = SpinnerColumn(style="green", spinner_name="dots")
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index 08ff846a8..740aa93c5 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -98,7 +98,7 @@ class ScrapeMapper:
async def start_real_debrid(self) -> None:
"""Starts RealDebrid."""
self.existing_crawlers["real-debrid"] = self.real_debrid = real = RealDebridCrawler(self.manager)
- await real.__async_init__()
+ await real.ready()
@classmethod
@contextlib.asynccontextmanager
@@ -141,7 +141,6 @@ class ScrapeMapper:
items_generator = self.load_links()
async for item in items_generator:
- await self.manager.states.RUNNING.wait()
item.children_limits = self.manager.config_manager.settings_data.download_options.maximum_number_of_children
if self.filter_items(item):
if item_limit and self.count >= item_limit:
@@ -234,8 +233,7 @@ class ScrapeMapper:
crawler_match = match_url_to_crawler(self.existing_crawlers, scrape_item.url)
if crawler_match:
- if not crawler_match.ready:
- await crawler_match.__async_init__()
+ await crawler_match.ready()
self.manager.task_group.create_task(crawler_match.run(scrape_item))
return
diff --git a/tests/conftest.py b/tests/conftest.py
index 599b256dc..373c88766 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -79,7 +79,5 @@ def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Man
async def running_manager(manager: Manager) -> AsyncGenerator[Manager]:
scrape_mapper.existing_crawlers.clear()
await manager.async_startup()
- manager.states.RUNNING.set()
yield manager
- manager.states.RUNNING.clear()
await manager.close()
diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.py
index 136674c0d..97adfe692 100644
--- a/tests/crawlers/test_crawlers.py
+++ b/tests/crawlers/test_crawlers.py
@@ -106,7 +106,7 @@ async def test_crawler(running_manager: Manager, crawler_test_case: CrawlerTestC
None,
)
assert crawler, f"{test_case.domain} is not a valid crawler domain. Test case is invalid"
- await crawler.__async_init__()
+ await crawler.ready()
item = ScrapeItem(url=crawler.parse_url(test_case.input_url))
await crawler.run(item)