Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: remove manager states (#93)
NTFSvolume84fe7c481e20
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.pyindex 44dc5a6db..8ad74e06e 100644--- a/cyberdrop_dl/clients/__init__.py+++ b/cyberdrop_dl/clients/__init__.py@@ -55,7 +55,6 @@ class HTTPClient:with self.client_manager.request_context(domain):domain_limiter = self.client_manager.get_rate_limiter(domain)async with self.client_manager.global_rate_limiter, domain_limiter:- await self.client_manager.manager.states.RUNNING.wait()yielddef _prepare_headers(self, headers: Mapping[str, str] | None = None) -> CIMultiDict[str]:diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex 277bf880a..cbf3fc831 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -64,7 +64,6 @@ class DownloadClient:@contextlib.asynccontextmanagerasync def _track_errors(self, domain: str):with self.client_manager.request_context(domain):- await self.client_manager.manager.states.RUNNING.wait()yielddef _get_download_headers(self, domain: str, referer: AbsoluteHttpURL) -> dict[str, str]:@@ -207,7 +206,7 @@ class DownloadClient:process_response: Callable[[aiohttp.ClientResponse | AbstractResponse], Coroutine[None, None, bool]],) -> bool:download_url = media_item.debrid_link or media_item.url- await self.manager.states.RUNNING.wait()+fallback_url_generator = _fallback_generator(media_item)fallback_count = 0@@ -251,7 +250,6 @@ class DownloadClient:async with aiofiles.open(media_item.partial_file, mode="ab") as f:async for chunk in content.iter_chunked(self.client_manager.speed_limiter.chunk_size):- await self.manager.states.RUNNING.wait()await check_free_space()chunk_size = len(chunk)await self.client_manager.speed_limiter.acquire(chunk_size)diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.pyindex 712006c97..c1142de88 100644--- a/cyberdrop_dl/clients/hash_client.py+++ b/cyberdrop_dl/clients/hash_client.py@@ -81,7 +81,7 @@ class HashClient:returnif self.manager.config_manager.settings_data.dupe_cleanup_options.hashing != Hashing.IN_PLACE:return- await self.manager.states.RUNNING.wait()+try:assert media_item.original_filenamehash = await self.update_db_and_retrive_hash(diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.pyindex 4d4cae1f7..5a94f5f99 100644--- a/cyberdrop_dl/crawlers/bunkrr.py+++ b/cyberdrop_dl/crawlers/bunkrr.py@@ -264,7 +264,7 @@ class BunkrrCrawler(Crawler):if self._known_good_host:return await self.request_soup(url.with_host(self._known_good_host))- async with self.startup_lock:+ async with self._startup_lock:if url.host not in known_bad_hosts:if soup := await self._try_request_soup(url):return soupdiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex e22d11f79..45a7fdd40 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -78,8 +78,7 @@ SupportedDomains: TypeAlias = OneOrTuple[str]RateLimit = tuple[float, float]-HASH_PREFIXES = "md5:", "sha1:", "sha256:", "xxh128:"-VALID_RESOLUTION_NAMES = "4K", "8K", "HQ", "Unknown"+_HASH_PREFIXES = "md5:", "sha1:", "sha256:", "xxh128:"@dataclasses.dataclass(slots=True, frozen=True)@@ -171,21 +170,27 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):@finaldef __init__(self, manager: Manager) -> None:- self.manager = manager+ self.manager: Manager = managerself.downloader: Downloader = dataclasses.field(init=False)self.client: HTTPClient = dataclasses.field(init=False)- self.startup_lock = asyncio.Lock()- self.ready: bool = False+ self._startup_lock: asyncio.Lock = asyncio.Lock()+ self._ready: bool = Falseself.disabled: bool = Falseself.logged_in: bool = Falseself.scraped_items: set[str] = set()self.RATE_LIMIT = AsyncLimiter(*self._RATE_LIMIT)- self.waiting_items = 0+self.log = logself.log_debug = log_debug- self._semaphore = asyncio.Semaphore(20)+ self._semaphore: asyncio.Semaphore = asyncio.Semaphore(20)self.__post_init__()+ @property+ def waiting_items(self) -> int:+ if self._semaphore._waiters is None:+ return 0+ return len(self._semaphore._waiters)+def __post_init__(self) -> None:"""Override in subclasses to add custom init logic@@ -194,8 +199,8 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):@finalasync def __async_init__(self) -> None:"""Starts the crawler."""- async with self.startup_lock:- if self.ready:+ async with self._startup_lock:+ if self._ready:returnself.client = self.manager.client_manager.scraper_clientself.manager.client_manager.rate_limits[self.DOMAIN] = self.RATE_LIMIT@@ -205,7 +210,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):self.manager.client_manager.download_client.server_locked_domains.add(self.DOMAIN)self.downloader = self._init_downloader()await self.__async_post_init__()- self.ready = True+ self._ready = Trueasync def __async_post_init__(self) -> None:"""Perform additional setup that requires I/O@@ -214,6 +219,11 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):This method its called once and only if the crawler is actually going to be scrape something"""+ @final+ async def ready(self) -> None:+ if not self._ready:+ await self.__async_init__()+def __init_subclass__(cls,is_abc: bool = False,@@ -223,13 +233,15 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):) -> None:super().__init_subclass__(**kwargs)- msg = (- f"Subclass {cls.__name__} must not override __init__/_async_init_ method,"- "use __post_init__ for additional setup"- "use _async_post_init_ for setup that requires manipulating cookies or any IO (database access, https requests, etc...)"+ assert cls.__init__ is Crawler.__init__, (+ f"Subclass {cls.__name__} must not override __init__,use __post_init__ for additional setup")- assert cls.__init__ is Crawler.__init__, msg- assert cls.__async_init__ is Crawler.__async_init__, msg++ assert cls.__async_init__ is Crawler.__async_init__, (+ f"Subclass {cls.__name__} must not override __async_init__,"+ "use __async_post_init__ for setup that requires manipulating cookies or any IO (database access, HTTP requests, etc...)"+ )+cls.NAME: str = cls.__name__.removesuffix("Crawler")cls.IS_GENERIC: bool = is_genericcls.SUPPORTED_PATHS = _sort_supported_paths(cls.SUPPORTED_PATHS) # pyright: ignore[reportConstantRedefinition]@@ -316,10 +328,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):if self.disabled:return- self.waiting_items += 1async with self._semaphore:- await self.manager.states.RUNNING.wait()- self.waiting_items -= 1with scrape_item.track_changes():scrape_item.url = url = self.transform_url(scrape_item.url)@@ -474,7 +483,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):return check_completeasync def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.Rendition | None = None) -> None:- await self.manager.states.RUNNING.wait()+check_complete = await self.check_complete(media_item.url, media_item.referer)if check_complete:if media_item.album_id:@@ -873,7 +882,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):yield res.nameif _include.hash and hash_string:- assert any(hash_string.startswith(x) for x in HASH_PREFIXES), f"Invalid: {hash_string = }"+ assert any(hash_string.startswith(x) for x in _HASH_PREFIXES), f"Invalid: {hash_string = }"yield hash_stringfilename, had_invalid_chars = _make_custom_filename(stem, ext, list(extra_info()), only_truncate_stem)@@ -1032,7 +1041,7 @@ def auto_task_id(@wraps(func)async def wrapper(self: _CrawlerT, scrape_item: ScrapeItem, *args: _P.args, **kwargs: _P.kwargs) -> _R:- await self.manager.states.RUNNING.wait()+with self.new_task_id(scrape_item.url):result = func(self, scrape_item, *args, **kwargs)if inspect.isawaitable(result):diff --git a/cyberdrop_dl/crawlers/ehentai.py b/cyberdrop_dl/crawlers/ehentai.pyindex 47b2fdce3..3c120231f 100644--- a/cyberdrop_dl/crawlers/ehentai.py+++ b/cyberdrop_dl/crawlers/ehentai.py@@ -51,7 +51,7 @@ class EHentaiCrawler(Crawler):@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:- async with self.startup_lock:+ async with self._startup_lock:if not self._warnings_set:await self.set_no_warnings(scrape_item)diff --git a/cyberdrop_dl/crawlers/hitomi_la.py b/cyberdrop_dl/crawlers/hitomi_la.pyindex cfc456c90..9eff8f3b5 100644--- a/cyberdrop_dl/crawlers/hitomi_la.py+++ b/cyberdrop_dl/crawlers/hitomi_la.py@@ -166,7 +166,7 @@ class HitomiLaCrawler(Crawler):return json.loads(js_text.split("=", 1)[-1])async def get_servers(self) -> Servers:- async with self.startup_lock:+ async with self._startup_lock:if self._servers is None or (datetime.now() - self._servers.fetch_datetime > SERVERS_EXPIRE_AFTER):self._servers = await self._get_servers()return self._serversdiff --git a/cyberdrop_dl/crawlers/send_now.py b/cyberdrop_dl/crawlers/send_now.pyindex a8ac9b288..ace6272fc 100644--- a/cyberdrop_dl/crawlers/send_now.py+++ b/cyberdrop_dl/crawlers/send_now.py@@ -51,7 +51,7 @@ class SendNowCrawler(Crawler):async def _get_cookies(self, scrape_item: ScrapeItem) -> None:if self.got_cookies:return- async with self.startup_lock:+ async with self._startup_lock:if self.got_cookies:returndiff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.pyindex dd21cf945..3f82e4d38 100644--- a/cyberdrop_dl/director.py+++ b/cyberdrop_dl/director.py@@ -1,6 +1,5 @@from __future__ import annotations-import asyncioimport contextlibimport loggingfrom datetime import datetime@@ -91,20 +90,12 @@ async def _run_manager(manager: Manager) -> None:async def _scheduler(manager: Manager) -> None:for func in (_runtime, _post_runtime):- if manager.states.SHUTTING_DOWN.is_set():- return-- try:- await func(manager)- except asyncio.CancelledError:- if not manager.states.SHUTTING_DOWN.is_set():- raise+ await func(manager)async def _runtime(manager: Manager) -> None:"""Main runtime loop for the program, this will run until all scraping and downloading is complete."""- manager.states.RUNNING.set()with manager.live_manager.get_main_live(stop=True):async with ScrapeMapper.managed(manager) as scrape_mapper:await scrape_mapper.run()diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 2bd7839a1..9d38fc0e5 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -155,7 +155,7 @@ class Downloader:@contextlib.asynccontextmanagerasync def _download_context(self, media_item: MediaItem):- await self.manager.states.RUNNING.wait()+media_item.attempts = 0await self.client.mark_incomplete(media_item, self.domain)if media_item.is_segment:@@ -173,7 +173,6 @@ class Downloader:)async with server_limit, domain_limit, global_limit:- await self.manager.states.RUNNING.wait()self.processed_items.add(media_item.db_path)self.update_queued_files(increase_total=False)self.waiting_items -= 1@@ -448,7 +447,6 @@ class Downloader:if url_as_str in KNOWN_BAD_URLS:raise DownloadError(KNOWN_BAD_URLS[url_as_str])try:- await self.manager.states.RUNNING.wait()self.client.client_manager.check_domain_errors(self.domain)media_item.attempts = media_item.attempts or 1if not media_item.is_segment:diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.pyindex 828a4571c..ea3d398d8 100644--- a/cyberdrop_dl/downloader/mega_nz.py+++ b/cyberdrop_dl/downloader/mega_nz.py@@ -41,7 +41,6 @@ class MegaDownloadClient(DownloadClient):async with aiofiles.open(media_item.partial_file, mode="ab") as f:for _, chunk_size in get_chunks(file_size):- await self.manager.states.RUNNING.wait()raw_chunk = await content.readexactly(chunk_size)chunk = chunk_decryptor.read(raw_chunk)await check_free_space()diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex 7e5e8d944..75b353609 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -7,7 +7,7 @@ import loggingfrom dataclasses import Field, fieldfrom pathlib import Pathfrom time import perf_counter-from typing import TYPE_CHECKING, Any, NamedTuple, Self, TypeVar+from typing import TYPE_CHECKING, Any, Self, TypeVarfrom pydantic import BaseModel@@ -33,11 +33,6 @@ if TYPE_CHECKING:from cyberdrop_dl.scraper.scrape_mapper import ScrapeMapper-class AsyncioEvents(NamedTuple):- SHUTTING_DOWN: asyncio.Event- RUNNING: asyncio.Event--logger = logging.getLogger(__name__)@@ -68,7 +63,7 @@ class Manager:self.downloaded_data: int = 0self.loggers: dict[str, QueuedLogger] = {}self.args = args- self.states: AsyncioEvents+self._appdata: AppData | None = Noneself._completed_downloads: list[MediaItem] = []@@ -127,7 +122,7 @@ class Manager:async def async_startup(self) -> None:"""Async startup process for the manager."""- self.states = AsyncioEvents(asyncio.Event(), asyncio.Event())+self.args_logging()if not isinstance(self.client_manager, ClientManager):@@ -217,7 +212,6 @@ class Manager:async def close(self) -> None:"""Closes the manager."""- self.states.RUNNING.clear()await self.async_db_close()diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.pyindex 6cb867ddf..e15ae2d74 100644--- a/cyberdrop_dl/managers/progress_manager.py+++ b/cyberdrop_dl/managers/progress_manager.py@@ -75,21 +75,6 @@ class ProgressManager:finally:self.status_message.update(self.status_message_task_id, visible=False)- def pause_or_resume(self):- if self.manager.states.RUNNING.is_set():- self.pause()- else:- self.resume()-- def pause(self, msg: str = ""):- self.manager.states.RUNNING.clear()- suffix = f" [{msg}]" if msg else ""- self.activity.update(self.activity_task_id, description=f"Paused{suffix}")-- def resume(self):- self.manager.states.RUNNING.set()- self.activity.update(self.activity_task_id, description="Running Cyberdrop-DL")-def startup(self) -> None:"""Startup process for the progress manager."""spinner = SpinnerColumn(style="green", spinner_name="dots")diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex 08ff846a8..740aa93c5 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -98,7 +98,7 @@ class ScrapeMapper:async def start_real_debrid(self) -> None:"""Starts RealDebrid."""self.existing_crawlers["real-debrid"] = self.real_debrid = real = RealDebridCrawler(self.manager)- await real.__async_init__()+ await real.ready()@classmethod@contextlib.asynccontextmanager@@ -141,7 +141,6 @@ class ScrapeMapper:items_generator = self.load_links()async for item in items_generator:- await self.manager.states.RUNNING.wait()item.children_limits = self.manager.config_manager.settings_data.download_options.maximum_number_of_childrenif self.filter_items(item):if item_limit and self.count >= item_limit:@@ -234,8 +233,7 @@ class ScrapeMapper:crawler_match = match_url_to_crawler(self.existing_crawlers, scrape_item.url)if crawler_match:- if not crawler_match.ready:- await crawler_match.__async_init__()+ await crawler_match.ready()self.manager.task_group.create_task(crawler_match.run(scrape_item))returndiff --git a/tests/conftest.py b/tests/conftest.pyindex 599b256dc..373c88766 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -79,7 +79,5 @@ def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Manasync def running_manager(manager: Manager) -> AsyncGenerator[Manager]:scrape_mapper.existing_crawlers.clear()await manager.async_startup()- manager.states.RUNNING.set()yield manager- manager.states.RUNNING.clear()await manager.close()diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.pyindex 136674c0d..97adfe692 100644--- a/tests/crawlers/test_crawlers.py+++ b/tests/crawlers/test_crawlers.py@@ -106,7 +106,7 @@ async def test_crawler(running_manager: Manager, crawler_test_case: CrawlerTestCNone,)assert crawler, f"{test_case.domain} is not a valid crawler domain. Test case is invalid"- await crawler.__async_init__()+ await crawler.ready()item = ScrapeItem(url=crawler.parse_url(test_case.input_url))await crawler.run(item)