Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: allow no source for scraping (for tests)
NTFSvolume0f29ce911dbe
diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.pyindex 6292ef9a3..11a49747d 100644--- a/cyberdrop_dl/scrape_mapper.py+++ b/cyberdrop_dl/scrape_mapper.py@@ -209,8 +209,11 @@ class ScrapeMapper:await self._direct_http.__async_post_init__()source_name, source = _source(self.manager)+ stats = ScrapeStats(source_name)+ if source is None:+ return stats+async with contextlib.aclosing(source) as items:- stats = ScrapeStats(source_name)async def wait_until_scrape_is_done() -> None:_ = await self._done.wait()@@ -337,13 +340,15 @@ class ScrapeMapper:return crawler-def _source(manager: Manager) -> tuple[str, AsyncGenerator[ScrapeItem]]:+def _source(manager: Manager) -> tuple[str, AsyncGenerator[ScrapeItem] | None]:cli_args = manager.cli_argsif cli_args.urls:return "--links (CLI args)", _load_cli_links(cli_args.urls)- assert manager.input_file- return str(manager.input_file), _load_urls_from_file(manager.input_file)+ if manager.input_file:+ return str(manager.input_file), _load_urls_from_file(manager.input_file)++ return "", Noneasync def _load_urls_from_file(file: Path) -> AsyncGenerator[ScrapeItem]:diff --git a/pyproject.toml b/pyproject.tomlindex 751e8874c..f29ed38cf 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -53,7 +53,7 @@ license-files = ["LICENSE"]maintainers = [{name = "NTFSvolume", email = "[email protected]"}]readme = "README.md"requires-python = ">=3.12"-version = "10.0.0rc1"+version = "10.0.0rc2"[project.optional-dependencies]apprise = [diff --git a/uv.lock b/uv.lockindex efa6a8108..8ace1e481 100644--- a/uv.lock+++ b/uv.lock@@ -607,7 +607,7 @@ wheels = [[[package]]name = "cyberdrop-dl-patched"-version = "10.0.0rc1"+version = "10.0.0rc2"source = { editable = "." }dependencies = [{ name = "aiodns", marker = "sys_platform != 'android' and sys_platform != 'win32'" },