Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: allow no source for scraping (for tests)

NTFSvolume0f29ce911dbe
diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.py
index 6292ef9a3..11a49747d 100644
--- a/cyberdrop_dl/scrape_mapper.py
+++ b/cyberdrop_dl/scrape_mapper.py
@@ -209,8 +209,11 @@ class ScrapeMapper:
await self._direct_http.__async_post_init__()
source_name, source = _source(self.manager)
+ stats = ScrapeStats(source_name)
+ if source is None:
+ return stats
+
async with contextlib.aclosing(source) as items:
- stats = ScrapeStats(source_name)
async def wait_until_scrape_is_done() -> None:
_ = await self._done.wait()
@@ -337,13 +340,15 @@ class ScrapeMapper:
return crawler
-def _source(manager: Manager) -> tuple[str, AsyncGenerator[ScrapeItem]]:
+def _source(manager: Manager) -> tuple[str, AsyncGenerator[ScrapeItem] | None]:
cli_args = manager.cli_args
if cli_args.urls:
return "--links (CLI args)", _load_cli_links(cli_args.urls)
- assert manager.input_file
- return str(manager.input_file), _load_urls_from_file(manager.input_file)
+ if manager.input_file:
+ return str(manager.input_file), _load_urls_from_file(manager.input_file)
+
+ return "", None
async def _load_urls_from_file(file: Path) -> AsyncGenerator[ScrapeItem]:
diff --git a/pyproject.toml b/pyproject.toml
index 751e8874c..f29ed38cf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -53,7 +53,7 @@ license-files = ["LICENSE"]
maintainers = [{name = "NTFSvolume", email = "[email protected]"}]
readme = "README.md"
requires-python = ">=3.12"
-version = "10.0.0rc1"
+version = "10.0.0rc2"
[project.optional-dependencies]
apprise = [
diff --git a/uv.lock b/uv.lock
index efa6a8108..8ace1e481 100644
--- a/uv.lock
+++ b/uv.lock
@@ -607,7 +607,7 @@ wheels = [
[[package]]
name = "cyberdrop-dl-patched"
-version = "10.0.0rc1"
+version = "10.0.0rc2"
source = { editable = "." }
dependencies = [
{ name = "aiodns", marker = "sys_platform != 'android' and sys_platform != 'win32'" },