Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: allow a custom scrape source

NTFSvolumecaa38545aa81
diff --git a/cyberdrop_dl/cli.py b/cyberdrop_dl/cli.py
index 7958d7969..fbf6a1692 100644
--- a/cyberdrop_dl/cli.py
+++ b/cyberdrop_dl/cli.py
@@ -37,14 +37,15 @@ def main_menu(
show_default=False,
),
] = Path("URLs.txt"), # pyright: ignore[reportCallInDefaultInitializer]
- cli: CLIarguments | None = None,
+ cli_args: CLIarguments | None = None,
) -> None:
"Show a TUI menu equivalent to the CLI commands"
- with prepare_manager((), input_file, cli, cli_overrides=None)() as manager:
+ input_file = input_file.resolve().absolute()
+ with prepare_manager(cli_args, cli_overrides=None)() as manager:
from cyberdrop_dl import program_ui
- program_ui.run(manager)
- scrape(manager)
+ program_ui.run(manager, input_file)
+ scrape(manager, input_file)
@app.command
diff --git a/cyberdrop_dl/commands/__init__.py b/cyberdrop_dl/commands/__init__.py
index e641cc4f7..cebc2cfca 100644
--- a/cyberdrop_dl/commands/__init__.py
+++ b/cyberdrop_dl/commands/__init__.py
@@ -5,8 +5,6 @@ from typing import Annotated, Any
from cyclopts import Parameter, Token, validators
-from cyberdrop_dl.models.types import HttpURL
-
def _resolve_path(type_: type[Path], tokens: Sequence[Token]) -> Path:
assert len(tokens) == 1
@@ -25,8 +23,6 @@ type SQLiteFile = Annotated[Path, file_validator(".db")]
@Parameter(name="*")
@dataclasses.dataclass(slots=True)
class CLIarguments:
- urls: Annotated[tuple[HttpURL, ...], Parameter(show=False)] = ()
-
config_file: YAMLFile | None = None
"YAML file to use as config"
@@ -37,6 +33,4 @@ class CLIarguments:
"SQLite file to use as database"
def __json__(self) -> dict[str, Any]:
- me: dict[str, Any] = {k: None if v is None else str(v) for k, v in dataclasses.asdict(self).items()}
- me["urls"] = tuple(map(str, self.urls))
- return me
+ return {k: None if v is None else str(v) for k, v in dataclasses.asdict(self).items()}
diff --git a/cyberdrop_dl/commands/scrape.py b/cyberdrop_dl/commands/scrape.py
index 28e9422bf..97fd84c70 100644
--- a/cyberdrop_dl/commands/scrape.py
+++ b/cyberdrop_dl/commands/scrape.py
@@ -16,6 +16,7 @@ from cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroup
from cyberdrop_dl.logs import log_spacer, set_console_level, setup_file_logging
from cyberdrop_dl.models import merge_models
from cyberdrop_dl.models.types import HttpURL # noqa: TC001
+from cyberdrop_dl.scrape_source import URLSource
from cyberdrop_dl.utils import cleanup
logger = logging.getLogger(__name__)
@@ -24,9 +25,17 @@ if TYPE_CHECKING:
from cyclopts.argument import ArgumentCollection
from cyberdrop_dl.manager import Manager
+ from cyberdrop_dl.scrape_source import URLSource
-async def _scrape(manager: Manager) -> None:
+def scrape(manager: Manager, source: URLSource) -> None:
+ from cyberdrop_dl import aio
+
+ with setup_file_logging(manager.config.logs.files.main, level=manager.config.logs.effective_level):
+ aio.run(_scrape(manager, source))
+
+
+async def _scrape(manager: Manager, source: URLSource) -> None:
from cyberdrop_dl import ffmpeg
from cyberdrop_dl.scrape_mapper import ScrapeMapper
@@ -39,7 +48,7 @@ async def _scrape(manager: Manager) -> None:
log_spacer()
logger.info("Starting CDL...")
async with ScrapeMapper(manager)() as scrape_mapper:
- stats = await scrape_mapper.run()
+ stats = await scrape_mapper.run(source)
await _post_runtime(manager)
stats_summary = manager.print_stats(stats)
@@ -89,13 +98,6 @@ async def _notify(config: Config, stats_summary: str) -> None:
await apprise.notify(urls, stats_summary)
-def scrape(manager: Manager) -> None:
- from cyberdrop_dl import aio
-
- with setup_file_logging(manager.config.logs.files.main, level=manager.config.logs.effective_level):
- aio.run(_scrape(manager))
-
-
def _validate_inputs(args: ArgumentCollection) -> None:
try:
cyclopts.validators.LimitedChoice(min=1, max=1)(args)
@@ -128,27 +130,24 @@ def download(
validator=cyclopts.validators.Path(exists=True, dir_okay=False),
),
] = None,
- cli: CLIarguments | None = None,
+ cli_args: CLIarguments | None = None,
cli_overrides: Config | None = None,
) -> None:
"Download URLs"
- with prepare_manager(urls, input_file, cli, cli_overrides)() as manager:
- scrape(manager)
+ if input_file:
+ input_file = input_file.resolve().absolute()
+ with prepare_manager(cli_args, cli_overrides)() as manager:
+ scrape(manager, input_file or urls)
-def prepare_manager(
- urls: tuple[HttpURL, ...], input_file: Path | None, cli: CLIarguments | None, cli_overrides: Config | None
-) -> Manager:
+def prepare_manager(cli_args: CLIarguments | None, cli_overrides: Config | None) -> Manager:
check_for_v9_files()
- if input_file:
- input_file = input_file.resolve().absolute()
from cyberdrop_dl.manager import Manager
- cli = cli or CLIarguments()
- cli.urls = urls
- appdata, config = _prepare_appdata_and_config(cli, cli_overrides)
- manager = Manager(cli, appdata, config, input_file)
+ cli_args = cli_args or CLIarguments()
+ appdata, config = _prepare_appdata_and_config(cli_args, cli_overrides)
+ manager = Manager(cli_args, appdata, config)
set_console_level(manager.config.logs.effective_console_level)
manager.appdata.mkdirs()
return manager
diff --git a/cyberdrop_dl/manager.py b/cyberdrop_dl/manager.py
index 83592c523..2244f797f 100644
--- a/cyberdrop_dl/manager.py
+++ b/cyberdrop_dl/manager.py
@@ -44,7 +44,6 @@ class Manager:
cli_args: CLIarguments | None = None,
appdata: AppData | None = None,
config: Config | None = None,
- input_file: Path | None = None,
) -> None:
from cyberdrop_dl.commands import CLIarguments
@@ -52,7 +51,6 @@ class Manager:
self._appdata: AppData | None = appdata
self.cli_args: CLIarguments = cli_args or CLIarguments()
self._config: Config | None = config
- self.input_file: Path | None = input_file
self._completed_downloads: list[MediaItem] = []
self._hasher: Hasher | None = None
diff --git a/cyberdrop_dl/program_ui.py b/cyberdrop_dl/program_ui.py
index a68104321..1c42e8017 100644
--- a/cyberdrop_dl/program_ui.py
+++ b/cyberdrop_dl/program_ui.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import sys
+from contextvars import ContextVar
from typing import TYPE_CHECKING
import aiohttp
@@ -22,10 +23,11 @@ from cyberdrop_dl.utils import text_editor
if TYPE_CHECKING:
from collections.abc import Callable
+ from pathlib import Path
from cyberdrop_dl.manager import Manager
-
+_INPUT_FILE: ContextVar[Path] = ContextVar("_INPUT_FILE")
_CHANGELOG_URL = "https://raw.githubusercontent.com/Cyberdrop-DL/cyberdrop-dl/refs/heads/main/CHANGELOG.md"
_changelog_content: str = ""
@@ -38,12 +40,13 @@ def _changelog() -> str:
return _changelog_content
-def run(manager: Manager) -> None:
+def run(manager: Manager, input_file: Path) -> None:
+ _INPUT_FILE.set(input_file)
choices: dict[str, Callable[[Manager], bool | None]] = {
"Download": lambda _: True,
"Create file hashes": _scan_and_create_hashes,
"Sort files in download folder": _sort_files,
- "Edit URLs.txt": _edit_urls,
+ "Edit URLs.txt": lambda _: _edit_urls(),
"Edit config": _edit_config,
"View changelog": lambda _: _view_changelog(),
"Exit": lambda _: sys.exit(0),
@@ -93,10 +96,9 @@ def _edit_config(manager: Manager) -> None:
enter_to_continue()
-def _edit_urls(manager: Manager) -> None:
- assert manager.input_file
+def _edit_urls() -> None:
try:
- text_editor.open(manager.input_file)
+ text_editor.open(_INPUT_FILE.get())
except ValueError as e:
console.error(e)
@@ -132,7 +134,7 @@ def _app_header(manager: Manager) -> None:
paths = {
"Config file": manager.config.source,
"Database file": manager.appdata.db_file,
- "URLs file": manager.input_file,
+ "URLs file": _INPUT_FILE.get(),
"Cache file": manager.appdata.cache_file,
"Logs": manager.config.logs.effective_log_folder,
"Main log file": manager.config.logs.files.main,
diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.py
index 88825895c..696f9b773 100644
--- a/cyberdrop_dl/scrape_mapper.py
+++ b/cyberdrop_dl/scrape_mapper.py
@@ -4,23 +4,23 @@ import asyncio
import contextlib
import dataclasses
import logging
-import re
import time
from typing import TYPE_CHECKING, Any, Literal, Self
from pydantic.types import ByteSize
-from cyberdrop_dl import aio, env, filepath, storage
+from cyberdrop_dl import env, filepath, storage
from cyberdrop_dl.constants import BlockedDomains
from cyberdrop_dl.crawlers import ALLOW_NO_EXT, create_crawlers
from cyberdrop_dl.exceptions import JDownloaderError, NoExtensionError
from cyberdrop_dl.logs import log_spacer
from cyberdrop_dl.progress.scraping import ScrapingUI
+from cyberdrop_dl.scrape_source import ScrapeSource, URLSource
from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem, ScrapeItemType
from cyberdrop_dl.utils import remove_trailing_slash
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Coroutine, Generator, Iterable, Iterator, Sequence
+ from collections.abc import AsyncGenerator, Coroutine, Generator, Iterator, Sequence
from pathlib import Path
from cyberdrop_dl.clients.jdownloader import JDownloader
@@ -35,9 +35,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
-REGEX_LINKS = re.compile(r"(?:http.*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]))")
-
-
def _filter_by_domain(scrape_item: ScrapeItem, domain_list: Sequence[str]) -> bool:
return any(domain in scrape_item.url.host for domain in domain_list)
@@ -110,6 +107,7 @@ class ScrapeMapper:
_factory: CrawlerFactory = dataclasses.field(init=False)
tui: ScrapingUI = dataclasses.field(init=False, default_factory=ScrapingUI)
_done: asyncio.Event = dataclasses.field(init=False, default_factory=asyncio.Event)
+ _ready: bool = dataclasses.field(init=False, default=False)
def __repr__(self) -> str:
return f"<{type(self).__name__}(seen_url={len(self._seen_urls):,}, done={self._done!r})>"
@@ -199,7 +197,9 @@ class ScrapeMapper:
# The done event signals that all scraping is done, but there may still be downloads pending
self._done.set()
- async def run(self) -> ScrapeStats:
+ async def __async_init__(self) -> None:
+ if self._ready:
+ return
self._init_crawlers()
try:
await self._jdownloader.connect()
@@ -208,13 +208,16 @@ class ScrapeMapper:
await self._real_debrid.__async_init__()
await self._direct_http.__async_post_init__()
+ self._ready = True
- source_name, source = _source(self.manager)
- stats = ScrapeStats(source_name)
- if source is None:
- return stats
+ async def run(self, src: URLSource | None = None) -> ScrapeStats:
+ await self.__async_init__()
+ if src is None:
+ return ScrapeStats("")
- async with contextlib.aclosing(source) as items:
+ source = ScrapeSource(src)
+ stats = ScrapeStats(source.name)
+ async with source.items() as items:
async def wait_until_scrape_is_done() -> None:
_ = await self._done.wait()
@@ -341,72 +344,6 @@ class ScrapeMapper:
return crawler
-def _source(manager: Manager) -> tuple[str, AsyncGenerator[ScrapeItem] | None]:
- cli_args = manager.cli_args
- if cli_args.urls:
- return "--links (CLI args)", _load_cli_links(cli_args.urls)
-
- if manager.input_file:
- return str(manager.input_file), _load_urls_from_file(manager.input_file)
-
- return "", None
-
-
-async def _load_urls_from_file(file: Path) -> AsyncGenerator[ScrapeItem]:
- async for group_name, urls in _parse_input_file_groups(file):
- for url in urls:
- item = ScrapeItem.from_url(url)
- if group_name:
- item.append_folders(group_name)
- item.part_of_album = True
- yield item
-
-
-async def _parse_input_file_groups(input_file: Path) -> AsyncGenerator[tuple[str, list[AbsoluteHttpURL]]]:
- if not await aio.is_file(input_file):
- yield ("", [])
- return
-
- block_quote = False
- current_group_name = ""
- async with aio.open(input_file, encoding="utf8") as f:
- async for line in f:
- if line.startswith(("---", "===")): # New group begins here
- current_group_name = line.replace("---", "").replace("===", "").strip()
-
- if current_group_name:
- yield (current_group_name, list(_regex_links(line)))
- continue
-
- block_quote = not block_quote if line == "#\n" else block_quote
- if not block_quote:
- yield ("", list(_regex_links(line)))
-
-
-async def _load_cli_links(links: Iterable[AbsoluteHttpURL]) -> AsyncGenerator[ScrapeItem]:
- for url in links:
- yield ScrapeItem.from_url(url)
-
-
-def _regex_links(line: str) -> Generator[AbsoluteHttpURL]:
- """Regex grab the links from the URLs.txt file.
-
- This allows code blocks or full paragraphs to be copy and pasted into the URLs.txt.
- """
-
- line = line.strip()
- if line.startswith("#"):
- return
-
- http_urls = (url.group().replace(".md.", ".") for url in re.finditer(REGEX_LINKS, line))
- for link in http_urls:
- try:
- encoded = "%" in link
- yield AbsoluteHttpURL(link, encoded=encoded)
- except Exception as e: # noqa: BLE001
- logger.error(f"Unable to parse URL from input file: {link} {e:!r}")
-
-
def get_crawlers_mapping() -> dict[str, type[Crawler]]:
from cyberdrop_dl.crawlers import Registry
diff --git a/cyberdrop_dl/scrape_source.py b/cyberdrop_dl/scrape_source.py
new file mode 100644
index 000000000..58f81b8fb
--- /dev/null
+++ b/cyberdrop_dl/scrape_source.py
@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import contextlib
+import dataclasses
+import logging
+import re
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from cyberdrop_dl import aio
+from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator, Generator, Iterable
+
+
+type URLSource = Path | Iterable[AbsoluteHttpURL]
+REGEX_LINKS = re.compile(r"(?:http.*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]))")
+
+logger = logging.getLogger(__name__)
+
+
[email protected](slots=True)
+class ScrapeSource:
+ source: URLSource
+ name: str = ""
+
+ def __post_init__(self) -> None:
+ if not self.name:
+ self.name = str(self.source) if isinstance(self.source, Path) else "--links (CLI args)"
+
+ def _items(self) -> AsyncGenerator[ScrapeItem]:
+ if isinstance(self.source, Path):
+ return _load_urls_from_file(self.source)
+
+ return _load_cli_links(self.source)
+
+ def items(self) -> contextlib.aclosing[AsyncGenerator[ScrapeItem]]:
+ return contextlib.aclosing(self._items())
+
+
+async def _load_urls_from_file(file: Path) -> AsyncGenerator[ScrapeItem]:
+ async for group_name, urls in _parse_input_file_groups(file):
+ for url in urls:
+ item = ScrapeItem.from_url(url)
+ if group_name:
+ item.append_folders(group_name)
+ item.part_of_album = True
+ yield item
+
+
+async def _parse_input_file_groups(input_file: Path) -> AsyncGenerator[tuple[str, list[AbsoluteHttpURL]]]:
+ if not await aio.is_file(input_file):
+ yield ("", [])
+ return
+
+ block_quote = False
+ current_group_name = ""
+ async with aio.open(input_file, encoding="utf8") as f:
+ async for line in f:
+ if line.startswith(("---", "===")): # New group begins here
+ current_group_name = line.replace("---", "").replace("===", "").strip()
+
+ if current_group_name:
+ yield (current_group_name, list(_regex_links(line)))
+ continue
+
+ block_quote = not block_quote if line == "#\n" else block_quote
+ if not block_quote:
+ yield ("", list(_regex_links(line)))
+
+
+async def _load_cli_links(links: Iterable[AbsoluteHttpURL]) -> AsyncGenerator[ScrapeItem]:
+ for url in links:
+ yield ScrapeItem.from_url(url)
+
+
+def _regex_links(line: str) -> Generator[AbsoluteHttpURL]:
+ """Regex grab the links from the URLs.txt file.
+
+ This allows code blocks or full paragraphs to be copy and pasted into the URLs.txt.
+ """
+
+ line = line.strip()
+ if line.startswith("#"):
+ return
+
+ http_urls = (url.group().replace(".md.", ".") for url in re.finditer(REGEX_LINKS, line))
+ for link in http_urls:
+ try:
+ encoded = "%" in link
+ yield AbsoluteHttpURL(link, encoded=encoded)
+ except Exception as e: # noqa: BLE001
+ logger.error(f"Unable to parse URL from input file: {link} {e:!r}")