Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: allow a custom scrape source
NTFSvolumecaa38545aa81
diff --git a/cyberdrop_dl/cli.py b/cyberdrop_dl/cli.pyindex 7958d7969..fbf6a1692 100644--- a/cyberdrop_dl/cli.py+++ b/cyberdrop_dl/cli.py@@ -37,14 +37,15 @@ def main_menu(show_default=False,),] = Path("URLs.txt"), # pyright: ignore[reportCallInDefaultInitializer]- cli: CLIarguments | None = None,+ cli_args: CLIarguments | None = None,) -> None:"Show a TUI menu equivalent to the CLI commands"- with prepare_manager((), input_file, cli, cli_overrides=None)() as manager:+ input_file = input_file.resolve().absolute()+ with prepare_manager(cli_args, cli_overrides=None)() as manager:from cyberdrop_dl import program_ui- program_ui.run(manager)- scrape(manager)+ program_ui.run(manager, input_file)+ scrape(manager, input_file)@app.commanddiff --git a/cyberdrop_dl/commands/__init__.py b/cyberdrop_dl/commands/__init__.pyindex e641cc4f7..cebc2cfca 100644--- a/cyberdrop_dl/commands/__init__.py+++ b/cyberdrop_dl/commands/__init__.py@@ -5,8 +5,6 @@ from typing import Annotated, Anyfrom cyclopts import Parameter, Token, validators-from cyberdrop_dl.models.types import HttpURL-def _resolve_path(type_: type[Path], tokens: Sequence[Token]) -> Path:assert len(tokens) == 1@@ -25,8 +23,6 @@ type SQLiteFile = Annotated[Path, file_validator(".db")]@Parameter(name="*")@dataclasses.dataclass(slots=True)class CLIarguments:- urls: Annotated[tuple[HttpURL, ...], Parameter(show=False)] = ()-config_file: YAMLFile | None = None"YAML file to use as config"@@ -37,6 +33,4 @@ class CLIarguments:"SQLite file to use as database"def __json__(self) -> dict[str, Any]:- me: dict[str, Any] = {k: None if v is None else str(v) for k, v in dataclasses.asdict(self).items()}- me["urls"] = tuple(map(str, self.urls))- return me+ return {k: None if v is None else str(v) for k, v in dataclasses.asdict(self).items()}diff --git a/cyberdrop_dl/commands/scrape.py b/cyberdrop_dl/commands/scrape.pyindex 28e9422bf..97fd84c70 100644--- a/cyberdrop_dl/commands/scrape.py+++ b/cyberdrop_dl/commands/scrape.py@@ -16,6 +16,7 @@ from cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroupfrom cyberdrop_dl.logs import log_spacer, set_console_level, setup_file_loggingfrom cyberdrop_dl.models import merge_modelsfrom cyberdrop_dl.models.types import HttpURL # noqa: TC001+from cyberdrop_dl.scrape_source import URLSourcefrom cyberdrop_dl.utils import cleanuplogger = logging.getLogger(__name__)@@ -24,9 +25,17 @@ if TYPE_CHECKING:from cyclopts.argument import ArgumentCollectionfrom cyberdrop_dl.manager import Manager+ from cyberdrop_dl.scrape_source import URLSource-async def _scrape(manager: Manager) -> None:+def scrape(manager: Manager, source: URLSource) -> None:+ from cyberdrop_dl import aio++ with setup_file_logging(manager.config.logs.files.main, level=manager.config.logs.effective_level):+ aio.run(_scrape(manager, source))+++async def _scrape(manager: Manager, source: URLSource) -> None:from cyberdrop_dl import ffmpegfrom cyberdrop_dl.scrape_mapper import ScrapeMapper@@ -39,7 +48,7 @@ async def _scrape(manager: Manager) -> None:log_spacer()logger.info("Starting CDL...")async with ScrapeMapper(manager)() as scrape_mapper:- stats = await scrape_mapper.run()+ stats = await scrape_mapper.run(source)await _post_runtime(manager)stats_summary = manager.print_stats(stats)@@ -89,13 +98,6 @@ async def _notify(config: Config, stats_summary: str) -> None:await apprise.notify(urls, stats_summary)-def scrape(manager: Manager) -> None:- from cyberdrop_dl import aio-- with setup_file_logging(manager.config.logs.files.main, level=manager.config.logs.effective_level):- aio.run(_scrape(manager))--def _validate_inputs(args: ArgumentCollection) -> None:try:cyclopts.validators.LimitedChoice(min=1, max=1)(args)@@ -128,27 +130,24 @@ def download(validator=cyclopts.validators.Path(exists=True, dir_okay=False),),] = None,- cli: CLIarguments | None = None,+ cli_args: CLIarguments | None = None,cli_overrides: Config | None = None,) -> None:"Download URLs"- with prepare_manager(urls, input_file, cli, cli_overrides)() as manager:- scrape(manager)+ if input_file:+ input_file = input_file.resolve().absolute()+ with prepare_manager(cli_args, cli_overrides)() as manager:+ scrape(manager, input_file or urls)-def prepare_manager(- urls: tuple[HttpURL, ...], input_file: Path | None, cli: CLIarguments | None, cli_overrides: Config | None-) -> Manager:+def prepare_manager(cli_args: CLIarguments | None, cli_overrides: Config | None) -> Manager:check_for_v9_files()- if input_file:- input_file = input_file.resolve().absolute()from cyberdrop_dl.manager import Manager- cli = cli or CLIarguments()- cli.urls = urls- appdata, config = _prepare_appdata_and_config(cli, cli_overrides)- manager = Manager(cli, appdata, config, input_file)+ cli_args = cli_args or CLIarguments()+ appdata, config = _prepare_appdata_and_config(cli_args, cli_overrides)+ manager = Manager(cli_args, appdata, config)set_console_level(manager.config.logs.effective_console_level)manager.appdata.mkdirs()return managerdiff --git a/cyberdrop_dl/manager.py b/cyberdrop_dl/manager.pyindex 83592c523..2244f797f 100644--- a/cyberdrop_dl/manager.py+++ b/cyberdrop_dl/manager.py@@ -44,7 +44,6 @@ class Manager:cli_args: CLIarguments | None = None,appdata: AppData | None = None,config: Config | None = None,- input_file: Path | None = None,) -> None:from cyberdrop_dl.commands import CLIarguments@@ -52,7 +51,6 @@ class Manager:self._appdata: AppData | None = appdataself.cli_args: CLIarguments = cli_args or CLIarguments()self._config: Config | None = config- self.input_file: Path | None = input_fileself._completed_downloads: list[MediaItem] = []self._hasher: Hasher | None = Nonediff --git a/cyberdrop_dl/program_ui.py b/cyberdrop_dl/program_ui.pyindex a68104321..1c42e8017 100644--- a/cyberdrop_dl/program_ui.py+++ b/cyberdrop_dl/program_ui.py@@ -1,6 +1,7 @@from __future__ import annotationsimport sys+from contextvars import ContextVarfrom typing import TYPE_CHECKINGimport aiohttp@@ -22,10 +23,11 @@ from cyberdrop_dl.utils import text_editorif TYPE_CHECKING:from collections.abc import Callable+ from pathlib import Pathfrom cyberdrop_dl.manager import Manager-+_INPUT_FILE: ContextVar[Path] = ContextVar("_INPUT_FILE")_CHANGELOG_URL = "https://raw.githubusercontent.com/Cyberdrop-DL/cyberdrop-dl/refs/heads/main/CHANGELOG.md"_changelog_content: str = ""@@ -38,12 +40,13 @@ def _changelog() -> str:return _changelog_content-def run(manager: Manager) -> None:+def run(manager: Manager, input_file: Path) -> None:+ _INPUT_FILE.set(input_file)choices: dict[str, Callable[[Manager], bool | None]] = {"Download": lambda _: True,"Create file hashes": _scan_and_create_hashes,"Sort files in download folder": _sort_files,- "Edit URLs.txt": _edit_urls,+ "Edit URLs.txt": lambda _: _edit_urls(),"Edit config": _edit_config,"View changelog": lambda _: _view_changelog(),"Exit": lambda _: sys.exit(0),@@ -93,10 +96,9 @@ def _edit_config(manager: Manager) -> None:enter_to_continue()-def _edit_urls(manager: Manager) -> None:- assert manager.input_file+def _edit_urls() -> None:try:- text_editor.open(manager.input_file)+ text_editor.open(_INPUT_FILE.get())except ValueError as e:console.error(e)@@ -132,7 +134,7 @@ def _app_header(manager: Manager) -> None:paths = {"Config file": manager.config.source,"Database file": manager.appdata.db_file,- "URLs file": manager.input_file,+ "URLs file": _INPUT_FILE.get(),"Cache file": manager.appdata.cache_file,"Logs": manager.config.logs.effective_log_folder,"Main log file": manager.config.logs.files.main,diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.pyindex 88825895c..696f9b773 100644--- a/cyberdrop_dl/scrape_mapper.py+++ b/cyberdrop_dl/scrape_mapper.py@@ -4,23 +4,23 @@ import asyncioimport contextlibimport dataclassesimport logging-import reimport timefrom typing import TYPE_CHECKING, Any, Literal, Selffrom pydantic.types import ByteSize-from cyberdrop_dl import aio, env, filepath, storage+from cyberdrop_dl import env, filepath, storagefrom cyberdrop_dl.constants import BlockedDomainsfrom cyberdrop_dl.crawlers import ALLOW_NO_EXT, create_crawlersfrom cyberdrop_dl.exceptions import JDownloaderError, NoExtensionErrorfrom cyberdrop_dl.logs import log_spacerfrom cyberdrop_dl.progress.scraping import ScrapingUI+from cyberdrop_dl.scrape_source import ScrapeSource, URLSourcefrom cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem, ScrapeItemTypefrom cyberdrop_dl.utils import remove_trailing_slashif TYPE_CHECKING:- from collections.abc import AsyncGenerator, Coroutine, Generator, Iterable, Iterator, Sequence+ from collections.abc import AsyncGenerator, Coroutine, Generator, Iterator, Sequencefrom pathlib import Pathfrom cyberdrop_dl.clients.jdownloader import JDownloader@@ -35,9 +35,6 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)-REGEX_LINKS = re.compile(r"(?:http.*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]))")--def _filter_by_domain(scrape_item: ScrapeItem, domain_list: Sequence[str]) -> bool:return any(domain in scrape_item.url.host for domain in domain_list)@@ -110,6 +107,7 @@ class ScrapeMapper:_factory: CrawlerFactory = dataclasses.field(init=False)tui: ScrapingUI = dataclasses.field(init=False, default_factory=ScrapingUI)_done: asyncio.Event = dataclasses.field(init=False, default_factory=asyncio.Event)+ _ready: bool = dataclasses.field(init=False, default=False)def __repr__(self) -> str:return f"<{type(self).__name__}(seen_url={len(self._seen_urls):,}, done={self._done!r})>"@@ -199,7 +197,9 @@ class ScrapeMapper:# The done event signals that all scraping is done, but there may still be downloads pendingself._done.set()- async def run(self) -> ScrapeStats:+ async def __async_init__(self) -> None:+ if self._ready:+ returnself._init_crawlers()try:await self._jdownloader.connect()@@ -208,13 +208,16 @@ class ScrapeMapper:await self._real_debrid.__async_init__()await self._direct_http.__async_post_init__()+ self._ready = True- source_name, source = _source(self.manager)- stats = ScrapeStats(source_name)- if source is None:- return stats+ async def run(self, src: URLSource | None = None) -> ScrapeStats:+ await self.__async_init__()+ if src is None:+ return ScrapeStats("")- async with contextlib.aclosing(source) as items:+ source = ScrapeSource(src)+ stats = ScrapeStats(source.name)+ async with source.items() as items:async def wait_until_scrape_is_done() -> None:_ = await self._done.wait()@@ -341,72 +344,6 @@ class ScrapeMapper:return crawler-def _source(manager: Manager) -> tuple[str, AsyncGenerator[ScrapeItem] | None]:- cli_args = manager.cli_args- if cli_args.urls:- return "--links (CLI args)", _load_cli_links(cli_args.urls)-- if manager.input_file:- return str(manager.input_file), _load_urls_from_file(manager.input_file)-- return "", None---async def _load_urls_from_file(file: Path) -> AsyncGenerator[ScrapeItem]:- async for group_name, urls in _parse_input_file_groups(file):- for url in urls:- item = ScrapeItem.from_url(url)- if group_name:- item.append_folders(group_name)- item.part_of_album = True- yield item---async def _parse_input_file_groups(input_file: Path) -> AsyncGenerator[tuple[str, list[AbsoluteHttpURL]]]:- if not await aio.is_file(input_file):- yield ("", [])- return-- block_quote = False- current_group_name = ""- async with aio.open(input_file, encoding="utf8") as f:- async for line in f:- if line.startswith(("---", "===")): # New group begins here- current_group_name = line.replace("---", "").replace("===", "").strip()-- if current_group_name:- yield (current_group_name, list(_regex_links(line)))- continue-- block_quote = not block_quote if line == "#\n" else block_quote- if not block_quote:- yield ("", list(_regex_links(line)))---async def _load_cli_links(links: Iterable[AbsoluteHttpURL]) -> AsyncGenerator[ScrapeItem]:- for url in links:- yield ScrapeItem.from_url(url)---def _regex_links(line: str) -> Generator[AbsoluteHttpURL]:- """Regex grab the links from the URLs.txt file.-- This allows code blocks or full paragraphs to be copy and pasted into the URLs.txt.- """-- line = line.strip()- if line.startswith("#"):- return-- http_urls = (url.group().replace(".md.", ".") for url in re.finditer(REGEX_LINKS, line))- for link in http_urls:- try:- encoded = "%" in link- yield AbsoluteHttpURL(link, encoded=encoded)- except Exception as e: # noqa: BLE001- logger.error(f"Unable to parse URL from input file: {link} {e:!r}")--def get_crawlers_mapping() -> dict[str, type[Crawler]]:from cyberdrop_dl.crawlers import Registrydiff --git a/cyberdrop_dl/scrape_source.py b/cyberdrop_dl/scrape_source.pynew file mode 100644index 000000000..58f81b8fb--- /dev/null+++ b/cyberdrop_dl/scrape_source.py@@ -0,0 +1,94 @@+from __future__ import annotations++import contextlib+import dataclasses+import logging+import re+from pathlib import Path+from typing import TYPE_CHECKING++from cyberdrop_dl import aio+from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem++if TYPE_CHECKING:+ from collections.abc import AsyncGenerator, Generator, Iterable+++type URLSource = Path | Iterable[AbsoluteHttpURL]+REGEX_LINKS = re.compile(r"(?:http.*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]))")++logger = logging.getLogger(__name__)++[email protected](slots=True)+class ScrapeSource:+ source: URLSource+ name: str = ""++ def __post_init__(self) -> None:+ if not self.name:+ self.name = str(self.source) if isinstance(self.source, Path) else "--links (CLI args)"++ def _items(self) -> AsyncGenerator[ScrapeItem]:+ if isinstance(self.source, Path):+ return _load_urls_from_file(self.source)++ return _load_cli_links(self.source)++ def items(self) -> contextlib.aclosing[AsyncGenerator[ScrapeItem]]:+ return contextlib.aclosing(self._items())+++async def _load_urls_from_file(file: Path) -> AsyncGenerator[ScrapeItem]:+ async for group_name, urls in _parse_input_file_groups(file):+ for url in urls:+ item = ScrapeItem.from_url(url)+ if group_name:+ item.append_folders(group_name)+ item.part_of_album = True+ yield item+++async def _parse_input_file_groups(input_file: Path) -> AsyncGenerator[tuple[str, list[AbsoluteHttpURL]]]:+ if not await aio.is_file(input_file):+ yield ("", [])+ return++ block_quote = False+ current_group_name = ""+ async with aio.open(input_file, encoding="utf8") as f:+ async for line in f:+ if line.startswith(("---", "===")): # New group begins here+ current_group_name = line.replace("---", "").replace("===", "").strip()++ if current_group_name:+ yield (current_group_name, list(_regex_links(line)))+ continue++ block_quote = not block_quote if line == "#\n" else block_quote+ if not block_quote:+ yield ("", list(_regex_links(line)))+++async def _load_cli_links(links: Iterable[AbsoluteHttpURL]) -> AsyncGenerator[ScrapeItem]:+ for url in links:+ yield ScrapeItem.from_url(url)+++def _regex_links(line: str) -> Generator[AbsoluteHttpURL]:+ """Regex grab the links from the URLs.txt file.++ This allows code blocks or full paragraphs to be copy and pasted into the URLs.txt.+ """++ line = line.strip()+ if line.startswith("#"):+ return++ http_urls = (url.group().replace(".md.", ".") for url in re.finditer(REGEX_LINKS, line))+ for link in http_urls:+ try:+ encoded = "%" in link+ yield AbsoluteHttpURL(link, encoded=encoded)+ except Exception as e: # noqa: BLE001+ logger.error(f"Unable to parse URL from input file: {link} {e:!r}")