Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: accept none values for timedeltas
NTFSvolume0555b9f8db60
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 84c6d7baf..9026ddc8a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -57,6 +57,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- `--input-file` is now a CLI only arg- If supplied, `--input-file` needs to be a valid file that already exists- Refuse to run if both URLs and `--input-file` are passed as arguments+- Using number for any config option that expects a timedelta will be parsed as seconds instead of days- Detect and report BasedFlare anti-bot protection- Limit max queued downloads of a single site to the config concurrency limit x10 (capped at 50). All scraping for a site will be paused if its queue is full- Always remove generated id from filenames (Cyberdrop)diff --git a/cyberdrop_dl/clients/downloads.py b/cyberdrop_dl/clients/downloads.pyindex b4b873852..29a181a6c 100644--- a/cyberdrop_dl/clients/downloads.py+++ b/cyberdrop_dl/clients/downloads.py@@ -376,13 +376,13 @@ class DownloadClient:assert media.size is not Noneif media.ext in FileExt.IMAGE:- return media.size in limits.image+ return not limits.image or media.size in limits.imageif media.ext in FileExt.VIDEO:- return media.size in limits.video+ return not limits.video or media.size in limits.videoif media.ext in FileExt.AUDIO:- return media.size in limits.audio+ return not limits.image or media.size in limits.image- return media.size in limits.non_media+ return not limits.non_media or media.size in limits.non_mediadef _check_content_type(content_type: str, ext: str) -> str | None:diff --git a/cyberdrop_dl/clients/jdownloader.py b/cyberdrop_dl/clients/jdownloader.pyindex 5a1712ecb..4880900aa 100644--- a/cyberdrop_dl/clients/jdownloader.py+++ b/cyberdrop_dl/clients/jdownloader.py@@ -40,7 +40,7 @@ class JDownloader:@classmethoddef from_config(cls, config: Config, /) -> Self:- download_dir = config.jdownloader.download_dir or config.download_folder+ download_dir = config.jdownloader.download_folder or config.download_folderjd_config = JDConfig(enabled=config.jdownloader.enabled,device=config.auth.jdownloader.device,diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yamlindex fa4f32119..b8e89c76a 100644--- a/cyberdrop_dl/config/default.yaml+++ b/cyberdrop_dl/config/default.yaml@@ -64,11 +64,11 @@ filters:before: nullduration:audio:- max: 0:00:00- min: 0:00:00+ max: null+ min: nullvideo:- max: 0:00:00- min: 0:00:00+ max: null+ min: nullfilename_regex: nullfiles:audio: true@@ -78,17 +78,17 @@ filters:only_hosts: []sizes:audio:- max: 0B- min: 0B+ max: null+ min: nullimage:- max: 0B- min: 0B+ max: null+ min: nullnon_media:- max: 0B- min: 0B+ max: null+ min: nullvideo:- max: 0B- min: 0B+ max: null+ min: nullskip_hosts: []hashing:algorithms:@@ -102,7 +102,7 @@ hashing:ignore_history: falsejdownloader:autostart: false- download_dir: null+ download_folder: nullenabled: falsewhitelist: []logs:diff --git a/cyberdrop_dl/config/filters.py b/cyberdrop_dl/config/filters.pyindex 1d163e463..f7b6a5585 100644--- a/cyberdrop_dl/config/filters.py+++ b/cyberdrop_dl/config/filters.py@@ -1,12 +1,11 @@from __future__ import annotationsimport dataclasses-import datetimeimport re # noqa: TC003from typing import TYPE_CHECKING, Selffrom cyclopts import Parameter-from pydantic import ByteSize, Field+from pydantic import Fieldfrom cyberdrop_dl.models import ConfigGroup, ConfigModelfrom cyberdrop_dl.models.types import ( # noqa: TC001@@ -17,6 +16,7 @@ from cyberdrop_dl.models.types import ( # noqa: TC001)if TYPE_CHECKING:+ import datetimefrom collections.abc import Callable@@ -40,23 +40,23 @@ class _FloatRange:return self.min <= value <= self.max@classmethod- def parse(cls, min: float, max: float | None) -> Self | None: # noqa: A002+ def parse(cls, min: float | None, max: float | None) -> Self | None: # noqa: A002if not min and not max:return None- return cls(min, max or float("inf"))+ return cls(min or 0, max or float("inf"))@dataclasses.dataclass(slots=True, frozen=True)class _FileSizeRanges:- video: _FloatRange- image: _FloatRange- audio: _FloatRange- non_media: _FloatRange+ video: _FloatRange | None+ image: _FloatRange | None+ audio: _FloatRange | None+ non_media: _FloatRange | Noneclass _SizeLimit(ConfigModel):- min: ByteSizeSerilized = ByteSize(0)- max: ByteSizeSerilized = ByteSize(0)+ min: ByteSizeSerilized | None = None+ max: ByteSizeSerilized | None = None@Parameter(name="*", name_transform=_limit_suffix("size"))@@ -71,19 +71,19 @@ class _FileSizes(ConfigModel):def ranges(self) -> _FileSizeRanges:if self._ranges is None:self._ranges = _FileSizeRanges(- video=_FloatRange(+ video=_FloatRange.parse(self.video.min,self.video.max,),- image=_FloatRange(+ image=_FloatRange.parse(self.image.min,self.image.max,),- non_media=_FloatRange(+ non_media=_FloatRange.parse(self.non_media.min,self.non_media.max,),- audio=_FloatRange(+ audio=_FloatRange.parse(self.audio.min,self.audio.max,),@@ -98,8 +98,8 @@ class _DurationRanges:class _DurationLimit(ConfigModel):- min: Timedelta = datetime.timedelta(seconds=0)- max: Timedelta = datetime.timedelta(seconds=0)+ min: Timedelta | None = None+ max: Timedelta | None = None@Parameter(name="*", name_transform=_limit_suffix("duration"))@@ -117,12 +117,12 @@ class _DurationLimits(ConfigModel):if self._ranges is None:self._ranges = _DurationRanges(video=_FloatRange.parse(- self.video.min.total_seconds(),- self.video.max.total_seconds(),+ self.video.min.total_seconds() if self.video.min else None,+ self.video.max.total_seconds() if self.video.max else None,),audio=_FloatRange.parse(- self.audio.min.total_seconds(),- self.audio.max.total_seconds(),+ self.audio.min.total_seconds() if self.audio.min else None,+ self.audio.max.total_seconds() if self.audio.max else None,),)return self._rangesdiff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.jsonindex 594118a6e..4da97a3d9 100644--- a/cyberdrop_dl/config/schema.json+++ b/cyberdrop_dl/config/schema.json@@ -595,7 +595,7 @@"title": "Autostart","type": "boolean"},- "download_dir": {+ "download_folder": {"$ref": "#/$defs/FalsyAsNone_Path_","default": null},@@ -943,12 +943,26 @@"additionalProperties": false,"properties": {"min": {- "$ref": "#/$defs/Timedelta",- "default": "PT0S"+ "anyOf": [+ {+ "$ref": "#/$defs/Timedelta"+ },+ {+ "type": "null"+ }+ ],+ "default": null},"max": {- "$ref": "#/$defs/Timedelta",- "default": "PT0S"+ "anyOf": [+ {+ "$ref": "#/$defs/Timedelta"+ },+ {+ "type": "null"+ }+ ],+ "default": null}},"title": "_DurationLimit",@@ -1017,12 +1031,26 @@"additionalProperties": false,"properties": {"min": {- "$ref": "#/$defs/ByteSizeSerilized",- "default": 0+ "anyOf": [+ {+ "$ref": "#/$defs/ByteSizeSerilized"+ },+ {+ "type": "null"+ }+ ],+ "default": null},"max": {- "$ref": "#/$defs/ByteSizeSerilized",- "default": 0+ "anyOf": [+ {+ "$ref": "#/$defs/ByteSizeSerilized"+ },+ {+ "type": "null"+ }+ ],+ "default": null}},"title": "_SizeLimit",diff --git a/cyberdrop_dl/config/settings.py b/cyberdrop_dl/config/settings.pyindex 3c88e6ab5..96b4be368 100644--- a/cyberdrop_dl/config/settings.py+++ b/cyberdrop_dl/config/settings.py@@ -148,7 +148,7 @@ class Jdownloader(ConfigGroup, name=None):autostart: bool = False"Immediately start downloads as soon as they are sent"- download_dir: FalsyAsNone[Path] = None+ download_folder: FalsyAsNone[Path] = None"Output path for Jdownloader. Defaults to `--download-folder`"whitelist: set[NonEmptyStr] = Field(default_factory=set)diff --git a/cyberdrop_dl/models/validators/__init__.py b/cyberdrop_dl/models/validators/__init__.pyindex 48e8bbe58..4beca53ed 100644--- a/cyberdrop_dl/models/validators/__init__.py+++ b/cyberdrop_dl/models/validators/__init__.py@@ -86,7 +86,7 @@ def to_timedelta(input_date: datetime.timedelta | str | int | None) -> datetime.if isinstance(input_date, datetime.timedelta):return input_dateif isinstance(input_date, int):- return datetime.timedelta(days=input_date)+ return datetime.timedelta(seconds=input_date)try:return _str_to_timedelta(input_date)except Exception: # noqa: BLE001