Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: accept none values for timedeltas

NTFSvolume0555b9f8db60
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 84c6d7baf..9026ddc8a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -57,6 +57,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `--input-file` is now a CLI only arg
- If supplied, `--input-file` needs to be a valid file that already exists
- Refuse to run if both URLs and `--input-file` are passed as arguments
+- Using number for any config option that expects a timedelta will be parsed as seconds instead of days
- Detect and report BasedFlare anti-bot protection
- Limit max queued downloads of a single site to the config concurrency limit x10 (capped at 50). All scraping for a site will be paused if its queue is full
- Always remove generated id from filenames (Cyberdrop)
diff --git a/cyberdrop_dl/clients/downloads.py b/cyberdrop_dl/clients/downloads.py
index b4b873852..29a181a6c 100644
--- a/cyberdrop_dl/clients/downloads.py
+++ b/cyberdrop_dl/clients/downloads.py
@@ -376,13 +376,13 @@ class DownloadClient:
assert media.size is not None
if media.ext in FileExt.IMAGE:
- return media.size in limits.image
+ return not limits.image or media.size in limits.image
if media.ext in FileExt.VIDEO:
- return media.size in limits.video
+ return not limits.video or media.size in limits.video
if media.ext in FileExt.AUDIO:
- return media.size in limits.audio
+ return not limits.image or media.size in limits.image
- return media.size in limits.non_media
+ return not limits.non_media or media.size in limits.non_media
def _check_content_type(content_type: str, ext: str) -> str | None:
diff --git a/cyberdrop_dl/clients/jdownloader.py b/cyberdrop_dl/clients/jdownloader.py
index 5a1712ecb..4880900aa 100644
--- a/cyberdrop_dl/clients/jdownloader.py
+++ b/cyberdrop_dl/clients/jdownloader.py
@@ -40,7 +40,7 @@ class JDownloader:
@classmethod
def from_config(cls, config: Config, /) -> Self:
- download_dir = config.jdownloader.download_dir or config.download_folder
+ download_dir = config.jdownloader.download_folder or config.download_folder
jd_config = JDConfig(
enabled=config.jdownloader.enabled,
device=config.auth.jdownloader.device,
diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yaml
index fa4f32119..b8e89c76a 100644
--- a/cyberdrop_dl/config/default.yaml
+++ b/cyberdrop_dl/config/default.yaml
@@ -64,11 +64,11 @@ filters:
before: null
duration:
audio:
- max: 0:00:00
- min: 0:00:00
+ max: null
+ min: null
video:
- max: 0:00:00
- min: 0:00:00
+ max: null
+ min: null
filename_regex: null
files:
audio: true
@@ -78,17 +78,17 @@ filters:
only_hosts: []
sizes:
audio:
- max: 0B
- min: 0B
+ max: null
+ min: null
image:
- max: 0B
- min: 0B
+ max: null
+ min: null
non_media:
- max: 0B
- min: 0B
+ max: null
+ min: null
video:
- max: 0B
- min: 0B
+ max: null
+ min: null
skip_hosts: []
hashing:
algorithms:
@@ -102,7 +102,7 @@ hashing:
ignore_history: false
jdownloader:
autostart: false
- download_dir: null
+ download_folder: null
enabled: false
whitelist: []
logs:
diff --git a/cyberdrop_dl/config/filters.py b/cyberdrop_dl/config/filters.py
index 1d163e463..f7b6a5585 100644
--- a/cyberdrop_dl/config/filters.py
+++ b/cyberdrop_dl/config/filters.py
@@ -1,12 +1,11 @@
from __future__ import annotations
import dataclasses
-import datetime
import re # noqa: TC003
from typing import TYPE_CHECKING, Self
from cyclopts import Parameter
-from pydantic import ByteSize, Field
+from pydantic import Field
from cyberdrop_dl.models import ConfigGroup, ConfigModel
from cyberdrop_dl.models.types import ( # noqa: TC001
@@ -17,6 +16,7 @@ from cyberdrop_dl.models.types import ( # noqa: TC001
)
if TYPE_CHECKING:
+ import datetime
from collections.abc import Callable
@@ -40,23 +40,23 @@ class _FloatRange:
return self.min <= value <= self.max
@classmethod
- def parse(cls, min: float, max: float | None) -> Self | None: # noqa: A002
+ def parse(cls, min: float | None, max: float | None) -> Self | None: # noqa: A002
if not min and not max:
return None
- return cls(min, max or float("inf"))
+ return cls(min or 0, max or float("inf"))
@dataclasses.dataclass(slots=True, frozen=True)
class _FileSizeRanges:
- video: _FloatRange
- image: _FloatRange
- audio: _FloatRange
- non_media: _FloatRange
+ video: _FloatRange | None
+ image: _FloatRange | None
+ audio: _FloatRange | None
+ non_media: _FloatRange | None
class _SizeLimit(ConfigModel):
- min: ByteSizeSerilized = ByteSize(0)
- max: ByteSizeSerilized = ByteSize(0)
+ min: ByteSizeSerilized | None = None
+ max: ByteSizeSerilized | None = None
@Parameter(name="*", name_transform=_limit_suffix("size"))
@@ -71,19 +71,19 @@ class _FileSizes(ConfigModel):
def ranges(self) -> _FileSizeRanges:
if self._ranges is None:
self._ranges = _FileSizeRanges(
- video=_FloatRange(
+ video=_FloatRange.parse(
self.video.min,
self.video.max,
),
- image=_FloatRange(
+ image=_FloatRange.parse(
self.image.min,
self.image.max,
),
- non_media=_FloatRange(
+ non_media=_FloatRange.parse(
self.non_media.min,
self.non_media.max,
),
- audio=_FloatRange(
+ audio=_FloatRange.parse(
self.audio.min,
self.audio.max,
),
@@ -98,8 +98,8 @@ class _DurationRanges:
class _DurationLimit(ConfigModel):
- min: Timedelta = datetime.timedelta(seconds=0)
- max: Timedelta = datetime.timedelta(seconds=0)
+ min: Timedelta | None = None
+ max: Timedelta | None = None
@Parameter(name="*", name_transform=_limit_suffix("duration"))
@@ -117,12 +117,12 @@ class _DurationLimits(ConfigModel):
if self._ranges is None:
self._ranges = _DurationRanges(
video=_FloatRange.parse(
- self.video.min.total_seconds(),
- self.video.max.total_seconds(),
+ self.video.min.total_seconds() if self.video.min else None,
+ self.video.max.total_seconds() if self.video.max else None,
),
audio=_FloatRange.parse(
- self.audio.min.total_seconds(),
- self.audio.max.total_seconds(),
+ self.audio.min.total_seconds() if self.audio.min else None,
+ self.audio.max.total_seconds() if self.audio.max else None,
),
)
return self._ranges
diff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.json
index 594118a6e..4da97a3d9 100644
--- a/cyberdrop_dl/config/schema.json
+++ b/cyberdrop_dl/config/schema.json
@@ -595,7 +595,7 @@
"title": "Autostart",
"type": "boolean"
},
- "download_dir": {
+ "download_folder": {
"$ref": "#/$defs/FalsyAsNone_Path_",
"default": null
},
@@ -943,12 +943,26 @@
"additionalProperties": false,
"properties": {
"min": {
- "$ref": "#/$defs/Timedelta",
- "default": "PT0S"
+ "anyOf": [
+ {
+ "$ref": "#/$defs/Timedelta"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null
},
"max": {
- "$ref": "#/$defs/Timedelta",
- "default": "PT0S"
+ "anyOf": [
+ {
+ "$ref": "#/$defs/Timedelta"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null
}
},
"title": "_DurationLimit",
@@ -1017,12 +1031,26 @@
"additionalProperties": false,
"properties": {
"min": {
- "$ref": "#/$defs/ByteSizeSerilized",
- "default": 0
+ "anyOf": [
+ {
+ "$ref": "#/$defs/ByteSizeSerilized"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null
},
"max": {
- "$ref": "#/$defs/ByteSizeSerilized",
- "default": 0
+ "anyOf": [
+ {
+ "$ref": "#/$defs/ByteSizeSerilized"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null
}
},
"title": "_SizeLimit",
diff --git a/cyberdrop_dl/config/settings.py b/cyberdrop_dl/config/settings.py
index 3c88e6ab5..96b4be368 100644
--- a/cyberdrop_dl/config/settings.py
+++ b/cyberdrop_dl/config/settings.py
@@ -148,7 +148,7 @@ class Jdownloader(ConfigGroup, name=None):
autostart: bool = False
"Immediately start downloads as soon as they are sent"
- download_dir: FalsyAsNone[Path] = None
+ download_folder: FalsyAsNone[Path] = None
"Output path for Jdownloader. Defaults to `--download-folder`"
whitelist: set[NonEmptyStr] = Field(default_factory=set)
diff --git a/cyberdrop_dl/models/validators/__init__.py b/cyberdrop_dl/models/validators/__init__.py
index 48e8bbe58..4beca53ed 100644
--- a/cyberdrop_dl/models/validators/__init__.py
+++ b/cyberdrop_dl/models/validators/__init__.py
@@ -86,7 +86,7 @@ def to_timedelta(input_date: datetime.timedelta | str | int | None) -> datetime.
if isinstance(input_date, datetime.timedelta):
return input_date
if isinstance(input_date, int):
- return datetime.timedelta(days=input_date)
+ return datetime.timedelta(seconds=input_date)
try:
return _str_to_timedelta(input_date)
except Exception: # noqa: BLE001