Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add `--ignore-hashes` option

NTFSvolume6e6e858236cf
diff --git a/cyberdrop_dl/config/__init__.py b/cyberdrop_dl/config/__init__.py
index 3825ebef9..8bd0d3905 100644
--- a/cyberdrop_dl/config/__init__.py
+++ b/cyberdrop_dl/config/__init__.py
@@ -75,6 +75,9 @@ class Config(ConfigModel, title="cyberdrop-dl config"):
ignore_history: bool = False
"Download files even if the already are marked as downloaded on the database"
+ ignore_hashes: bool = False
+ "Download files even if their hash matches a file downloaded on the database"
+
jdownloader: Jdownloader = Field(default_factory=Jdownloader)
logs: Logs = Field(default_factory=Logs)
max_children: MaxChildren = Field(default_factory=MaxChildren)
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 2ff87ae9e..de689e6ab 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -653,6 +653,8 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
self: Crawler, url: AbsoluteHttpURL, hash_algo: Literal["md5", "sha256"], checksum: str
) -> bool:
"""Returns `True` if at least 1 file with this hash is recorded on the database"""
+ if self.config.ignore_hashes:
+ return False
expected_len = 32 if hash_algo == "md5" else 64
if len(checksum) != expected_len:
diff --git a/docs/reference/config/general.md b/docs/reference/config/general.md
index 974177062..b5fbdf6fd 100644
--- a/docs/reference/config/general.md
+++ b/docs/reference/config/general.md
@@ -151,6 +151,22 @@ Set `true` will delete any `.part` and `.cdl_hls` files in the download folder a
delete_partial_files: false
```
+# `ignore_hashes`
+
+| Type | Default |
+| ------ | ------- |
+| `bool` | `false` |
+
+Download files even if their hash matches a file downloaded on the database.
+
+{% hint style="info" %}
+Not all file hosts provide hash information before download
+{% endhint %}
+
+```yaml
+ignore_hashes: false
+```
+
# `ignore_history`
| Type | Default |