Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add `--ignore-hashes` option
NTFSvolume6e6e858236cf
diff --git a/cyberdrop_dl/config/__init__.py b/cyberdrop_dl/config/__init__.pyindex 3825ebef9..8bd0d3905 100644--- a/cyberdrop_dl/config/__init__.py+++ b/cyberdrop_dl/config/__init__.py@@ -75,6 +75,9 @@ class Config(ConfigModel, title="cyberdrop-dl config"):ignore_history: bool = False"Download files even if the already are marked as downloaded on the database"+ ignore_hashes: bool = False+ "Download files even if their hash matches a file downloaded on the database"+jdownloader: Jdownloader = Field(default_factory=Jdownloader)logs: Logs = Field(default_factory=Logs)max_children: MaxChildren = Field(default_factory=MaxChildren)diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 2ff87ae9e..de689e6ab 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -653,6 +653,8 @@ class Crawler(HTTPMixin, HLSMixin, ABC):self: Crawler, url: AbsoluteHttpURL, hash_algo: Literal["md5", "sha256"], checksum: str) -> bool:"""Returns `True` if at least 1 file with this hash is recorded on the database"""+ if self.config.ignore_hashes:+ return Falseexpected_len = 32 if hash_algo == "md5" else 64if len(checksum) != expected_len:diff --git a/docs/reference/config/general.md b/docs/reference/config/general.mdindex 974177062..b5fbdf6fd 100644--- a/docs/reference/config/general.md+++ b/docs/reference/config/general.md@@ -151,6 +151,22 @@ Set `true` will delete any `.part` and `.cdl_hls` files in the download folder adelete_partial_files: false```+# `ignore_hashes`++| Type | Default |+| ------ | ------- |+| `bool` | `false` |++Download files even if their hash matches a file downloaded on the database.++{% hint style="info" %}+Not all file hosts provide hash information before download+{% endhint %}++```yaml+ignore_hashes: false+```+# `ignore_history`| Type | Default |