Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: do not re-export dataclass utils
NTFSvolume2cef26cb2dc5
diff --git a/cyberdrop_dl/crawlers/_chibisafe.py b/cyberdrop_dl/crawlers/_chibisafe.pyindex 355e3cbb1..21086de06 100644--- a/cyberdrop_dl/crawlers/_chibisafe.py+++ b/cyberdrop_dl/crawlers/_chibisafe.py@@ -17,7 +17,8 @@ from pydantic import Fieldfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.models import DeferedModel-from cyberdrop_dl.utils import DictDataclass, error_handling_wrapper+from cyberdrop_dl.utils import error_handling_wrapper+from cyberdrop_dl.utils.dataclass import DictDataclassfrom cyberdrop_dl.utils.dates import to_timestampif TYPE_CHECKING:diff --git a/cyberdrop_dl/crawlers/acast.py b/cyberdrop_dl/crawlers/acast.pyindex 45c9e390f..fcfe87807 100644--- a/cyberdrop_dl/crawlers/acast.py+++ b/cyberdrop_dl/crawlers/acast.py@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItemfrom cyberdrop_dl.utils import error_handling_wrapper-from cyberdrop_dl.utils._dataclasses import DictDataclass+from cyberdrop_dl.utils.dataclass import DictDataclassif TYPE_CHECKING:from collections.abc import Iterablediff --git a/cyberdrop_dl/crawlers/archive_org.py b/cyberdrop_dl/crawlers/archive_org.pyindex 668b3da82..6edf951e8 100644--- a/cyberdrop_dl/crawlers/archive_org.py+++ b/cyberdrop_dl/crawlers/archive_org.py@@ -12,7 +12,8 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import API, Crawler, RateLimit, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import DictDataclass, error_handling_wrapper+from cyberdrop_dl.utils import error_handling_wrapper+from cyberdrop_dl.utils.dataclass import DictDataclassif TYPE_CHECKING:from collections.abc import Generator, Iterablediff --git a/cyberdrop_dl/crawlers/beeg.py b/cyberdrop_dl/crawlers/beeg.pyindex 8ed265ac3..5e5b56712 100644--- a/cyberdrop_dl/crawlers/beeg.py+++ b/cyberdrop_dl/crawlers/beeg.py@@ -6,7 +6,8 @@ from typing import TYPE_CHECKING, Any, ClassVar, Literalfrom cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import DictDataclass, error_handling_wrapper+from cyberdrop_dl.utils import error_handling_wrapper+from cyberdrop_dl.utils.dataclass import DictDataclassif TYPE_CHECKING:from collections.abc import Generatordiff --git a/cyberdrop_dl/crawlers/dropbox.py b/cyberdrop_dl/crawlers/dropbox.pyindex 40f984536..e2e86eddf 100644--- a/cyberdrop_dl/crawlers/dropbox.py+++ b/cyberdrop_dl/crawlers/dropbox.py@@ -6,7 +6,8 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import LoginError, ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import DictDataclass, error_handling_wrapper+from cyberdrop_dl.utils import error_handling_wrapper+from cyberdrop_dl.utils.dataclass import DictDataclassif TYPE_CHECKING:from collections.abc import AsyncGeneratordiff --git a/cyberdrop_dl/crawlers/hitomi_la.py b/cyberdrop_dl/crawlers/hitomi_la.pyindex 2eb467ea0..148a3e436 100644--- a/cyberdrop_dl/crawlers/hitomi_la.py+++ b/cyberdrop_dl/crawlers/hitomi_la.py@@ -15,7 +15,8 @@ from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.filepath import get_filename_and_extfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import DictDataclass, error_handling_wrapper+from cyberdrop_dl.utils import error_handling_wrapper+from cyberdrop_dl.utils.dataclass import DictDataclassif TYPE_CHECKING:from collections.abc import AsyncGenerator, Generator, Iterable, Mappingdiff --git a/cyberdrop_dl/crawlers/livestreamfails.py b/cyberdrop_dl/crawlers/livestreamfails.pyindex b98781a04..38a14b55a 100644--- a/cyberdrop_dl/crawlers/livestreamfails.py+++ b/cyberdrop_dl/crawlers/livestreamfails.py@@ -5,7 +5,8 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import API, Crawler, RateLimit, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import deserialize, error_handling_wrapper+from cyberdrop_dl.utils import error_handling_wrapper+from cyberdrop_dl.utils.dataclass import deserializeif TYPE_CHECKING:from collections.abc import AsyncGeneratordiff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.pyindex 2d6e158b0..aedb39f2e 100644--- a/cyberdrop_dl/crawlers/mediafire.py+++ b/cyberdrop_dl/crawlers/mediafire.py@@ -11,7 +11,8 @@ from typing import TYPE_CHECKING, Any, ClassVar, Literalfrom cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, auto_task_idfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import DictDataclass, css, error_handling_wrapper, is_blob_or_svg+from cyberdrop_dl.utils import css, error_handling_wrapper, is_blob_or_svg+from cyberdrop_dl.utils.dataclass import DictDataclassif TYPE_CHECKING:from collections.abc import AsyncGeneratordiff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.pyindex d31f0287f..8dfd5097c 100644--- a/cyberdrop_dl/crawlers/pmvhaven.py+++ b/cyberdrop_dl/crawlers/pmvhaven.py@@ -8,7 +8,8 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import DictDataclass, error_handling_wrapper, nuxt+from cyberdrop_dl.utils import error_handling_wrapper, nuxt+from cyberdrop_dl.utils.dataclass import DictDataclassif TYPE_CHECKING:from collections.abc import AsyncGenerator, Generator, Iterablediff --git a/cyberdrop_dl/crawlers/rule34vault.py b/cyberdrop_dl/crawlers/rule34vault.pyindex 47e43164f..4fa782574 100644--- a/cyberdrop_dl/crawlers/rule34vault.py+++ b/cyberdrop_dl/crawlers/rule34vault.py@@ -5,7 +5,8 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import DictDataclass, error_handling_wrapper+from cyberdrop_dl.utils import error_handling_wrapper+from cyberdrop_dl.utils.dataclass import DictDataclassif TYPE_CHECKING:from collections.abc import AsyncGenerator, Iterablediff --git a/cyberdrop_dl/crawlers/tiktok.py b/cyberdrop_dl/crawlers/tiktok.pyindex 1d572061d..c45afecdb 100644--- a/cyberdrop_dl/crawlers/tiktok.py+++ b/cyberdrop_dl/crawlers/tiktok.py@@ -7,7 +7,8 @@ from typing import TYPE_CHECKING, Any, ClassVar, Selffrom cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPaths, auto_task_idfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem-from cyberdrop_dl.utils import DictDataclass, error_handling_wrapper+from cyberdrop_dl.utils import error_handling_wrapper+from cyberdrop_dl.utils.dataclass import DictDataclassif TYPE_CHECKING:from collections.abc import AsyncGenerator, Mappingdiff --git a/cyberdrop_dl/crawlers/whyp_it.py b/cyberdrop_dl/crawlers/whyp_it.pyindex 29fef6f9b..55ef024a6 100644--- a/cyberdrop_dl/crawlers/whyp_it.py+++ b/cyberdrop_dl/crawlers/whyp_it.py@@ -5,7 +5,8 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import API, Crawler, RateLimit, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import deserialize, error_handling_wrapper, parse_url+from cyberdrop_dl.utils import error_handling_wrapper, parse_url+from cyberdrop_dl.utils.dataclass import deserializeif TYPE_CHECKING:from collections.abc import AsyncGenerator, Iterablediff --git a/cyberdrop_dl/crawlers/yandex_disk.py b/cyberdrop_dl/crawlers/yandex_disk.pyindex 19c0abaa3..de7564924 100644--- a/cyberdrop_dl/crawlers/yandex_disk.py+++ b/cyberdrop_dl/crawlers/yandex_disk.py@@ -8,7 +8,8 @@ from typing import TYPE_CHECKING, Any, ClassVar, Literal, Selffrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPathsfrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import DictDataclass, css, error_handling_wrapper+from cyberdrop_dl.utils import css, error_handling_wrapper+from cyberdrop_dl.utils.dataclass import DictDataclassif TYPE_CHECKING:from collections.abc import Generatordiff --git a/cyberdrop_dl/crawlers/yurivan.py b/cyberdrop_dl/crawlers/yurivan.pyindex f61a10c51..d3f82b379 100644--- a/cyberdrop_dl/crawlers/yurivan.py+++ b/cyberdrop_dl/crawlers/yurivan.py@@ -7,7 +7,8 @@ from pydantic import dataclassesfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, deserialize, error_handling_wrapper, next_js+from cyberdrop_dl.utils import css, error_handling_wrapper, next_js+from cyberdrop_dl.utils.dataclass import deserializeif TYPE_CHECKING:from bs4 import BeautifulSoupdiff --git a/cyberdrop_dl/ffmpeg.py b/cyberdrop_dl/ffmpeg.pyindex 26b1c0b40..abda7a066 100644--- a/cyberdrop_dl/ffmpeg.py+++ b/cyberdrop_dl/ffmpeg.py@@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any, Literal, Self, TypedDictfrom multidict import CIMultiDict, CIMultiDictProxyfrom cyberdrop_dl import aio-from cyberdrop_dl.utils import DictDataclass+from cyberdrop_dl.utils.dataclass import DictDataclassif TYPE_CHECKING:from collections.abc import Generator, Iterable, Iterator, Mapping, Sequencediff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.pyindex bc9632160..53b5dbbf1 100644--- a/cyberdrop_dl/utils/__init__.py+++ b/cyberdrop_dl/utils/__init__.py@@ -9,7 +9,6 @@ import sysfrom typing import TYPE_CHECKING, Any, castfrom cyberdrop_dl.constants import MISSING-from cyberdrop_dl.utils._dataclasses import DictDataclass, deserialize, filter_data # noqa: F401from cyberdrop_dl.utils._errors import error_handling_context, error_handling_wrapper # noqa: F401from cyberdrop_dl.utils._url import parse_http_url as parse_url # noqa: F401from cyberdrop_dl.utils._url import remove_trailing_slash # noqa: F401diff --git a/cyberdrop_dl/utils/_dataclasses.py b/cyberdrop_dl/utils/dataclass.pysimilarity index 100%rename from cyberdrop_dl/utils/_dataclasses.pyrename to cyberdrop_dl/utils/dataclass.pydiff --git a/pyproject.toml b/pyproject.tomlindex 7c690bda8..7a5b03c2d 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -171,6 +171,7 @@ banned-from = ["base64","cyberdrop_dl.aio","cyberdrop_dl.stats",+ "dataclasses","datetime","functools","hashlib",