Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: circular imports (#1005)
NTFSvolume609f1332d6f0
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex ec1d8dee0..35dbc674a 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -16,8 +16,8 @@ from dateutil import parserfrom videoprops import get_audio_properties, get_video_propertiesfrom yarl import URL-from cyberdrop_dl.clients.errors import DDOSGuardError, DownloadError, InvalidContentTypeError, SlowDownloadError-from cyberdrop_dl.utils.constants import FILE_FORMATS+from cyberdrop_dl.constants import FILE_FORMATS+from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError, SlowDownloadErrorfrom cyberdrop_dl.utils.logger import log, log_debugif TYPE_CHECKING:@@ -27,9 +27,9 @@ if TYPE_CHECKING:from multidict import CIMultiDictProxy+ from cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.managers.client_manager import ClientManagerfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItemP = ParamSpec("P")diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.pyindex af6ebd408..6b7a7c856 100644--- a/cyberdrop_dl/clients/hash_client.py+++ b/cyberdrop_dl/clients/hash_client.py@@ -9,16 +9,16 @@ from typing import TYPE_CHECKINGimport aiofiles.osfrom send2trash import send2trash+from cyberdrop_dl.data_structures.hash import Hashingfrom cyberdrop_dl.ui.prompts.basic_prompts import enter_to_continue-from cyberdrop_dl.utils.data_enums_classes.hash import Hashingfrom cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import get_size_or_noneif TYPE_CHECKING:from yarl import URL+ from cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItemdef hash_directory_scanner(manager: Manager, path: Path) -> None:diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.pyindex 20bdd9d85..fc2f9ba5f 100644--- a/cyberdrop_dl/clients/scraper_client.py+++ b/cyberdrop_dl/clients/scraper_client.py@@ -18,9 +18,9 @@ from aiohttp_client_cache.response import AnyResponsefrom aiohttp_client_cache.session import CachedSessionfrom bs4 import BeautifulSoup-import cyberdrop_dl.utils.constants as constants+import cyberdrop_dl.constants as constantsfrom cyberdrop_dl import env-from cyberdrop_dl.clients.errors import DDOSGuardError, DownloadError, InvalidContentTypeError, ScrapeError+from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError, ScrapeErrorfrom cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import get_soup_no_error, sanitize_filenamediff --git a/cyberdrop_dl/config_definitions/authentication_settings.py b/cyberdrop_dl/config_definitions/authentication_settings.pyindex c5cda7e8c..e7a6b1fad 100644--- a/cyberdrop_dl/config_definitions/authentication_settings.py+++ b/cyberdrop_dl/config_definitions/authentication_settings.py@@ -1,6 +1,6 @@from pydantic import BaseModel, Field-from .custom.types import AliasModel+from cyberdrop_dl.types import AliasModelclass CoomerAuth(BaseModel):diff --git a/cyberdrop_dl/config_definitions/config_settings.py b/cyberdrop_dl/config_definitions/config_settings.pyindex 9340a1349..0bf68ca5d 100644--- a/cyberdrop_dl/config_definitions/config_settings.py+++ b/cyberdrop_dl/config_definitions/config_settings.py@@ -5,11 +5,10 @@ from pathlib import Pathfrom pydantic import BaseModel, ByteSize, Field, NonNegativeInt, PositiveInt, field_serializer, field_validator-from cyberdrop_dl.utils.constants import APP_STORAGE, BROWSERS, DOWNLOAD_STORAGE-from cyberdrop_dl.utils.data_enums_classes.hash import Hashing-from cyberdrop_dl.utils.data_enums_classes.supported_domains import SUPPORTED_SITES_DOMAINS--from .custom.types import (+from cyberdrop_dl.constants import APP_STORAGE, BROWSERS, DOWNLOAD_STORAGE+from cyberdrop_dl.data_structures.hash import Hashing+from cyberdrop_dl.data_structures.supported_domains import SUPPORTED_SITES_DOMAINS+from cyberdrop_dl.types import (AliasModel,ByteSizeSerilized,HttpAppriseURL,@@ -21,7 +20,7 @@ from .custom.types import (NonEmptyStrOrNone,PathOrNone,)-from .custom.validators import parse_duration_as_timedelta, parse_falsy_as+from cyberdrop_dl.utils.validators import parse_duration_as_timedelta, parse_falsy_asALL_SUPPORTED_SITES = ["<<ALL_SUPPORTED_SITES>>"]diff --git a/cyberdrop_dl/config_definitions/global_settings.py b/cyberdrop_dl/config_definitions/global_settings.pyindex 1777b3e29..425ac5dd9 100644--- a/cyberdrop_dl/config_definitions/global_settings.py+++ b/cyberdrop_dl/config_definitions/global_settings.py@@ -4,9 +4,9 @@ from datetime import timedeltafrom pydantic import BaseModel, ByteSize, Field, NonNegativeFloat, PositiveInt, field_serializer, field_validatorfrom yarl import URL-from .custom.converters import convert_to_byte_size-from .custom.types import AliasModel, ByteSizeSerilized, HttpURL, NonEmptyStr-from .custom.validators import parse_duration_as_timedelta, parse_falsy_as+from cyberdrop_dl.types import AliasModel, ByteSizeSerilized, HttpURL, NonEmptyStr+from cyberdrop_dl.utils.converters import convert_to_byte_size+from cyberdrop_dl.utils.validators import parse_duration_as_timedelta, parse_falsy_asMIN_REQUIRED_FREE_SPACE = convert_to_byte_size("512MB")DEFAULT_REQUIRED_FREE_SPACE = convert_to_byte_size("5GB")diff --git a/cyberdrop_dl/utils/constants.py b/cyberdrop_dl/constants.pysimilarity index 100%rename from cyberdrop_dl/utils/constants.pyrename to cyberdrop_dl/constants.pydiff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex 55df64960..368b22b17 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -6,14 +6,14 @@ from typing import TYPE_CHECKINGfrom bs4 import BeautifulSoup-from cyberdrop_dl.clients.errors import PasswordProtectedError, ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import PasswordProtectedError, ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from yarl import URL- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemITEM_DESCRIPTION_SELECTOR = "p[class*=description-meta]"diff --git a/cyberdrop_dl/crawlers/acidimg.py b/cyberdrop_dl/crawlers/acidimg.pyindex 10d5250fb..13c364f9b 100644--- a/cyberdrop_dl/crawlers/acidimg.py+++ b/cyberdrop_dl/crawlers/acidimg.py@@ -8,8 +8,8 @@ from cyberdrop_dl.crawlers.crawler import create_task_idfrom cyberdrop_dl.crawlers.imx_to import ImxToCrawlerif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass AcidImgCrawler(ImxToCrawler):diff --git a/cyberdrop_dl/crawlers/archivebate.py b/cyberdrop_dl/crawlers/archivebate.pyindex 54fc2ada0..27d765ba1 100644--- a/cyberdrop_dl/crawlers/archivebate.py+++ b/cyberdrop_dl/crawlers/archivebate.py@@ -5,16 +5,16 @@ from typing import TYPE_CHECKING, ClassVarfrom aiolimiter import AsyncLimiterfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import create_task_idfrom cyberdrop_dl.crawlers.mixdrop import MixDropCrawler+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_og_properties, get_text_betweenif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass Selectors:diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.pyindex 3f0ce1075..8cc597e27 100644--- a/cyberdrop_dl/crawlers/ashemaletube.py+++ b/cyberdrop_dl/crawlers/ashemaletube.py@@ -8,15 +8,15 @@ from typing import TYPE_CHECKING, NamedTuplefrom aiolimiter import AsyncLimiterfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:from bs4 import BeautifulSoup, Tag+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass Selectors:diff --git a/cyberdrop_dl/crawlers/bestprettygirl.py b/cyberdrop_dl/crawlers/bestprettygirl.pyindex 715fb5e7f..72f0e4a50 100644--- a/cyberdrop_dl/crawlers/bestprettygirl.py+++ b/cyberdrop_dl/crawlers/bestprettygirl.py@@ -5,15 +5,15 @@ from typing import TYPE_CHECKINGfrom aiolimiter import AsyncLimiterfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass Selectors:diff --git a/cyberdrop_dl/crawlers/box_dot_com.py b/cyberdrop_dl/crawlers/box_dot_com.pyindex 97472fba4..62f91af5d 100644--- a/cyberdrop_dl/crawlers/box_dot_com.py+++ b/cyberdrop_dl/crawlers/box_dot_com.py@@ -9,16 +9,16 @@ from typing import TYPE_CHECKING, Anyfrom pydantic import Fieldfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeError-from cyberdrop_dl.config_definitions.custom.types import AliasModelfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.types import AliasModelfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass ItemType(StrEnum):diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.pyindex 195bea04c..24385978d 100644--- a/cyberdrop_dl/crawlers/bunkrr.py+++ b/cyberdrop_dl/crawlers/bunkrr.py@@ -14,16 +14,16 @@ from typing import TYPE_CHECKING, ClassVar, NamedTuplefrom aiohttp import ClientConnectorErrorfrom yarl import URL-from cyberdrop_dl.clients.errors import DDOSGuardError, NoExtensionError, ScrapeError+from cyberdrop_dl.constants import FILE_FORMATSfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id-from cyberdrop_dl.utils.constants import FILE_FORMATS+from cyberdrop_dl.exceptions import DDOSGuardError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_og_properties, get_text_between, parse_urlif TYPE_CHECKING:from bs4 import BeautifulSoup, Tag+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem# CDNsdiff --git a/cyberdrop_dl/crawlers/bunkrr_albums_io.py b/cyberdrop_dl/crawlers/bunkrr_albums_io.pyindex 9af74472f..b55a40479 100644--- a/cyberdrop_dl/crawlers/bunkrr_albums_io.py+++ b/cyberdrop_dl/crawlers/bunkrr_albums_io.py@@ -12,8 +12,8 @@ if TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemALBUM_SELECTOR = "main div.auto-rows-max a"diff --git a/cyberdrop_dl/crawlers/buzzheavier.py b/cyberdrop_dl/crawlers/buzzheavier.pyindex 18e6e63f4..6e6e66d56 100644--- a/cyberdrop_dl/crawlers/buzzheavier.py+++ b/cyberdrop_dl/crawlers/buzzheavier.py@@ -8,8 +8,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_idfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_from_headersif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass BuzzHeavierCrawler(Crawler):diff --git a/cyberdrop_dl/crawlers/catbox.py b/cyberdrop_dl/crawlers/catbox.pyindex 4995e1ab9..602358d97 100644--- a/cyberdrop_dl/crawlers/catbox.py+++ b/cyberdrop_dl/crawlers/catbox.py@@ -7,8 +7,8 @@ from yarl import URLfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_idif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemCDN_HOSTS = "litter.catbox.moe", "files.catbox.moe"diff --git a/cyberdrop_dl/crawlers/coomer.py b/cyberdrop_dl/crawlers/coomer.pyindex 68167d3fa..b08097c4a 100644--- a/cyberdrop_dl/crawlers/coomer.py+++ b/cyberdrop_dl/crawlers/coomer.py@@ -11,8 +11,8 @@ from cyberdrop_dl.crawlers.kemono import KemonoCrawler, UserPostif TYPE_CHECKING:from aiohttp_client_cache.response import AnyResponse+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass CoomerCrawler(KemonoCrawler):diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex e1980a23a..444ea71bf 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -14,10 +14,10 @@ from aiolimiter import AsyncLimiterfrom dateutil import parserfrom yarl import URL+from cyberdrop_dl.data_structures.url_objects import MediaItem, ScrapeItemfrom cyberdrop_dl.downloader.downloader import Downloaderfrom cyberdrop_dl.scraper import filtersfrom cyberdrop_dl.utils import utilities-from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem, ScrapeItemfrom cyberdrop_dl.utils.database.tables.history_table import get_db_pathfrom cyberdrop_dl.utils.logger import log, log_debugfrom cyberdrop_dl.utils.utilities import (diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.pyindex da8693037..7b6b42521 100644--- a/cyberdrop_dl/crawlers/cyberdrop.py+++ b/cyberdrop_dl/crawlers/cyberdrop.py@@ -6,15 +6,15 @@ from aiolimiter import AsyncLimiterfrom bs4 import BeautifulSoupfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemAPI_ENTRYPOINT = URL("https://api.cyberdrop.me/api/")diff --git a/cyberdrop_dl/crawlers/cyberfile.py b/cyberdrop_dl/crawlers/cyberfile.pyindex fcd8b96e8..1d04b4fc6 100644--- a/cyberdrop_dl/crawlers/cyberfile.py+++ b/cyberdrop_dl/crawlers/cyberfile.py@@ -7,13 +7,13 @@ from aiolimiter import AsyncLimiterfrom bs4 import BeautifulSoupfrom yarl import URL-from cyberdrop_dl.clients.errors import PasswordProtectedError, ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import PasswordProtectedError, ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemSOUP_ERRORS = {diff --git a/cyberdrop_dl/crawlers/dirtyship.py b/cyberdrop_dl/crawlers/dirtyship.pyindex d2e4081ee..9b0dfd186 100644--- a/cyberdrop_dl/crawlers/dirtyship.py+++ b/cyberdrop_dl/crawlers/dirtyship.py@@ -5,15 +5,15 @@ from typing import TYPE_CHECKING, NamedTuplefrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemVIDEO_SELECTOR = "video#fp-video-0 > source"diff --git a/cyberdrop_dl/crawlers/doodstream.py b/cyberdrop_dl/crawlers/doodstream.pyindex 2b72b1c75..dabcddf0c 100644--- a/cyberdrop_dl/crawlers/doodstream.py+++ b/cyberdrop_dl/crawlers/doodstream.py@@ -7,15 +7,15 @@ from typing import TYPE_CHECKING, ClassVarfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass Selectors:diff --git a/cyberdrop_dl/crawlers/dropbox.py b/cyberdrop_dl/crawlers/dropbox.pyindex 2c0344107..458dedef0 100644--- a/cyberdrop_dl/crawlers/dropbox.py+++ b/cyberdrop_dl/crawlers/dropbox.py@@ -6,15 +6,15 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_from_headersif TYPE_CHECKING:from collections.abc import Mapping+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass DropboxCrawler(Crawler):diff --git a/cyberdrop_dl/crawlers/e621.py b/cyberdrop_dl/crawlers/e621.pyindex d1bebe137..c1ba12157 100644--- a/cyberdrop_dl/crawlers/e621.py+++ b/cyberdrop_dl/crawlers/e621.py@@ -7,15 +7,15 @@ from aiolimiter import AsyncLimiterfrom yarl import URLfrom cyberdrop_dl import __version__-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from collections.abc import AsyncGenerator+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass E621Crawler(Crawler):diff --git a/cyberdrop_dl/crawlers/ehentai.py b/cyberdrop_dl/crawlers/ehentai.pyindex b71133837..75b2fd769 100644--- a/cyberdrop_dl/crawlers/ehentai.py+++ b/cyberdrop_dl/crawlers/ehentai.py@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass Selectors:diff --git a/cyberdrop_dl/crawlers/eightmuses.py b/cyberdrop_dl/crawlers/eightmuses.pyindex 638ddf484..3af810c28 100644--- a/cyberdrop_dl/crawlers/eightmuses.py+++ b/cyberdrop_dl/crawlers/eightmuses.py@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemTILE_SELECTOR = "a[class*=c-tile]"diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.pyindex 1a8e2a285..36f287bd7 100644--- a/cyberdrop_dl/crawlers/eporner.py+++ b/cyberdrop_dl/crawlers/eporner.py@@ -4,17 +4,17 @@ from typing import TYPE_CHECKING, NamedTuplefrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.data_structures.url_objects import ScrapeItem+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils import javascript-from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup, Tag+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass Selectors:diff --git a/cyberdrop_dl/crawlers/erome.py b/cyberdrop_dl/crawlers/erome.pyindex 0c06df79a..76a2e88b4 100644--- a/cyberdrop_dl/crawlers/erome.py+++ b/cyberdrop_dl/crawlers/erome.py@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass Selectors:diff --git a/cyberdrop_dl/crawlers/fapello.py b/cyberdrop_dl/crawlers/fapello.pyindex 1af268cd9..875913206 100644--- a/cyberdrop_dl/crawlers/fapello.py+++ b/cyberdrop_dl/crawlers/fapello.py@@ -5,15 +5,15 @@ from typing import TYPE_CHECKINGfrom aiolimiter import AsyncLimiterfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemCONTENT_SELECTOR = "div[id=content] a"diff --git a/cyberdrop_dl/crawlers/fileditch.py b/cyberdrop_dl/crawlers/fileditch.pyindex e429ff0f5..1c9db7b04 100644--- a/cyberdrop_dl/crawlers/fileditch.py+++ b/cyberdrop_dl/crawlers/fileditch.py@@ -4,15 +4,15 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemDOWNLOAD_SELECTOR = "a[class*='download-button']"HOMEPAGE_CATCHALL_FILE = "/s21/FHVZKQyAZlIsrneDAsp.jpeg"diff --git a/cyberdrop_dl/crawlers/files_vc.py b/cyberdrop_dl/crawlers/files_vc.pyindex ea3cd70db..fc253a479 100644--- a/cyberdrop_dl/crawlers/files_vc.py+++ b/cyberdrop_dl/crawlers/files_vc.py@@ -9,8 +9,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_idfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemAPI_ENTRYPOINT = URL("https://api.files.vc/api")diff --git a/cyberdrop_dl/crawlers/fourchan.py b/cyberdrop_dl/crawlers/fourchan.pyindex 87b0d75f2..7e7991833 100644--- a/cyberdrop_dl/crawlers/fourchan.py+++ b/cyberdrop_dl/crawlers/fourchan.py@@ -6,13 +6,13 @@ from aiolimiter import AsyncLimiterfrom bs4 import BeautifulSoupfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemAPI_ENTRYPOINT = URL("https://a.4cdn.org/")FILES_BASE_URL = URL("https://i.4cdn.org/")diff --git a/cyberdrop_dl/crawlers/generic.py b/cyberdrop_dl/crawlers/generic.pyindex cc977ee59..2119d04a8 100644--- a/cyberdrop_dl/crawlers/generic.py+++ b/cyberdrop_dl/crawlers/generic.py@@ -6,8 +6,8 @@ from functools import wrapsfrom pathlib import Pathfrom typing import TYPE_CHECKING, ParamSpec, TypeVar-from cyberdrop_dl.clients.errors import InvalidContentTypeError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import InvalidContentTypeError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.scraper.filters import has_valid_extensionfrom cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_and_ext@@ -18,8 +18,8 @@ if TYPE_CHECKING:from bs4 import BeautifulSoupfrom yarl import URL+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemP = ParamSpec("P")diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.pyindex 6fd8dea8f..89014e270 100644--- a/cyberdrop_dl/crawlers/gofile.py+++ b/cyberdrop_dl/crawlers/gofile.py@@ -9,9 +9,9 @@ from typing import TYPE_CHECKING, Literal, NotRequired, TypedDict, castfrom aiolimiter import AsyncLimiterfrom yarl import URL-from cyberdrop_dl.clients.errors import DownloadError, PasswordProtectedError, ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id-from cyberdrop_dl.utils.data_enums_classes.url_objects import FILE_HOST_ALBUM, ScrapeItem+from cyberdrop_dl.data_structures.url_objects import FILE_HOST_ALBUM, ScrapeItem+from cyberdrop_dl.exceptions import DownloadError, PasswordProtectedError, ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.pyindex 23a2b0a82..60febbda5 100644--- a/cyberdrop_dl/crawlers/google_drive.py+++ b/cyberdrop_dl/crawlers/google_drive.py@@ -38,8 +38,8 @@ from typing import TYPE_CHECKING, ClassVarfrom aiolimiter import AsyncLimiterfrom yarl import URL-from cyberdrop_dl.clients.errors import DownloadError, ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import DownloadError, ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_from_headersif TYPE_CHECKING:@@ -47,8 +47,8 @@ if TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemVALID_FILE_URL_PARTS = "file", "document", "presentation", "spreadsheets"diff --git a/cyberdrop_dl/crawlers/hotpic.py b/cyberdrop_dl/crawlers/hotpic.pyindex a4f49f2c2..f20de92e0 100644--- a/cyberdrop_dl/crawlers/hotpic.py+++ b/cyberdrop_dl/crawlers/hotpic.py@@ -5,15 +5,15 @@ from typing import TYPE_CHECKING, ClassVarfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemIMAGE_SELECTOR = "img[id*=main-image]"VIDEO_SELECTOR = "video > source"diff --git a/cyberdrop_dl/crawlers/imagebam.py b/cyberdrop_dl/crawlers/imagebam.pyindex 09ba5b14d..da88fa569 100644--- a/cyberdrop_dl/crawlers/imagebam.py+++ b/cyberdrop_dl/crawlers/imagebam.py@@ -4,15 +4,15 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemGALLERY_TITLE_SELECTOR = "a#gallery-name"diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.pyindex 51d5216dd..82aa4a78d 100644--- a/cyberdrop_dl/crawlers/imgbb.py+++ b/cyberdrop_dl/crawlers/imgbb.py@@ -12,8 +12,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemIMAGE_PAGE_SELECTOR = "a[class*=image-container]"diff --git a/cyberdrop_dl/crawlers/imgbox.py b/cyberdrop_dl/crawlers/imgbox.pyindex 6ec73370b..a95eaab51 100644--- a/cyberdrop_dl/crawlers/imgbox.py+++ b/cyberdrop_dl/crawlers/imgbox.py@@ -6,9 +6,9 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id-from cyberdrop_dl.utils.data_enums_classes.url_objects import FILE_HOST_ALBUM, ScrapeItem+from cyberdrop_dl.data_structures.url_objects import FILE_HOST_ALBUM, ScrapeItem+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.pyindex 591ba92b0..2afd36f7a 100644--- a/cyberdrop_dl/crawlers/imgur.py+++ b/cyberdrop_dl/crawlers/imgur.py@@ -5,13 +5,13 @@ from typing import TYPE_CHECKING, Anyfrom yarl import URL-from cyberdrop_dl.clients.errors import LoginError, ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import LoginError, ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemAPI_ENTRYPOINT = URL("https://api.imgur.com/3/")diff --git a/cyberdrop_dl/crawlers/imx_to.py b/cyberdrop_dl/crawlers/imx_to.pyindex 24f261250..0b3348388 100644--- a/cyberdrop_dl/crawlers/imx_to.py+++ b/cyberdrop_dl/crawlers/imx_to.py@@ -9,8 +9,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_idfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemIMG_SELECTOR = "div#container a > img"diff --git a/cyberdrop_dl/crawlers/incestflix.py b/cyberdrop_dl/crawlers/incestflix.pyindex 0348d3944..743cd12e7 100644--- a/cyberdrop_dl/crawlers/incestflix.py+++ b/cyberdrop_dl/crawlers/incestflix.py@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass Selectors:diff --git a/cyberdrop_dl/crawlers/influencer_bitches.py b/cyberdrop_dl/crawlers/influencer_bitches.pyindex f2e421534..18d79b0a4 100644--- a/cyberdrop_dl/crawlers/influencer_bitches.py+++ b/cyberdrop_dl/crawlers/influencer_bitches.py@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass Selectors:diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.pyindex d9fc17e4d..09e5b50b0 100644--- a/cyberdrop_dl/crawlers/jpg5.py+++ b/cyberdrop_dl/crawlers/jpg5.py@@ -12,8 +12,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperfrom ._chevereto import CheveretoCrawlerif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemPRIMARY_BASE_DOMAIN = URL("https://jpg5.su")JPG5_REPLACE_HOST_REGEX = re.compile(r"(jpg\.fish/)|(jpg\.fishing/)|(jpg\.church/)")diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.pyindex 21915c366..8bbb4bca1 100644--- a/cyberdrop_dl/crawlers/kemono.py+++ b/cyberdrop_dl/crawlers/kemono.py@@ -13,11 +13,11 @@ from pydantic import AliasChoices, BeforeValidator, Fieldfrom typing_extensions import TypedDict # Import from typing is not compatible with pydanticfrom yarl import URL-from cyberdrop_dl.clients.errors import NoExtensionError, ScrapeError-from cyberdrop_dl.config_definitions.custom.types import AliasModel-from cyberdrop_dl.config_definitions.custom.validators import parse_falsy_as_nonefrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import NoExtensionError, ScrapeError+from cyberdrop_dl.types import AliasModelfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_parts+from cyberdrop_dl.utils.validators import parse_falsy_as_noneif TYPE_CHECKING:from collections.abc import AsyncGenerator, Callable, Coroutine, Generator@@ -25,8 +25,8 @@ if TYPE_CHECKING:from aiohttp_client_cache.response import AnyResponsefrom bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemMAX_OFFSET_PER_CALL = 50LINK_REGEX = re.compile(r"(?:http(?!.*\.\.)[^ ]*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]|</|'))")diff --git a/cyberdrop_dl/crawlers/luscious.py b/cyberdrop_dl/crawlers/luscious.pyindex f010cb02d..ebcb57ee9 100644--- a/cyberdrop_dl/crawlers/luscious.py+++ b/cyberdrop_dl/crawlers/luscious.py@@ -6,16 +6,16 @@ from typing import TYPE_CHECKING, Anyfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from collections.abc import AsyncGenerator+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemGRAPHQL_URL = URL("https://members.luscious.net/graphql/nobatch/")diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.pyindex 5c9a0ac2d..8544a36a3 100644--- a/cyberdrop_dl/crawlers/mediafire.py+++ b/cyberdrop_dl/crawlers/mediafire.py@@ -9,15 +9,15 @@ from aiolimiter import AsyncLimiterfrom mediafire import MediaFireApi, apifrom yarl import URL-from cyberdrop_dl.clients.errors import MediaFireError, ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import MediaFireError, ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemDOWNLOAD_BUTTON_SELECTOR = "a[id=downloadButton]"DATE_SELECTOR = "ul[class=details] li span"diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.pyindex 09fa6946f..7d36745a3 100644--- a/cyberdrop_dl/crawlers/mega_nz.py+++ b/cyberdrop_dl/crawlers/mega_nz.py@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, NamedTuple, castfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_idfrom cyberdrop_dl.downloader.mega_nz import (DecryptData,@@ -16,11 +15,12 @@ from cyberdrop_dl.downloader.mega_nz import (base64_url_decode,decrypt_attr,)+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass FileTuple(NamedTuple):diff --git a/cyberdrop_dl/crawlers/missav.py b/cyberdrop_dl/crawlers/missav.pyindex a4055fdbc..b816a3e32 100644--- a/cyberdrop_dl/crawlers/missav.py+++ b/cyberdrop_dl/crawlers/missav.py@@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, NamedTuplefrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -16,8 +16,8 @@ if TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemM3U8_SERVER = URL("https://surrit.com/")diff --git a/cyberdrop_dl/crawlers/mixdrop.py b/cyberdrop_dl/crawlers/mixdrop.pyindex e8e2aa144..31c712e69 100644--- a/cyberdrop_dl/crawlers/mixdrop.py+++ b/cyberdrop_dl/crawlers/mixdrop.py@@ -11,8 +11,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweeif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass Selectors:diff --git a/cyberdrop_dl/crawlers/motherless.py b/cyberdrop_dl/crawlers/motherless.pyindex 125e113de..fc27c786e 100644--- a/cyberdrop_dl/crawlers/motherless.py+++ b/cyberdrop_dl/crawlers/motherless.py@@ -6,15 +6,15 @@ from typing import TYPE_CHECKING, Literal, NamedTuplefrom aiolimiter import AsyncLimiterfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemMEDIA_INFO_JS_SELECTOR = "script:contains('__fileurl')"diff --git a/cyberdrop_dl/crawlers/nekohouse.py b/cyberdrop_dl/crawlers/nekohouse.pyindex 9da6de522..da2385146 100644--- a/cyberdrop_dl/crawlers/nekohouse.py+++ b/cyberdrop_dl/crawlers/nekohouse.py@@ -8,8 +8,8 @@ from cyberdrop_dl.crawlers.crawler import create_task_idfrom cyberdrop_dl.crawlers.kemono import KemonoCrawlerif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass NekohouseCrawler(KemonoCrawler):diff --git a/cyberdrop_dl/crawlers/nhentai.py b/cyberdrop_dl/crawlers/nhentai.pyindex 109d23edf..f2984d784 100644--- a/cyberdrop_dl/crawlers/nhentai.py+++ b/cyberdrop_dl/crawlers/nhentai.py@@ -6,16 +6,16 @@ from typing import TYPE_CHECKINGfrom aiolimiter import AsyncLimiterfrom yarl import URL-from cyberdrop_dl.clients.errors import LoginError, ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import LoginError, ScrapeErrorfrom cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from collections.abc import Generator+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemPRIMARY_BASE_DOMAIN = URL("https://nhentai.net/")diff --git a/cyberdrop_dl/crawlers/noodle_magazine.py b/cyberdrop_dl/crawlers/noodle_magazine.pyindex 2f3d0873a..9dc8babbd 100644--- a/cyberdrop_dl/crawlers/noodle_magazine.py+++ b/cyberdrop_dl/crawlers/noodle_magazine.py@@ -8,15 +8,15 @@ from aiolimiter import AsyncLimiterfrom bs4 import BeautifulSoupfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemPLAYLIST_SELECTOR = "script:contains('window.playlist')"diff --git a/cyberdrop_dl/crawlers/nudostartv.py b/cyberdrop_dl/crawlers/nudostartv.pyindex 1a955a29e..8e4f44791 100644--- a/cyberdrop_dl/crawlers/nudostartv.py+++ b/cyberdrop_dl/crawlers/nudostartv.py@@ -4,15 +4,15 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemIMAGE_SELECTOR = "div[class=block-video] a img"CONTENT_SELECTOR = "div[id=list_videos_common_videos_list_items] div a"diff --git a/cyberdrop_dl/crawlers/omegascans.py b/cyberdrop_dl/crawlers/omegascans.pyindex cf8ee793b..035fa57fa 100644--- a/cyberdrop_dl/crawlers/omegascans.py+++ b/cyberdrop_dl/crawlers/omegascans.py@@ -7,15 +7,15 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemDATE_SELECTOR = 'h2[class="font-semibold font-sans text-muted-foreground text-xs"]'diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.pyindex 9b157a252..30f9b9dc7 100644--- a/cyberdrop_dl/crawlers/onedrive.py+++ b/cyberdrop_dl/crawlers/onedrive.py@@ -13,13 +13,13 @@ from typing import TYPE_CHECKING, Any, ClassVar, Selffrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemAPI_ENTRYPOINT = URL("https://api.onedrive.com/v1.0/drives/")diff --git a/cyberdrop_dl/crawlers/pimpandhost.py b/cyberdrop_dl/crawlers/pimpandhost.pyindex a675bc514..14f3394e0 100644--- a/cyberdrop_dl/crawlers/pimpandhost.py+++ b/cyberdrop_dl/crawlers/pimpandhost.py@@ -12,8 +12,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemALBUM_TITLE_SELECTOR = "span[class=author-header__album-name]"DATE_SELECTOR = "span[class=date-time]"diff --git a/cyberdrop_dl/crawlers/pixeldrain.py b/cyberdrop_dl/crawlers/pixeldrain.pyindex b7c5ec8ad..e0b607d66 100644--- a/cyberdrop_dl/crawlers/pixeldrain.py+++ b/cyberdrop_dl/crawlers/pixeldrain.py@@ -7,16 +7,16 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import DownloadError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import DownloadError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_og_properties, get_text_betweenif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemAPI_ENTRYPOINT = URL("https://pixeldrain.com/api/")JS_SELECTOR = 'script:contains("window.initial_node")'diff --git a/cyberdrop_dl/crawlers/pixhost.py b/cyberdrop_dl/crawlers/pixhost.pyindex e5fddccd3..960e9e365 100644--- a/cyberdrop_dl/crawlers/pixhost.py+++ b/cyberdrop_dl/crawlers/pixhost.py@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemPRIMARY_BASE_DOMAIN = URL("https://pixhost.to/")diff --git a/cyberdrop_dl/crawlers/pkmncards.py b/cyberdrop_dl/crawlers/pkmncards.pyindex 485ee679d..2ba175214 100644--- a/cyberdrop_dl/crawlers/pkmncards.py+++ b/cyberdrop_dl/crawlers/pkmncards.py@@ -11,15 +11,15 @@ from typing import TYPE_CHECKING, NewTypefrom bs4 import BeautifulSoupfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup, Tag+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemTimeStamp = NewType("TimeStamp", int)diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.pyindex e4d0526da..2d381eaea 100644--- a/cyberdrop_dl/crawlers/pmvhaven.py+++ b/cyberdrop_dl/crawlers/pmvhaven.py@@ -7,8 +7,8 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils import javascriptfrom cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapper@@ -18,8 +18,8 @@ if TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemJS_VIDEO_INFO_SELECTOR = "script#__NUXT_DATA__"diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.pyindex 3e2d6468f..be9504bb6 100644--- a/cyberdrop_dl/crawlers/pornhub.py+++ b/cyberdrop_dl/crawlers/pornhub.py@@ -6,8 +6,8 @@ from typing import TYPE_CHECKING, Any, Literal, NamedTuple, TypedDictfrom yarl import URL-from cyberdrop_dl.clients.errors import NoExtensionError, ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import NoExtensionError, ScrapeErrorfrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between@@ -16,8 +16,8 @@ if TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemPRIMARY_BASE_DOMAIN = URL("https://www.pornhub.com")ALBUM_API_URL = PRIMARY_BASE_DOMAIN / "album/show_album_json"diff --git a/cyberdrop_dl/crawlers/pornpics.py b/cyberdrop_dl/crawlers/pornpics.pyindex 4151364d2..6b4a714be 100644--- a/cyberdrop_dl/crawlers/pornpics.py+++ b/cyberdrop_dl/crawlers/pornpics.py@@ -13,8 +13,8 @@ if TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemCOLLECTION_PARTS = "search", "channel", "pornstar", "tag", "category"IMAGE_SELECTOR = "div#main a.rel-link"diff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.pyindex 62a55aa52..8d102bac9 100644--- a/cyberdrop_dl/crawlers/porntrex.py+++ b/cyberdrop_dl/crawlers/porntrex.py@@ -7,15 +7,15 @@ from typing import TYPE_CHECKING, NamedTuplefrom aiolimiter import AsyncLimiterfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass Video(NamedTuple):diff --git a/cyberdrop_dl/crawlers/postimg.py b/cyberdrop_dl/crawlers/postimg.pyindex 710dd060a..9920f1cdb 100644--- a/cyberdrop_dl/crawlers/postimg.py+++ b/cyberdrop_dl/crawlers/postimg.py@@ -11,8 +11,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemDOWNLOAD_BUTTON_SELECTOR = "a[id=download]"diff --git a/cyberdrop_dl/crawlers/realbooru.py b/cyberdrop_dl/crawlers/realbooru.pyindex 3b1078dee..e08620799 100644--- a/cyberdrop_dl/crawlers/realbooru.py+++ b/cyberdrop_dl/crawlers/realbooru.py@@ -4,15 +4,15 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemCONTENT_SELECTOR = "div[class=items] div a"VIDEO_SELECTOR = "video source"diff --git a/cyberdrop_dl/crawlers/realdebrid.py b/cyberdrop_dl/crawlers/realdebrid.pyindex 2338f986f..ee38f9e17 100644--- a/cyberdrop_dl/crawlers/realdebrid.py+++ b/cyberdrop_dl/crawlers/realdebrid.py@@ -12,8 +12,8 @@ from cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass RealDebridCrawler(Crawler):diff --git a/cyberdrop_dl/crawlers/reddit.py b/cyberdrop_dl/crawlers/reddit.pyindex c697aafc7..dddb19155 100644--- a/cyberdrop_dl/crawlers/reddit.py+++ b/cyberdrop_dl/crawlers/reddit.py@@ -8,9 +8,9 @@ from aiolimiter import AsyncLimiterfrom asyncpraw import Redditfrom yarl import URL-from cyberdrop_dl.clients.errors import LoginError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.clients.scraper_client import cache_control_managerfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import LoginError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import error_handling_wrapper@@ -19,8 +19,8 @@ if TYPE_CHECKING:from asyncpraw.models import Redditor, Submission, Subreddit+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass MediaFile(TypedDict):diff --git a/cyberdrop_dl/crawlers/redgifs.py b/cyberdrop_dl/crawlers/redgifs.pyindex e9d2963bb..6649fc32a 100644--- a/cyberdrop_dl/crawlers/redgifs.py+++ b/cyberdrop_dl/crawlers/redgifs.py@@ -8,8 +8,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_idfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemAPI_ENTRYPOINT = URL("https://api.redgifs.com/")diff --git a/cyberdrop_dl/crawlers/rule34vault.py b/cyberdrop_dl/crawlers/rule34vault.pyindex 983792bdf..70bd55236 100644--- a/cyberdrop_dl/crawlers/rule34vault.py+++ b/cyberdrop_dl/crawlers/rule34vault.py@@ -11,8 +11,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass Selectors:diff --git a/cyberdrop_dl/crawlers/rule34video.py b/cyberdrop_dl/crawlers/rule34video.pyindex c5f6a79fd..61246b968 100644--- a/cyberdrop_dl/crawlers/rule34video.py+++ b/cyberdrop_dl/crawlers/rule34video.py@@ -6,10 +6,10 @@ from typing import TYPE_CHECKING, NamedTuplefrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.data_structures.url_objects import ScrapeItem+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils import javascript-from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemfrom cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapper@@ -18,8 +18,8 @@ if TYPE_CHECKING:from bs4 import BeautifulSoup, Tag+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemRESOLUTIONS = ["4k", "2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"] # best to worstdiff --git a/cyberdrop_dl/crawlers/rule34xxx.py b/cyberdrop_dl/crawlers/rule34xxx.pyindex 8bd450e50..f324717fa 100644--- a/cyberdrop_dl/crawlers/rule34xxx.py+++ b/cyberdrop_dl/crawlers/rule34xxx.py@@ -4,15 +4,15 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass Selector:diff --git a/cyberdrop_dl/crawlers/saint.py b/cyberdrop_dl/crawlers/saint.pyindex 23b50585f..78f20c609 100644--- a/cyberdrop_dl/crawlers/saint.py+++ b/cyberdrop_dl/crawlers/saint.py@@ -6,15 +6,15 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemVIDEOS_SELECTOR = "a.btn-primary.action.download"diff --git a/cyberdrop_dl/crawlers/scrolller.py b/cyberdrop_dl/crawlers/scrolller.pyindex 53e183e46..109fc3e2b 100644--- a/cyberdrop_dl/crawlers/scrolller.py+++ b/cyberdrop_dl/crawlers/scrolller.py@@ -9,8 +9,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_idfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemDEFAULT_QUERY = """query SubredditQuery(diff --git a/cyberdrop_dl/crawlers/send_now.py b/cyberdrop_dl/crawlers/send_now.pyindex 938a4b2f8..22305ef68 100644--- a/cyberdrop_dl/crawlers/send_now.py+++ b/cyberdrop_dl/crawlers/send_now.py@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from curl_cffi.requests.models import Response as CurlResponse+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass SendNowCrawler(Crawler):diff --git a/cyberdrop_dl/crawlers/sendvid.py b/cyberdrop_dl/crawlers/sendvid.pyindex adeb25b1c..5f3cdae9e 100644--- a/cyberdrop_dl/crawlers/sendvid.py+++ b/cyberdrop_dl/crawlers/sendvid.py@@ -4,15 +4,15 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_partsif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemVIDEO_SRC_SELECTOR = "video > source#video_source"diff --git a/cyberdrop_dl/crawlers/sex_dot_com.py b/cyberdrop_dl/crawlers/sex_dot_com.pyindex 4df17f2a4..1149f0a71 100644--- a/cyberdrop_dl/crawlers/sex_dot_com.py+++ b/cyberdrop_dl/crawlers/sex_dot_com.py@@ -12,8 +12,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from collections.abc import AsyncGenerator+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemAPI_URL = URL("https://iframe.sex.com/api/")diff --git a/cyberdrop_dl/crawlers/spankbang.py b/cyberdrop_dl/crawlers/spankbang.pyindex bff94ec8e..3d9bc6b88 100644--- a/cyberdrop_dl/crawlers/spankbang.py+++ b/cyberdrop_dl/crawlers/spankbang.py@@ -8,18 +8,18 @@ from typing import TYPE_CHECKING, Any, NamedTuplefrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.data_structures.url_objects import ScrapeItem+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils import javascript-from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemfrom cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemPRIMARY_BASE_DOMAIN = URL("https://spankbang.com/")diff --git a/cyberdrop_dl/crawlers/streamable.py b/cyberdrop_dl/crawlers/streamable.pyindex 56b15e51c..7f5a292c1 100644--- a/cyberdrop_dl/crawlers/streamable.py+++ b/cyberdrop_dl/crawlers/streamable.py@@ -5,16 +5,16 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemAJAX_ENTRYPOINT = URL("https://ajax.streamable.com/videos/")diff --git a/cyberdrop_dl/crawlers/thisvid.py b/cyberdrop_dl/crawlers/thisvid.pyindex 596ca2f89..1da8af5c0 100644--- a/cyberdrop_dl/crawlers/thisvid.py+++ b/cyberdrop_dl/crawlers/thisvid.py@@ -6,15 +6,15 @@ from typing import TYPE_CHECKING, NamedTuplefrom aiolimiter import AsyncLimiterfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem# SelectorsUNAUTHORIZED_SELECTOR = "div.video-holder:contains('This video is a private video')"diff --git a/cyberdrop_dl/crawlers/tiktok.py b/cyberdrop_dl/crawlers/tiktok.pyindex a4075a04b..00541924f 100644--- a/cyberdrop_dl/crawlers/tiktok.py+++ b/cyberdrop_dl/crawlers/tiktok.py@@ -11,8 +11,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from collections.abc import AsyncGenerator+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemVIDEO_PARTS = "video", "photo", "v"API_URL = URL("https://www.tikwm.com/api/")diff --git a/cyberdrop_dl/crawlers/tokyomotion.py b/cyberdrop_dl/crawlers/tokyomotion.pyindex c458dc00f..93584995a 100644--- a/cyberdrop_dl/crawlers/tokyomotion.py+++ b/cyberdrop_dl/crawlers/tokyomotion.py@@ -8,15 +8,15 @@ from typing import TYPE_CHECKINGfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_partsif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemDATE_PATTERN = re.compile(r"(\d+)\s*(weeks?|days?|hours?|minutes?|seconds?)", re.IGNORECASE)diff --git a/cyberdrop_dl/crawlers/toonily.py b/cyberdrop_dl/crawlers/toonily.pyindex 523c26c63..d712b2295 100644--- a/cyberdrop_dl/crawlers/toonily.py+++ b/cyberdrop_dl/crawlers/toonily.py@@ -7,7 +7,7 @@ from typing import TYPE_CHECKINGfrom yarl import URLfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id-from cyberdrop_dl.utils.data_enums_classes.url_objects import FILE_HOST_PROFILE, ScrapeItem+from cyberdrop_dl.data_structures.url_objects import FILE_HOST_PROFILE, ScrapeItemfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:diff --git a/cyberdrop_dl/crawlers/twitter_images.py b/cyberdrop_dl/crawlers/twitter_images.pyindex 4a06c530c..5778264d5 100644--- a/cyberdrop_dl/crawlers/twitter_images.py+++ b/cyberdrop_dl/crawlers/twitter_images.py@@ -8,8 +8,8 @@ from yarl import URLfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_idif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemCDN_HOST = "pbs.twimg.com"diff --git a/cyberdrop_dl/crawlers/vipr_dot_im.py b/cyberdrop_dl/crawlers/vipr_dot_im.pyindex 9914f14f7..b8b5c6330 100644--- a/cyberdrop_dl/crawlers/vipr_dot_im.py+++ b/cyberdrop_dl/crawlers/vipr_dot_im.py@@ -9,8 +9,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_idfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemIMG_SELECTOR = "div#body a > img"diff --git a/cyberdrop_dl/crawlers/wetransfer.py b/cyberdrop_dl/crawlers/wetransfer.pyindex 22c0897de..ea0f62086 100644--- a/cyberdrop_dl/crawlers/wetransfer.py+++ b/cyberdrop_dl/crawlers/wetransfer.py@@ -6,13 +6,13 @@ from typing import TYPE_CHECKING, ClassVarfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemAPI_ENTRYPOINT = URL("https://wetransfer.com/api/v4/transfers")diff --git a/cyberdrop_dl/crawlers/xbunkr.py b/cyberdrop_dl/crawlers/xbunkr.pyindex 8c0e75a7a..cadc82c10 100644--- a/cyberdrop_dl/crawlers/xbunkr.py+++ b/cyberdrop_dl/crawlers/xbunkr.py@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemIMAGE_SELECTOR = "a[class=image]"diff --git a/cyberdrop_dl/crawlers/xenforo/f95zone.py b/cyberdrop_dl/crawlers/xenforo/f95zone.pyindex f84c7864e..d551ce0bc 100644--- a/cyberdrop_dl/crawlers/xenforo/f95zone.py+++ b/cyberdrop_dl/crawlers/xenforo/f95zone.py@@ -9,8 +9,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperfrom .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectorsif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemclass F95ZoneCrawler(XenforoCrawler):diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.pyindex bd1f69e2a..f51dc534b 100644--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py@@ -12,9 +12,9 @@ from typing import TYPE_CHECKINGfrom bs4 import BeautifulSoup, Tagfrom yarl import URL-from cyberdrop_dl.clients.errors import InvalidURLError, LoginError, ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id-from cyberdrop_dl.utils.data_enums_classes.url_objects import FORUM, ScrapeItem+from cyberdrop_dl.data_structures.url_objects import FORUM, ScrapeItem+from cyberdrop_dl.exceptions import InvalidURLError, LoginError, ScrapeErrorfrom cyberdrop_dl.utils.logger import log, log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_trailing_slashdiff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.pyindex d5994b015..93579dcb9 100644--- a/cyberdrop_dl/crawlers/xhamster.py+++ b/cyberdrop_dl/crawlers/xhamster.py@@ -6,8 +6,8 @@ from typing import TYPE_CHECKING, Annotated, Any, NamedTuplefrom pydantic import AliasPath, Field, PlainValidatorfrom yarl import URL-from cyberdrop_dl.config_definitions.custom.types import AliasModelfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.types import AliasModelfrom cyberdrop_dl.utils import javascriptfrom cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, parse_url@@ -17,8 +17,8 @@ if TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemPRIMARY_BASE_DOMAIN = URL("https://xhamster.com/")JS_VIDEO_INFO_SELECTOR = "script#initials-script"diff --git a/cyberdrop_dl/crawlers/xxxbunker.py b/cyberdrop_dl/crawlers/xxxbunker.pyindex 89d3a4225..16bc051ef 100644--- a/cyberdrop_dl/crawlers/xxxbunker.py+++ b/cyberdrop_dl/crawlers/xxxbunker.py@@ -10,16 +10,16 @@ from aiolimiter import AsyncLimiterfrom bs4 import BeautifulSoupfrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from collections.abc import AsyncGenerator+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemDATE_PATTERN = re.compile(r"(\d+)\s*(weeks?|days?|hours?|minutes?|seconds?)", re.IGNORECASE)MIN_RATE_LIMIT = 4 # per minutediff --git a/cyberdrop_dl/crawlers/yandex_disk.py b/cyberdrop_dl/crawlers/yandex_disk.pyindex ed48d56f9..f53ece4a2 100644--- a/cyberdrop_dl/crawlers/yandex_disk.py+++ b/cyberdrop_dl/crawlers/yandex_disk.py@@ -8,8 +8,8 @@ from typing import TYPE_CHECKING, Any, Literal, Selffrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils import javascriptfrom cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapper@@ -19,8 +19,8 @@ if TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemJS_SELECTOR = "script#store-prefetch"diff --git a/cyberdrop_dl/crawlers/youjizz.py b/cyberdrop_dl/crawlers/youjizz.pyindex d2b2f0c8a..ebe570ee1 100644--- a/cyberdrop_dl/crawlers/youjizz.py+++ b/cyberdrop_dl/crawlers/youjizz.py@@ -8,8 +8,8 @@ from typing import TYPE_CHECKING, NamedTuplefrom yarl import URL-from cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils import javascriptfrom cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapper@@ -17,8 +17,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoup+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemDEFAULT_QUALITY = "Auto"diff --git a/cyberdrop_dl/utils/data_enums_classes/__init__.py b/cyberdrop_dl/data_structures/__init__.pysimilarity index 100%rename from cyberdrop_dl/utils/data_enums_classes/__init__.pyrename to cyberdrop_dl/data_structures/__init__.pydiff --git a/cyberdrop_dl/utils/data_enums_classes/hash.py b/cyberdrop_dl/data_structures/hash.pysimilarity index 100%rename from cyberdrop_dl/utils/data_enums_classes/hash.pyrename to cyberdrop_dl/data_structures/hash.pydiff --git a/cyberdrop_dl/utils/data_enums_classes/supported_domains.py b/cyberdrop_dl/data_structures/supported_domains.pysimilarity index 100%rename from cyberdrop_dl/utils/data_enums_classes/supported_domains.pyrename to cyberdrop_dl/data_structures/supported_domains.pydiff --git a/cyberdrop_dl/utils/data_enums_classes/url_objects.py b/cyberdrop_dl/data_structures/url_objects.pysimilarity index 99%rename from cyberdrop_dl/utils/data_enums_classes/url_objects.pyrename to cyberdrop_dl/data_structures/url_objects.pyindex 9aad39c20..c6e49dac2 100644--- a/cyberdrop_dl/utils/data_enums_classes/url_objects.py+++ b/cyberdrop_dl/data_structures/url_objects.py@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, NamedTuplefrom yarl import URL-from cyberdrop_dl.clients.errors import MaxChildrenError+from cyberdrop_dl.exceptions import MaxChildrenErrorfrom cyberdrop_dl.utils.utilities import sanitize_folderif TYPE_CHECKING:diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.pyindex 595cd7ea6..4dcc12268 100644--- a/cyberdrop_dl/director.py+++ b/cyberdrop_dl/director.py@@ -16,13 +16,12 @@ import browser_cookie3from pydantic import ValidationErrorfrom rich import print as rich_print-from cyberdrop_dl import env-from cyberdrop_dl.clients.errors import InvalidYamlError+from cyberdrop_dl import constants, env+from cyberdrop_dl.exceptions import InvalidYamlErrorfrom cyberdrop_dl.managers.manager import Managerfrom cyberdrop_dl.scraper.scrape_mapper import ScrapeMapperfrom cyberdrop_dl.ui.program_ui import ProgramUIfrom cyberdrop_dl.ui.textual import textual_ui-from cyberdrop_dl.utils import constantsfrom cyberdrop_dl.utils.apprise import send_apprise_notificationsfrom cyberdrop_dl.utils.dumper import Dumperfrom cyberdrop_dl.utils.logger import LogHandler, QueuedLogger, TextualLogQueueHandler, log, log_spacer, log_with_colordiff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex b0425bf9b..97a7138a8 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -15,15 +15,15 @@ from typing import TYPE_CHECKING, ParamSpec, TypeVarfrom aiohttp import ClientConnectorError, ClientError, ClientResponseError-from cyberdrop_dl.clients.errors import (+from cyberdrop_dl.constants import CustomHTTPStatus+from cyberdrop_dl.data_structures.url_objects import HlsSegment, MediaItem+from cyberdrop_dl.exceptions import (DownloadError,DurationError,ErrorLogMessage,InvalidContentTypeError,RestrictedFiletypeError,)-from cyberdrop_dl.utils.constants import CustomHTTPStatus-from cyberdrop_dl.utils.data_enums_classes.url_objects import HlsSegment, MediaItemfrom cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_size_or_nonediff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.pyindex 95bfa5e67..644b04fd0 100644--- a/cyberdrop_dl/downloader/mega_nz.py+++ b/cyberdrop_dl/downloader/mega_nz.py@@ -74,8 +74,8 @@ from Crypto.PublicKey import RSAfrom Crypto.Util import Counterfrom cyberdrop_dl.clients.download_client import DownloadClient-from cyberdrop_dl.clients.errors import CDLBaseError, DownloadError, SlowDownloadErrorfrom cyberdrop_dl.downloader.downloader import Downloader+from cyberdrop_dl.exceptions import CDLBaseError, DownloadError, SlowDownloadErrorfrom cyberdrop_dl.utils.logger import logif TYPE_CHECKING:@@ -83,8 +83,8 @@ if TYPE_CHECKING:from yarl import URL+ from cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItemclass MegaNzError(CDLBaseError): ...diff --git a/cyberdrop_dl/clients/errors.py b/cyberdrop_dl/exceptions.pysimilarity index 98%rename from cyberdrop_dl/clients/errors.pyrename to cyberdrop_dl/exceptions.pyindex 3abdf680e..4328b6890 100644--- a/cyberdrop_dl/clients/errors.py+++ b/cyberdrop_dl/exceptions.py@@ -8,12 +8,12 @@ from typing import TYPE_CHECKINGfrom yaml import YAMLErrorfrom yarl import URL-from cyberdrop_dl.utils.constants import VALIDATION_ERROR_FOOTER+from cyberdrop_dl.constants import VALIDATION_ERROR_FOOTERif TYPE_CHECKING:from requests import Response- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem, ScrapeItem+ from cyberdrop_dl.data_structures.url_objects import MediaItem, ScrapeItem# See: https://developers.cloudflare.com/support/troubleshooting/cloudflare-errors/troubleshooting-cloudflare-5xx-errors/diff --git a/cyberdrop_dl/managers/cache_manager.py b/cyberdrop_dl/managers/cache_manager.pyindex 4335c1849..1844a932c 100644--- a/cyberdrop_dl/managers/cache_manager.py+++ b/cyberdrop_dl/managers/cache_manager.py@@ -8,9 +8,9 @@ from typing import TYPE_CHECKING, Anyfrom aiohttp_client_cache import CacheBackend, SQLiteBackendfrom cyberdrop_dl import __version__ as current_version+from cyberdrop_dl.data_structures.supported_domains import SUPPORTED_FORUMS, SUPPORTED_WEBSITESfrom cyberdrop_dl.scraper.filters import cache_filter_fnfrom cyberdrop_dl.utils import yaml-from cyberdrop_dl.utils.data_enums_classes.supported_domains import SUPPORTED_FORUMS, SUPPORTED_WEBSITESif TYPE_CHECKING:from pathlib import Pathdiff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex 4e121c894..168e99883 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -17,8 +17,8 @@ from bs4 import BeautifulSoupfrom yarl import URLfrom cyberdrop_dl.clients.download_client import DownloadClient-from cyberdrop_dl.clients.errors import DDOSGuardError, DownloadError, ScrapeErrorfrom cyberdrop_dl.clients.scraper_client import ScraperClient+from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeErrorfrom cyberdrop_dl.managers.download_speed_manager import DownloadSpeedLimiterfrom cyberdrop_dl.ui.prompts.user_prompts import get_cookies_from_browsersfrom cyberdrop_dl.utils.logger import log, log_spacer@@ -28,8 +28,8 @@ if TYPE_CHECKING:from aiohttp_client_cache import CachedResponsefrom curl_cffi.requests.models import Response as CurlResponse+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItemDOWNLOAD_ERROR_ETAGS = {"d835884373f4d6c8f24742ceabe74946": "Imgur image has been removed",diff --git a/cyberdrop_dl/managers/config_manager.py b/cyberdrop_dl/managers/config_manager.pyindex 8060ba79f..5e0b7cc0f 100644--- a/cyberdrop_dl/managers/config_manager.py+++ b/cyberdrop_dl/managers/config_manager.py@@ -6,8 +6,8 @@ from dataclasses import fieldfrom time import sleepfrom typing import TYPE_CHECKING-from cyberdrop_dl.clients.errors import InvalidYamlErrorfrom cyberdrop_dl.config_definitions import AuthSettings, ConfigSettings, GlobalSettings+from cyberdrop_dl.exceptions import InvalidYamlErrorfrom cyberdrop_dl.managers.log_manager import LogManagerfrom cyberdrop_dl.utils import yamlfrom cyberdrop_dl.utils.apprise import get_apprise_urls@@ -114,7 +114,7 @@ class ConfigManager:if posible_fields == set_fields and not needs_update and self.pydantic_config:returnelse:- from cyberdrop_dl.utils import constants+ from cyberdrop_dl import constantsself.settings_data = ConfigSettings()self.settings_data.files.input_file = constants.APP_STORAGE / "Configs" / self.loaded_config / "URLs.txt"diff --git a/cyberdrop_dl/managers/download_manager.py b/cyberdrop_dl/managers/download_manager.pyindex dcc0a9b7a..09a1f7a64 100644--- a/cyberdrop_dl/managers/download_manager.py+++ b/cyberdrop_dl/managers/download_manager.py@@ -6,14 +6,14 @@ from contextlib import asynccontextmanagerfrom typing import TYPE_CHECKINGfrom cyberdrop_dl.clients.download_client import check_file_duration-from cyberdrop_dl.utils.constants import FILE_FORMATS+from cyberdrop_dl.constants import FILE_FORMATSfrom cyberdrop_dl.utils.logger import log_debugif TYPE_CHECKING:from collections.abc import AsyncGenerator+ from cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItemclass FileLocksVault:diff --git a/cyberdrop_dl/managers/live_manager.py b/cyberdrop_dl/managers/live_manager.pyindex 77cda2d22..40f493c34 100644--- a/cyberdrop_dl/managers/live_manager.py+++ b/cyberdrop_dl/managers/live_manager.py@@ -8,7 +8,7 @@ from typing import TYPE_CHECKINGfrom rich.live import Live-from cyberdrop_dl.utils import constants+from cyberdrop_dl import constantsfrom cyberdrop_dl.utils.args import is_terminal_in_portraitfrom cyberdrop_dl.utils.logger import consolediff --git a/cyberdrop_dl/managers/log_manager.py b/cyberdrop_dl/managers/log_manager.pyindex 7736a0bed..a13f22d51 100644--- a/cyberdrop_dl/managers/log_manager.py+++ b/cyberdrop_dl/managers/log_manager.py@@ -8,15 +8,15 @@ from typing import TYPE_CHECKINGimport aiofiles-from cyberdrop_dl.clients.errors import get_origin-from cyberdrop_dl.utils.constants import CSV_DELIMITER+from cyberdrop_dl.constants import CSV_DELIMITER+from cyberdrop_dl.exceptions import get_originfrom cyberdrop_dl.utils.logger import log, log_spacerif TYPE_CHECKING:from yarl import URL+ from cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItemclass LogManager:diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex c0f2bb1d7..f4ffca0ff 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -8,7 +8,7 @@ from dataclasses import Field, fieldfrom time import perf_counterfrom typing import TYPE_CHECKING, Any, Literal, NamedTuple, Protocol, TypeVar-from cyberdrop_dl import __version__+from cyberdrop_dl import __version__, constantsfrom cyberdrop_dl.config_definitions import ConfigSettings, GlobalSettingsfrom cyberdrop_dl.managers.cache_manager import CacheManagerfrom cyberdrop_dl.managers.client_manager import ClientManager@@ -23,7 +23,6 @@ from cyberdrop_dl.managers.progress_manager import ProgressManagerfrom cyberdrop_dl.managers.realdebrid_manager import RealDebridManagerfrom cyberdrop_dl.managers.storage_manager import StorageManagerfrom cyberdrop_dl.ui.textual import TextualUI-from cyberdrop_dl.utils import constantsfrom cyberdrop_dl.utils.args import ParsedArgsfrom cyberdrop_dl.utils.ffmpeg import FFmpeg, get_ffmpeg_versionfrom cyberdrop_dl.utils.logger import QueuedLogger, logdiff --git a/cyberdrop_dl/managers/path_manager.py b/cyberdrop_dl/managers/path_manager.pyindex 810832a1b..4a30912e4 100644--- a/cyberdrop_dl/managers/path_manager.py+++ b/cyberdrop_dl/managers/path_manager.py@@ -5,13 +5,12 @@ from datetime import datetimefrom pathlib import Pathfrom typing import TYPE_CHECKING-from cyberdrop_dl import env-from cyberdrop_dl.utils import constants+from cyberdrop_dl import constants, envfrom cyberdrop_dl.utils.utilities import purge_dir_treeif TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItemif env.RUNNING_IN_IDE and Path.cwd().name == "cyberdrop_dl":diff --git a/cyberdrop_dl/managers/real_debrid/api.py b/cyberdrop_dl/managers/real_debrid/api.pyindex 43c660e33..cc94059d5 100644--- a/cyberdrop_dl/managers/real_debrid/api.py+++ b/cyberdrop_dl/managers/real_debrid/api.py@@ -9,7 +9,7 @@ from requests import Sessionfrom requests.exceptions import RequestExceptionfrom yarl import URL-from cyberdrop_dl.clients.errors import RealDebridError+from cyberdrop_dl.exceptions import RealDebridErrorfrom cyberdrop_dl.managers.real_debrid.errors import ERROR_CODESif TYPE_CHECKING:diff --git a/cyberdrop_dl/managers/realdebrid_manager.py b/cyberdrop_dl/managers/realdebrid_manager.pyindex 505a8e4b5..2bee0d28f 100644--- a/cyberdrop_dl/managers/realdebrid_manager.py+++ b/cyberdrop_dl/managers/realdebrid_manager.py@@ -6,7 +6,7 @@ from dataclasses import fieldfrom re import Patternfrom typing import TYPE_CHECKING-from cyberdrop_dl.clients.errors import RealDebridError+from cyberdrop_dl.exceptions import RealDebridErrorfrom cyberdrop_dl.managers.real_debrid.api import RealDebridApifrom cyberdrop_dl.utils.logger import logdiff --git a/cyberdrop_dl/managers/storage_manager.py b/cyberdrop_dl/managers/storage_manager.pyindex 1e46286d6..36d925759 100644--- a/cyberdrop_dl/managers/storage_manager.py+++ b/cyberdrop_dl/managers/storage_manager.py@@ -12,14 +12,14 @@ from typing import TYPE_CHECKING, NamedTupleimport psutilfrom pydantic import ByteSize-from cyberdrop_dl.clients.errors import InsufficientFreeSpaceError+from cyberdrop_dl.exceptions import InsufficientFreeSpaceErrorfrom cyberdrop_dl.utils.logger import log, log_debugif TYPE_CHECKING:from psutil._common import sdiskpart+ from cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem@dataclass(frozen=True, slots=True, order=True)diff --git a/cyberdrop_dl/profiling.py b/cyberdrop_dl/profiling.pyindex f11368a75..bf059e211 100644--- a/cyberdrop_dl/profiling.py+++ b/cyberdrop_dl/profiling.py@@ -12,8 +12,7 @@ from pathlib import Pathfrom tempfile import TemporaryDirectoryfrom typing import TYPE_CHECKING-from cyberdrop_dl import env-from cyberdrop_dl.utils import constants+from cyberdrop_dl import constants, envif TYPE_CHECKING:from collections.abc import Callablediff --git a/cyberdrop_dl/scraper/filters.py b/cyberdrop_dl/scraper/filters.pyindex d55085b98..ac92c2b13 100644--- a/cyberdrop_dl/scraper/filters.py+++ b/cyberdrop_dl/scraper/filters.py@@ -7,9 +7,9 @@ from typing import TYPE_CHECKINGimport arrowfrom yarl import URL-import cyberdrop_dl.utils.constants as constants-from cyberdrop_dl.clients.errors import NoExtensionError-from cyberdrop_dl.utils.constants import FILE_FORMATS+import cyberdrop_dl.constants as constants+from cyberdrop_dl.constants import FILE_FORMATS+from cyberdrop_dl.exceptions import NoExtensionErrorfrom cyberdrop_dl.utils.utilities import get_filename_and_extif TYPE_CHECKING:@@ -18,7 +18,7 @@ if TYPE_CHECKING:from aiohttp import ClientResponse- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem+ from cyberdrop_dl.data_structures.url_objects import ScrapeItemreturn_values: dict[URL | str, tuple] = {}diff --git a/cyberdrop_dl/scraper/jdownloader.py b/cyberdrop_dl/scraper/jdownloader.pyindex 48d21e8d6..780adb368 100644--- a/cyberdrop_dl/scraper/jdownloader.py+++ b/cyberdrop_dl/scraper/jdownloader.py@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, ParamSpec, TypeVarfrom myjdapi import myjdapi-from cyberdrop_dl.clients.errors import JDownloaderError+from cyberdrop_dl.exceptions import JDownloaderErrorfrom cyberdrop_dl.utils.logger import logif TYPE_CHECKING:diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex b06983b4a..645b462dc 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -11,13 +11,13 @@ import aiofilesimport arrowfrom yarl import URL-from cyberdrop_dl.clients.errors import JDownloaderError, NoExtensionError+from cyberdrop_dl.constants import BLOCKED_DOMAINS, REGEX_LINKSfrom cyberdrop_dl.crawlers import CRAWLERS+from cyberdrop_dl.data_structures.url_objects import MediaItem, ScrapeItemfrom cyberdrop_dl.downloader.downloader import Downloader+from cyberdrop_dl.exceptions import JDownloaderError, NoExtensionErrorfrom cyberdrop_dl.scraper.filters import has_valid_extension, is_in_domain_list, is_outside_date_range, is_valid_urlfrom cyberdrop_dl.scraper.jdownloader import JDownloader-from cyberdrop_dl.utils.constants import BLOCKED_DOMAINS, REGEX_LINKS-from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem, ScrapeItemfrom cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import get_download_path, get_filename_and_ext, remove_trailing_slashdiff --git a/cyberdrop_dl/config_definitions/custom/types.py b/cyberdrop_dl/types.pysimilarity index 64%rename from cyberdrop_dl/config_definitions/custom/types.pyrename to cyberdrop_dl/types.pyindex 1a577c4ba..de7059cd2 100644--- a/cyberdrop_dl/config_definitions/custom/types.py+++ b/cyberdrop_dl/types.py@@ -1,8 +1,16 @@+"""Custom types for type annotations+++1. Only add types here if they do NOT depend on any runtime import from `cyberdrop_dl` itself, except utils+2. Only add types here if they are going to be used across multiple modules+"""+from __future__ import annotations+from collections.abc import Sequencefrom functools import partialfrom pathlib import Path-from typing import Annotated+from typing import Annotated, Any, TypeAlias, TypeGuard, TypeVar, finalimport yarlfrom pydantic import (@@ -22,8 +30,38 @@ from pydantic import (model_validator,)-from .converters import change_path_suffix, convert_byte_size_to_str-from .validators import parse_apprise_url, parse_falsy_as_none, parse_list, pydantyc_yarl_url+from cyberdrop_dl.utils.converters import change_path_suffix, convert_byte_size_to_str+from cyberdrop_dl.utils.validators import (+ parse_apprise_url,+ parse_falsy_as_none,+ parse_list,+ pydantyc_yarl_url,+)+++def do_nothing(cls=None) -> Any: ...+++og_init_subclass = yarl.URL.__init_subclass__++yarl.URL.__init_subclass__ = do_nothing # type: ignore+++@final+class AbsoluteHttpURL(yarl.URL):+ def __init__(*args, **kwargs) -> None:+ raise RuntimeError("Do not create instances, call yarl.URL and them assert validate")++ def host(self) -> str: # type: ignore+ """Decoded host part of URL."""++ @staticmethod+ def validate(url: yarl.URL) -> TypeGuard[AbsoluteHttpURL]:+ return url.absolute and url.scheme.startswith("http")+++yarl.URL.__init_subclass__ = og_init_subclass+# ~~~~~ Strings ~~~~~~~StrSerializer = PlainSerializer(str, return_type=str, when_used="json-unless-none")@@ -78,3 +116,12 @@ class HttpAppriseURL(AppriseURLModel):# DEPRECATED# HttpURL = Annotated[HttpUrl, AfterValidator(convert_to_yarl), StrSerializer]+++T = TypeVar("T")+Array: TypeAlias = list[T] | tuple[T, ...]+CMD: TypeAlias = Array[str]+U32Int: TypeAlias = int+U32IntArray: TypeAlias = Array[U32Int]+U32IntSequence: TypeAlias = Sequence[U32Int]+AnyDict: TypeAlias = dict[str, Any]diff --git a/cyberdrop_dl/ui/prompts/user_prompts.py b/cyberdrop_dl/ui/prompts/user_prompts.pyindex 5e3029db7..8a44fcf16 100644--- a/cyberdrop_dl/ui/prompts/user_prompts.py+++ b/cyberdrop_dl/ui/prompts/user_prompts.py@@ -14,15 +14,15 @@ from InquirerPy.enum import (from rich.console import Consolefrom cyberdrop_dl import __version__-from cyberdrop_dl.ui.prompts import basic_prompts-from cyberdrop_dl.ui.prompts.defaults import ALL_CHOICE, DONE_CHOICE, EXIT_CHOICE-from cyberdrop_dl.utils.constants import BROWSERS, RESERVED_CONFIG_NAMES-from cyberdrop_dl.utils.cookie_management import get_cookies_from_browsers-from cyberdrop_dl.utils.data_enums_classes.supported_domains import (+from cyberdrop_dl.constants import BROWSERS, RESERVED_CONFIG_NAMES+from cyberdrop_dl.data_structures.supported_domains import (SUPPORTED_FORUMS,SUPPORTED_SITES_DOMAINS,SUPPORTED_WEBSITES,)+from cyberdrop_dl.ui.prompts import basic_prompts+from cyberdrop_dl.ui.prompts.defaults import ALL_CHOICE, DONE_CHOICE, EXIT_CHOICE+from cyberdrop_dl.utils.cookie_management import get_cookies_from_browsersfrom cyberdrop_dl.utils.utilities import clear_termif TYPE_CHECKING:diff --git a/cyberdrop_dl/utils/apprise.py b/cyberdrop_dl/utils/apprise.pyindex 444c99897..c138fe24d 100644--- a/cyberdrop_dl/utils/apprise.py+++ b/cyberdrop_dl/utils/apprise.py@@ -14,8 +14,8 @@ import richfrom pydantic import ValidationErrorfrom rich.text import Text-from cyberdrop_dl.config_definitions.custom.types import AppriseURLModel-from cyberdrop_dl.utils import constants+from cyberdrop_dl import constants+from cyberdrop_dl.types import AppriseURLModelfrom cyberdrop_dl.utils.logger import log, log_debug, log_spacerfrom cyberdrop_dl.utils.yaml import handle_validation_errordiff --git a/cyberdrop_dl/utils/args.py b/cyberdrop_dl/utils/args.pyindex e91c5727c..fe30d7a4e 100644--- a/cyberdrop_dl/utils/args.py+++ b/cyberdrop_dl/utils/args.py@@ -14,7 +14,7 @@ from pydantic import BaseModel, Field, ValidationError, computed_field, field_vafrom cyberdrop_dl import __version__, envfrom cyberdrop_dl.config_definitions import ConfigSettings, GlobalSettings-from cyberdrop_dl.config_definitions.custom.types import AliasModel, HttpURL+from cyberdrop_dl.types import AliasModel, HttpURLfrom cyberdrop_dl.utils.yaml import handle_validation_errorif TYPE_CHECKING:diff --git a/cyberdrop_dl/config_definitions/custom/converters.py b/cyberdrop_dl/utils/converters.pysimilarity index 91%rename from cyberdrop_dl/config_definitions/custom/converters.pyrename to cyberdrop_dl/utils/converters.pyindex 8d3350252..54ad8f67a 100644--- a/cyberdrop_dl/config_definitions/custom/converters.py+++ b/cyberdrop_dl/utils/converters.py@@ -2,16 +2,22 @@Functions to use with `AfterValidator`, `field_validator(mode="after")` or `model_validator(mode="after")`"""+from __future__ import annotations+import refrom datetime import timedelta-from pathlib import Path+from typing import TYPE_CHECKING-import yarlfrom pydantic import AnyUrl, ByteSize, TypeAdapter-from cyberdrop_dl.clients.errors import InvalidURLError+from cyberdrop_dl.exceptions import InvalidURLErrorfrom cyberdrop_dl.utils.utilities import parse_url+if TYPE_CHECKING:+ from pathlib import Path++ import yarl+DATE_PATTERN_REGEX = r"(\d+)\s*(second|seconds|minute|minutes|hour|hours|day|days|week|weeks|month|months|year|years)"DATE_PATTERN = re.compile(DATE_PATTERN_REGEX, re.IGNORECASE)diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.pyindex dd303d858..2d714ba7b 100644--- a/cyberdrop_dl/utils/cookie_management.py+++ b/cyberdrop_dl/utils/cookie_management.py@@ -11,8 +11,8 @@ import browser_cookie3if TYPE_CHECKING:from collections.abc import Callable, Generator+ from cyberdrop_dl.constants import BROWSERSfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.constants import BROWSERSP = ParamSpec("P")R = TypeVar("R")diff --git a/cyberdrop_dl/utils/database/tables/history_table.py b/cyberdrop_dl/utils/database/tables/history_table.pyindex 869dc67c8..576c0856b 100644--- a/cyberdrop_dl/utils/database/tables/history_table.py+++ b/cyberdrop_dl/utils/database/tables/history_table.py@@ -15,7 +15,7 @@ if TYPE_CHECKING:from yarl import URLfrom cyberdrop_dl.crawlers import Crawler- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem+ from cyberdrop_dl.data_structures.url_objects import MediaItemdef get_db_path(url: URL, referer: str = "") -> str:diff --git a/cyberdrop_dl/utils/dumper.py b/cyberdrop_dl/utils/dumper.pyindex aadcf6716..ac5cdee60 100644--- a/cyberdrop_dl/utils/dumper.py+++ b/cyberdrop_dl/utils/dumper.py@@ -10,8 +10,8 @@ if TYPE_CHECKING:from collections.abc import Generatorfrom pathlib import Path+ from cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItemKEYS_TO_REMOVE = "file_lock_reference_name", "task_id"diff --git a/cyberdrop_dl/utils/logger.py b/cyberdrop_dl/utils/logger.pyindex e7e0144d9..cc317c85e 100644--- a/cyberdrop_dl/utils/logger.py+++ b/cyberdrop_dl/utils/logger.py@@ -15,8 +15,7 @@ from rich.measure import Measurementfrom rich.padding import Paddingfrom rich.text import Text, TextType-from cyberdrop_dl import env-from cyberdrop_dl.utils import constants+from cyberdrop_dl import constants, envlogger = logging.getLogger("cyberdrop_dl")logger_debug = logging.getLogger("cyberdrop_dl_debug")diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.pyindex 13e29874b..6213d60d5 100644--- a/cyberdrop_dl/utils/sorting.py+++ b/cyberdrop_dl/utils/sorting.py@@ -13,7 +13,7 @@ import PILfrom PIL import Imagefrom videoprops import get_audio_properties, get_video_properties-from cyberdrop_dl.utils.constants import FILE_FORMATS+from cyberdrop_dl.constants import FILE_FORMATSfrom cyberdrop_dl.utils.logger import log_with_colorfrom cyberdrop_dl.utils.utilities import purge_dir_treediff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex bd2bc6a75..b968a177e 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -21,7 +21,8 @@ from aiohttp_client_cache.response import AnyResponsefrom bs4 import BeautifulSoupfrom yarl import URL-from cyberdrop_dl.clients.errors import (+from cyberdrop_dl import constants+from cyberdrop_dl.exceptions import (CDLBaseError,ErrorLogMessage,InvalidExtensionError,@@ -29,7 +30,6 @@ from cyberdrop_dl.clients.errors import (NoExtensionError,get_origin,)-from cyberdrop_dl.utils import constantsfrom cyberdrop_dl.utils.logger import log, log_debug, log_spacer, log_with_colorif TYPE_CHECKING:@@ -39,9 +39,9 @@ if TYPE_CHECKING:from rich.text import Textfrom cyberdrop_dl.crawlers import Crawler+ from cyberdrop_dl.data_structures.url_objects import MediaItem, ScrapeItemfrom cyberdrop_dl.downloader.downloader import Downloaderfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem, ScrapeItemP = ParamSpec("P")diff --git a/cyberdrop_dl/config_definitions/custom/validators.py b/cyberdrop_dl/utils/validators.pysimilarity index 91%rename from cyberdrop_dl/config_definitions/custom/validators.pyrename to cyberdrop_dl/utils/validators.pyindex b1e492438..c1b197913 100644--- a/cyberdrop_dl/config_definitions/custom/validators.py+++ b/cyberdrop_dl/utils/validators.py@@ -2,15 +2,20 @@Functions to use with `BeforeValidator`, `field_validator(mode="before")` or `model_validator(mode="before")`"""-from collections.abc import Callable+from __future__ import annotations+from datetime import timedeltafrom functools import singledispatch-from typing import Any+from typing import TYPE_CHECKING, Anyfrom pydantic import HttpUrl-from yarl import URL-from .converters import convert_str_to_timedelta, convert_to_yarl+from cyberdrop_dl.utils.converters import convert_str_to_timedelta, convert_to_yarl++if TYPE_CHECKING:+ from collections.abc import Callable++ from yarl import URLdef pydantyc_yarl_url(value: str) -> URL:diff --git a/cyberdrop_dl/utils/yaml.py b/cyberdrop_dl/utils/yaml.pyindex da0ff4b48..d5830d798 100644--- a/cyberdrop_dl/utils/yaml.py+++ b/cyberdrop_dl/utils/yaml.py@@ -11,8 +11,8 @@ import yamlfrom pydantic import BaseModel, ValidationErrorfrom yarl import URL-from cyberdrop_dl.clients.errors import InvalidYamlError-from cyberdrop_dl.utils.constants import CLI_VALIDATION_ERROR_FOOTER, VALIDATION_ERROR_FOOTER+from cyberdrop_dl.constants import CLI_VALIDATION_ERROR_FOOTER, VALIDATION_ERROR_FOOTER+from cyberdrop_dl.exceptions import InvalidYamlErrorif TYPE_CHECKING:from pydantic_core import ErrorDetailsdiff --git a/tests/test_apprise.py b/tests/test_apprise.pyindex 4e98e83e1..c083b0461 100644--- a/tests/test_apprise.py+++ b/tests/test_apprise.py@@ -6,11 +6,12 @@ from pathlib import Pathimport pytestfrom rich.text import Text+from cyberdrop_dl import constants+from cyberdrop_dl.constants import NotificationResultfrom cyberdrop_dl.managers.config_manager import ConfigManagerfrom cyberdrop_dl.managers.manager import Managerfrom cyberdrop_dl.managers.path_manager import PathManager-from cyberdrop_dl.utils import apprise, constants-from cyberdrop_dl.utils.constants import NotificationResult+from cyberdrop_dl.utils import apprisefrom tests.fake_classes.managers import FakeCacheManagerTEST_FILES_PATH = Path(__file__).parent / "test_files/apprise"