Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: circular imports (#1005)

NTFSvolume609f1332d6f0
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index ec1d8dee0..35dbc674a 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -16,8 +16,8 @@ from dateutil import parser
from videoprops import get_audio_properties, get_video_properties
from yarl import URL
-from cyberdrop_dl.clients.errors import DDOSGuardError, DownloadError, InvalidContentTypeError, SlowDownloadError
-from cyberdrop_dl.utils.constants import FILE_FORMATS
+from cyberdrop_dl.constants import FILE_FORMATS
+from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError, SlowDownloadError
from cyberdrop_dl.utils.logger import log, log_debug
if TYPE_CHECKING:
@@ -27,9 +27,9 @@ if TYPE_CHECKING:
from multidict import CIMultiDictProxy
+ from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.managers.client_manager import ClientManager
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem
P = ParamSpec("P")
diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.py
index af6ebd408..6b7a7c856 100644
--- a/cyberdrop_dl/clients/hash_client.py
+++ b/cyberdrop_dl/clients/hash_client.py
@@ -9,16 +9,16 @@ from typing import TYPE_CHECKING
import aiofiles.os
from send2trash import send2trash
+from cyberdrop_dl.data_structures.hash import Hashing
from cyberdrop_dl.ui.prompts.basic_prompts import enter_to_continue
-from cyberdrop_dl.utils.data_enums_classes.hash import Hashing
from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import get_size_or_none
if TYPE_CHECKING:
from yarl import URL
+ from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem
def hash_directory_scanner(manager: Manager, path: Path) -> None:
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.py
index 20bdd9d85..fc2f9ba5f 100644
--- a/cyberdrop_dl/clients/scraper_client.py
+++ b/cyberdrop_dl/clients/scraper_client.py
@@ -18,9 +18,9 @@ from aiohttp_client_cache.response import AnyResponse
from aiohttp_client_cache.session import CachedSession
from bs4 import BeautifulSoup
-import cyberdrop_dl.utils.constants as constants
+import cyberdrop_dl.constants as constants
from cyberdrop_dl import env
-from cyberdrop_dl.clients.errors import DDOSGuardError, DownloadError, InvalidContentTypeError, ScrapeError
+from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError, ScrapeError
from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import get_soup_no_error, sanitize_filename
diff --git a/cyberdrop_dl/config_definitions/authentication_settings.py b/cyberdrop_dl/config_definitions/authentication_settings.py
index c5cda7e8c..e7a6b1fad 100644
--- a/cyberdrop_dl/config_definitions/authentication_settings.py
+++ b/cyberdrop_dl/config_definitions/authentication_settings.py
@@ -1,6 +1,6 @@
from pydantic import BaseModel, Field
-from .custom.types import AliasModel
+from cyberdrop_dl.types import AliasModel
class CoomerAuth(BaseModel):
diff --git a/cyberdrop_dl/config_definitions/config_settings.py b/cyberdrop_dl/config_definitions/config_settings.py
index 9340a1349..0bf68ca5d 100644
--- a/cyberdrop_dl/config_definitions/config_settings.py
+++ b/cyberdrop_dl/config_definitions/config_settings.py
@@ -5,11 +5,10 @@ from pathlib import Path
from pydantic import BaseModel, ByteSize, Field, NonNegativeInt, PositiveInt, field_serializer, field_validator
-from cyberdrop_dl.utils.constants import APP_STORAGE, BROWSERS, DOWNLOAD_STORAGE
-from cyberdrop_dl.utils.data_enums_classes.hash import Hashing
-from cyberdrop_dl.utils.data_enums_classes.supported_domains import SUPPORTED_SITES_DOMAINS
-
-from .custom.types import (
+from cyberdrop_dl.constants import APP_STORAGE, BROWSERS, DOWNLOAD_STORAGE
+from cyberdrop_dl.data_structures.hash import Hashing
+from cyberdrop_dl.data_structures.supported_domains import SUPPORTED_SITES_DOMAINS
+from cyberdrop_dl.types import (
AliasModel,
ByteSizeSerilized,
HttpAppriseURL,
@@ -21,7 +20,7 @@ from .custom.types import (
NonEmptyStrOrNone,
PathOrNone,
)
-from .custom.validators import parse_duration_as_timedelta, parse_falsy_as
+from cyberdrop_dl.utils.validators import parse_duration_as_timedelta, parse_falsy_as
ALL_SUPPORTED_SITES = ["<<ALL_SUPPORTED_SITES>>"]
diff --git a/cyberdrop_dl/config_definitions/global_settings.py b/cyberdrop_dl/config_definitions/global_settings.py
index 1777b3e29..425ac5dd9 100644
--- a/cyberdrop_dl/config_definitions/global_settings.py
+++ b/cyberdrop_dl/config_definitions/global_settings.py
@@ -4,9 +4,9 @@ from datetime import timedelta
from pydantic import BaseModel, ByteSize, Field, NonNegativeFloat, PositiveInt, field_serializer, field_validator
from yarl import URL
-from .custom.converters import convert_to_byte_size
-from .custom.types import AliasModel, ByteSizeSerilized, HttpURL, NonEmptyStr
-from .custom.validators import parse_duration_as_timedelta, parse_falsy_as
+from cyberdrop_dl.types import AliasModel, ByteSizeSerilized, HttpURL, NonEmptyStr
+from cyberdrop_dl.utils.converters import convert_to_byte_size
+from cyberdrop_dl.utils.validators import parse_duration_as_timedelta, parse_falsy_as
MIN_REQUIRED_FREE_SPACE = convert_to_byte_size("512MB")
DEFAULT_REQUIRED_FREE_SPACE = convert_to_byte_size("5GB")
diff --git a/cyberdrop_dl/utils/constants.py b/cyberdrop_dl/constants.py
similarity index 100%
rename from cyberdrop_dl/utils/constants.py
rename to cyberdrop_dl/constants.py
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index 55df64960..368b22b17 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -6,14 +6,14 @@ from typing import TYPE_CHECKING
from bs4 import BeautifulSoup
-from cyberdrop_dl.clients.errors import PasswordProtectedError, ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import PasswordProtectedError, ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from yarl import URL
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
ITEM_DESCRIPTION_SELECTOR = "p[class*=description-meta]"
diff --git a/cyberdrop_dl/crawlers/acidimg.py b/cyberdrop_dl/crawlers/acidimg.py
index 10d5250fb..13c364f9b 100644
--- a/cyberdrop_dl/crawlers/acidimg.py
+++ b/cyberdrop_dl/crawlers/acidimg.py
@@ -8,8 +8,8 @@ from cyberdrop_dl.crawlers.crawler import create_task_id
from cyberdrop_dl.crawlers.imx_to import ImxToCrawler
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class AcidImgCrawler(ImxToCrawler):
diff --git a/cyberdrop_dl/crawlers/archivebate.py b/cyberdrop_dl/crawlers/archivebate.py
index 54fc2ada0..27d765ba1 100644
--- a/cyberdrop_dl/crawlers/archivebate.py
+++ b/cyberdrop_dl/crawlers/archivebate.py
@@ -5,16 +5,16 @@ from typing import TYPE_CHECKING, ClassVar
from aiolimiter import AsyncLimiter
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import create_task_id
from cyberdrop_dl.crawlers.mixdrop import MixDropCrawler
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_og_properties, get_text_between
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class Selectors:
diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.py
index 3f0ce1075..8cc597e27 100644
--- a/cyberdrop_dl/crawlers/ashemaletube.py
+++ b/cyberdrop_dl/crawlers/ashemaletube.py
@@ -8,15 +8,15 @@ from typing import TYPE_CHECKING, NamedTuple
from aiolimiter import AsyncLimiter
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
from bs4 import BeautifulSoup, Tag
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class Selectors:
diff --git a/cyberdrop_dl/crawlers/bestprettygirl.py b/cyberdrop_dl/crawlers/bestprettygirl.py
index 715fb5e7f..72f0e4a50 100644
--- a/cyberdrop_dl/crawlers/bestprettygirl.py
+++ b/cyberdrop_dl/crawlers/bestprettygirl.py
@@ -5,15 +5,15 @@ from typing import TYPE_CHECKING
from aiolimiter import AsyncLimiter
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class Selectors:
diff --git a/cyberdrop_dl/crawlers/box_dot_com.py b/cyberdrop_dl/crawlers/box_dot_com.py
index 97472fba4..62f91af5d 100644
--- a/cyberdrop_dl/crawlers/box_dot_com.py
+++ b/cyberdrop_dl/crawlers/box_dot_com.py
@@ -9,16 +9,16 @@ from typing import TYPE_CHECKING, Any
from pydantic import Field
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
-from cyberdrop_dl.config_definitions.custom.types import AliasModel
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
+from cyberdrop_dl.types import AliasModel
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class ItemType(StrEnum):
diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.py
index 195bea04c..24385978d 100644
--- a/cyberdrop_dl/crawlers/bunkrr.py
+++ b/cyberdrop_dl/crawlers/bunkrr.py
@@ -14,16 +14,16 @@ from typing import TYPE_CHECKING, ClassVar, NamedTuple
from aiohttp import ClientConnectorError
from yarl import URL
-from cyberdrop_dl.clients.errors import DDOSGuardError, NoExtensionError, ScrapeError
+from cyberdrop_dl.constants import FILE_FORMATS
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
-from cyberdrop_dl.utils.constants import FILE_FORMATS
+from cyberdrop_dl.exceptions import DDOSGuardError, NoExtensionError, ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_og_properties, get_text_between, parse_url
if TYPE_CHECKING:
from bs4 import BeautifulSoup, Tag
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
# CDNs
diff --git a/cyberdrop_dl/crawlers/bunkrr_albums_io.py b/cyberdrop_dl/crawlers/bunkrr_albums_io.py
index 9af74472f..b55a40479 100644
--- a/cyberdrop_dl/crawlers/bunkrr_albums_io.py
+++ b/cyberdrop_dl/crawlers/bunkrr_albums_io.py
@@ -12,8 +12,8 @@ if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
ALBUM_SELECTOR = "main div.auto-rows-max a"
diff --git a/cyberdrop_dl/crawlers/buzzheavier.py b/cyberdrop_dl/crawlers/buzzheavier.py
index 18e6e63f4..6e6e66d56 100644
--- a/cyberdrop_dl/crawlers/buzzheavier.py
+++ b/cyberdrop_dl/crawlers/buzzheavier.py
@@ -8,8 +8,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_from_headers
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class BuzzHeavierCrawler(Crawler):
diff --git a/cyberdrop_dl/crawlers/catbox.py b/cyberdrop_dl/crawlers/catbox.py
index 4995e1ab9..602358d97 100644
--- a/cyberdrop_dl/crawlers/catbox.py
+++ b/cyberdrop_dl/crawlers/catbox.py
@@ -7,8 +7,8 @@ from yarl import URL
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
CDN_HOSTS = "litter.catbox.moe", "files.catbox.moe"
diff --git a/cyberdrop_dl/crawlers/coomer.py b/cyberdrop_dl/crawlers/coomer.py
index 68167d3fa..b08097c4a 100644
--- a/cyberdrop_dl/crawlers/coomer.py
+++ b/cyberdrop_dl/crawlers/coomer.py
@@ -11,8 +11,8 @@ from cyberdrop_dl.crawlers.kemono import KemonoCrawler, UserPost
if TYPE_CHECKING:
from aiohttp_client_cache.response import AnyResponse
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class CoomerCrawler(KemonoCrawler):
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index e1980a23a..444ea71bf 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -14,10 +14,10 @@ from aiolimiter import AsyncLimiter
from dateutil import parser
from yarl import URL
+from cyberdrop_dl.data_structures.url_objects import MediaItem, ScrapeItem
from cyberdrop_dl.downloader.downloader import Downloader
from cyberdrop_dl.scraper import filters
from cyberdrop_dl.utils import utilities
-from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem, ScrapeItem
from cyberdrop_dl.utils.database.tables.history_table import get_db_path
from cyberdrop_dl.utils.logger import log, log_debug
from cyberdrop_dl.utils.utilities import (
diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.py
index da8693037..7b6b42521 100644
--- a/cyberdrop_dl/crawlers/cyberdrop.py
+++ b/cyberdrop_dl/crawlers/cyberdrop.py
@@ -6,15 +6,15 @@ from aiolimiter import AsyncLimiter
from bs4 import BeautifulSoup
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
API_ENTRYPOINT = URL("https://api.cyberdrop.me/api/")
diff --git a/cyberdrop_dl/crawlers/cyberfile.py b/cyberdrop_dl/crawlers/cyberfile.py
index fcd8b96e8..1d04b4fc6 100644
--- a/cyberdrop_dl/crawlers/cyberfile.py
+++ b/cyberdrop_dl/crawlers/cyberfile.py
@@ -7,13 +7,13 @@ from aiolimiter import AsyncLimiter
from bs4 import BeautifulSoup
from yarl import URL
-from cyberdrop_dl.clients.errors import PasswordProtectedError, ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import PasswordProtectedError, ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
SOUP_ERRORS = {
diff --git a/cyberdrop_dl/crawlers/dirtyship.py b/cyberdrop_dl/crawlers/dirtyship.py
index d2e4081ee..9b0dfd186 100644
--- a/cyberdrop_dl/crawlers/dirtyship.py
+++ b/cyberdrop_dl/crawlers/dirtyship.py
@@ -5,15 +5,15 @@ from typing import TYPE_CHECKING, NamedTuple
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
VIDEO_SELECTOR = "video#fp-video-0 > source"
diff --git a/cyberdrop_dl/crawlers/doodstream.py b/cyberdrop_dl/crawlers/doodstream.py
index 2b72b1c75..dabcddf0c 100644
--- a/cyberdrop_dl/crawlers/doodstream.py
+++ b/cyberdrop_dl/crawlers/doodstream.py
@@ -7,15 +7,15 @@ from typing import TYPE_CHECKING, ClassVar
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class Selectors:
diff --git a/cyberdrop_dl/crawlers/dropbox.py b/cyberdrop_dl/crawlers/dropbox.py
index 2c0344107..458dedef0 100644
--- a/cyberdrop_dl/crawlers/dropbox.py
+++ b/cyberdrop_dl/crawlers/dropbox.py
@@ -6,15 +6,15 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_from_headers
if TYPE_CHECKING:
from collections.abc import Mapping
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class DropboxCrawler(Crawler):
diff --git a/cyberdrop_dl/crawlers/e621.py b/cyberdrop_dl/crawlers/e621.py
index d1bebe137..c1ba12157 100644
--- a/cyberdrop_dl/crawlers/e621.py
+++ b/cyberdrop_dl/crawlers/e621.py
@@ -7,15 +7,15 @@ from aiolimiter import AsyncLimiter
from yarl import URL
from cyberdrop_dl import __version__
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class E621Crawler(Crawler):
diff --git a/cyberdrop_dl/crawlers/ehentai.py b/cyberdrop_dl/crawlers/ehentai.py
index b71133837..75b2fd769 100644
--- a/cyberdrop_dl/crawlers/ehentai.py
+++ b/cyberdrop_dl/crawlers/ehentai.py
@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class Selectors:
diff --git a/cyberdrop_dl/crawlers/eightmuses.py b/cyberdrop_dl/crawlers/eightmuses.py
index 638ddf484..3af810c28 100644
--- a/cyberdrop_dl/crawlers/eightmuses.py
+++ b/cyberdrop_dl/crawlers/eightmuses.py
@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
TILE_SELECTOR = "a[class*=c-tile]"
diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.py
index 1a8e2a285..36f287bd7 100644
--- a/cyberdrop_dl/crawlers/eporner.py
+++ b/cyberdrop_dl/crawlers/eporner.py
@@ -4,17 +4,17 @@ from typing import TYPE_CHECKING, NamedTuple
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils import javascript
-from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup, Tag
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class Selectors:
diff --git a/cyberdrop_dl/crawlers/erome.py b/cyberdrop_dl/crawlers/erome.py
index 0c06df79a..76a2e88b4 100644
--- a/cyberdrop_dl/crawlers/erome.py
+++ b/cyberdrop_dl/crawlers/erome.py
@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class Selectors:
diff --git a/cyberdrop_dl/crawlers/fapello.py b/cyberdrop_dl/crawlers/fapello.py
index 1af268cd9..875913206 100644
--- a/cyberdrop_dl/crawlers/fapello.py
+++ b/cyberdrop_dl/crawlers/fapello.py
@@ -5,15 +5,15 @@ from typing import TYPE_CHECKING
from aiolimiter import AsyncLimiter
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
CONTENT_SELECTOR = "div[id=content] a"
diff --git a/cyberdrop_dl/crawlers/fileditch.py b/cyberdrop_dl/crawlers/fileditch.py
index e429ff0f5..1c9db7b04 100644
--- a/cyberdrop_dl/crawlers/fileditch.py
+++ b/cyberdrop_dl/crawlers/fileditch.py
@@ -4,15 +4,15 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
DOWNLOAD_SELECTOR = "a[class*='download-button']"
HOMEPAGE_CATCHALL_FILE = "/s21/FHVZKQyAZlIsrneDAsp.jpeg"
diff --git a/cyberdrop_dl/crawlers/files_vc.py b/cyberdrop_dl/crawlers/files_vc.py
index ea3cd70db..fc253a479 100644
--- a/cyberdrop_dl/crawlers/files_vc.py
+++ b/cyberdrop_dl/crawlers/files_vc.py
@@ -9,8 +9,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
API_ENTRYPOINT = URL("https://api.files.vc/api")
diff --git a/cyberdrop_dl/crawlers/fourchan.py b/cyberdrop_dl/crawlers/fourchan.py
index 87b0d75f2..7e7991833 100644
--- a/cyberdrop_dl/crawlers/fourchan.py
+++ b/cyberdrop_dl/crawlers/fourchan.py
@@ -6,13 +6,13 @@ from aiolimiter import AsyncLimiter
from bs4 import BeautifulSoup
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
API_ENTRYPOINT = URL("https://a.4cdn.org/")
FILES_BASE_URL = URL("https://i.4cdn.org/")
diff --git a/cyberdrop_dl/crawlers/generic.py b/cyberdrop_dl/crawlers/generic.py
index cc977ee59..2119d04a8 100644
--- a/cyberdrop_dl/crawlers/generic.py
+++ b/cyberdrop_dl/crawlers/generic.py
@@ -6,8 +6,8 @@ from functools import wraps
from pathlib import Path
from typing import TYPE_CHECKING, ParamSpec, TypeVar
-from cyberdrop_dl.clients.errors import InvalidContentTypeError, NoExtensionError, ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import InvalidContentTypeError, NoExtensionError, ScrapeError
from cyberdrop_dl.scraper.filters import has_valid_extension
from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_and_ext
@@ -18,8 +18,8 @@ if TYPE_CHECKING:
from bs4 import BeautifulSoup
from yarl import URL
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
P = ParamSpec("P")
diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.py
index 6fd8dea8f..89014e270 100644
--- a/cyberdrop_dl/crawlers/gofile.py
+++ b/cyberdrop_dl/crawlers/gofile.py
@@ -9,9 +9,9 @@ from typing import TYPE_CHECKING, Literal, NotRequired, TypedDict, cast
from aiolimiter import AsyncLimiter
from yarl import URL
-from cyberdrop_dl.clients.errors import DownloadError, PasswordProtectedError, ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
-from cyberdrop_dl.utils.data_enums_classes.url_objects import FILE_HOST_ALBUM, ScrapeItem
+from cyberdrop_dl.data_structures.url_objects import FILE_HOST_ALBUM, ScrapeItem
+from cyberdrop_dl.exceptions import DownloadError, PasswordProtectedError, ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.py
index 23a2b0a82..60febbda5 100644
--- a/cyberdrop_dl/crawlers/google_drive.py
+++ b/cyberdrop_dl/crawlers/google_drive.py
@@ -38,8 +38,8 @@ from typing import TYPE_CHECKING, ClassVar
from aiolimiter import AsyncLimiter
from yarl import URL
-from cyberdrop_dl.clients.errors import DownloadError, ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import DownloadError, ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_from_headers
if TYPE_CHECKING:
@@ -47,8 +47,8 @@ if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
VALID_FILE_URL_PARTS = "file", "document", "presentation", "spreadsheets"
diff --git a/cyberdrop_dl/crawlers/hotpic.py b/cyberdrop_dl/crawlers/hotpic.py
index a4f49f2c2..f20de92e0 100644
--- a/cyberdrop_dl/crawlers/hotpic.py
+++ b/cyberdrop_dl/crawlers/hotpic.py
@@ -5,15 +5,15 @@ from typing import TYPE_CHECKING, ClassVar
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
IMAGE_SELECTOR = "img[id*=main-image]"
VIDEO_SELECTOR = "video > source"
diff --git a/cyberdrop_dl/crawlers/imagebam.py b/cyberdrop_dl/crawlers/imagebam.py
index 09ba5b14d..da88fa569 100644
--- a/cyberdrop_dl/crawlers/imagebam.py
+++ b/cyberdrop_dl/crawlers/imagebam.py
@@ -4,15 +4,15 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
GALLERY_TITLE_SELECTOR = "a#gallery-name"
diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.py
index 51d5216dd..82aa4a78d 100644
--- a/cyberdrop_dl/crawlers/imgbb.py
+++ b/cyberdrop_dl/crawlers/imgbb.py
@@ -12,8 +12,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
IMAGE_PAGE_SELECTOR = "a[class*=image-container]"
diff --git a/cyberdrop_dl/crawlers/imgbox.py b/cyberdrop_dl/crawlers/imgbox.py
index 6ec73370b..a95eaab51 100644
--- a/cyberdrop_dl/crawlers/imgbox.py
+++ b/cyberdrop_dl/crawlers/imgbox.py
@@ -6,9 +6,9 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
-from cyberdrop_dl.utils.data_enums_classes.url_objects import FILE_HOST_ALBUM, ScrapeItem
+from cyberdrop_dl.data_structures.url_objects import FILE_HOST_ALBUM, ScrapeItem
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.py
index 591ba92b0..2afd36f7a 100644
--- a/cyberdrop_dl/crawlers/imgur.py
+++ b/cyberdrop_dl/crawlers/imgur.py
@@ -5,13 +5,13 @@ from typing import TYPE_CHECKING, Any
from yarl import URL
-from cyberdrop_dl.clients.errors import LoginError, ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import LoginError, ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
API_ENTRYPOINT = URL("https://api.imgur.com/3/")
diff --git a/cyberdrop_dl/crawlers/imx_to.py b/cyberdrop_dl/crawlers/imx_to.py
index 24f261250..0b3348388 100644
--- a/cyberdrop_dl/crawlers/imx_to.py
+++ b/cyberdrop_dl/crawlers/imx_to.py
@@ -9,8 +9,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
IMG_SELECTOR = "div#container a > img"
diff --git a/cyberdrop_dl/crawlers/incestflix.py b/cyberdrop_dl/crawlers/incestflix.py
index 0348d3944..743cd12e7 100644
--- a/cyberdrop_dl/crawlers/incestflix.py
+++ b/cyberdrop_dl/crawlers/incestflix.py
@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class Selectors:
diff --git a/cyberdrop_dl/crawlers/influencer_bitches.py b/cyberdrop_dl/crawlers/influencer_bitches.py
index f2e421534..18d79b0a4 100644
--- a/cyberdrop_dl/crawlers/influencer_bitches.py
+++ b/cyberdrop_dl/crawlers/influencer_bitches.py
@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class Selectors:
diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.py
index d9fc17e4d..09e5b50b0 100644
--- a/cyberdrop_dl/crawlers/jpg5.py
+++ b/cyberdrop_dl/crawlers/jpg5.py
@@ -12,8 +12,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
from ._chevereto import CheveretoCrawler
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
PRIMARY_BASE_DOMAIN = URL("https://jpg5.su")
JPG5_REPLACE_HOST_REGEX = re.compile(r"(jpg\.fish/)|(jpg\.fishing/)|(jpg\.church/)")
diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.py
index 21915c366..8bbb4bca1 100644
--- a/cyberdrop_dl/crawlers/kemono.py
+++ b/cyberdrop_dl/crawlers/kemono.py
@@ -13,11 +13,11 @@ from pydantic import AliasChoices, BeforeValidator, Field
from typing_extensions import TypedDict # Import from typing is not compatible with pydantic
from yarl import URL
-from cyberdrop_dl.clients.errors import NoExtensionError, ScrapeError
-from cyberdrop_dl.config_definitions.custom.types import AliasModel
-from cyberdrop_dl.config_definitions.custom.validators import parse_falsy_as_none
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import NoExtensionError, ScrapeError
+from cyberdrop_dl.types import AliasModel
from cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_parts
+from cyberdrop_dl.utils.validators import parse_falsy_as_none
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable, Coroutine, Generator
@@ -25,8 +25,8 @@ if TYPE_CHECKING:
from aiohttp_client_cache.response import AnyResponse
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
MAX_OFFSET_PER_CALL = 50
LINK_REGEX = re.compile(r"(?:http(?!.*\.\.)[^ ]*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]|</|'))")
diff --git a/cyberdrop_dl/crawlers/luscious.py b/cyberdrop_dl/crawlers/luscious.py
index f010cb02d..ebcb57ee9 100644
--- a/cyberdrop_dl/crawlers/luscious.py
+++ b/cyberdrop_dl/crawlers/luscious.py
@@ -6,16 +6,16 @@ from typing import TYPE_CHECKING, Any
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
GRAPHQL_URL = URL("https://members.luscious.net/graphql/nobatch/")
diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.py
index 5c9a0ac2d..8544a36a3 100644
--- a/cyberdrop_dl/crawlers/mediafire.py
+++ b/cyberdrop_dl/crawlers/mediafire.py
@@ -9,15 +9,15 @@ from aiolimiter import AsyncLimiter
from mediafire import MediaFireApi, api
from yarl import URL
-from cyberdrop_dl.clients.errors import MediaFireError, ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import MediaFireError, ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
DOWNLOAD_BUTTON_SELECTOR = "a[id=downloadButton]"
DATE_SELECTOR = "ul[class=details] li span"
diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.py
index 09fa6946f..7d36745a3 100644
--- a/cyberdrop_dl/crawlers/mega_nz.py
+++ b/cyberdrop_dl/crawlers/mega_nz.py
@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, NamedTuple, cast
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
from cyberdrop_dl.downloader.mega_nz import (
DecryptData,
@@ -16,11 +15,12 @@ from cyberdrop_dl.downloader.mega_nz import (
base64_url_decode,
decrypt_attr,
)
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class FileTuple(NamedTuple):
diff --git a/cyberdrop_dl/crawlers/missav.py b/cyberdrop_dl/crawlers/missav.py
index a4055fdbc..b816a3e32 100644
--- a/cyberdrop_dl/crawlers/missav.py
+++ b/cyberdrop_dl/crawlers/missav.py
@@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, NamedTuple
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -16,8 +16,8 @@ if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
M3U8_SERVER = URL("https://surrit.com/")
diff --git a/cyberdrop_dl/crawlers/mixdrop.py b/cyberdrop_dl/crawlers/mixdrop.py
index e8e2aa144..31c712e69 100644
--- a/cyberdrop_dl/crawlers/mixdrop.py
+++ b/cyberdrop_dl/crawlers/mixdrop.py
@@ -11,8 +11,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betwee
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class Selectors:
diff --git a/cyberdrop_dl/crawlers/motherless.py b/cyberdrop_dl/crawlers/motherless.py
index 125e113de..fc27c786e 100644
--- a/cyberdrop_dl/crawlers/motherless.py
+++ b/cyberdrop_dl/crawlers/motherless.py
@@ -6,15 +6,15 @@ from typing import TYPE_CHECKING, Literal, NamedTuple
from aiolimiter import AsyncLimiter
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
MEDIA_INFO_JS_SELECTOR = "script:contains('__fileurl')"
diff --git a/cyberdrop_dl/crawlers/nekohouse.py b/cyberdrop_dl/crawlers/nekohouse.py
index 9da6de522..da2385146 100644
--- a/cyberdrop_dl/crawlers/nekohouse.py
+++ b/cyberdrop_dl/crawlers/nekohouse.py
@@ -8,8 +8,8 @@ from cyberdrop_dl.crawlers.crawler import create_task_id
from cyberdrop_dl.crawlers.kemono import KemonoCrawler
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class NekohouseCrawler(KemonoCrawler):
diff --git a/cyberdrop_dl/crawlers/nhentai.py b/cyberdrop_dl/crawlers/nhentai.py
index 109d23edf..f2984d784 100644
--- a/cyberdrop_dl/crawlers/nhentai.py
+++ b/cyberdrop_dl/crawlers/nhentai.py
@@ -6,16 +6,16 @@ from typing import TYPE_CHECKING
from aiolimiter import AsyncLimiter
from yarl import URL
-from cyberdrop_dl.clients.errors import LoginError, ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import LoginError, ScrapeError
from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from collections.abc import Generator
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
PRIMARY_BASE_DOMAIN = URL("https://nhentai.net/")
diff --git a/cyberdrop_dl/crawlers/noodle_magazine.py b/cyberdrop_dl/crawlers/noodle_magazine.py
index 2f3d0873a..9dc8babbd 100644
--- a/cyberdrop_dl/crawlers/noodle_magazine.py
+++ b/cyberdrop_dl/crawlers/noodle_magazine.py
@@ -8,15 +8,15 @@ from aiolimiter import AsyncLimiter
from bs4 import BeautifulSoup
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
PLAYLIST_SELECTOR = "script:contains('window.playlist')"
diff --git a/cyberdrop_dl/crawlers/nudostartv.py b/cyberdrop_dl/crawlers/nudostartv.py
index 1a955a29e..8e4f44791 100644
--- a/cyberdrop_dl/crawlers/nudostartv.py
+++ b/cyberdrop_dl/crawlers/nudostartv.py
@@ -4,15 +4,15 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
IMAGE_SELECTOR = "div[class=block-video] a img"
CONTENT_SELECTOR = "div[id=list_videos_common_videos_list_items] div a"
diff --git a/cyberdrop_dl/crawlers/omegascans.py b/cyberdrop_dl/crawlers/omegascans.py
index cf8ee793b..035fa57fa 100644
--- a/cyberdrop_dl/crawlers/omegascans.py
+++ b/cyberdrop_dl/crawlers/omegascans.py
@@ -7,15 +7,15 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
DATE_SELECTOR = 'h2[class="font-semibold font-sans text-muted-foreground text-xs"]'
diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.py
index 9b157a252..30f9b9dc7 100644
--- a/cyberdrop_dl/crawlers/onedrive.py
+++ b/cyberdrop_dl/crawlers/onedrive.py
@@ -13,13 +13,13 @@ from typing import TYPE_CHECKING, Any, ClassVar, Self
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
API_ENTRYPOINT = URL("https://api.onedrive.com/v1.0/drives/")
diff --git a/cyberdrop_dl/crawlers/pimpandhost.py b/cyberdrop_dl/crawlers/pimpandhost.py
index a675bc514..14f3394e0 100644
--- a/cyberdrop_dl/crawlers/pimpandhost.py
+++ b/cyberdrop_dl/crawlers/pimpandhost.py
@@ -12,8 +12,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
ALBUM_TITLE_SELECTOR = "span[class=author-header__album-name]"
DATE_SELECTOR = "span[class=date-time]"
diff --git a/cyberdrop_dl/crawlers/pixeldrain.py b/cyberdrop_dl/crawlers/pixeldrain.py
index b7c5ec8ad..e0b607d66 100644
--- a/cyberdrop_dl/crawlers/pixeldrain.py
+++ b/cyberdrop_dl/crawlers/pixeldrain.py
@@ -7,16 +7,16 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import DownloadError, NoExtensionError, ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import DownloadError, NoExtensionError, ScrapeError
from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_og_properties, get_text_between
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
API_ENTRYPOINT = URL("https://pixeldrain.com/api/")
JS_SELECTOR = 'script:contains("window.initial_node")'
diff --git a/cyberdrop_dl/crawlers/pixhost.py b/cyberdrop_dl/crawlers/pixhost.py
index e5fddccd3..960e9e365 100644
--- a/cyberdrop_dl/crawlers/pixhost.py
+++ b/cyberdrop_dl/crawlers/pixhost.py
@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
PRIMARY_BASE_DOMAIN = URL("https://pixhost.to/")
diff --git a/cyberdrop_dl/crawlers/pkmncards.py b/cyberdrop_dl/crawlers/pkmncards.py
index 485ee679d..2ba175214 100644
--- a/cyberdrop_dl/crawlers/pkmncards.py
+++ b/cyberdrop_dl/crawlers/pkmncards.py
@@ -11,15 +11,15 @@ from typing import TYPE_CHECKING, NewType
from bs4 import BeautifulSoup
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup, Tag
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
TimeStamp = NewType("TimeStamp", int)
diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.py
index e4d0526da..2d381eaea 100644
--- a/cyberdrop_dl/crawlers/pmvhaven.py
+++ b/cyberdrop_dl/crawlers/pmvhaven.py
@@ -7,8 +7,8 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils import javascript
from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper
@@ -18,8 +18,8 @@ if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
JS_VIDEO_INFO_SELECTOR = "script#__NUXT_DATA__"
diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.py
index 3e2d6468f..be9504bb6 100644
--- a/cyberdrop_dl/crawlers/pornhub.py
+++ b/cyberdrop_dl/crawlers/pornhub.py
@@ -6,8 +6,8 @@ from typing import TYPE_CHECKING, Any, Literal, NamedTuple, TypedDict
from yarl import URL
-from cyberdrop_dl.clients.errors import NoExtensionError, ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import NoExtensionError, ScrapeError
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
@@ -16,8 +16,8 @@ if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
PRIMARY_BASE_DOMAIN = URL("https://www.pornhub.com")
ALBUM_API_URL = PRIMARY_BASE_DOMAIN / "album/show_album_json"
diff --git a/cyberdrop_dl/crawlers/pornpics.py b/cyberdrop_dl/crawlers/pornpics.py
index 4151364d2..6b4a714be 100644
--- a/cyberdrop_dl/crawlers/pornpics.py
+++ b/cyberdrop_dl/crawlers/pornpics.py
@@ -13,8 +13,8 @@ if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
COLLECTION_PARTS = "search", "channel", "pornstar", "tag", "category"
IMAGE_SELECTOR = "div#main a.rel-link"
diff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.py
index 62a55aa52..8d102bac9 100644
--- a/cyberdrop_dl/crawlers/porntrex.py
+++ b/cyberdrop_dl/crawlers/porntrex.py
@@ -7,15 +7,15 @@ from typing import TYPE_CHECKING, NamedTuple
from aiolimiter import AsyncLimiter
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class Video(NamedTuple):
diff --git a/cyberdrop_dl/crawlers/postimg.py b/cyberdrop_dl/crawlers/postimg.py
index 710dd060a..9920f1cdb 100644
--- a/cyberdrop_dl/crawlers/postimg.py
+++ b/cyberdrop_dl/crawlers/postimg.py
@@ -11,8 +11,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
DOWNLOAD_BUTTON_SELECTOR = "a[id=download]"
diff --git a/cyberdrop_dl/crawlers/realbooru.py b/cyberdrop_dl/crawlers/realbooru.py
index 3b1078dee..e08620799 100644
--- a/cyberdrop_dl/crawlers/realbooru.py
+++ b/cyberdrop_dl/crawlers/realbooru.py
@@ -4,15 +4,15 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
CONTENT_SELECTOR = "div[class=items] div a"
VIDEO_SELECTOR = "video source"
diff --git a/cyberdrop_dl/crawlers/realdebrid.py b/cyberdrop_dl/crawlers/realdebrid.py
index 2338f986f..ee38f9e17 100644
--- a/cyberdrop_dl/crawlers/realdebrid.py
+++ b/cyberdrop_dl/crawlers/realdebrid.py
@@ -12,8 +12,8 @@ from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class RealDebridCrawler(Crawler):
diff --git a/cyberdrop_dl/crawlers/reddit.py b/cyberdrop_dl/crawlers/reddit.py
index c697aafc7..dddb19155 100644
--- a/cyberdrop_dl/crawlers/reddit.py
+++ b/cyberdrop_dl/crawlers/reddit.py
@@ -8,9 +8,9 @@ from aiolimiter import AsyncLimiter
from asyncpraw import Reddit
from yarl import URL
-from cyberdrop_dl.clients.errors import LoginError, NoExtensionError, ScrapeError
from cyberdrop_dl.clients.scraper_client import cache_control_manager
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import LoginError, NoExtensionError, ScrapeError
from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import error_handling_wrapper
@@ -19,8 +19,8 @@ if TYPE_CHECKING:
from asyncpraw.models import Redditor, Submission, Subreddit
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class MediaFile(TypedDict):
diff --git a/cyberdrop_dl/crawlers/redgifs.py b/cyberdrop_dl/crawlers/redgifs.py
index e9d2963bb..6649fc32a 100644
--- a/cyberdrop_dl/crawlers/redgifs.py
+++ b/cyberdrop_dl/crawlers/redgifs.py
@@ -8,8 +8,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
API_ENTRYPOINT = URL("https://api.redgifs.com/")
diff --git a/cyberdrop_dl/crawlers/rule34vault.py b/cyberdrop_dl/crawlers/rule34vault.py
index 983792bdf..70bd55236 100644
--- a/cyberdrop_dl/crawlers/rule34vault.py
+++ b/cyberdrop_dl/crawlers/rule34vault.py
@@ -11,8 +11,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class Selectors:
diff --git a/cyberdrop_dl/crawlers/rule34video.py b/cyberdrop_dl/crawlers/rule34video.py
index c5f6a79fd..61246b968 100644
--- a/cyberdrop_dl/crawlers/rule34video.py
+++ b/cyberdrop_dl/crawlers/rule34video.py
@@ -6,10 +6,10 @@ from typing import TYPE_CHECKING, NamedTuple
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils import javascript
-from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper
@@ -18,8 +18,8 @@ if TYPE_CHECKING:
from bs4 import BeautifulSoup, Tag
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
RESOLUTIONS = ["4k", "2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"] # best to worst
diff --git a/cyberdrop_dl/crawlers/rule34xxx.py b/cyberdrop_dl/crawlers/rule34xxx.py
index 8bd450e50..f324717fa 100644
--- a/cyberdrop_dl/crawlers/rule34xxx.py
+++ b/cyberdrop_dl/crawlers/rule34xxx.py
@@ -4,15 +4,15 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class Selector:
diff --git a/cyberdrop_dl/crawlers/saint.py b/cyberdrop_dl/crawlers/saint.py
index 23b50585f..78f20c609 100644
--- a/cyberdrop_dl/crawlers/saint.py
+++ b/cyberdrop_dl/crawlers/saint.py
@@ -6,15 +6,15 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
VIDEOS_SELECTOR = "a.btn-primary.action.download"
diff --git a/cyberdrop_dl/crawlers/scrolller.py b/cyberdrop_dl/crawlers/scrolller.py
index 53e183e46..109fc3e2b 100644
--- a/cyberdrop_dl/crawlers/scrolller.py
+++ b/cyberdrop_dl/crawlers/scrolller.py
@@ -9,8 +9,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
DEFAULT_QUERY = """
query SubredditQuery(
diff --git a/cyberdrop_dl/crawlers/send_now.py b/cyberdrop_dl/crawlers/send_now.py
index 938a4b2f8..22305ef68 100644
--- a/cyberdrop_dl/crawlers/send_now.py
+++ b/cyberdrop_dl/crawlers/send_now.py
@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from curl_cffi.requests.models import Response as CurlResponse
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class SendNowCrawler(Crawler):
diff --git a/cyberdrop_dl/crawlers/sendvid.py b/cyberdrop_dl/crawlers/sendvid.py
index adeb25b1c..5f3cdae9e 100644
--- a/cyberdrop_dl/crawlers/sendvid.py
+++ b/cyberdrop_dl/crawlers/sendvid.py
@@ -4,15 +4,15 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_parts
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
VIDEO_SRC_SELECTOR = "video > source#video_source"
diff --git a/cyberdrop_dl/crawlers/sex_dot_com.py b/cyberdrop_dl/crawlers/sex_dot_com.py
index 4df17f2a4..1149f0a71 100644
--- a/cyberdrop_dl/crawlers/sex_dot_com.py
+++ b/cyberdrop_dl/crawlers/sex_dot_com.py
@@ -12,8 +12,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
API_URL = URL("https://iframe.sex.com/api/")
diff --git a/cyberdrop_dl/crawlers/spankbang.py b/cyberdrop_dl/crawlers/spankbang.py
index bff94ec8e..3d9bc6b88 100644
--- a/cyberdrop_dl/crawlers/spankbang.py
+++ b/cyberdrop_dl/crawlers/spankbang.py
@@ -8,18 +8,18 @@ from typing import TYPE_CHECKING, Any, NamedTuple
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils import javascript
-from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
PRIMARY_BASE_DOMAIN = URL("https://spankbang.com/")
diff --git a/cyberdrop_dl/crawlers/streamable.py b/cyberdrop_dl/crawlers/streamable.py
index 56b15e51c..7f5a292c1 100644
--- a/cyberdrop_dl/crawlers/streamable.py
+++ b/cyberdrop_dl/crawlers/streamable.py
@@ -5,16 +5,16 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
AJAX_ENTRYPOINT = URL("https://ajax.streamable.com/videos/")
diff --git a/cyberdrop_dl/crawlers/thisvid.py b/cyberdrop_dl/crawlers/thisvid.py
index 596ca2f89..1da8af5c0 100644
--- a/cyberdrop_dl/crawlers/thisvid.py
+++ b/cyberdrop_dl/crawlers/thisvid.py
@@ -6,15 +6,15 @@ from typing import TYPE_CHECKING, NamedTuple
from aiolimiter import AsyncLimiter
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
# Selectors
UNAUTHORIZED_SELECTOR = "div.video-holder:contains('This video is a private video')"
diff --git a/cyberdrop_dl/crawlers/tiktok.py b/cyberdrop_dl/crawlers/tiktok.py
index a4075a04b..00541924f 100644
--- a/cyberdrop_dl/crawlers/tiktok.py
+++ b/cyberdrop_dl/crawlers/tiktok.py
@@ -11,8 +11,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
VIDEO_PARTS = "video", "photo", "v"
API_URL = URL("https://www.tikwm.com/api/")
diff --git a/cyberdrop_dl/crawlers/tokyomotion.py b/cyberdrop_dl/crawlers/tokyomotion.py
index c458dc00f..93584995a 100644
--- a/cyberdrop_dl/crawlers/tokyomotion.py
+++ b/cyberdrop_dl/crawlers/tokyomotion.py
@@ -8,15 +8,15 @@ from typing import TYPE_CHECKING
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_parts
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
DATE_PATTERN = re.compile(r"(\d+)\s*(weeks?|days?|hours?|minutes?|seconds?)", re.IGNORECASE)
diff --git a/cyberdrop_dl/crawlers/toonily.py b/cyberdrop_dl/crawlers/toonily.py
index 523c26c63..d712b2295 100644
--- a/cyberdrop_dl/crawlers/toonily.py
+++ b/cyberdrop_dl/crawlers/toonily.py
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
from yarl import URL
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
-from cyberdrop_dl.utils.data_enums_classes.url_objects import FILE_HOST_PROFILE, ScrapeItem
+from cyberdrop_dl.data_structures.url_objects import FILE_HOST_PROFILE, ScrapeItem
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
diff --git a/cyberdrop_dl/crawlers/twitter_images.py b/cyberdrop_dl/crawlers/twitter_images.py
index 4a06c530c..5778264d5 100644
--- a/cyberdrop_dl/crawlers/twitter_images.py
+++ b/cyberdrop_dl/crawlers/twitter_images.py
@@ -8,8 +8,8 @@ from yarl import URL
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
CDN_HOST = "pbs.twimg.com"
diff --git a/cyberdrop_dl/crawlers/vipr_dot_im.py b/cyberdrop_dl/crawlers/vipr_dot_im.py
index 9914f14f7..b8b5c6330 100644
--- a/cyberdrop_dl/crawlers/vipr_dot_im.py
+++ b/cyberdrop_dl/crawlers/vipr_dot_im.py
@@ -9,8 +9,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
IMG_SELECTOR = "div#body a > img"
diff --git a/cyberdrop_dl/crawlers/wetransfer.py b/cyberdrop_dl/crawlers/wetransfer.py
index 22c0897de..ea0f62086 100644
--- a/cyberdrop_dl/crawlers/wetransfer.py
+++ b/cyberdrop_dl/crawlers/wetransfer.py
@@ -6,13 +6,13 @@ from typing import TYPE_CHECKING, ClassVar
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
API_ENTRYPOINT = URL("https://wetransfer.com/api/v4/transfers")
diff --git a/cyberdrop_dl/crawlers/xbunkr.py b/cyberdrop_dl/crawlers/xbunkr.py
index 8c0e75a7a..cadc82c10 100644
--- a/cyberdrop_dl/crawlers/xbunkr.py
+++ b/cyberdrop_dl/crawlers/xbunkr.py
@@ -10,8 +10,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
IMAGE_SELECTOR = "a[class=image]"
diff --git a/cyberdrop_dl/crawlers/xenforo/f95zone.py b/cyberdrop_dl/crawlers/xenforo/f95zone.py
index f84c7864e..d551ce0bc 100644
--- a/cyberdrop_dl/crawlers/xenforo/f95zone.py
+++ b/cyberdrop_dl/crawlers/xenforo/f95zone.py
@@ -9,8 +9,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
class F95ZoneCrawler(XenforoCrawler):
diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.py
index bd1f69e2a..f51dc534b 100644
--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py
+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py
@@ -12,9 +12,9 @@ from typing import TYPE_CHECKING
from bs4 import BeautifulSoup, Tag
from yarl import URL
-from cyberdrop_dl.clients.errors import InvalidURLError, LoginError, ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
-from cyberdrop_dl.utils.data_enums_classes.url_objects import FORUM, ScrapeItem
+from cyberdrop_dl.data_structures.url_objects import FORUM, ScrapeItem
+from cyberdrop_dl.exceptions import InvalidURLError, LoginError, ScrapeError
from cyberdrop_dl.utils.logger import log, log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_trailing_slash
diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.py
index d5994b015..93579dcb9 100644
--- a/cyberdrop_dl/crawlers/xhamster.py
+++ b/cyberdrop_dl/crawlers/xhamster.py
@@ -6,8 +6,8 @@ from typing import TYPE_CHECKING, Annotated, Any, NamedTuple
from pydantic import AliasPath, Field, PlainValidator
from yarl import URL
-from cyberdrop_dl.config_definitions.custom.types import AliasModel
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.types import AliasModel
from cyberdrop_dl.utils import javascript
from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, parse_url
@@ -17,8 +17,8 @@ if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
PRIMARY_BASE_DOMAIN = URL("https://xhamster.com/")
JS_VIDEO_INFO_SELECTOR = "script#initials-script"
diff --git a/cyberdrop_dl/crawlers/xxxbunker.py b/cyberdrop_dl/crawlers/xxxbunker.py
index 89d3a4225..16bc051ef 100644
--- a/cyberdrop_dl/crawlers/xxxbunker.py
+++ b/cyberdrop_dl/crawlers/xxxbunker.py
@@ -10,16 +10,16 @@ from aiolimiter import AsyncLimiter
from bs4 import BeautifulSoup
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
DATE_PATTERN = re.compile(r"(\d+)\s*(weeks?|days?|hours?|minutes?|seconds?)", re.IGNORECASE)
MIN_RATE_LIMIT = 4 # per minute
diff --git a/cyberdrop_dl/crawlers/yandex_disk.py b/cyberdrop_dl/crawlers/yandex_disk.py
index ed48d56f9..f53ece4a2 100644
--- a/cyberdrop_dl/crawlers/yandex_disk.py
+++ b/cyberdrop_dl/crawlers/yandex_disk.py
@@ -8,8 +8,8 @@ from typing import TYPE_CHECKING, Any, Literal, Self
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils import javascript
from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper
@@ -19,8 +19,8 @@ if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
JS_SELECTOR = "script#store-prefetch"
diff --git a/cyberdrop_dl/crawlers/youjizz.py b/cyberdrop_dl/crawlers/youjizz.py
index d2b2f0c8a..ebe570ee1 100644
--- a/cyberdrop_dl/crawlers/youjizz.py
+++ b/cyberdrop_dl/crawlers/youjizz.py
@@ -8,8 +8,8 @@ from typing import TYPE_CHECKING, NamedTuple
from yarl import URL
-from cyberdrop_dl.clients.errors import ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils import javascript
from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper
@@ -17,8 +17,8 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
DEFAULT_QUALITY = "Auto"
diff --git a/cyberdrop_dl/utils/data_enums_classes/__init__.py b/cyberdrop_dl/data_structures/__init__.py
similarity index 100%
rename from cyberdrop_dl/utils/data_enums_classes/__init__.py
rename to cyberdrop_dl/data_structures/__init__.py
diff --git a/cyberdrop_dl/utils/data_enums_classes/hash.py b/cyberdrop_dl/data_structures/hash.py
similarity index 100%
rename from cyberdrop_dl/utils/data_enums_classes/hash.py
rename to cyberdrop_dl/data_structures/hash.py
diff --git a/cyberdrop_dl/utils/data_enums_classes/supported_domains.py b/cyberdrop_dl/data_structures/supported_domains.py
similarity index 100%
rename from cyberdrop_dl/utils/data_enums_classes/supported_domains.py
rename to cyberdrop_dl/data_structures/supported_domains.py
diff --git a/cyberdrop_dl/utils/data_enums_classes/url_objects.py b/cyberdrop_dl/data_structures/url_objects.py
similarity index 99%
rename from cyberdrop_dl/utils/data_enums_classes/url_objects.py
rename to cyberdrop_dl/data_structures/url_objects.py
index 9aad39c20..c6e49dac2 100644
--- a/cyberdrop_dl/utils/data_enums_classes/url_objects.py
+++ b/cyberdrop_dl/data_structures/url_objects.py
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, NamedTuple
from yarl import URL
-from cyberdrop_dl.clients.errors import MaxChildrenError
+from cyberdrop_dl.exceptions import MaxChildrenError
from cyberdrop_dl.utils.utilities import sanitize_folder
if TYPE_CHECKING:
diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.py
index 595cd7ea6..4dcc12268 100644
--- a/cyberdrop_dl/director.py
+++ b/cyberdrop_dl/director.py
@@ -16,13 +16,12 @@ import browser_cookie3
from pydantic import ValidationError
from rich import print as rich_print
-from cyberdrop_dl import env
-from cyberdrop_dl.clients.errors import InvalidYamlError
+from cyberdrop_dl import constants, env
+from cyberdrop_dl.exceptions import InvalidYamlError
from cyberdrop_dl.managers.manager import Manager
from cyberdrop_dl.scraper.scrape_mapper import ScrapeMapper
from cyberdrop_dl.ui.program_ui import ProgramUI
from cyberdrop_dl.ui.textual import textual_ui
-from cyberdrop_dl.utils import constants
from cyberdrop_dl.utils.apprise import send_apprise_notifications
from cyberdrop_dl.utils.dumper import Dumper
from cyberdrop_dl.utils.logger import LogHandler, QueuedLogger, TextualLogQueueHandler, log, log_spacer, log_with_color
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index b0425bf9b..97a7138a8 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -15,15 +15,15 @@ from typing import TYPE_CHECKING, ParamSpec, TypeVar
from aiohttp import ClientConnectorError, ClientError, ClientResponseError
-from cyberdrop_dl.clients.errors import (
+from cyberdrop_dl.constants import CustomHTTPStatus
+from cyberdrop_dl.data_structures.url_objects import HlsSegment, MediaItem
+from cyberdrop_dl.exceptions import (
DownloadError,
DurationError,
ErrorLogMessage,
InvalidContentTypeError,
RestrictedFiletypeError,
)
-from cyberdrop_dl.utils.constants import CustomHTTPStatus
-from cyberdrop_dl.utils.data_enums_classes.url_objects import HlsSegment, MediaItem
from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_size_or_none
diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.py
index 95bfa5e67..644b04fd0 100644
--- a/cyberdrop_dl/downloader/mega_nz.py
+++ b/cyberdrop_dl/downloader/mega_nz.py
@@ -74,8 +74,8 @@ from Crypto.PublicKey import RSA
from Crypto.Util import Counter
from cyberdrop_dl.clients.download_client import DownloadClient
-from cyberdrop_dl.clients.errors import CDLBaseError, DownloadError, SlowDownloadError
from cyberdrop_dl.downloader.downloader import Downloader
+from cyberdrop_dl.exceptions import CDLBaseError, DownloadError, SlowDownloadError
from cyberdrop_dl.utils.logger import log
if TYPE_CHECKING:
@@ -83,8 +83,8 @@ if TYPE_CHECKING:
from yarl import URL
+ from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem
class MegaNzError(CDLBaseError): ...
diff --git a/cyberdrop_dl/clients/errors.py b/cyberdrop_dl/exceptions.py
similarity index 98%
rename from cyberdrop_dl/clients/errors.py
rename to cyberdrop_dl/exceptions.py
index 3abdf680e..4328b6890 100644
--- a/cyberdrop_dl/clients/errors.py
+++ b/cyberdrop_dl/exceptions.py
@@ -8,12 +8,12 @@ from typing import TYPE_CHECKING
from yaml import YAMLError
from yarl import URL
-from cyberdrop_dl.utils.constants import VALIDATION_ERROR_FOOTER
+from cyberdrop_dl.constants import VALIDATION_ERROR_FOOTER
if TYPE_CHECKING:
from requests import Response
- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem, ScrapeItem
+ from cyberdrop_dl.data_structures.url_objects import MediaItem, ScrapeItem
# See: https://developers.cloudflare.com/support/troubleshooting/cloudflare-errors/troubleshooting-cloudflare-5xx-errors/
diff --git a/cyberdrop_dl/managers/cache_manager.py b/cyberdrop_dl/managers/cache_manager.py
index 4335c1849..1844a932c 100644
--- a/cyberdrop_dl/managers/cache_manager.py
+++ b/cyberdrop_dl/managers/cache_manager.py
@@ -8,9 +8,9 @@ from typing import TYPE_CHECKING, Any
from aiohttp_client_cache import CacheBackend, SQLiteBackend
from cyberdrop_dl import __version__ as current_version
+from cyberdrop_dl.data_structures.supported_domains import SUPPORTED_FORUMS, SUPPORTED_WEBSITES
from cyberdrop_dl.scraper.filters import cache_filter_fn
from cyberdrop_dl.utils import yaml
-from cyberdrop_dl.utils.data_enums_classes.supported_domains import SUPPORTED_FORUMS, SUPPORTED_WEBSITES
if TYPE_CHECKING:
from pathlib import Path
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index 4e121c894..168e99883 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -17,8 +17,8 @@ from bs4 import BeautifulSoup
from yarl import URL
from cyberdrop_dl.clients.download_client import DownloadClient
-from cyberdrop_dl.clients.errors import DDOSGuardError, DownloadError, ScrapeError
from cyberdrop_dl.clients.scraper_client import ScraperClient
+from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError
from cyberdrop_dl.managers.download_speed_manager import DownloadSpeedLimiter
from cyberdrop_dl.ui.prompts.user_prompts import get_cookies_from_browsers
from cyberdrop_dl.utils.logger import log, log_spacer
@@ -28,8 +28,8 @@ if TYPE_CHECKING:
from aiohttp_client_cache import CachedResponse
from curl_cffi.requests.models import Response as CurlResponse
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
DOWNLOAD_ERROR_ETAGS = {
"d835884373f4d6c8f24742ceabe74946": "Imgur image has been removed",
diff --git a/cyberdrop_dl/managers/config_manager.py b/cyberdrop_dl/managers/config_manager.py
index 8060ba79f..5e0b7cc0f 100644
--- a/cyberdrop_dl/managers/config_manager.py
+++ b/cyberdrop_dl/managers/config_manager.py
@@ -6,8 +6,8 @@ from dataclasses import field
from time import sleep
from typing import TYPE_CHECKING
-from cyberdrop_dl.clients.errors import InvalidYamlError
from cyberdrop_dl.config_definitions import AuthSettings, ConfigSettings, GlobalSettings
+from cyberdrop_dl.exceptions import InvalidYamlError
from cyberdrop_dl.managers.log_manager import LogManager
from cyberdrop_dl.utils import yaml
from cyberdrop_dl.utils.apprise import get_apprise_urls
@@ -114,7 +114,7 @@ class ConfigManager:
if posible_fields == set_fields and not needs_update and self.pydantic_config:
return
else:
- from cyberdrop_dl.utils import constants
+ from cyberdrop_dl import constants
self.settings_data = ConfigSettings()
self.settings_data.files.input_file = constants.APP_STORAGE / "Configs" / self.loaded_config / "URLs.txt"
diff --git a/cyberdrop_dl/managers/download_manager.py b/cyberdrop_dl/managers/download_manager.py
index dcc0a9b7a..09a1f7a64 100644
--- a/cyberdrop_dl/managers/download_manager.py
+++ b/cyberdrop_dl/managers/download_manager.py
@@ -6,14 +6,14 @@ from contextlib import asynccontextmanager
from typing import TYPE_CHECKING
from cyberdrop_dl.clients.download_client import check_file_duration
-from cyberdrop_dl.utils.constants import FILE_FORMATS
+from cyberdrop_dl.constants import FILE_FORMATS
from cyberdrop_dl.utils.logger import log_debug
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
+ from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem
class FileLocksVault:
diff --git a/cyberdrop_dl/managers/live_manager.py b/cyberdrop_dl/managers/live_manager.py
index 77cda2d22..40f493c34 100644
--- a/cyberdrop_dl/managers/live_manager.py
+++ b/cyberdrop_dl/managers/live_manager.py
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
from rich.live import Live
-from cyberdrop_dl.utils import constants
+from cyberdrop_dl import constants
from cyberdrop_dl.utils.args import is_terminal_in_portrait
from cyberdrop_dl.utils.logger import console
diff --git a/cyberdrop_dl/managers/log_manager.py b/cyberdrop_dl/managers/log_manager.py
index 7736a0bed..a13f22d51 100644
--- a/cyberdrop_dl/managers/log_manager.py
+++ b/cyberdrop_dl/managers/log_manager.py
@@ -8,15 +8,15 @@ from typing import TYPE_CHECKING
import aiofiles
-from cyberdrop_dl.clients.errors import get_origin
-from cyberdrop_dl.utils.constants import CSV_DELIMITER
+from cyberdrop_dl.constants import CSV_DELIMITER
+from cyberdrop_dl.exceptions import get_origin
from cyberdrop_dl.utils.logger import log, log_spacer
if TYPE_CHECKING:
from yarl import URL
+ from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem
class LogManager:
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index c0f2bb1d7..f4ffca0ff 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -8,7 +8,7 @@ from dataclasses import Field, field
from time import perf_counter
from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Protocol, TypeVar
-from cyberdrop_dl import __version__
+from cyberdrop_dl import __version__, constants
from cyberdrop_dl.config_definitions import ConfigSettings, GlobalSettings
from cyberdrop_dl.managers.cache_manager import CacheManager
from cyberdrop_dl.managers.client_manager import ClientManager
@@ -23,7 +23,6 @@ from cyberdrop_dl.managers.progress_manager import ProgressManager
from cyberdrop_dl.managers.realdebrid_manager import RealDebridManager
from cyberdrop_dl.managers.storage_manager import StorageManager
from cyberdrop_dl.ui.textual import TextualUI
-from cyberdrop_dl.utils import constants
from cyberdrop_dl.utils.args import ParsedArgs
from cyberdrop_dl.utils.ffmpeg import FFmpeg, get_ffmpeg_version
from cyberdrop_dl.utils.logger import QueuedLogger, log
diff --git a/cyberdrop_dl/managers/path_manager.py b/cyberdrop_dl/managers/path_manager.py
index 810832a1b..4a30912e4 100644
--- a/cyberdrop_dl/managers/path_manager.py
+++ b/cyberdrop_dl/managers/path_manager.py
@@ -5,13 +5,12 @@ from datetime import datetime
from pathlib import Path
from typing import TYPE_CHECKING
-from cyberdrop_dl import env
-from cyberdrop_dl.utils import constants
+from cyberdrop_dl import constants, env
from cyberdrop_dl.utils.utilities import purge_dir_tree
if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem
if env.RUNNING_IN_IDE and Path.cwd().name == "cyberdrop_dl":
diff --git a/cyberdrop_dl/managers/real_debrid/api.py b/cyberdrop_dl/managers/real_debrid/api.py
index 43c660e33..cc94059d5 100644
--- a/cyberdrop_dl/managers/real_debrid/api.py
+++ b/cyberdrop_dl/managers/real_debrid/api.py
@@ -9,7 +9,7 @@ from requests import Session
from requests.exceptions import RequestException
from yarl import URL
-from cyberdrop_dl.clients.errors import RealDebridError
+from cyberdrop_dl.exceptions import RealDebridError
from cyberdrop_dl.managers.real_debrid.errors import ERROR_CODES
if TYPE_CHECKING:
diff --git a/cyberdrop_dl/managers/realdebrid_manager.py b/cyberdrop_dl/managers/realdebrid_manager.py
index 505a8e4b5..2bee0d28f 100644
--- a/cyberdrop_dl/managers/realdebrid_manager.py
+++ b/cyberdrop_dl/managers/realdebrid_manager.py
@@ -6,7 +6,7 @@ from dataclasses import field
from re import Pattern
from typing import TYPE_CHECKING
-from cyberdrop_dl.clients.errors import RealDebridError
+from cyberdrop_dl.exceptions import RealDebridError
from cyberdrop_dl.managers.real_debrid.api import RealDebridApi
from cyberdrop_dl.utils.logger import log
diff --git a/cyberdrop_dl/managers/storage_manager.py b/cyberdrop_dl/managers/storage_manager.py
index 1e46286d6..36d925759 100644
--- a/cyberdrop_dl/managers/storage_manager.py
+++ b/cyberdrop_dl/managers/storage_manager.py
@@ -12,14 +12,14 @@ from typing import TYPE_CHECKING, NamedTuple
import psutil
from pydantic import ByteSize
-from cyberdrop_dl.clients.errors import InsufficientFreeSpaceError
+from cyberdrop_dl.exceptions import InsufficientFreeSpaceError
from cyberdrop_dl.utils.logger import log, log_debug
if TYPE_CHECKING:
from psutil._common import sdiskpart
+ from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem
@dataclass(frozen=True, slots=True, order=True)
diff --git a/cyberdrop_dl/profiling.py b/cyberdrop_dl/profiling.py
index f11368a75..bf059e211 100644
--- a/cyberdrop_dl/profiling.py
+++ b/cyberdrop_dl/profiling.py
@@ -12,8 +12,7 @@ from pathlib import Path
from tempfile import TemporaryDirectory
from typing import TYPE_CHECKING
-from cyberdrop_dl import env
-from cyberdrop_dl.utils import constants
+from cyberdrop_dl import constants, env
if TYPE_CHECKING:
from collections.abc import Callable
diff --git a/cyberdrop_dl/scraper/filters.py b/cyberdrop_dl/scraper/filters.py
index d55085b98..ac92c2b13 100644
--- a/cyberdrop_dl/scraper/filters.py
+++ b/cyberdrop_dl/scraper/filters.py
@@ -7,9 +7,9 @@ from typing import TYPE_CHECKING
import arrow
from yarl import URL
-import cyberdrop_dl.utils.constants as constants
-from cyberdrop_dl.clients.errors import NoExtensionError
-from cyberdrop_dl.utils.constants import FILE_FORMATS
+import cyberdrop_dl.constants as constants
+from cyberdrop_dl.constants import FILE_FORMATS
+from cyberdrop_dl.exceptions import NoExtensionError
from cyberdrop_dl.utils.utilities import get_filename_and_ext
if TYPE_CHECKING:
@@ -18,7 +18,7 @@ if TYPE_CHECKING:
from aiohttp import ClientResponse
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
return_values: dict[URL | str, tuple] = {}
diff --git a/cyberdrop_dl/scraper/jdownloader.py b/cyberdrop_dl/scraper/jdownloader.py
index 48d21e8d6..780adb368 100644
--- a/cyberdrop_dl/scraper/jdownloader.py
+++ b/cyberdrop_dl/scraper/jdownloader.py
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, ParamSpec, TypeVar
from myjdapi import myjdapi
-from cyberdrop_dl.clients.errors import JDownloaderError
+from cyberdrop_dl.exceptions import JDownloaderError
from cyberdrop_dl.utils.logger import log
if TYPE_CHECKING:
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index b06983b4a..645b462dc 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -11,13 +11,13 @@ import aiofiles
import arrow
from yarl import URL
-from cyberdrop_dl.clients.errors import JDownloaderError, NoExtensionError
+from cyberdrop_dl.constants import BLOCKED_DOMAINS, REGEX_LINKS
from cyberdrop_dl.crawlers import CRAWLERS
+from cyberdrop_dl.data_structures.url_objects import MediaItem, ScrapeItem
from cyberdrop_dl.downloader.downloader import Downloader
+from cyberdrop_dl.exceptions import JDownloaderError, NoExtensionError
from cyberdrop_dl.scraper.filters import has_valid_extension, is_in_domain_list, is_outside_date_range, is_valid_url
from cyberdrop_dl.scraper.jdownloader import JDownloader
-from cyberdrop_dl.utils.constants import BLOCKED_DOMAINS, REGEX_LINKS
-from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem, ScrapeItem
from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import get_download_path, get_filename_and_ext, remove_trailing_slash
diff --git a/cyberdrop_dl/config_definitions/custom/types.py b/cyberdrop_dl/types.py
similarity index 64%
rename from cyberdrop_dl/config_definitions/custom/types.py
rename to cyberdrop_dl/types.py
index 1a577c4ba..de7059cd2 100644
--- a/cyberdrop_dl/config_definitions/custom/types.py
+++ b/cyberdrop_dl/types.py
@@ -1,8 +1,16 @@
+"""Custom types for type annotations
+
+
+1. Only add types here if they do NOT depend on any runtime import from `cyberdrop_dl` itself, except utils
+2. Only add types here if they are going to be used across multiple modules
+"""
+
from __future__ import annotations
+from collections.abc import Sequence
from functools import partial
from pathlib import Path
-from typing import Annotated
+from typing import Annotated, Any, TypeAlias, TypeGuard, TypeVar, final
import yarl
from pydantic import (
@@ -22,8 +30,38 @@ from pydantic import (
model_validator,
)
-from .converters import change_path_suffix, convert_byte_size_to_str
-from .validators import parse_apprise_url, parse_falsy_as_none, parse_list, pydantyc_yarl_url
+from cyberdrop_dl.utils.converters import change_path_suffix, convert_byte_size_to_str
+from cyberdrop_dl.utils.validators import (
+ parse_apprise_url,
+ parse_falsy_as_none,
+ parse_list,
+ pydantyc_yarl_url,
+)
+
+
+def do_nothing(cls=None) -> Any: ...
+
+
+og_init_subclass = yarl.URL.__init_subclass__
+
+yarl.URL.__init_subclass__ = do_nothing # type: ignore
+
+
+@final
+class AbsoluteHttpURL(yarl.URL):
+ def __init__(*args, **kwargs) -> None:
+ raise RuntimeError("Do not create instances, call yarl.URL and them assert validate")
+
+ def host(self) -> str: # type: ignore
+ """Decoded host part of URL."""
+
+ @staticmethod
+ def validate(url: yarl.URL) -> TypeGuard[AbsoluteHttpURL]:
+ return url.absolute and url.scheme.startswith("http")
+
+
+yarl.URL.__init_subclass__ = og_init_subclass
+
# ~~~~~ Strings ~~~~~~~
StrSerializer = PlainSerializer(str, return_type=str, when_used="json-unless-none")
@@ -78,3 +116,12 @@ class HttpAppriseURL(AppriseURLModel):
# DEPRECATED
# HttpURL = Annotated[HttpUrl, AfterValidator(convert_to_yarl), StrSerializer]
+
+
+T = TypeVar("T")
+Array: TypeAlias = list[T] | tuple[T, ...]
+CMD: TypeAlias = Array[str]
+U32Int: TypeAlias = int
+U32IntArray: TypeAlias = Array[U32Int]
+U32IntSequence: TypeAlias = Sequence[U32Int]
+AnyDict: TypeAlias = dict[str, Any]
diff --git a/cyberdrop_dl/ui/prompts/user_prompts.py b/cyberdrop_dl/ui/prompts/user_prompts.py
index 5e3029db7..8a44fcf16 100644
--- a/cyberdrop_dl/ui/prompts/user_prompts.py
+++ b/cyberdrop_dl/ui/prompts/user_prompts.py
@@ -14,15 +14,15 @@ from InquirerPy.enum import (
from rich.console import Console
from cyberdrop_dl import __version__
-from cyberdrop_dl.ui.prompts import basic_prompts
-from cyberdrop_dl.ui.prompts.defaults import ALL_CHOICE, DONE_CHOICE, EXIT_CHOICE
-from cyberdrop_dl.utils.constants import BROWSERS, RESERVED_CONFIG_NAMES
-from cyberdrop_dl.utils.cookie_management import get_cookies_from_browsers
-from cyberdrop_dl.utils.data_enums_classes.supported_domains import (
+from cyberdrop_dl.constants import BROWSERS, RESERVED_CONFIG_NAMES
+from cyberdrop_dl.data_structures.supported_domains import (
SUPPORTED_FORUMS,
SUPPORTED_SITES_DOMAINS,
SUPPORTED_WEBSITES,
)
+from cyberdrop_dl.ui.prompts import basic_prompts
+from cyberdrop_dl.ui.prompts.defaults import ALL_CHOICE, DONE_CHOICE, EXIT_CHOICE
+from cyberdrop_dl.utils.cookie_management import get_cookies_from_browsers
from cyberdrop_dl.utils.utilities import clear_term
if TYPE_CHECKING:
diff --git a/cyberdrop_dl/utils/apprise.py b/cyberdrop_dl/utils/apprise.py
index 444c99897..c138fe24d 100644
--- a/cyberdrop_dl/utils/apprise.py
+++ b/cyberdrop_dl/utils/apprise.py
@@ -14,8 +14,8 @@ import rich
from pydantic import ValidationError
from rich.text import Text
-from cyberdrop_dl.config_definitions.custom.types import AppriseURLModel
-from cyberdrop_dl.utils import constants
+from cyberdrop_dl import constants
+from cyberdrop_dl.types import AppriseURLModel
from cyberdrop_dl.utils.logger import log, log_debug, log_spacer
from cyberdrop_dl.utils.yaml import handle_validation_error
diff --git a/cyberdrop_dl/utils/args.py b/cyberdrop_dl/utils/args.py
index e91c5727c..fe30d7a4e 100644
--- a/cyberdrop_dl/utils/args.py
+++ b/cyberdrop_dl/utils/args.py
@@ -14,7 +14,7 @@ from pydantic import BaseModel, Field, ValidationError, computed_field, field_va
from cyberdrop_dl import __version__, env
from cyberdrop_dl.config_definitions import ConfigSettings, GlobalSettings
-from cyberdrop_dl.config_definitions.custom.types import AliasModel, HttpURL
+from cyberdrop_dl.types import AliasModel, HttpURL
from cyberdrop_dl.utils.yaml import handle_validation_error
if TYPE_CHECKING:
diff --git a/cyberdrop_dl/config_definitions/custom/converters.py b/cyberdrop_dl/utils/converters.py
similarity index 91%
rename from cyberdrop_dl/config_definitions/custom/converters.py
rename to cyberdrop_dl/utils/converters.py
index 8d3350252..54ad8f67a 100644
--- a/cyberdrop_dl/config_definitions/custom/converters.py
+++ b/cyberdrop_dl/utils/converters.py
@@ -2,16 +2,22 @@
Functions to use with `AfterValidator`, `field_validator(mode="after")` or `model_validator(mode="after")`
"""
+from __future__ import annotations
+
import re
from datetime import timedelta
-from pathlib import Path
+from typing import TYPE_CHECKING
-import yarl
from pydantic import AnyUrl, ByteSize, TypeAdapter
-from cyberdrop_dl.clients.errors import InvalidURLError
+from cyberdrop_dl.exceptions import InvalidURLError
from cyberdrop_dl.utils.utilities import parse_url
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ import yarl
+
DATE_PATTERN_REGEX = r"(\d+)\s*(second|seconds|minute|minutes|hour|hours|day|days|week|weeks|month|months|year|years)"
DATE_PATTERN = re.compile(DATE_PATTERN_REGEX, re.IGNORECASE)
diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.py
index dd303d858..2d714ba7b 100644
--- a/cyberdrop_dl/utils/cookie_management.py
+++ b/cyberdrop_dl/utils/cookie_management.py
@@ -11,8 +11,8 @@ import browser_cookie3
if TYPE_CHECKING:
from collections.abc import Callable, Generator
+ from cyberdrop_dl.constants import BROWSERS
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.constants import BROWSERS
P = ParamSpec("P")
R = TypeVar("R")
diff --git a/cyberdrop_dl/utils/database/tables/history_table.py b/cyberdrop_dl/utils/database/tables/history_table.py
index 869dc67c8..576c0856b 100644
--- a/cyberdrop_dl/utils/database/tables/history_table.py
+++ b/cyberdrop_dl/utils/database/tables/history_table.py
@@ -15,7 +15,7 @@ if TYPE_CHECKING:
from yarl import URL
from cyberdrop_dl.crawlers import Crawler
- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem
+ from cyberdrop_dl.data_structures.url_objects import MediaItem
def get_db_path(url: URL, referer: str = "") -> str:
diff --git a/cyberdrop_dl/utils/dumper.py b/cyberdrop_dl/utils/dumper.py
index aadcf6716..ac5cdee60 100644
--- a/cyberdrop_dl/utils/dumper.py
+++ b/cyberdrop_dl/utils/dumper.py
@@ -10,8 +10,8 @@ if TYPE_CHECKING:
from collections.abc import Generator
from pathlib import Path
+ from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem
KEYS_TO_REMOVE = "file_lock_reference_name", "task_id"
diff --git a/cyberdrop_dl/utils/logger.py b/cyberdrop_dl/utils/logger.py
index e7e0144d9..cc317c85e 100644
--- a/cyberdrop_dl/utils/logger.py
+++ b/cyberdrop_dl/utils/logger.py
@@ -15,8 +15,7 @@ from rich.measure import Measurement
from rich.padding import Padding
from rich.text import Text, TextType
-from cyberdrop_dl import env
-from cyberdrop_dl.utils import constants
+from cyberdrop_dl import constants, env
logger = logging.getLogger("cyberdrop_dl")
logger_debug = logging.getLogger("cyberdrop_dl_debug")
diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py
index 13e29874b..6213d60d5 100644
--- a/cyberdrop_dl/utils/sorting.py
+++ b/cyberdrop_dl/utils/sorting.py
@@ -13,7 +13,7 @@ import PIL
from PIL import Image
from videoprops import get_audio_properties, get_video_properties
-from cyberdrop_dl.utils.constants import FILE_FORMATS
+from cyberdrop_dl.constants import FILE_FORMATS
from cyberdrop_dl.utils.logger import log_with_color
from cyberdrop_dl.utils.utilities import purge_dir_tree
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index bd2bc6a75..b968a177e 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -21,7 +21,8 @@ from aiohttp_client_cache.response import AnyResponse
from bs4 import BeautifulSoup
from yarl import URL
-from cyberdrop_dl.clients.errors import (
+from cyberdrop_dl import constants
+from cyberdrop_dl.exceptions import (
CDLBaseError,
ErrorLogMessage,
InvalidExtensionError,
@@ -29,7 +30,6 @@ from cyberdrop_dl.clients.errors import (
NoExtensionError,
get_origin,
)
-from cyberdrop_dl.utils import constants
from cyberdrop_dl.utils.logger import log, log_debug, log_spacer, log_with_color
if TYPE_CHECKING:
@@ -39,9 +39,9 @@ if TYPE_CHECKING:
from rich.text import Text
from cyberdrop_dl.crawlers import Crawler
+ from cyberdrop_dl.data_structures.url_objects import MediaItem, ScrapeItem
from cyberdrop_dl.downloader.downloader import Downloader
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem, ScrapeItem
P = ParamSpec("P")
diff --git a/cyberdrop_dl/config_definitions/custom/validators.py b/cyberdrop_dl/utils/validators.py
similarity index 91%
rename from cyberdrop_dl/config_definitions/custom/validators.py
rename to cyberdrop_dl/utils/validators.py
index b1e492438..c1b197913 100644
--- a/cyberdrop_dl/config_definitions/custom/validators.py
+++ b/cyberdrop_dl/utils/validators.py
@@ -2,15 +2,20 @@
Functions to use with `BeforeValidator`, `field_validator(mode="before")` or `model_validator(mode="before")`
"""
-from collections.abc import Callable
+from __future__ import annotations
+
from datetime import timedelta
from functools import singledispatch
-from typing import Any
+from typing import TYPE_CHECKING, Any
from pydantic import HttpUrl
-from yarl import URL
-from .converters import convert_str_to_timedelta, convert_to_yarl
+from cyberdrop_dl.utils.converters import convert_str_to_timedelta, convert_to_yarl
+
+if TYPE_CHECKING:
+ from collections.abc import Callable
+
+ from yarl import URL
def pydantyc_yarl_url(value: str) -> URL:
diff --git a/cyberdrop_dl/utils/yaml.py b/cyberdrop_dl/utils/yaml.py
index da0ff4b48..d5830d798 100644
--- a/cyberdrop_dl/utils/yaml.py
+++ b/cyberdrop_dl/utils/yaml.py
@@ -11,8 +11,8 @@ import yaml
from pydantic import BaseModel, ValidationError
from yarl import URL
-from cyberdrop_dl.clients.errors import InvalidYamlError
-from cyberdrop_dl.utils.constants import CLI_VALIDATION_ERROR_FOOTER, VALIDATION_ERROR_FOOTER
+from cyberdrop_dl.constants import CLI_VALIDATION_ERROR_FOOTER, VALIDATION_ERROR_FOOTER
+from cyberdrop_dl.exceptions import InvalidYamlError
if TYPE_CHECKING:
from pydantic_core import ErrorDetails
diff --git a/tests/test_apprise.py b/tests/test_apprise.py
index 4e98e83e1..c083b0461 100644
--- a/tests/test_apprise.py
+++ b/tests/test_apprise.py
@@ -6,11 +6,12 @@ from pathlib import Path
import pytest
from rich.text import Text
+from cyberdrop_dl import constants
+from cyberdrop_dl.constants import NotificationResult
from cyberdrop_dl.managers.config_manager import ConfigManager
from cyberdrop_dl.managers.manager import Manager
from cyberdrop_dl.managers.path_manager import PathManager
-from cyberdrop_dl.utils import apprise, constants
-from cyberdrop_dl.utils.constants import NotificationResult
+from cyberdrop_dl.utils import apprise
from tests.fake_classes.managers import FakeCacheManager
TEST_FILES_PATH = Path(__file__).parent / "test_files/apprise"