Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add jpg6 support (#1174)

NTFSvolume1c5421ea06d6
diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.py
index d364f9a1a..b6f8274f3 100644
--- a/cyberdrop_dl/crawlers/jpg5.py
+++ b/cyberdrop_dl/crawlers/jpg5.py
@@ -14,8 +14,24 @@ if TYPE_CHECKING:
from cyberdrop_dl.crawlers.crawler import SupportedDomains, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
-JPG5_REPLACE_HOST_REGEX = re.compile(r"(jpg\.fish/)|(jpg\.fishing/)|(jpg\.church/)")
-DECRYPTION_KEY = b"seltilovessimpcity@simpcityhatesscrapers"
+
+_DECRYPTION_KEY = b"seltilovessimpcity@simpcityhatesscrapers"
+_OLD_DOMAINS = (
+ "jpg.homes",
+ "jpg.church",
+ "jpg.fish",
+ "jpg.fishing",
+ "jpg.pet",
+ "jpeg.pet",
+ "jpg1.su",
+ "jpg2.su",
+ "jpg3.su",
+ "jpg4.su",
+ "jpg5.su",
+ "host.church",
+)
+
+_REPLACE_OLD_HOST_REGEX = re.compile("|".join(_OLD_DOMAINS))
class JPG5Crawler(CheveretoCrawler):
@@ -26,24 +42,11 @@ class JPG5Crawler(CheveretoCrawler):
"Direct links": "",
}
- SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (
- "jpg5.su",
- "jpg.homes",
- "jpg.church",
- "jpg.fish",
- "jpg.fishing",
- "jpg.pet",
- "jpeg.pet",
- "jpg1.su",
- "jpg2.su",
- "jpg3.su",
- "jpg4.su",
- "host.church",
- )
+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (*_OLD_DOMAINS, "jpg6.su")
DOMAIN: ClassVar[str] = "jpg5.su"
FOLDER_DOMAIN: ClassVar[str] = "JPG5"
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://jpg5.su")
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://jpg6.su")
def __post_init__(self) -> None:
self.request_limiter = AsyncLimiter(1, 1)
@@ -59,7 +62,7 @@ class JPG5Crawler(CheveretoCrawler):
self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True
) -> AbsoluteHttpURL:
if not link_str.startswith("https") and not link_str.startswith("/"):
- link_str = decrypt_xor(link_str, DECRYPTION_KEY)
+ link_str = decrypt_xor(link_str, _DECRYPTION_KEY)
return super().parse_url(link_str, relative_to, trim=trim)
async def handle_direct_link(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:
@@ -73,7 +76,11 @@ class JPG5Crawler(CheveretoCrawler):
def fix_host(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
- new_host = re.sub(JPG5_REPLACE_HOST_REGEX, r"jpg5.su", f"{url.host}/").removesuffix("/")
+ new_host = re.sub(_REPLACE_OLD_HOST_REGEX, r"jpg5.su", url.host)
+ if new_host.removeprefix("www.") == "jpg5.su":
+ # replace only if it is matches the second level domain exactly
+ # old jpg5 subdomains are still valid. ex: simp4.jpg5.su
+ return url.with_host("jpg6.su")
return url.with_host(new_host)
@@ -81,3 +88,8 @@ def decrypt_xor(encrypted_str: str, key: bytes) -> str:
div = len(key)
encrypted = bytes.fromhex(binascii.a2b_base64(encrypted_str).decode())
return bytes([encrypted[i] ^ key[i % div] for i in range(len(encrypted))]).decode()
+
+
+def fix_db_referer(referer: str) -> str:
+ url = AbsoluteHttpURL(referer)
+ return str(fix_host(url))
diff --git a/cyberdrop_dl/utils/database/tables/history_table.py b/cyberdrop_dl/utils/database/tables/history_table.py
index 1ea4d54e2..aae5409b4 100644
--- a/cyberdrop_dl/utils/database/tables/history_table.py
+++ b/cyberdrop_dl/utils/database/tables/history_table.py
@@ -21,6 +21,7 @@ DB_UPDATES = (
"UPDATE OR REPLACE media SET domain = 'jpg5.su' WHERE domain = 'sharex'",
"UPDATE OR REPLACE media SET domain = 'nudostar.tv' WHERE domain = 'nudostartv'",
"UPDATE OR REPLACE media SET referer = FIX_REDGIFS_REFERER(referer) WHERE domain = 'redgifs';",
+ "UPDATE OR REPLACE media SET referer = FIX_JPG5_REFERER(referer) WHERE domain = 'jpg5.su';",
)
@@ -44,9 +45,10 @@ class HistoryTable:
async def startup(self) -> None:
"""Startup process for the HistoryTable."""
- from cyberdrop_dl.crawlers import redgifs
+ from cyberdrop_dl.crawlers import jpg5, redgifs
await self.db_conn.create_function("FIX_REDGIFS_REFERER", 1, redgifs.fix_db_referer, deterministic=True)
+ await self.db_conn.create_function("FIX_JPG5_REFERER", 1, jpg5.fix_db_referer, deterministic=True)
await self.db_conn.execute(create_history)
await self.db_conn.commit()
await self.fix_primary_keys()