Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add jpg6 support (#1174)
NTFSvolume1c5421ea06d6
diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.pyindex d364f9a1a..b6f8274f3 100644--- a/cyberdrop_dl/crawlers/jpg5.py+++ b/cyberdrop_dl/crawlers/jpg5.py@@ -14,8 +14,24 @@ if TYPE_CHECKING:from cyberdrop_dl.crawlers.crawler import SupportedDomains, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import ScrapeItem-JPG5_REPLACE_HOST_REGEX = re.compile(r"(jpg\.fish/)|(jpg\.fishing/)|(jpg\.church/)")-DECRYPTION_KEY = b"seltilovessimpcity@simpcityhatesscrapers"++_DECRYPTION_KEY = b"seltilovessimpcity@simpcityhatesscrapers"+_OLD_DOMAINS = (+ "jpg.homes",+ "jpg.church",+ "jpg.fish",+ "jpg.fishing",+ "jpg.pet",+ "jpeg.pet",+ "jpg1.su",+ "jpg2.su",+ "jpg3.su",+ "jpg4.su",+ "jpg5.su",+ "host.church",+)++_REPLACE_OLD_HOST_REGEX = re.compile("|".join(_OLD_DOMAINS))class JPG5Crawler(CheveretoCrawler):@@ -26,24 +42,11 @@ class JPG5Crawler(CheveretoCrawler):"Direct links": "",}- SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (- "jpg5.su",- "jpg.homes",- "jpg.church",- "jpg.fish",- "jpg.fishing",- "jpg.pet",- "jpeg.pet",- "jpg1.su",- "jpg2.su",- "jpg3.su",- "jpg4.su",- "host.church",- )+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (*_OLD_DOMAINS, "jpg6.su")DOMAIN: ClassVar[str] = "jpg5.su"FOLDER_DOMAIN: ClassVar[str] = "JPG5"- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://jpg5.su")+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://jpg6.su")def __post_init__(self) -> None:self.request_limiter = AsyncLimiter(1, 1)@@ -59,7 +62,7 @@ class JPG5Crawler(CheveretoCrawler):self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True) -> AbsoluteHttpURL:if not link_str.startswith("https") and not link_str.startswith("/"):- link_str = decrypt_xor(link_str, DECRYPTION_KEY)+ link_str = decrypt_xor(link_str, _DECRYPTION_KEY)return super().parse_url(link_str, relative_to, trim=trim)async def handle_direct_link(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:@@ -73,7 +76,11 @@ class JPG5Crawler(CheveretoCrawler):def fix_host(url: AbsoluteHttpURL) -> AbsoluteHttpURL:- new_host = re.sub(JPG5_REPLACE_HOST_REGEX, r"jpg5.su", f"{url.host}/").removesuffix("/")+ new_host = re.sub(_REPLACE_OLD_HOST_REGEX, r"jpg5.su", url.host)+ if new_host.removeprefix("www.") == "jpg5.su":+ # replace only if it is matches the second level domain exactly+ # old jpg5 subdomains are still valid. ex: simp4.jpg5.su+ return url.with_host("jpg6.su")return url.with_host(new_host)@@ -81,3 +88,8 @@ def decrypt_xor(encrypted_str: str, key: bytes) -> str:div = len(key)encrypted = bytes.fromhex(binascii.a2b_base64(encrypted_str).decode())return bytes([encrypted[i] ^ key[i % div] for i in range(len(encrypted))]).decode()+++def fix_db_referer(referer: str) -> str:+ url = AbsoluteHttpURL(referer)+ return str(fix_host(url))diff --git a/cyberdrop_dl/utils/database/tables/history_table.py b/cyberdrop_dl/utils/database/tables/history_table.pyindex 1ea4d54e2..aae5409b4 100644--- a/cyberdrop_dl/utils/database/tables/history_table.py+++ b/cyberdrop_dl/utils/database/tables/history_table.py@@ -21,6 +21,7 @@ DB_UPDATES = ("UPDATE OR REPLACE media SET domain = 'jpg5.su' WHERE domain = 'sharex'","UPDATE OR REPLACE media SET domain = 'nudostar.tv' WHERE domain = 'nudostartv'","UPDATE OR REPLACE media SET referer = FIX_REDGIFS_REFERER(referer) WHERE domain = 'redgifs';",+ "UPDATE OR REPLACE media SET referer = FIX_JPG5_REFERER(referer) WHERE domain = 'jpg5.su';",)@@ -44,9 +45,10 @@ class HistoryTable:async def startup(self) -> None:"""Startup process for the HistoryTable."""- from cyberdrop_dl.crawlers import redgifs+ from cyberdrop_dl.crawlers import jpg5, redgifsawait self.db_conn.create_function("FIX_REDGIFS_REFERER", 1, redgifs.fix_db_referer, deterministic=True)+ await self.db_conn.create_function("FIX_JPG5_REFERER", 1, jpg5.fix_db_referer, deterministic=True)await self.db_conn.execute(create_history)await self.db_conn.commit()await self.fix_primary_keys()