Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: update old domains (#1872)
NTFSvolume5cc292f5b2c2
diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.pyindex d02e60e92..5f4601723 100644--- a/cyberdrop_dl/crawlers/cyberdrop.py+++ b/cyberdrop_dl/crawlers/cyberdrop.py@@ -83,8 +83,3 @@ class CyberdropCrawler(Crawler):filename, ext = self.get_filename_and_ext(name)link = self.parse_url(auth["url"])await self.handle_file(link, scrape_item, name, ext, custom_filename=filename)---def fix_db_referer(referer: str) -> str:- url = AbsoluteHttpURL(referer)- return str(CyberdropCrawler.transform_url(url))diff --git a/cyberdrop_dl/crawlers/imagepond.py b/cyberdrop_dl/crawlers/imagepond.pyindex 6a298e2b9..7ccac77ac 100644--- a/cyberdrop_dl/crawlers/imagepond.py+++ b/cyberdrop_dl/crawlers/imagepond.py@@ -70,7 +70,7 @@ class ImagePondCrawler(Crawler):),"Direct links": "/media/<slug>",}- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://imagepond.net")+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.imagepond.net")DOMAIN: ClassVar[str] = "imagepond.net"FOLDER_DOMAIN: ClassVar[str] = "ImagePond"DEFAULT_TRIM_URLS: ClassVar[bool] = Falsediff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.pyindex 4ab4e54c9..935ecaa98 100644--- a/cyberdrop_dl/crawlers/jpg5.py+++ b/cyberdrop_dl/crawlers/jpg5.py@@ -68,8 +68,3 @@ class JPG5Crawler(CheveretoCrawler):encrypted_url = bytes.fromhex(base64.b64decode(link_str).decode())link_str = xor_decrypt(encrypted_url, _DECRYPTION_KEY)return super().parse_url(link_str, relative_to, trim=trim)---def fix_db_referer(referer: str) -> str:- url = AbsoluteHttpURL(referer)- return str(JPG5Crawler.transform_url(url))diff --git a/cyberdrop_dl/crawlers/xenforo/xbunker.py b/cyberdrop_dl/crawlers/xenforo/xbunker.pyindex 3b57bbbff..382f2baf4 100644--- a/cyberdrop_dl/crawlers/xenforo/xbunker.py+++ b/cyberdrop_dl/crawlers/xenforo/xbunker.py@@ -8,6 +8,6 @@ from .xenforo import XenforoCrawlerclass XBunkerCrawler(XenforoCrawler):- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://xbunker.nu/")+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://xbunker.cc")DOMAIN: ClassVar[str] = "xbunker"FOLDER_DOMAIN: ClassVar[str] = "XBunker"diff --git a/cyberdrop_dl/database/tables/history.py b/cyberdrop_dl/database/tables/history.pyindex 6caebe6f4..0f22db09f 100644--- a/cyberdrop_dl/database/tables/history.py+++ b/cyberdrop_dl/database/tables/history.py@@ -14,6 +14,7 @@ if TYPE_CHECKING:import aiosqlitefrom yarl import URL+ from cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.database import Databasefrom cyberdrop_dl.url_objects import MediaItem@@ -325,8 +326,8 @@ async def fix_referers(db_conn: aiosqlite.Connection) -> None:for name, fn in [("FIX_REDGIFS_REFERER", redgifs.fix_db_referer),- ("FIX_JPG5_REFERER", jpg5.fix_db_referer),- ("FIX_CYBERDROP_REFERER", cyberdrop.fix_db_referer),+ ("FIX_JPG5_REFERER", _generic_fix_referer(jpg5.JPG5Crawler)),+ ("FIX_CYBERDROP_REFERER", _generic_fix_referer(cyberdrop.CyberdropCrawler)),("FIX_TURBOVID_REFERER", turbovid.fix_db_referer),]:await db_conn.create_function(name, 1, try_wrap(fn), deterministic=True)@@ -340,3 +341,11 @@ async def fix_referers(db_conn: aiosqlite.Connection) -> None:await db_conn.executescript(updates)await db_conn.commit()+++def _generic_fix_referer(crawler: type[Crawler]) -> Callable[[str], str]:+ def fix_db_referer(referer: str) -> str:+ url = crawler.parse_url(referer, trim=False)+ return str(crawler.transform_url(url))++ return fix_db_refererdiff --git a/cyberdrop_dl/database/tables/schema.py b/cyberdrop_dl/database/tables/schema.pyindex eb6086d15..26fe0ea3f 100644--- a/cyberdrop_dl/database/tables/schema.py+++ b/cyberdrop_dl/database/tables/schema.py@@ -27,7 +27,7 @@ class Version(NamedTuple):return ".".join(map(str, self))-CURRENT_APP_SCHEMA_VERSION = Version(9, 10, 1)+CURRENT_APP_SCHEMA_VERSION = Version(9, 10, 2)logger = logging.getLogger(__name__)