Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: update old domains (#1872)

NTFSvolume5cc292f5b2c2
diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.py
index d02e60e92..5f4601723 100644
--- a/cyberdrop_dl/crawlers/cyberdrop.py
+++ b/cyberdrop_dl/crawlers/cyberdrop.py
@@ -83,8 +83,3 @@ class CyberdropCrawler(Crawler):
filename, ext = self.get_filename_and_ext(name)
link = self.parse_url(auth["url"])
await self.handle_file(link, scrape_item, name, ext, custom_filename=filename)
-
-
-def fix_db_referer(referer: str) -> str:
- url = AbsoluteHttpURL(referer)
- return str(CyberdropCrawler.transform_url(url))
diff --git a/cyberdrop_dl/crawlers/imagepond.py b/cyberdrop_dl/crawlers/imagepond.py
index 6a298e2b9..7ccac77ac 100644
--- a/cyberdrop_dl/crawlers/imagepond.py
+++ b/cyberdrop_dl/crawlers/imagepond.py
@@ -70,7 +70,7 @@ class ImagePondCrawler(Crawler):
),
"Direct links": "/media/<slug>",
}
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://imagepond.net")
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.imagepond.net")
DOMAIN: ClassVar[str] = "imagepond.net"
FOLDER_DOMAIN: ClassVar[str] = "ImagePond"
DEFAULT_TRIM_URLS: ClassVar[bool] = False
diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.py
index 4ab4e54c9..935ecaa98 100644
--- a/cyberdrop_dl/crawlers/jpg5.py
+++ b/cyberdrop_dl/crawlers/jpg5.py
@@ -68,8 +68,3 @@ class JPG5Crawler(CheveretoCrawler):
encrypted_url = bytes.fromhex(base64.b64decode(link_str).decode())
link_str = xor_decrypt(encrypted_url, _DECRYPTION_KEY)
return super().parse_url(link_str, relative_to, trim=trim)
-
-
-def fix_db_referer(referer: str) -> str:
- url = AbsoluteHttpURL(referer)
- return str(JPG5Crawler.transform_url(url))
diff --git a/cyberdrop_dl/crawlers/xenforo/xbunker.py b/cyberdrop_dl/crawlers/xenforo/xbunker.py
index 3b57bbbff..382f2baf4 100644
--- a/cyberdrop_dl/crawlers/xenforo/xbunker.py
+++ b/cyberdrop_dl/crawlers/xenforo/xbunker.py
@@ -8,6 +8,6 @@ from .xenforo import XenforoCrawler
class XBunkerCrawler(XenforoCrawler):
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://xbunker.nu/")
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://xbunker.cc")
DOMAIN: ClassVar[str] = "xbunker"
FOLDER_DOMAIN: ClassVar[str] = "XBunker"
diff --git a/cyberdrop_dl/database/tables/history.py b/cyberdrop_dl/database/tables/history.py
index 6caebe6f4..0f22db09f 100644
--- a/cyberdrop_dl/database/tables/history.py
+++ b/cyberdrop_dl/database/tables/history.py
@@ -14,6 +14,7 @@ if TYPE_CHECKING:
import aiosqlite
from yarl import URL
+ from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.database import Database
from cyberdrop_dl.url_objects import MediaItem
@@ -325,8 +326,8 @@ async def fix_referers(db_conn: aiosqlite.Connection) -> None:
for name, fn in [
("FIX_REDGIFS_REFERER", redgifs.fix_db_referer),
- ("FIX_JPG5_REFERER", jpg5.fix_db_referer),
- ("FIX_CYBERDROP_REFERER", cyberdrop.fix_db_referer),
+ ("FIX_JPG5_REFERER", _generic_fix_referer(jpg5.JPG5Crawler)),
+ ("FIX_CYBERDROP_REFERER", _generic_fix_referer(cyberdrop.CyberdropCrawler)),
("FIX_TURBOVID_REFERER", turbovid.fix_db_referer),
]:
await db_conn.create_function(name, 1, try_wrap(fn), deterministic=True)
@@ -340,3 +341,11 @@ async def fix_referers(db_conn: aiosqlite.Connection) -> None:
await db_conn.executescript(updates)
await db_conn.commit()
+
+
+def _generic_fix_referer(crawler: type[Crawler]) -> Callable[[str], str]:
+ def fix_db_referer(referer: str) -> str:
+ url = crawler.parse_url(referer, trim=False)
+ return str(crawler.transform_url(url))
+
+ return fix_db_referer
diff --git a/cyberdrop_dl/database/tables/schema.py b/cyberdrop_dl/database/tables/schema.py
index eb6086d15..26fe0ea3f 100644
--- a/cyberdrop_dl/database/tables/schema.py
+++ b/cyberdrop_dl/database/tables/schema.py
@@ -27,7 +27,7 @@ class Version(NamedTuple):
return ".".join(map(str, self))
-CURRENT_APP_SCHEMA_VERSION = Version(9, 10, 1)
+CURRENT_APP_SCHEMA_VERSION = Version(9, 10, 2)
logger = logging.getLogger(__name__)