Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: support new encrypted URLs (xHamster) (#1392)
NTFSvolumec30e901b6f1f
diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.pyindex 47696757d..87aa6ade4 100644--- a/cyberdrop_dl/crawlers/xhamster.py+++ b/cyberdrop_dl/crawlers/xhamster.py@@ -33,11 +33,9 @@ class Selector:def _decrypt_url(raw_url: str) -> str | None:if raw_url.startswith("http") or raw_url.startswith("/"):return raw_url- try:- if _is_hex(raw_url):- return _decode_hex_url(raw_url)- except ValueError:- pass++ if _is_hex(raw_url):+ return _decode_hex_url(raw_url)try:decoded_url = base64.b64decode(raw_url)@@ -350,9 +348,7 @@ def _make_decoder(algo: int, seed: int) -> Callable[[], int]:current_step = _ensure_signed_32int(current_step * 1664525) + 1013904223return current_step & 255- return decode_next-- if algo == 2:+ elif algo == 2:def decode_next() -> int:nonlocal current_step@@ -365,9 +361,7 @@ def _make_decoder(algo: int, seed: int) -> Callable[[], int]:return current_step & 255- return decode_next-- if algo == 3:+ elif algo == 3:def decode_next() -> int:nonlocal current_step@@ -380,9 +374,32 @@ def _make_decoder(algo: int, seed: int) -> Callable[[], int]:return val & 255- return decode_next+ elif algo == 4:- raise ValueError(f"Unknown crypto algo: {algo}")+ def decode_next() -> int:+ nonlocal current_step+ val = current_step = _ensure_signed_32int(current_step + 0x6D2B79F5)+ val = _ensure_signed_32int((val << 7) | ((val & 0xFFFFFFFF) >> 25))+ val = _ensure_signed_32int(val + 0x9E3779B9)+ val = _ensure_signed_32int(val ^ ((val & 0xFFFFFFFF) >> 11))+ val = _ensure_signed_32int(val * 0x27D4EB2D)+ return 255 & val++ elif algo == 5:++ def decode_next() -> int:+ nonlocal current_step++ current_step = _ensure_signed_32int(current_step ^ (current_step << 7))+ current_step = _ensure_signed_32int(current_step ^ ((current_step & 0xFFFFFFFF) >> 9))+ current_step = _ensure_signed_32int(current_step ^ (current_step << 8))+ current_step = _ensure_signed_32int(current_step + 0xA5A5A5A5)+ return current_step++ else:+ raise ValueError(f"Unknown crypto algo: {algo}")++ return decode_nextdef _is_hex(hex_string: str) -> bool:@@ -397,6 +414,9 @@ def _decode_hex_url(encrypted_url: str) -> str:array = bytearray.fromhex(encrypted_url)algo = array[0]seed = _ensure_signed_32int(array[1] | (array[2] << 8) | (array[3] << 16) | (array[4] << 24))- decode_next = _make_decoder(algo, seed)+ try:+ decode_next = _make_decoder(algo, seed)+ except ValueError:+ raise ValueError(f"Unknown encrypted URL {encrypted_url} with {algo = } and {seed = }") from Nonedecoded_array = bytearray([(array[idx + 5] ^ decode_next()) & 255 for idx in range(len(array) - 5)])return decoded_array.decode("utf-8")diff --git a/tests/crawlers/test_xhamster.py b/tests/crawlers/test_xhamster.pyindex f88868a29..d74fa5b27 100644--- a/tests/crawlers/test_xhamster.py+++ b/tests/crawlers/test_xhamster.py@@ -10,10 +10,30 @@ from cyberdrop_dl.crawlers.xhamster import _decode_hex_url"01fe0bef492d94cb623a2e4c49fbe1239f7e1105bda61daecca10c105273f634a5cca31dc107eaa9d731f0ec43f44d54f4439ee517bc9cacc3c439bfe70a4d4f5ff915a90eb50f9237ed3dc29ba5a4f40e7e11de62611932102c92cb47692cad3bd5f8cbe489f42dc4d1c6f776d75a93ea0126e796002457f60174e9224588c7902d5de946f51808a0f9aed4d43d78fd56a085d97e2d817f5a69dec286a568a7700d3fc32ad5786d03df0439b41c9ed76a8b733bdc427f5b374aab206289d7a8be11e0449064d64efc64a41708207b7282","https://video-nss-a.xhcdn.com/f_bjveXDoEYe3nrBnyILCA==,1758589200/media=hls4/multi=256x144:144p:,426x240:240p:,854x480:480p:,1280x720:720p:,1920x1080:1080p:,3840x2160:2160p:/027/453/344/_TPL_.av1.mp4.m3u8",),+ (+ "026378c22deccdb1e36654485790e343dbccb1ca73f6cdb0f928de23410785d5c9bb7369acbbe2c749ddc19a89fdcf13d6586411ccb808067f8df83776ce4e794e1b3b3ba6091b1d0fd20ab30fcbd9eec847dd160fb59885965d720cb74eee49abaccd7fa5fb190c72f67f1a92e03c53dfd856958f05ee905de14d6f57e59f25d9d3be0800623a63b05bce15918835f3086b2f",+ "https://video-a.xhcdn.com/key=6Syo47cqICONgqkWcfrP5Q,end=1760554800,limit=3/data=2a02:6ea0:da08:3028::12-dvp/speed=0/016/033/213/144p.h264.mp4",+ ),+ (+ "017fe4c13bba7da05355776728ccb8984ee1b891a1daf1475798f3fb78e74e275e271801aa3c511c12bf2aa95369b5f2231ffdd29f21f79602ba98565317b0db6dce70d58b2a79244c02ecfc26c7235d5bfcef06aa83e95e17572de06df991be43a45fb1ed9f0dc2f329df4d01ff43b9f953dabd8ae69065aebe510b4345ac59f74aec1dcd67bf2c9292f9f2b60e650c91222142395a2b90834c6f2ca77adbe083eb91b82bca99cef77edd189df6402ef39e4d271df20373e1260d82989a305af44ef65310a0f8a7ccb959588b25bfdd9d7f7bdd3f567789c0c3a5",+ "https://video-a.xhcdn.com/key=A5NxhQyGAt3Dnh1HB0cNRQ,end=1760554800/data=2a02:6ea0:da08:3028::12-dvp/referer=/media=hls4/multi=256x144:144p:,426x240:240p:,854x480:480p:,1280x720:720p:/016/033/213/_TPL_.av1.mp4.m3u8",+ ),("030251bf078b3c34108b5ec467b21781c6e865fcbe2f0fec4fa03c7bedf6634783bb816af8803b19971f0360b2d33dfa77e7e170544e6f3a69d77e93526ba71710858822c5b0c7371fe8af1aa21f7db5e46a1b329eceeceeadbb845e6398c431a107cdee5c478335d3e0b46e9be59e4b492a702f0c257339238c63607de262e7c8d30bba40d8febad01b1c33cf2b177b1a3e04d826286b9aaf45bf6c4229b2ea31de015312067881b8e97e25df7fdc","https://video-nss-a.xhcdn.com/if6CPa9MjylU2jNYhxtjWg==,1758596400/media=hls4/multi=256x144:144p:,426x240:240p:,854x480:480p:,1280x720:720p:/017/537/816/_TPL_.av1.mp4.m3u8",),+ (+ "03f3ad0c480495f537d177073095dc4782be7c3cf459faa0abc387b06c390b432c65905ad99211c35e89578fddd8e131bf68650af80956367f90986f842d7b5d9c9a7eb3ba996823bbbb33d051a855485427b7c2bbac594de94b46dd1d8e00070d09747456b9aec4a800be8b32fdb402188b75f2854e3171e01aebc612efafbd6e3f1e6ae661788522513f5e9c8c9be1c3fe0ba72f9548880f4a307af8825436ad6359698a87ac0fb2d47a860051d1152144695821d6bcaaf284d5941042ee9d84a10a36d1bd",+ "https://video-cf-a.xhcdn.com/qnuR8f8q7%2F5zwQw0sOyCDOVRZbTFdzsuDw1be6MkumY%3D/83/1760554800/media=hls4/multi=256x144:144p,426x240:240p,854x480:480p,1280x720:720p/016/033/213/_TPL_.h264.mp4.m3u8",+ ),+ (+ "04ef4ab00261eb0e53ae45c2446f06f0496d972516e50cd02e5737071f0976e435e8ef889446347802f4e7ed7563b61b10d85c0a9cd2812ea689e2eef6094279b9721e6aad4c7df5141058bbedad3b2f8878184925ea66824d9542d4d869b276a1dd1d1fbf4bdc149dac0e94f03a8192c3523ef1b4823f646c5767e58b0c07ed08238b7a73b6042e5854c6567d76cc3f01b3c906c5ebca9f1cb95f4a6e31237653d4082f2b9da02142e218691df1ffeadce1657cc0a745312bbf472596f9d464e81c03c58540",+ "https://video-cf-a.xhcdn.com/qnuR8f8q7%2F5zwQw0sOyCDOVRZbTFdzsuDw1be6MkumY%3D/83/1760554800/media=hls4/multi=256x144:144p,426x240:240p,854x480:480p,1280x720:720p/016/033/213/_TPL_.h264.mp4.m3u8",+ ),+ (+ "051e2cfbfd5cca82bd301b716943c9450b62bff13c1f6d26be15dccce34d519a4cb0cd2be88874ff1b027fed601033440f8bebc4ae99b404a9c2d47ec2d13ef4de47257bff8f007ba24350e33b24bfff6c92cd0cb1db63b5b6ea5df26f4574ace4e81799bee282fbfcec78c5a5276050545446a1ddfd791e97144583b187405ac2c9e3433e528c343b015248e1f492c1666262e8e68b002c1bf2c0498b57ec17bbc469250a98577d13d3f08c28be2e2498b5d7c2525bc2b58062cbf28e5b6d3b6dc81d488c9cec649b845d",+ "https://video-cf-a.xhcdn.com/ZkCq1RTUhoKb4%2BqgcEkQ6m7bqkxMui3N%2FDOMGFwMjyI%3D/87/1760554800/media=hls4/multi=256x144:144p:,426x240:240p:,854x480:480p:,1280x720:720p:/016/033/213/_TPL_.av1.mp4.m3u8",+ ),],)def test_decode_hex_url(raw_url: str, expected_output: str) -> None: