Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: copy headers for HLS segments (#151)

NTFSvolumeae2ff9065b4b
diff --git a/.zed/debug.json b/.zed/debug.json
index 8dc1175b7..052abf570 100644
--- a/.zed/debug.json
+++ b/.zed/debug.json
@@ -1,30 +1,30 @@
[
- {
- "label": "Run CDL debug mode",
- "adapter": "Debugpy",
- "request": "launch",
- "program": "cyberdrop_dl/__main__.py",
- "justMyCode": false,
- "purpose": ["debug-test"],
- "args": [
- "--download",
- "--ignore-history",
- "--dump-json",
- "--required-free-space",
- "512MB",
- "--no-ignore-coomer-post-content",
- "--max-simultaneous-downloads",
- "2",
- "--flaresolverr",
- "http://localhost:8191",
- "--input-file",
- "debug_run.test",
- "--main-log",
- "debug_run.log"
- ],
- "env": {
- "PYTHONASYNCIODEBUG": "1",
- "CDL_ENABLE_TWITTER": "true"
- }
- }
+ {
+ "label": "Run CDL debug mode",
+ "adapter": "Debugpy",
+ "request": "launch",
+ "program": "cyberdrop_dl/__main__.py",
+ "justMyCode": false,
+ "purpose": ["debug-test"],
+ "args": [
+ "--download",
+ "--ignore-history",
+ "--dump-json",
+ "--required-free-space",
+ "512MB",
+ "--no-ignore-coomer-post-content",
+ "--max-simultaneous-downloads",
+ "2",
+ "--flaresolverr",
+ "http://localhost:8191",
+ "--input-file",
+ "debug_run.test",
+ "--main-log",
+ "debug_run.log",
+ ],
+ "env": {
+ "PYTHONASYNCIODEBUG": "1",
+ "CDL_ENABLE_TWITTER": "true",
+ },
+ },
]
diff --git a/cyberdrop_dl/crawlers/missav.py b/cyberdrop_dl/crawlers/missav.py
index 43a395d87..05d2cd528 100644
--- a/cyberdrop_dl/crawlers/missav.py
+++ b/cyberdrop_dl/crawlers/missav.py
@@ -14,7 +14,6 @@ if TYPE_CHECKING:
_M3U8_SERVER = AbsoluteHttpURL("https://surrit.com/")
-_PRIMARY_URL = AbsoluteHttpURL("https://missav.ws")
_COLLECTION_TYPES = "makers", "search", "genres", "labels", "tags"
@@ -31,57 +30,57 @@ class MissAVCrawler(Crawler):
"Video": "/...",
**{name.capitalize(): f"/{name}/<{name.removesuffix('s')}>" for name in _COLLECTION_TYPES},
}
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URL
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://missav.ws")
DOMAIN: ClassVar[str] = "missav"
FOLDER_DOMAIN: ClassVar[str] = "MissAV"
NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE
+ _IMPERSONATE: ClassVar[str | bool | None] = True
async def fetch(self, scrape_item: ScrapeItem) -> None:
- n_parts = len(scrape_item.url.parts)
- for part in _COLLECTION_TYPES:
- if part in scrape_item.url.parts and n_parts == scrape_item.url.parts.index(part) + 2:
- return await self.collection(scrape_item, part)
- return await self.video(scrape_item)
+ match scrape_item.url.parts[1:]:
+ case [*_, collection_type, name] if collection_type in _COLLECTION_TYPES:
+ return await self.collection(scrape_item, collection_type, name)
+ case [*_, video_id]:
+ return await self.video(scrape_item, video_id)
+ case _:
+ raise ValueError
@error_handling_wrapper
- async def collection(self, scrape_item: ScrapeItem, collection_type: str) -> None:
- name = scrape_item.url.name
+ async def collection(self, scrape_item: ScrapeItem, collection_type: str, name: str) -> None:
title = self.create_title(f"{name} [{collection_type}]")
scrape_item.setup_as_album(title)
- async for soup in self.web_pager(scrape_item.url.update_query(page=1), impersonate=True):
+ async for soup in self.web_pager(scrape_item.url.update_query(page=1)):
for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ITEM):
self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
- async def video(self, scrape_item: ScrapeItem) -> None:
- canonical_url = _PRIMARY_URL / "en" / scrape_item.url.name
- scrape_item.url = canonical_url
+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
+ scrape_item.url = canonical_url = self.PRIMARY_URL / "en" / video_id
if await self.check_complete_from_referer(canonical_url):
return
- soup = await self.request_soup(scrape_item.url, impersonate=True)
+ soup = await self.request_soup(scrape_item.url)
- title, date_str = open_graph.title(soup), open_graph.get("video_release_date", soup)
+ title = open_graph.title(soup)
if dvd_code_tag := soup.select_one(Selector.DVD_CODE):
title = _fix_title(title, dvd_code_tag)
- if date_str:
- scrape_item.uploaded_at = self.parse_iso_date(date_str)
- elif date_tag := soup.select_one(Selector.DATE):
- scrape_item.uploaded_at = self.parse_iso_date(css.attr(date_tag, "datetime"))
- else:
- _ = self.parse_date("") # Trigger warning
+ scrape_item.uploaded_at = self.parse_iso_date(
+ open_graph.get("video_release_date", soup) or css.select(soup, Selector.DATE, "datetime")
+ )
- uuid = _get_uuid(soup)
- m3u8_playlist_url = _M3U8_SERVER / uuid / "playlist.m3u8"
- m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_playlist_url)
- ext = ".mp4"
- filename = self.create_custom_filename(title, ext, resolution=info.resolution)
- await self.handle_file(m3u8_playlist_url, scrape_item, title, ext, m3u8=m3u8, custom_filename=filename)
+ uuid = _extract_uuid(soup)
+ m3u8_url = _M3U8_SERVER / uuid / "playlist.m3u8"
+ m3u8, info = await self.get_m3u8_from_playlist_url(
+ m3u8_url,
+ headers={"Referer": "https://missav.ws/"},
+ )
+ filename = self.create_custom_filename(title, ext := ".mp4", resolution=info.resolution)
+ await self.handle_file(m3u8_url, scrape_item, title, ext, m3u8=m3u8, custom_filename=filename)
-def _get_uuid(soup: BeautifulSoup) -> str:
+def _extract_uuid(soup: BeautifulSoup) -> str:
js_text = css.select_text(soup, Selector.UUID)
uuid_joined_parts = js_text.split("m3u8|", 1)[-1].split("|com|surrit", 1)[0]
uuid_parts = reversed(uuid_joined_parts.split("|"))
diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.py
index 9bd9bb35a..89ebd4003 100644
--- a/cyberdrop_dl/data_structures/url_objects.py
+++ b/cyberdrop_dl/data_structures/url_objects.py
@@ -221,7 +221,7 @@ class MediaItem:
me = asdict(self)
if self.hash:
me["hash"] = f"xxh128:{self.hash}"
- for name in ("fallbacks", "_task_id", "is_segment", "parent_media_item"):
+ for name in ("fallbacks", "is_segment"):
del me[name]
return me
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index efe9d3ebf..550c38770 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -264,6 +264,7 @@ class Downloader:
ext=media_item.ext,
)
seg_media_item.is_segment = True
+ seg_media_item.headers = media_item.headers.copy()
return SegmentDownloadResult(
seg_media_item,
await self.start_download(seg_media_item),
diff --git a/cyberdrop_dl/progress/__init__.py b/cyberdrop_dl/progress/__init__.py
index ef0bfa502..5e81923d7 100644
--- a/cyberdrop_dl/progress/__init__.py
+++ b/cyberdrop_dl/progress/__init__.py
@@ -94,7 +94,7 @@ class LiveUI(ABC):
@contextlib.contextmanager
def __call__(self, *, transient: bool = True) -> Generator[None]:
- if self.disabled:
+ if self.disabled or "pytest" in sys.modules:
yield None
return
@@ -102,10 +102,10 @@ class LiveUI(ABC):
try:
with Live(
refresh_per_second=REFRESH_RATE.get(),
- auto_refresh="pytest" not in sys.modules,
+ auto_refresh=True,
screen=transient,
transient=transient,
- get_renderable=self.__rich__ if "pytest" not in sys.modules else None,
+ get_renderable=self.__rich__,
):
yield
finally: