Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: copy headers for HLS segments (#151)
NTFSvolumeae2ff9065b4b
diff --git a/.zed/debug.json b/.zed/debug.jsonindex 8dc1175b7..052abf570 100644--- a/.zed/debug.json+++ b/.zed/debug.json@@ -1,30 +1,30 @@[- {- "label": "Run CDL debug mode",- "adapter": "Debugpy",- "request": "launch",- "program": "cyberdrop_dl/__main__.py",- "justMyCode": false,- "purpose": ["debug-test"],- "args": [- "--download",- "--ignore-history",- "--dump-json",- "--required-free-space",- "512MB",- "--no-ignore-coomer-post-content",- "--max-simultaneous-downloads",- "2",- "--flaresolverr",- "http://localhost:8191",- "--input-file",- "debug_run.test",- "--main-log",- "debug_run.log"- ],- "env": {- "PYTHONASYNCIODEBUG": "1",- "CDL_ENABLE_TWITTER": "true"- }- }+ {+ "label": "Run CDL debug mode",+ "adapter": "Debugpy",+ "request": "launch",+ "program": "cyberdrop_dl/__main__.py",+ "justMyCode": false,+ "purpose": ["debug-test"],+ "args": [+ "--download",+ "--ignore-history",+ "--dump-json",+ "--required-free-space",+ "512MB",+ "--no-ignore-coomer-post-content",+ "--max-simultaneous-downloads",+ "2",+ "--flaresolverr",+ "http://localhost:8191",+ "--input-file",+ "debug_run.test",+ "--main-log",+ "debug_run.log",+ ],+ "env": {+ "PYTHONASYNCIODEBUG": "1",+ "CDL_ENABLE_TWITTER": "true",+ },+ },]diff --git a/cyberdrop_dl/crawlers/missav.py b/cyberdrop_dl/crawlers/missav.pyindex 43a395d87..05d2cd528 100644--- a/cyberdrop_dl/crawlers/missav.py+++ b/cyberdrop_dl/crawlers/missav.py@@ -14,7 +14,6 @@ if TYPE_CHECKING:_M3U8_SERVER = AbsoluteHttpURL("https://surrit.com/")-_PRIMARY_URL = AbsoluteHttpURL("https://missav.ws")_COLLECTION_TYPES = "makers", "search", "genres", "labels", "tags"@@ -31,57 +30,57 @@ class MissAVCrawler(Crawler):"Video": "/...",**{name.capitalize(): f"/{name}/<{name.removesuffix('s')}>" for name in _COLLECTION_TYPES},}- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URL+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://missav.ws")DOMAIN: ClassVar[str] = "missav"FOLDER_DOMAIN: ClassVar[str] = "MissAV"NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE+ _IMPERSONATE: ClassVar[str | bool | None] = Trueasync def fetch(self, scrape_item: ScrapeItem) -> None:- n_parts = len(scrape_item.url.parts)- for part in _COLLECTION_TYPES:- if part in scrape_item.url.parts and n_parts == scrape_item.url.parts.index(part) + 2:- return await self.collection(scrape_item, part)- return await self.video(scrape_item)+ match scrape_item.url.parts[1:]:+ case [*_, collection_type, name] if collection_type in _COLLECTION_TYPES:+ return await self.collection(scrape_item, collection_type, name)+ case [*_, video_id]:+ return await self.video(scrape_item, video_id)+ case _:+ raise ValueError@error_handling_wrapper- async def collection(self, scrape_item: ScrapeItem, collection_type: str) -> None:- name = scrape_item.url.name+ async def collection(self, scrape_item: ScrapeItem, collection_type: str, name: str) -> None:title = self.create_title(f"{name} [{collection_type}]")scrape_item.setup_as_album(title)- async for soup in self.web_pager(scrape_item.url.update_query(page=1), impersonate=True):+ async for soup in self.web_pager(scrape_item.url.update_query(page=1)):for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ITEM):self.create_task(self.run(new_scrape_item))@error_handling_wrapper- async def video(self, scrape_item: ScrapeItem) -> None:- canonical_url = _PRIMARY_URL / "en" / scrape_item.url.name- scrape_item.url = canonical_url+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:+ scrape_item.url = canonical_url = self.PRIMARY_URL / "en" / video_idif await self.check_complete_from_referer(canonical_url):return- soup = await self.request_soup(scrape_item.url, impersonate=True)+ soup = await self.request_soup(scrape_item.url)- title, date_str = open_graph.title(soup), open_graph.get("video_release_date", soup)+ title = open_graph.title(soup)if dvd_code_tag := soup.select_one(Selector.DVD_CODE):title = _fix_title(title, dvd_code_tag)- if date_str:- scrape_item.uploaded_at = self.parse_iso_date(date_str)- elif date_tag := soup.select_one(Selector.DATE):- scrape_item.uploaded_at = self.parse_iso_date(css.attr(date_tag, "datetime"))- else:- _ = self.parse_date("") # Trigger warning+ scrape_item.uploaded_at = self.parse_iso_date(+ open_graph.get("video_release_date", soup) or css.select(soup, Selector.DATE, "datetime")+ )- uuid = _get_uuid(soup)- m3u8_playlist_url = _M3U8_SERVER / uuid / "playlist.m3u8"- m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_playlist_url)- ext = ".mp4"- filename = self.create_custom_filename(title, ext, resolution=info.resolution)- await self.handle_file(m3u8_playlist_url, scrape_item, title, ext, m3u8=m3u8, custom_filename=filename)+ uuid = _extract_uuid(soup)+ m3u8_url = _M3U8_SERVER / uuid / "playlist.m3u8"+ m3u8, info = await self.get_m3u8_from_playlist_url(+ m3u8_url,+ headers={"Referer": "https://missav.ws/"},+ )+ filename = self.create_custom_filename(title, ext := ".mp4", resolution=info.resolution)+ await self.handle_file(m3u8_url, scrape_item, title, ext, m3u8=m3u8, custom_filename=filename)-def _get_uuid(soup: BeautifulSoup) -> str:+def _extract_uuid(soup: BeautifulSoup) -> str:js_text = css.select_text(soup, Selector.UUID)uuid_joined_parts = js_text.split("m3u8|", 1)[-1].split("|com|surrit", 1)[0]uuid_parts = reversed(uuid_joined_parts.split("|"))diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.pyindex 9bd9bb35a..89ebd4003 100644--- a/cyberdrop_dl/data_structures/url_objects.py+++ b/cyberdrop_dl/data_structures/url_objects.py@@ -221,7 +221,7 @@ class MediaItem:me = asdict(self)if self.hash:me["hash"] = f"xxh128:{self.hash}"- for name in ("fallbacks", "_task_id", "is_segment", "parent_media_item"):+ for name in ("fallbacks", "is_segment"):del me[name]return mediff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex efe9d3ebf..550c38770 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -264,6 +264,7 @@ class Downloader:ext=media_item.ext,)seg_media_item.is_segment = True+ seg_media_item.headers = media_item.headers.copy()return SegmentDownloadResult(seg_media_item,await self.start_download(seg_media_item),diff --git a/cyberdrop_dl/progress/__init__.py b/cyberdrop_dl/progress/__init__.pyindex ef0bfa502..5e81923d7 100644--- a/cyberdrop_dl/progress/__init__.py+++ b/cyberdrop_dl/progress/__init__.py@@ -94,7 +94,7 @@ class LiveUI(ABC):@contextlib.contextmanagerdef __call__(self, *, transient: bool = True) -> Generator[None]:- if self.disabled:+ if self.disabled or "pytest" in sys.modules:yield Nonereturn@@ -102,10 +102,10 @@ class LiveUI(ABC):try:with Live(refresh_per_second=REFRESH_RATE.get(),- auto_refresh="pytest" not in sys.modules,+ auto_refresh=True,screen=transient,transient=transient,- get_renderable=self.__rich__ if "pytest" not in sys.modules else None,+ get_renderable=self.__rich__,):yieldfinally: