Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: profile selectors (ThisVid) (#1955)
NTFSvolumece99b69b6d36
diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.pyindex 3a140038c..d20a7ee82 100644--- a/cyberdrop_dl/crawlers/_kvs.py+++ b/cyberdrop_dl/crawlers/_kvs.py@@ -33,16 +33,19 @@ class Selector:UNAUTHORIZED = "div.video-holder:-soup-contains('This video is a private video')"FLASHVARS = "script:-soup-contains('video_id:')"USER_NAME = "div.headline > h2"- ALBUM_NAME = "div.headline > h1"- ALBUM_PICTURES = "div.album-list > a, .images a"- PICTURE = "div.photo-holder > img"- PUBLIC_VIDEOS = "div#list_videos_public_videos_items"- PRIVATE_VIDEOS = "div#list_videos_private_videos_items"- FAVOURITE_VIDEOS = "div#list_videos_favourite_videos_items"- COMMON_VIDEOS_TITLE = "div#list_videos_common_videos_list h1"- VIDEOS = "div#list_videos_common_videos_list_items a"- NEXT_PAGE = "li.pagination-next > a"- ALBUM_ID = "script:-soup-contains('album_id')"+ IMAGE = "div.photo-holder > img"+ THUMBNAILS = "a.tumbpu"++ class Album:+ NAME = "div.headline > h1"+ IMAGES = "div.album-list > a, .images a"+ ID = "script:-soup-contains('album_id')"++ class Videos:+ PUBLIC = "div#list_videos_public_videos_items"+ PRIVATE = "div#list_videos_private_videos_items"+ FAVOURITE = "div#list_videos_favourite_videos_items"+ TITLE = "div#list_videos_common_videos_list h1"class KernelVideoSharingCrawler(Crawler, is_abc=True):@@ -55,7 +58,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):"Videos": "/videos/<slug>","Members": "/members/<member_id>",}- NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE+ NEXT_PAGE_SELECTOR: ClassVar[str] = "li.pagination-next > a"_RATE_LIMIT: ClassVar[RateLimit] = 6, 5def __init_subclass__(cls, *, ensure_trailing_slash: bool = False, **kwargs: Any) -> None:@@ -78,12 +81,14 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):async def fetch(self, scrape_item: ScrapeItem) -> None: # noqa: PLR0911match scrape_item.url.parts[1:]:- case ["categories" | "tags", _]:- return await self.collection(scrape_item)+ case ["categories" | "tags" as type_, name]:+ return await self.collection(scrape_item, name, type_)case ["search", query]:- return await self.collection(scrape_item, query)- case ["members", _, *_]:- return await self.profile(scrape_item)+ return await self.search(scrape_item, query)+ case ["members", member_id, "public_videos" | "favourite_videos" | "private_videos", *_]:+ return await self.profile(scrape_item, member_id, entire_profile=False)+ case ["members", member_id, *_]:+ return await self.profile(scrape_item, member_id)case ["videos", _, *_]:return await self.video(scrape_item)case ["albums", _]:@@ -92,7 +97,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):return await self.picture(scrape_item)case _:if query := scrape_item.url.query.get("q"):- return await self.collection(scrape_item, query)+ return await self.search(scrape_item, query)raise ValueError@classmethod@@ -109,46 +114,43 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):@classmethoddef _collection_title(cls, soup: BeautifulSoup) -> str:- return cls._clean_title(css.select_text(soup, Selector.COMMON_VIDEOS_TITLE))+ return cls._clean_title(css.select_text(soup, Selector.Videos.TITLE))@error_handling_wrapper- async def collection(self, scrape_item: ScrapeItem, query: str | None = None) -> None:- soup = await self.request_soup(scrape_item.url)- if query:- title = f"{query} [search]"- else:- common_title = css.select_text(soup, Selector.COMMON_VIDEOS_TITLE)- if common_title.startswith("New Videos Tagged"):- common_title = common_title.split("Showing")[0].split("Tagged with")[1].strip()- title = f"{common_title} [tag]"- else:- common_title = common_title.split("New Videos")[0].strip()- title = f"{common_title} [category]"+ async def search(self, scrape_item: ScrapeItem, query: str) -> None:+ title = self.create_title(f"{query} [search]")+ scrape_item.setup_as_album(title)+ await self._iter_videos(scrape_item)+ @error_handling_wrapper+ async def collection(self, scrape_item: ScrapeItem, name: str, type_: str) -> None:+ title = f"{name} [{'tag' if type_ == 'tags' else 'category'}]"title = self.create_title(title)scrape_item.setup_as_album(title)await self._iter_videos(scrape_item)@error_handling_wrapper- async def profile(self, scrape_item: ScrapeItem) -> None:- soup = await self.request_soup(scrape_item.url)- user_name: str = (- css.select_text(soup, Selector.USER_NAME).split("'s Profile")[0].strip().removesuffix("'s Page")- )- title = self.create_title(f"{user_name} [user]")- scrape_item.setup_as_profile(title)-- if soup.select_one(Selector.PUBLIC_VIDEOS):- await self._iter_videos(scrape_item, "public_videos")- if soup.select_one(Selector.FAVOURITE_VIDEOS):- await self._iter_videos(scrape_item, "favourite_videos")- if soup.select_one(Selector.PRIVATE_VIDEOS):- await self._iter_videos(scrape_item, "private_videos")-- async def _iter_videos(self, scrape_item: ScrapeItem, video_category: str = "") -> None:- url = scrape_item.url / video_category if video_category else scrape_item.url- async for soup in self.web_pager(url):- for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.VIDEOS):+ async def profile(self, scrape_item: ScrapeItem, member_id: str, *, entire_profile: bool = False) -> None:+ profile_url = scrape_item.url.origin() / "members" / member_id+ soup = await self.request_soup(profile_url)+ user_name = _extract_user_name(soup)+ scrape_item.setup_as_profile(self.create_title(f"{user_name} [user]"))++ if not entire_profile:+ await self._iter_videos(scrape_item, scrape_item.url)+ return++ for selector, path in [+ (Selector.Videos.PUBLIC, "public_videos"),+ (Selector.Videos.FAVOURITE, "favourite_videos"),+ (Selector.Videos.PRIVATE, "private_videos"),+ ]:+ if soup.select_one(selector):+ await self._iter_videos(scrape_item, profile_url / path)++ async def _iter_videos(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:+ async for soup in self.web_pager(url or scrape_item.url):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.THUMBNAILS):self.create_task(self.run(new_scrape_item))@error_handling_wrapper@@ -186,16 +188,12 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem, album_id: str | None = None) -> None:soup = await self.request_soup(scrape_item.url)- if not album_id:- js_text = css.select_text(soup, Selector.ALBUM_ID)- album_id = extr_text(js_text, "params['album_id'] =", ";")-- results = await self.get_album_results(album_id)- title = css.select_text(soup, Selector.ALBUM_NAME)- title = self.create_title(f"{title} [album]", album_id)+ album_id = album_id or _extract_album_id(soup)+ name = css.select_text(soup, Selector.Album.NAME)+ title = self.create_title(f"{name} [album]", album_id)scrape_item.setup_as_album(title, album_id=album_id)- for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ALBUM_PICTURES, results=results):- self.create_task(self.run(new_scrape_item))+ for _, new_item in self.iter_children(scrape_item, soup, Selector.Album.IMAGES):+ self.create_task(self.run(new_item))@error_handling_wrapperasync def picture(self, scrape_item: ScrapeItem) -> None:@@ -203,8 +201,8 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):returnsoup = await self.request_soup(scrape_item.url)- src = self.parse_url(css.select(soup, Selector.PICTURE, "src"))- await self.direct_file(scrape_item, src)+ src = css.select(soup, Selector.IMAGE, "src")+ await self.direct_file(scrape_item, self.parse_url(src))async def _ajax_pagination( # noqa: PLR0913self,@@ -326,3 +324,12 @@ def _deobfuscate_url(video_url_str: str, license_token: Sequence[int]) -> Absolunew_parts = list(url.parts)new_parts[3] = "".join(checksum[index] for index in indices) + tailreturn url.with_path("/".join(new_parts[1:]), keep_query=True, keep_fragment=True)+++def _extract_user_name(soup: BeautifulSoup) -> str:+ return css.select_text(soup, Selector.USER_NAME).partition("'s Profile")[0].strip().removesuffix("'s Page")+++def _extract_album_id(soup: BeautifulSoup) -> str:+ js_text = css.select_text(soup, Selector.Album.ID)+ return extr_text(js_text, "params['album_id'] =", ";")diff --git a/tests/crawlers/test_cases/thisvid.py b/tests/crawlers/test_cases/thisvid.pyindex e53e65ce7..bba79a472 100644--- a/tests/crawlers/test_cases/thisvid.py+++ b/tests/crawlers/test_cases/thisvid.py@@ -27,7 +27,7 @@ TEST_CASES = ["url": "https://thisvid.com/albums/bruxel/","results": [{- "url": "re:https://media.thisvid.com/contents/albums/main/700x525/517000/51739",+ "url": "re:https://static.thisvid.com/contents/albums/main/","download_folder": "re:Julia Bruxel \\[album\\] \\(ThisVid\\)","referer": "re:https://thisvid.com/albums/bruxel","album_id": "517393",@@ -35,4 +35,110 @@ TEST_CASES = [],"count": 10,},+ {+ "url": "https://thisvid.com/members/1354738",+ "results": [+ {+ "url": "https://thisvid.com/videos/cake-dance-booty-cake",+ "filename": "Cake Dance (Booty Cake!) [14295498].mp4",+ "debrid_link": "https://thisvid.com/get_file/5/021865d76ffb8605044e6a141e23f8a205f10fe8cf/14295000/14295498/14295498.mp4",+ "original_filename": "14295498.mp4",+ "referer": "https://thisvid.com/videos/cake-dance-booty-cake",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:NoOneImportantHere [user] (ThisVid)",+ },+ {+ "url": "https://thisvid.com/videos/icing-on-the-cake5",+ "filename": "Icing On The Cake [13720173].mp4",+ "debrid_link": "https://thisvid.com/get_file/5/301aff8b86d3abb382e880631ab9173121be81d33c/13720000/13720173/13720173.mp4",+ "original_filename": "13720173.mp4",+ "referer": "https://thisvid.com/videos/icing-on-the-cake5",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:NoOneImportantHere [user] (ThisVid)",+ },+ {+ "url": "https://thisvid.com/videos/booty-cake-delivered",+ "filename": "Booty Cake Delivered [12835401].mp4",+ "debrid_link": "https://thisvid.com/get_file/5/f477da4ddb227967b05943b15b80148dffcefd54df/12835000/12835401/12835401.mp4",+ "original_filename": "12835401.mp4",+ "referer": "https://thisvid.com/videos/booty-cake-delivered",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:NoOneImportantHere [user] (ThisVid)",+ },+ {+ "url": "https://thisvid.com/videos/sudden-harsh-wet-sneeze-interupts",+ "filename": "Sudden Harsh Wet Sneeze Interupts! [12462474].mp4",+ "debrid_link": "https://thisvid.com/get_file/5/774153084e028c1765d9a29c0e6d488fb1d644b820/12462000/12462474/12462474.mp4",+ "original_filename": "12462474.mp4",+ "referer": "https://thisvid.com/videos/sudden-harsh-wet-sneeze-interupts",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:NoOneImportantHere [user] (ThisVid)",+ },+ {+ "url": "https://thisvid.com/videos/thicc-jiggly-booty-in-tight-pants",+ "filename": "Thicc Jiggly Booty In Tight Pants [12012010].mp4",+ "debrid_link": "https://thisvid.com/get_file/5/45280519865477661b8e1822342aa162abd5053095/12012000/12012010/12012010.mp4",+ "original_filename": "12012010.mp4",+ "referer": "https://thisvid.com/videos/thicc-jiggly-booty-in-tight-pants",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:NoOneImportantHere [user] (ThisVid)",+ },+ {+ "url": "https://thisvid.com/videos/thicc-booty-in-sweat-pants",+ "filename": "Thicc Booty In Sweat Pants [11700700].mp4",+ "debrid_link": "https://thisvid.com/get_file/4/57597de49a32da1e46d672cc18a21a05efe54c8ee8/11700000/11700700/11700700.mp4",+ "original_filename": "11700700.mp4",+ "referer": "https://thisvid.com/videos/thicc-booty-in-sweat-pants",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:NoOneImportantHere [user] (ThisVid)",+ },+ {+ "url": "https://thisvid.com/videos/sneezing-fit-in-the-car2",+ "filename": "Sneezing Fit In The Car [5658887].mp4",+ "debrid_link": "https://thisvid.com/get_file/4/782d1e37e18684900103fe5d354f9413f2e0b02226/5658000/5658887/5658887.mp4",+ "original_filename": "5658887.mp4",+ "referer": "https://thisvid.com/videos/sneezing-fit-in-the-car2",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:NoOneImportantHere [user] (ThisVid)",+ },+ {+ "url": "https://thisvid.com/videos/glasses-girl-mini-wet-sneeze-fit",+ "filename": "Glasses Girl Mini Wet Sneeze Fit [5658871].mp4",+ "debrid_link": "https://thisvid.com/get_file/4/55c3e64323410ac3e767cca14501d4972d8b1a7b3e/5658000/5658871/5658871.mp4",+ "original_filename": "5658871.mp4",+ "referer": "https://thisvid.com/videos/glasses-girl-mini-wet-sneeze-fit",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:NoOneImportantHere [user] (ThisVid)",+ },+ {+ "url": "https://thisvid.com/videos/black-haired-girl-sneezing-fit",+ "filename": "Black Haired Girl Sneezing Fit [5113285].mp4",+ "debrid_link": "https://thisvid.com/get_file/4/7260e109787f406756fa812a50fb6ba0140629bb23/5113000/5113285/5113285.mp4",+ "original_filename": "5113285.mp4",+ "referer": "https://thisvid.com/videos/black-haired-girl-sneezing-fit",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:NoOneImportantHere [user] (ThisVid)",+ },+ {+ "url": "https://thisvid.com/videos/brazilian-booty-rub-down",+ "filename": "Brazilian Booty Rub Down [5113273].mp4",+ "debrid_link": "https://thisvid.com/get_file/4/76ed0f803842a93fa5a6e08952aded8b273e381548/5113000/5113273/5113273.mp4",+ "original_filename": "5113273.mp4",+ "referer": "https://thisvid.com/videos/brazilian-booty-rub-down",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:NoOneImportantHere [user] (ThisVid)",+ },+ ],+ "count": 10,+ },]