Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: individual videos download (Porntrex) (#1530)
NTFSvolumefe563a35ab33
diff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.pyindex d7959e1e8..07436c1b5 100644--- a/cyberdrop_dl/crawlers/porntrex.py+++ b/cyberdrop_dl/crawlers/porntrex.py@@ -14,7 +14,7 @@ if TYPE_CHECKING:from cyberdrop_dl.data_structures.url_objects import ScrapeItem-class Selectors:+class Selector:VIDEO_JS = "div.video-holder script:-soup-contains('var flashvars')"NEXT_PAGE = "div#list_videos_videos_pagination li.next"USER_NAME = "div.user-name"@@ -30,10 +30,7 @@ class Selectors:VIDEOS_OR_ALBUMS = f"{VIDEOS}, {ALBUMS}"-COLLECTION_PARTS = "tags", "categories", "models", "playlists", "search", "members"TITLE_TRASH = "Free HD ", "Most Relevant ", "New ", "Videos", "Porn", "for:", "New Videos", "Tagged with"-_SELECTORS = Selectors()-PRIMARY_URL = AbsoluteHttpURL("https://www.porntrex.com")@@ -49,52 +46,49 @@ class PorntrexCrawler(Crawler):"Search": "/search/...",}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL- NEXT_PAGE_SELECTOR: ClassVar[str] = _SELECTORS.NEXT_PAGE+ NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGEDOMAIN: ClassVar[str] = "porntrex"- _RATE_LIMIT = 3, 10+ DEFAULT_TRIM_URLS: ClassVar[bool] = Falseasync def fetch(self, scrape_item: ScrapeItem) -> None:if scrape_item.url.name: # The ending slash is necessary or we get a 404 errorscrape_item.url = scrape_item.url / ""- if len(scrape_item.url.parts) >= 3:- if "video" in scrape_item.url.parts:- return await self.video(scrape_item)- if any(p in scrape_item.url.parts for p in COLLECTION_PARTS):- return await self.collection(scrape_item)- if "albums" in scrape_item.url.parts:- return await self.album(scrape_item)-- raise ValueError+ match scrape_item.url.parts[1:]:+ case ["video", video_id, *_]:+ return await self.video(scrape_item, video_id)+ case ["tags" | "categories" | "models" | "playlists" | "search" | "members" as type_, _, *_]:+ return await self.collection(scrape_item, type_)+ case ["albums", album_id, *_]:+ return await self.album(scrape_item, album_id)+ case _:+ raise ValueError@error_handling_wrapper- async def album(self, scrape_item: ScrapeItem) -> None:- album_id = scrape_item.url.parts[2]+ async def album(self, scrape_item: ScrapeItem, album_id: str) -> None:soup = await self.request_soup(scrape_item.url)- if "This album is a private album" in soup.text:+ if "This album is a private album" in soup.get_text(strip=True):raise ScrapeError(401, "Private album")- title = css.select_one_get_text(soup, _SELECTORS.ALBUM_TITLE)+ title = css.select_one_get_text(soup, Selector.ALBUM_TITLE)title = self.create_title(title, album_id)- scrape_item.setup_as_album(title)+ scrape_item.setup_as_album(title, album_id=album_id)- for _, link in self.iter_tags(soup, _SELECTORS.IMAGES):- filename, ext = self.get_filename_and_ext(link.name)- debrid_link = link / "" # iter_tags always trims URLs- canonical_url = PRIMARY_URL / "albums" / album_id / filename- await self.handle_file(canonical_url, scrape_item, filename, ext, debrid_link=debrid_link)+ for _, link in self.iter_tags(soup, Selector.IMAGES):+ await self.direct_file(scrape_item, link)scrape_item.add_children()@error_handling_wrapper- async def video(self, scrape_item: ScrapeItem) -> None:- video_id = scrape_item.url.parts[2]+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:+ if not scrape_item.url.name:+ scrape_item.url = scrape_item.url.parent+canonical_url = PRIMARY_URL / "video" / video_idif await self.check_complete_from_referer(canonical_url):returnsoup = await self.request_soup(scrape_item.url)-video = extract_kvs_video(self, soup)link = video.urlscrape_item.url = canonical_url@@ -106,37 +100,31 @@ class PorntrexCrawler(Crawler):)@error_handling_wrapper- async def collection(self, scrape_item: ScrapeItem) -> None:+ async def collection(self, scrape_item: ScrapeItem, collection_type: str) -> None:soup = await self.request_soup(scrape_item.url)if "models" in scrape_item.url.parts:- title: str = css.select_one_get_text(soup, _SELECTORS.MODEL_NAME).title()+ title: str = css.select_one_get_text(soup, Selector.MODEL_NAME).title()elif "members" in scrape_item.url.parts:- title: str = css.select_one_get_text(soup, _SELECTORS.USER_NAME)+ title: str = css.select_one_get_text(soup, Selector.USER_NAME)elif "latest-updates" in scrape_item.url.parts:title: str = "Latest Updates"else:- title = css.select_one_get_text(soup, _SELECTORS.TITLE)+ title = css.select_one_get_text(soup, Selector.TITLE)for trash in TITLE_TRASH:title = title.replace(trash, "").strip()- if "categories" in scrape_item.url.parts:- collection_type = "category"- elif "members" in scrape_item.url.parts:- collection_type = "user"- else:- collection_type = next(p for p in COLLECTION_PARTS if p in scrape_item.url.parts).removesuffix("s")-+ collection_type = collection_type.removesuffix("s")title, *_ = title.split(",Page")title = self.create_title(f"{title} [{collection_type}]")scrape_item.setup_as_album(title)- if last_page_tag := soup.select(_SELECTORS.LAST_PAGE):+ if last_page_tag := soup.select(Selector.LAST_PAGE):last_page = int(last_page_tag[-1].get_text(strip=True))else:last_page = 1- for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEOS_OR_ALBUMS):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.VIDEOS_OR_ALBUMS):self.create_task(self.run(new_scrape_item))await self.proccess_additional_pages(scrape_item, last_page)@@ -149,7 +137,7 @@ class PorntrexCrawler(Crawler):albums_url = scrape_item.url / "albums"soup = await self.request_soup(albums_url)- for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ALBUMS, new_title_part="albums"):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ALBUMS, new_title_part="albums"):self.create_task(self.run(new_scrape_item))async def proccess_additional_pages(self, scrape_item: ScrapeItem, last_page: int, **kwargs: str) -> None:@@ -189,5 +177,5 @@ class PorntrexCrawler(Crawler):page_url = page_url.update_query({from_param_name: page})soup = await self.request_soup(page_url)- for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEOS_OR_ALBUMS):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.VIDEOS_OR_ALBUMS):self.create_task(self.run(new_scrape_item))diff --git a/tests/crawlers/test_cases/porntrex.py b/tests/crawlers/test_cases/porntrex.pynew file mode 100644index 000000000..30ac99063--- /dev/null+++ b/tests/crawlers/test_cases/porntrex.py@@ -0,0 +1,41 @@+DOMAIN = "porntrex"+TEST_CASES = [+ (+ "https://www.porntrex.com/video/3001819/a-pandemic-will-not-stop-the-two-lovers",+ [+ {+ "url": "https://www.porntrex.com/video/3001819",+ "debrid_url": "re:/get_file/",+ "filename": "A pandemic will not stop the two lovers [3001819][1080p].mp4",+ "original_filename": "3001819_1080p.mp4",+ "download_folder": "re:Loose Files (Porntrex)",+ "referer": "https://www.porntrex.com/video/3001819",+ "datetime": None,+ }+ ],+ ),+ (+ "https://www.porntrex.com/search/song/",+ [+ {+ "url": "ANY",+ "download_folder": "re:song [search] (Porntrex)",+ "album_id": None,+ },+ ],+ range(255, 300),+ ),+ (+ "https://www.porntrex.com/albums/57737/brandi-love-set-35/",+ [+ {+ "url": "re:/get_image/",+ "referer": "https://www.porntrex.com/albums/57737/brandi-love-set-35/",+ "download_folder": "re:Brandi Love SET 35 (Porntrex)",+ "datetime": None,+ "album_id": "57737",+ },+ ],+ 376,+ ),+]diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.pyindex 2ace9c86a..aa5bd4c16 100644--- a/tests/crawlers/test_crawlers.py+++ b/tests/crawlers/test_crawlers.py@@ -3,8 +3,9 @@ from __future__ import annotationsimport dataclassesimport importlib.utilimport re+from collections.abc import Sequencefrom pathlib import Path-from typing import TYPE_CHECKING, Literal, NamedTuple, NotRequired+from typing import TYPE_CHECKING, NamedTuple, NotRequiredfrom unittest import mockimport pytest@@ -29,7 +30,7 @@ class Result(TypedDict):# Simplified version of media_itemurl: strfilename: NotRequired[str]- debrid_link: NotRequired[Literal["ANY"] | None]+ debrid_link: NotRequired[str | None]original_filename: NotRequired[str]referer: NotRequired[str]album_id: NotRequired[str | None]@@ -51,7 +52,7 @@ class CrawlerTestCase(NamedTuple):input_url: strresults: list[Result]# TODO: deprecated total, move to config- total: int | None = None+ total: Sequence[int] | int | None = Noneconfig: Config = _default_config@@ -79,7 +80,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:_load_test_data()if "crawler_test_case" in metafunc.fixturenames:valid_domains = sorted(_TEST_DATA)- domains_to_tests: list[str] = metafunc.config.test_crawlers_domains # type: ignore+ domains_to_tests: list[str] = getattr(metafunc.config, "test_crawlers_domains", [])for domain in domains_to_tests:assert domain in valid_domains, f"{domain = } is not a valid or has not tests defined"@@ -111,12 +112,20 @@ async def test_crawler(running_manager: Manager, crawler_test_case: CrawlerTestCresults: list[MediaItem] = sorted((call.args[0] for call in func.call_args_list), key=lambda x: str(x.url))total = test_case.total or len(test_case.results)- assert total == len(results)+ _assert_n_results(test_case, len(results))if total:func.assert_awaited()_validate_results(crawler, test_case, results)+def _assert_n_results(test_case: CrawlerTestCase, n_results: int) -> None:+ total = test_case.total or len(test_case.results)+ if isinstance(total, Sequence):+ assert n_results in total+ else:+ assert total == n_results++def _validate_results(crawler: Crawler, test_case: CrawlerTestCase, results: list[MediaItem]) -> None:expected_results = sorted(test_case.results, key=lambda x: x["url"])origin = getattr(crawler, "PRIMARY_URL", AbsoluteHttpURL("https://google.com"))@@ -170,6 +179,5 @@ async def test_direct_http_crawler(running_manager: Manager, url: str, filename:await crawler.fetch(item)results: list[MediaItem] = sorted((call.args[0] for call in func.call_args_list), key=lambda x: str(x.url))-func.assert_awaited()_validate_results(crawler, test_case, results)