Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: episode parsing (OnePace) (#1910)
NTFSvolumeef01587657bb
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex d02166b84..c6488a013 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Date extraction (Rule34vault)- `--json` output missing files skipped by host or filename regex- Update website salt (Gofile)+- Episode parsing (OnePace)### Securitydiff --git a/cyberdrop_dl/crawlers/onepace.py b/cyberdrop_dl/crawlers/onepace.pyindex 740f1db15..7f6185eb1 100644--- a/cyberdrop_dl/crawlers/onepace.py+++ b/cyberdrop_dl/crawlers/onepace.py@@ -7,9 +7,11 @@ from pydantic import dataclassesfrom cyberdrop_dl import envfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import DictDataclass, error_handling_wrapper, next_js+from cyberdrop_dl.utils import error_handling_wrapper, next_jsif TYPE_CHECKING:+ from collections.abc import Generator+from bs4 import BeautifulSoupfrom cyberdrop_dl.url_objects import ScrapeItem@@ -35,10 +37,10 @@ class Playlist:@dataclasses.dataclass(slots=True)-class Episode(DictDataclass):+class Episode:title: strslug: str- playGroups: list[PlayGroup] # noqa: N815+ playlistGroups: list[PlayGroup] # noqa: N815_PD_BASE = AbsoluteHttpURL("https://pixeldrain.com/l/")@@ -60,17 +62,19 @@ class OnePaceCrawler(Crawler):@error_handling_wrapperasync def all_episodes(self, scrape_item: ScrapeItem) -> None:soup = await self.request_soup(self.PRIMARY_URL / "en/watch")- episodes = _extract_episodes(soup)+ episodes = tuple(_extract_episodes(soup))await self.write_metadata(scrape_item, "one_pace_episodes", {"episodes": episodes})scrape_item.setup_as_profile(self.FOLDER_DOMAIN)- for episode in episodes:- self._episode(scrape_item.copy(), Episode.from_dict(episode))+ for episode in (Episode(**ep) for ep in episodes):+ url = scrape_item.url.with_fragment(episode.slug)+ new_item = scrape_item.create_child(url)+ self._episode(new_item, episode)scrape_item.add_children()+ @error_handling_wrapperdef _episode(self, scrape_item: ScrapeItem, ep: Episode) -> None:- scrape_item.url = scrape_item.url.with_fragment(ep.slug)- best_group = max(ep.playGroups, key=lambda x: x.score)+ best_group = max(ep.playlistGroups, key=lambda x: x.score)self.log.info("Downloading %s with subs=%s and lang=%s",scrape_item.url,@@ -81,15 +85,18 @@ class OnePaceCrawler(Crawler):self.handle_embed(scrape_item.create_child(best_playlist.url))-def _extract_episodes(soup: BeautifulSoup) -> tuple[dict[str, Any], ...]:- episodes: tuple[dict[str, Any], ...] = tuple(next_js.ifind(next_js.extract(soup), "slug", "title", "playGroups"))+def _extract_episodes(soup: BeautifulSoup) -> Generator[dict[str, Any]]:+ next_data = next_js.extract(soup)+ for ep in next_js.ifind(next_data, "slug", "title", "playlistGroups"):+ yield _flatten_episode(ep)+- for ep in episodes:- ep["backdrops"] = [- {name: v for name, v in backdrop.items() if not name.startswith("blur")} for backdrop in ep["backdrops"]- ]- for group in ep["playGroups"]:- for playlist in group["playlists"]:- playlist["url"] = _PD_BASE / playlist["id"]+def _flatten_episode(ep: dict[str, Any]) -> dict[str, Any]:+ ep["backdrops"] = [+ {name: v for name, v in backdrop.items() if not name.startswith("blur")} for backdrop in ep["backdrops"]+ ]+ for group in ep["playlistGroups"]:+ for playlist in group["playlists"]:+ playlist["url"] = _PD_BASE / playlist["id"]- return episodes+ return epdiff --git a/tests/crawlers/test_cases/onepace.py b/tests/crawlers/test_cases/onepace.pynew file mode 100644index 000000000..1f97c8c71--- /dev/null+++ b/tests/crawlers/test_cases/onepace.py@@ -0,0 +1,17 @@+DOMAIN = "onepace.net"+TEST_CASES = [+ {+ "url": "https://onepace.net/en/watch",+ "results": [+ {+ "url": "re:https://pixeldrain.com/api/file/",+ "debrid_link": None,+ "referer": "re:https://pixeldrain.com/u",+ "album_id": str,+ "uploaded_at": int,+ "download_folder": r"re:OnePace\/.*\(PixelDrain\)$",+ },+ ],+ "count": range(470, 500),+ }+]