Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: episode parsing (OnePace) (#1910)

NTFSvolumeef01587657bb
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d02166b84..c6488a013 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Date extraction (Rule34vault)
- `--json` output missing files skipped by host or filename regex
- Update website salt (Gofile)
+- Episode parsing (OnePace)
### Security
diff --git a/cyberdrop_dl/crawlers/onepace.py b/cyberdrop_dl/crawlers/onepace.py
index 740f1db15..7f6185eb1 100644
--- a/cyberdrop_dl/crawlers/onepace.py
+++ b/cyberdrop_dl/crawlers/onepace.py
@@ -7,9 +7,11 @@ from pydantic import dataclasses
from cyberdrop_dl import env
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import DictDataclass, error_handling_wrapper, next_js
+from cyberdrop_dl.utils import error_handling_wrapper, next_js
if TYPE_CHECKING:
+ from collections.abc import Generator
+
from bs4 import BeautifulSoup
from cyberdrop_dl.url_objects import ScrapeItem
@@ -35,10 +37,10 @@ class Playlist:
@dataclasses.dataclass(slots=True)
-class Episode(DictDataclass):
+class Episode:
title: str
slug: str
- playGroups: list[PlayGroup] # noqa: N815
+ playlistGroups: list[PlayGroup] # noqa: N815
_PD_BASE = AbsoluteHttpURL("https://pixeldrain.com/l/")
@@ -60,17 +62,19 @@ class OnePaceCrawler(Crawler):
@error_handling_wrapper
async def all_episodes(self, scrape_item: ScrapeItem) -> None:
soup = await self.request_soup(self.PRIMARY_URL / "en/watch")
- episodes = _extract_episodes(soup)
+ episodes = tuple(_extract_episodes(soup))
await self.write_metadata(scrape_item, "one_pace_episodes", {"episodes": episodes})
scrape_item.setup_as_profile(self.FOLDER_DOMAIN)
- for episode in episodes:
- self._episode(scrape_item.copy(), Episode.from_dict(episode))
+ for episode in (Episode(**ep) for ep in episodes):
+ url = scrape_item.url.with_fragment(episode.slug)
+ new_item = scrape_item.create_child(url)
+ self._episode(new_item, episode)
scrape_item.add_children()
+ @error_handling_wrapper
def _episode(self, scrape_item: ScrapeItem, ep: Episode) -> None:
- scrape_item.url = scrape_item.url.with_fragment(ep.slug)
- best_group = max(ep.playGroups, key=lambda x: x.score)
+ best_group = max(ep.playlistGroups, key=lambda x: x.score)
self.log.info(
"Downloading %s with subs=%s and lang=%s",
scrape_item.url,
@@ -81,15 +85,18 @@ class OnePaceCrawler(Crawler):
self.handle_embed(scrape_item.create_child(best_playlist.url))
-def _extract_episodes(soup: BeautifulSoup) -> tuple[dict[str, Any], ...]:
- episodes: tuple[dict[str, Any], ...] = tuple(next_js.ifind(next_js.extract(soup), "slug", "title", "playGroups"))
+def _extract_episodes(soup: BeautifulSoup) -> Generator[dict[str, Any]]:
+ next_data = next_js.extract(soup)
+ for ep in next_js.ifind(next_data, "slug", "title", "playlistGroups"):
+ yield _flatten_episode(ep)
+
- for ep in episodes:
- ep["backdrops"] = [
- {name: v for name, v in backdrop.items() if not name.startswith("blur")} for backdrop in ep["backdrops"]
- ]
- for group in ep["playGroups"]:
- for playlist in group["playlists"]:
- playlist["url"] = _PD_BASE / playlist["id"]
+def _flatten_episode(ep: dict[str, Any]) -> dict[str, Any]:
+ ep["backdrops"] = [
+ {name: v for name, v in backdrop.items() if not name.startswith("blur")} for backdrop in ep["backdrops"]
+ ]
+ for group in ep["playlistGroups"]:
+ for playlist in group["playlists"]:
+ playlist["url"] = _PD_BASE / playlist["id"]
- return episodes
+ return ep
diff --git a/tests/crawlers/test_cases/onepace.py b/tests/crawlers/test_cases/onepace.py
new file mode 100644
index 000000000..1f97c8c71
--- /dev/null
+++ b/tests/crawlers/test_cases/onepace.py
@@ -0,0 +1,17 @@
+DOMAIN = "onepace.net"
+TEST_CASES = [
+ {
+ "url": "https://onepace.net/en/watch",
+ "results": [
+ {
+ "url": "re:https://pixeldrain.com/api/file/",
+ "debrid_link": None,
+ "referer": "re:https://pixeldrain.com/u",
+ "album_id": str,
+ "uploaded_at": int,
+ "download_folder": r"re:OnePace\/.*\(PixelDrain\)$",
+ },
+ ],
+ "count": range(470, 500),
+ }
+]