Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add nextjs13+ parsing utils (#5)

NTFSvolume912d2bea2969
diff --git a/cyberdrop_dl/utils/next_js.py b/cyberdrop_dl/utils/next_js.py
new file mode 100644
index 000000000..a537109b7
--- /dev/null
+++ b/cyberdrop_dl/utils/next_js.py
@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import dataclasses
+import json
+import re
+from collections.abc import Generator
+from enum import IntEnum
+from typing import TYPE_CHECKING, Any, TypeAlias
+
+if TYPE_CHECKING:
+ from collections.abc import Generator
+
+ from bs4 import BeautifulSoup
+
+ChunkID: TypeAlias = str
+
+
+class FlightDataType(IntEnum):
+ BOOTSTRAP = 0
+ PAYLOAD = 1
+ FORM_STATE = 2
+ BINARY = 3
+
+
[email protected](slots=True, order=True)
+class FlightChunk:
+ index: int = dataclasses.field(init=False)
+ id: ChunkID
+ marker: str | None
+
+ raw_data: str
+ data: Any = dataclasses.field(init=False)
+ hints: list[str] = dataclasses.field(init=False)
+ resolved: bool = False
+
+ def __post_init__(self) -> None:
+ self.index = int(self.id, base=16)
+ self.data, *self.hints = self.raw_data.split("\n:H")
+
+
+# Map of chunk_id (hex index of the chunk) -> list of all the objects (components) created from that chunk
+NextJSFlight = dict[ChunkID, list[dict[str, Any]]]
+
+
+_find_chunks = re.compile(r'^(?<![0-9A-Za-z:"])([0-9a-f]+):([A-Z]{0,2})(["\[\{]|null)', re.MULTILINE).finditer
+
+
+def ifind(next_flight: NextJSFlight, *attrs: str) -> Generator[dict[str, Any]]:
+ """Yield every object within `next_flight` that have the required `attrs`."""
+ needed = frozenset(attrs)
+
+ def walk(obj: Any) -> Generator[dict[str, Any]]:
+ if isinstance(obj, dict):
+ if needed.issubset(obj):
+ yield obj
+ else:
+ for v in obj.values():
+ yield from walk(v)
+ elif isinstance(obj, list):
+ for v in obj:
+ yield from walk(v)
+
+ for ele in next_flight.values():
+ yield from walk(ele)
+
+
+def find(next_flight: NextJSFlight, *attrs: str) -> dict[str, Any]:
+ """Get the first object within `next_flight` that have the required `attrs`."""
+ return next(ifind(next_flight, *attrs))
+
+
+def extract(soup: BeautifulSoup) -> NextJSFlight:
+ return parse(_extract(soup))
+
+
+def parse(flight_data: str) -> NextJSFlight:
+ return {chunk.id: chunk.data for chunk in _parse_chunks(flight_data)}
+
+
+def _extract(soup: BeautifulSoup) -> str:
+ return "".join((data for _, data in _get_flight_chunks(soup)))
+
+
+def _get_flight_chunks(soup: BeautifulSoup) -> Generator[tuple[FlightDataType, str]]:
+ for script in soup.select("script:-soup-contains('self.__next_f.push(')"):
+ js_text = script.get_text()
+ raw_data = js_text[js_text.find("(") + 1 : js_text.rfind(")")]
+ type, data = json.loads(raw_data)
+ type = FlightDataType(type)
+ if type is FlightDataType.PAYLOAD:
+ yield type, data
+
+
+def _parse_chunks(flight_data: str) -> Generator[FlightChunk]:
+ chunks: dict[ChunkID, FlightChunk] = {}
+
+ def revive(value: Any) -> Any:
+ if not value:
+ return value
+
+ if isinstance(value, str):
+ if value[0] != "$":
+ return value
+
+ if value == "$":
+ return ""
+
+ match value[1]:
+ case "$":
+ return value[2:]
+ case "@" | "L":
+ chunk_id = value[2:]
+ return revive(chunks[chunk_id])
+ case "u":
+ return None
+ case "D":
+ return value[2:]
+ case "n":
+ return int(value[2:])
+ case _:
+ return value[1:]
+
+ elif isinstance(value, list):
+ for idx, obj in enumerate(value):
+ value[idx] = revive(obj)
+
+ elif isinstance(value, dict):
+ for key, obj in value.items():
+ value[key] = revive(obj)
+
+ elif isinstance(value, FlightChunk):
+ initialize(value)
+ return value.data
+
+ return value
+
+ def initialize(chunk: FlightChunk) -> None:
+ if chunk.resolved:
+ return
+ try:
+ raw_value = json.loads(chunk.data) if isinstance(chunk.data, str) else chunk.data
+ except json.decoder.JSONDecodeError:
+ chunk.data = "ERROR"
+ else:
+ chunk.data = revive(raw_value)
+ finally:
+ chunk.resolved = True
+
+ flight_data = flight_data.replace('"$undefined"', "null")
+ matches = tuple(_find_chunks(flight_data))
+
+ for idx, match in enumerate(matches):
+ chunk_id, marker, delimiter = match.groups()
+ try:
+ end = matches[idx + 1].start()
+ except IndexError:
+ end = -1
+
+ data = flight_data[match.end() - 1 : end].strip()
+ chunks[chunk_id] = chunk = FlightChunk(chunk_id, marker=marker or None, raw_data=data)
+ if delimiter == "null":
+ chunk.data = None
+
+ for chunk in sorted(chunks.values()):
+ initialize(chunk)
+ yield chunk
diff --git a/tests/test_nextjs.py b/tests/test_nextjs.py
new file mode 100644
index 000000000..916dece9b
--- /dev/null
+++ b/tests/test_nextjs.py
@@ -0,0 +1,34 @@
+from collections.abc import AsyncGenerator
+
+import aiohttp
+import pytest
+from bs4 import BeautifulSoup
+
+from cyberdrop_dl.utils import css, next_js
+
+
[email protected](name="soup", scope="module")
+async def onepace_soup() -> AsyncGenerator[BeautifulSoup]:
+ async with aiohttp.ClientSession() as session:
+ resp = await session.get("https://onepace.net/en/watch")
+ yield css.make_soup(await resp.text())
+
+
+def test_next_js_parser(soup: BeautifulSoup) -> None:
+ next_data = next_js.extract(soup)
+ assert isinstance(next_data, dict)
+ assert next_data["1"] == "Sreact.fragment"
+ assert len(next_data) > 10
+
+
+def test_next_js_find(soup: BeautifulSoup) -> None:
+ next_data = next_js.extract(soup)
+ episode_keys = "slug", "title", "playGroups"
+ ep = next_js.find(next_data, *episode_keys)
+ assert isinstance(ep, dict)
+ assert set(episode_keys).issubset(ep.keys())
+ all_episodes = list(next_js.ifind(next_data, *episode_keys))
+ assert len(all_episodes) > 74
+ for ep in all_episodes:
+ assert isinstance(ep, dict)
+ assert set(episode_keys).issubset(ep.keys())