Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
"""Utils to parse Next.JS v13 Flight Data
We only parse a subset of the spec (w/ regex instead of a state machine) and discard any non Payload chunk"""
from __future__ import annotations
import base64
import dataclasses
import logging
import re
from collections.abc import Generator
from enum import IntEnum, StrEnum
from typing import TYPE_CHECKING, Any, NewType, final
from cyberdrop_dl.utils import css, json
if TYPE_CHECKING:
from collections.abc import Generator, Iterable, Mapping
from bs4 import BeautifulSoup
logger = logging.getLogger(__name__)
type _ChunkID = str
FlightData = NewType("FlightData", str)
NextJSFlight = dict[_ChunkID, list[dict[str, Any]]]
# Map of chunk_id (hex index of the chunk) -> list of all the objects (components) created from that chunk
_match_init_push = re.compile(r"\(self\.__next_f\s?=\s?self\.__next_f\s?\|\|\s?\[\]\)\.push\((\[.+?\])\)").match
class _Magic(StrEnum):
ERROR = "<ERROR>"
INIT = "<INIT>"
class _FlightType(IntEnum):
BOOTSTRAP = 0
PAYLOAD = 1
FORM_STATE = 2
BINARY = 3
_Push = tuple[_FlightType, str]
@final
@dataclasses.dataclass(slots=True, order=True)
class _FlightChunk:
index: int = dataclasses.field(init=False)
id: _ChunkID
marker: str | None
data: str
decoded_data: Any = dataclasses.field(init=False) # pyright: ignore[reportAny]
resolved: bool = dataclasses.field(init=False, default=False)
def __post_init__(self) -> None:
self.index = int(self.id, base=16)
self.decoded_data = self.data
@final
@dataclasses.dataclass(slots=True)
class LazyChunk:
reference: _ChunkID
def _split_push(push: str) -> tuple[str, str, str]:
chunk_id, sep, data = push[1:-1].partition(",")
return chunk_id.strip(), sep, data.strip().rstrip(",").strip()
def _dedent_push(push: str) -> str:
push = push.strip()
if push.startswith(("{", "[")) and push.endswith(("}", "]")):
parts = filter(None, _split_push(push))
push = "".join((push[0], *parts, push[-1]))
return push
def _extract_raw_pushes(soup: BeautifulSoup) -> Generator[str]:
push = "self.__next_f.push("
for script in css.iselect(soup, "script:-soup-contains-own('self.__next_f')"):
content = script.get_text(strip=True).strip()
if m := _match_init_push(content):
yield _dedent_push(m.group(1))
else:
try:
start = content.index(push) + len(push)
except ValueError:
continue
yield _dedent_push(content[start : content.rindex(")")])
def _unquote(string: str) -> str:
if len(string) > 1:
for char in ("'", '"'):
if string.startswith(char) and string.endswith(char):
string = string[1:-1]
return string
def _decode_push(raw_push: str) -> _Push:
try:
push: list[Any] = json.loads(raw_push) # pyright: ignore[reportAny]
except ValueError:
f_type, _, data = raw_push.strip().partition(",")
push = [int(f_type[1:]), _unquote(data[:-1])]
match push:
case [_FlightType.BOOTSTRAP]:
return _FlightType.BOOTSTRAP, _Magic.INIT
case [_FlightType.PAYLOAD | _FlightType.FORM_STATE as type_, value]:
return _FlightType(type_), value
case [_FlightType.BINARY, value]:
return _FlightType.BINARY, base64.b64decode(value.encode()).decode()
case _:
raise RuntimeError(f"Invalid NextJS push found: {push!r}")
def _extract_flight_data(raw_pushes: Iterable[str]) -> Generator[str]:
found_init: bool = False
for flight_type, data in map(_decode_push, raw_pushes):
if flight_type is _FlightType.BOOTSTRAP:
if found_init:
raise RuntimeError("NextJS data was initialized multiple times")
found_init = True
elif flight_type is _FlightType.PAYLOAD:
if not found_init:
raise RuntimeError("Found NextJS push without initialized array")
yield data
def _parse_raw_chunks(flight_data: FlightData) -> Generator[_FlightChunk]:
for idx, line in enumerate(flight_data.splitlines(), 1):
if line.startswith(":HL"):
continue
m = re.match("^([0-9a-f]+):(T[0-9A-Fa-f]+,|[A-SU-Z]{0,1})", line)
if not m:
logger.warning(f"not match for line {idx}: {line}")
continue
chunk_id, marker = m.groups()
marker = marker or None
data = line[m.end() :].strip()
if marker and marker.startswith("T"):
length = int(marker[1:-1], 16)
data, rest = data[:length], FlightData(data[length:])
yield _FlightChunk(chunk_id, "T", data)
if rest:
yield from _parse_raw_chunks(rest)
else:
yield _FlightChunk(chunk_id, marker, data)
def _revive_str(value: str) -> str | int | LazyChunk | None: # noqa: PLR0911
if value[0] != "$":
return value
if value == "$":
return ""
match value[1]:
case "$":
return value[2:]
case "@" | "L":
chunk_id = value[2:]
return LazyChunk(chunk_id)
case "u":
return None
case "D":
return value[2:]
case "n":
return int(value[2:])
case _:
return value[1:]
def _revive(value: object, /, chunks: Mapping[_ChunkID, _FlightChunk]) -> Any: # noqa: C901, PLR0911
if not value:
return value
match value:
case str():
obj = _revive_str(value)
if type(obj) is LazyChunk:
try:
ref = chunks[obj.reference]
except LookupError:
return _Magic.ERROR
else:
return _revive(ref, chunks)
return obj
case list():
for idx, obj in enumerate(value):
value[idx] = _revive(obj, chunks)
return value
case dict():
for key, obj in value.items():
value[key] = _revive(obj, chunks)
return value
case _FlightChunk():
_initialize(value, chunks)
return value.decoded_data
case _:
return value
def _initialize(chunk: _FlightChunk, /, chunks: Mapping[_ChunkID, _FlightChunk]) -> None:
if chunk.resolved:
return
try:
raw_value = json.loads(chunk.decoded_data) if isinstance(chunk.decoded_data, str) else chunk.decoded_data
except json.JSONDecodeError:
chunk.decoded_data = _Magic.ERROR
else:
chunk.decoded_data = _revive(raw_value, chunks)
finally:
chunk.resolved = True
def _hidrate_chunks(raw_chunks: Iterable[_FlightChunk]) -> Generator[_FlightChunk]:
chunks_map = {chunk.id: chunk for chunk in sorted(raw_chunks)}
for chunk in chunks_map.values():
_initialize(chunk, chunks_map)
yield chunk
def extract_flight_data(soup: BeautifulSoup) -> FlightData:
data = "".join(_extract_flight_data(_extract_raw_pushes(soup))).replace('"$undefined"', "null")
esc = ">>>>>ESCAPED_NEW_LINE>>>>>>"
data = data.replace(r"\\n", esc).replace("\\n", "\n").replace(esc, r"\\n")
return FlightData(data)
def ifind(next_flight: NextJSFlight, attr: str, *attrs: str) -> Generator[dict[str, Any]]:
"""Yield every object within `next_flight` that have the required `attrs`."""
needed = frozenset([attr, *attrs])
def walk(obj: object) -> Generator[dict[str, Any]]:
if isinstance(obj, dict):
if needed.issubset(obj):
yield obj
else:
for v in obj.values():
yield from walk(v)
elif isinstance(obj, list):
for v in obj:
yield from walk(v)
for ele in next_flight.values():
yield from walk(ele)
def find(next_flight: NextJSFlight, attr: str, *attrs: str) -> dict[str, Any]:
"""Get the first object within `next_flight` that have the required `attrs`."""
return next(ifind(next_flight, attr, *attrs))
def extract(soup: BeautifulSoup) -> NextJSFlight:
return parse(extract_flight_data(soup))
def parse(flight_data: FlightData, /) -> NextJSFlight:
return {chunk.id: chunk.decoded_data for chunk in _hidrate_chunks(_parse_raw_chunks(flight_data))} # pyright: ignore[reportAny]
def data(tag: BeautifulSoup) -> Any:
return json.loads(css.select_text(tag, "#__NEXT_DATA__"))
if __name__ == "__main__":
import sys
from pathlib import Path
file = Path(sys.argv[1])
soup = css.soup(file.read_text(encoding="utf8"))
flight_data = extract_flight_data(soup)
Path("flight_data.txt").write_text(flight_data, encoding="utf8")
decoded = parse(flight_data)
Path("flight_data_decoded.json").write_text(json.dumps(decoded, indent=2, ensure_ascii=False), encoding="utf8")