Skip to content

misterdownloadmanager: The ultimate open-source, clean-room download manager written from scratch in pure Rust for Windows, Mac, and Linux

20.0 KB
Raw Blame
//! Adaptive streams for the browser tests: an HLS master playlist with two video variants and an
//! audio rendition, and a DASH manifest with the same tracks (and a copy marked as encrypted with
//! Widevine, for the extension's media list to show as protected, #38). Each track has `SEGMENTS` segments
//! of `SEGMENT_SECS` seconds. HLS segments are real MPEG-TS (`ts_segment`): one AAC stream of
//! silence each, so ffmpeg (yt-dlp's HLS downloader) finds a stream and remuxes it, as it does a
//! site's (#236: fake bytes made every ffmpeg stop with "could not find codec parameters" or
//! "Output file does not contain any stream"). The video tracks carry that sound too: nothing here
//! decodes pictures. DASH segments are deterministic fakes (init segments start with an `ftyp`
//! box), enough for a client that lists, fetches or joins them, not for one that decodes them.

use std::collections::HashMap;

use axum::extract::{Path, Query};
use axum::http::{StatusCode, header};
use axum::response::{IntoResponse, Response};
use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};

use crate::content;

pub const SEGMENTS: u32 = 3;
pub const SEGMENT_SECS: u32 = 4;
const TS_PACKET: usize = 188;

/// A track: its name in paths, video size (none for audio), bandwidth, bytes per segment.
struct Track {
    name: &'static str,
    video: Option<(u32, u32)>,
    bandwidth: u32,
    segment_len: usize,
}

const TRACKS: [Track; 3] = [
    Track {
        name: "720p",
        video: Some((1280, 720)),
        bandwidth: 1_280_000,
        segment_len: TS_PACKET * 256,
    },
    Track {
        name: "360p",
        video: Some((640, 360)),
        bandwidth: 640_000,
        segment_len: TS_PACKET * 128,
    },
    Track {
        name: "audio",
        video: None,
        bandwidth: 128_000,
        segment_len: TS_PACKET * 32,
    },
];

fn track(name: &str) -> Option<(usize, &'static Track)> {
    TRACKS.iter().enumerate().find(|(_, t)| t.name == name)
}

fn text(ty: &'static str, body: String) -> Response {
    ([(header::CONTENT_TYPE, ty)], body).into_response()
}

fn bytes(ty: &'static str, body: Vec<u8>) -> Response {
    ([(header::CONTENT_TYPE, ty)], body).into_response()
}

/// Segment `index` of track number `track`: seeded content, distinct per track and segment.
fn segment(track: usize, index: u32, len: usize) -> Vec<u8> {
    let mut buf = vec![0; len];
    content::fill(10_000 + track as u64 * 100 + index as u64, 0, &mut buf);
    buf
}

/// The PMT's PID, the sound's, and the null packets' (ISO/IEC 13818-1).
const PMT_PID: u16 = 0x1000;
const AUDIO_PID: u16 = 0x0100;
const NULL_PID: u16 = 0x1fff;
/// AAC-LC, 32 kHz, mono: 1024 samples a frame, so a 4 s segment is exactly 125 frames.
const RATE: u32 = 32_000;
const FRAMES: u32 = SEGMENT_SECS * RATE / 1024;
/// One AAC-LC frame of digital silence (a single channel element, no spectral data).
const SILENCE: [u8; 4] = [0x01, 0x18, 0x20, 0x07];
/// 90 kHz clock ticks per AAC frame.
const FRAME_TICKS: u64 = 1024 * 90_000 / RATE as u64;

/// Segment `index` of track number `track` as MPEG-TS, exactly `len` bytes (a multiple of 188):
/// a PAT, a PMT, one PES of `FRAMES` silent ADTS frames whose PTS goes on from the segment
/// before (with a PCR), then null packets with seeded payloads, so every segment of every track
/// differs. Continuity counters go on from the segment before, as in one stream cut in pieces.
fn ts_segment(track: usize, index: u32, len: usize) -> Vec<u8> {
    let pts = u64::from(index) * u64::from(FRAMES) * FRAME_TICKS;
    let mut pes = vec![0x00, 0x00, 0x01, 0xc0, 0, 0, 0x80, 0x80, 5];
    pes.extend(timestamp(0x2, pts));
    let frame_len = 7 + SILENCE.len();
    for _ in 0..FRAMES {
        pes.extend(adts_header(frame_len));
        pes.extend(SILENCE);
    }
    let pes_len = u16::try_from(pes.len() - 6).expect("a PES fits its length field");
    pes[4..6].copy_from_slice(&pes_len.to_be_bytes());

    let audio_packets = packet_count(pes.len());
    let cc = |per_segment: usize| ((index as usize * per_segment) % 16) as u8;
    let mut out = Vec::with_capacity(len);
    out.extend(psi_packet(0, cc(1), &pat()));
    out.extend(psi_packet(PMT_PID, cc(1), &pmt()));
    let mut rest = &pes[..];
    let mut n = 0;
    while !rest.is_empty() {
        let counter = (cc(audio_packets) + n as u8) & 15;
        let (packet, used) = pes_packet(n == 0, counter, (n == 0).then_some(pts), rest);
        out.extend(packet);
        rest = &rest[used..];
        n += 1;
    }
    debug_assert_eq!(n, audio_packets);
    assert!(out.len() <= len, "a {len}-byte segment holds its stream");
    let mut fill = segment(track, index, len - out.len());
    for (k, packet) in fill.chunks_mut(TS_PACKET).enumerate() {
        let head = [
            0x47,
            (NULL_PID >> 8) as u8,
            NULL_PID as u8,
            0x10 | (k % 16) as u8,
        ];
        packet[..4].copy_from_slice(&head);
    }
    out.extend(fill);
    out
}

/// TS packets a PES of `len` bytes takes: 176 bytes in the first (its adaptation field holds
/// the PCR), 184 in each after.
fn packet_count(len: usize) -> usize {
    1 + len.saturating_sub(176).div_ceil(184)
}

/// One TS packet of the sound's PES and how much of `rest` it carries: the first with
/// payload_unit_start (and the PCR), the last filled out with adaptation-field stuffing.
fn pes_packet(start: bool, cc: u8, pcr: Option<u64>, rest: &[u8]) -> (Vec<u8>, usize) {
    // The adaptation field's bytes after its length byte.
    let mut field = Vec::new();
    if let Some(base) = pcr {
        // Random access and a PCR (a 33-bit base, 6 reserved bits, a 9-bit extension of 0).
        field.extend([
            0x50,
            (base >> 25) as u8,
            (base >> 17) as u8,
            (base >> 9) as u8,
            (base >> 1) as u8,
            (((base & 1) as u8) << 7) | 0x7e,
            0,
        ]);
    }
    let mut has_field = pcr.is_some();
    let room = if has_field { 183 - field.len() } else { 184 };
    let used = rest.len().min(room);
    let mut short = room - used;
    if short > 0 && !has_field {
        // A new field: its length byte takes one, a flags byte the next.
        has_field = true;
        short -= 1;
        if short > 0 {
            field.push(0x00);
            short -= 1;
        }
    }
    field.extend(std::iter::repeat_n(0xff, short));
    let control = if has_field { 0x30 } else { 0x10 };
    let mut p = vec![
        0x47,
        (u8::from(start) << 6) | (AUDIO_PID >> 8) as u8,
        AUDIO_PID as u8,
        control | cc,
    ];
    if has_field {
        p.push(field.len() as u8);
        p.extend(field);
    }
    p.extend(&rest[..used]);
    debug_assert_eq!(p.len(), TS_PACKET);
    (p, used)
}

/// A packet holding one PSI section (a pointer field of 0, then 0xff to the end).
fn psi_packet(pid: u16, cc: u8, section: &[u8]) -> Vec<u8> {
    let mut p = vec![0x47, 0x40 | (pid >> 8) as u8, pid as u8, 0x10 | cc, 0];
    p.extend(section);
    p.resize(TS_PACKET, 0xff);
    p
}

/// A long-form section: `table_id`, its length, `body` and its CRC.
fn section(table_id: u8, body: &[u8]) -> Vec<u8> {
    let len = body.len() + 4;
    let mut s = vec![table_id, 0xb0 | (len >> 8) as u8, len as u8];
    s.extend(body);
    let crc = crc32_mpeg(&s);
    s.extend(crc.to_be_bytes());
    s
}

/// The PAT: program 1, its PMT at `PMT_PID`.
fn pat() -> Vec<u8> {
    let pmt = [0xe0 | (PMT_PID >> 8) as u8, PMT_PID as u8];
    section(0x00, &[0, 1, 0xc1, 0, 0, 0, 1, pmt[0], pmt[1]])
}

/// The PMT: one stream, AAC in ADTS (stream type 0x0f), which carries the PCR too.
fn pmt() -> Vec<u8> {
    let (hi, lo) = (0xe0 | (AUDIO_PID >> 8) as u8, AUDIO_PID as u8);
    section(
        0x02,
        &[0, 1, 0xc1, 0, 0, hi, lo, 0xf0, 0, 0x0f, hi, lo, 0xf0, 0],
    )
}

/// An ADTS header (MPEG-4 AAC-LC, 32 kHz, mono, no CRC) for a frame of `len` bytes.
fn adts_header(len: usize) -> [u8; 7] {
    const RATE_INDEX: u8 = 5; // 32 kHz
    [
        0xff,
        0xf1,
        (1 << 6) | (RATE_INDEX << 2),
        (1 << 6) | ((len >> 11) & 3) as u8,
        (len >> 3) as u8,
        (((len & 7) as u8) << 5) | 0x1f,
        0xfc,
    ]
}

/// A PES timestamp: 4 bits of `prefix`, then the 33 bits of `ts` with their marker bits.
fn timestamp(prefix: u8, ts: u64) -> [u8; 5] {
    [
        (prefix << 4) | ((((ts >> 30) & 7) as u8) << 1) | 1,
        (ts >> 22) as u8,
        ((((ts >> 15) & 0x7f) as u8) << 1) | 1,
        (ts >> 7) as u8,
        (((ts & 0x7f) as u8) << 1) | 1,
    ]
}

/// MPEG-2's CRC-32 (polynomial 0x04c11db7, not reflected, from all ones, no final xor).
fn crc32_mpeg(data: &[u8]) -> u32 {
    let mut crc = 0xffff_ffffu32;
    for &b in data {
        crc ^= u32::from(b) << 24;
        for _ in 0..8 {
            crc = if crc & 0x8000_0000 != 0 {
                (crc << 1) ^ 0x04c1_1db7
            } else {
                crc << 1
            };
        }
    }
    crc
}

const M3U8: &str = "application/vnd.apple.mpegurl";

/// `/hls/master.m3u8`.
pub(crate) async fn hls_master() -> Response {
    let mut s = String::from(
        "#EXTM3U\n#EXT-X-VERSION:3\n\
         #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"English\",LANGUAGE=\"en\",\
         DEFAULT=YES,AUTOSELECT=YES,URI=\"audio/index.m3u8\"\n",
    );
    for t in &TRACKS {
        if let Some((w, h)) = t.video {
            s += &format!(
                "#EXT-X-STREAM-INF:BANDWIDTH={},RESOLUTION={w}x{h},\
                 CODECS=\"avc1.4d401f,mp4a.40.2\",AUDIO=\"aud\"\n{}/index.m3u8\n",
                t.bandwidth, t.name
            );
        }
    }
    text(M3U8, s)
}

/// `/hls/{track}/index.m3u8` (a media playlist) and `/hls/{track}/seg{i}.ts`.
pub(crate) async fn hls_file(Path((name, file)): Path<(String, String)>) -> Response {
    let Some((ti, t)) = track(&name) else {
        return StatusCode::NOT_FOUND.into_response();
    };
    if file == "index.m3u8" {
        let mut s = format!(
            "#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:{SEGMENT_SECS}\n\
             #EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-PLAYLIST-TYPE:VOD\n"
        );
        for i in 0..SEGMENTS {
            s += &format!("#EXTINF:{SEGMENT_SECS}.000,\nseg{i}.ts\n");
        }
        s += "#EXT-X-ENDLIST\n";
        return text(M3U8, s);
    }
    // #769: the same pieces encrypted with AES-128 (`enc.m3u8`): the first with an IV of its
    // own, the others with their sequence number's, all with `key.bin`.
    if file == "enc.m3u8" {
        let mut s = format!(
            "#EXTM3U
#EXT-X-VERSION:3
#EXT-X-TARGETDURATION:{SEGMENT_SECS}
             #EXT-X-MEDIA-SEQUENCE:{ENC_SEQUENCE}
#EXT-X-PLAYLIST-TYPE:VOD
"
        );
        for i in 0..SEGMENTS {
            if i == 0 {
                s += "#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\",IV=0x000000000000000000000000000007E9
";
            } else if i == 1 {
                s += "#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"
";
            }
            s += &format!(
                "#EXTINF:{SEGMENT_SECS}.000,
enc{i}.ts
"
            );
        }
        s += "#EXT-X-ENDLIST
";
        return text(M3U8, s);
    }
    if file == "key.bin" {
        return bytes("application/octet-stream", ENC_KEY.to_vec());
    }
    if let Some(i) = numbered(&file, "enc", ".ts").filter(|i| *i < SEGMENTS) {
        let iv = if i == 0 {
            enc_iv(0x7E9)
        } else {
            enc_iv(ENC_SEQUENCE + u64::from(i))
        };
        return bytes(
            "video/mp2t",
            encrypt(&ts_segment(ti, i, t.segment_len), &iv),
        );
    }
    match numbered(&file, "seg", ".ts") {
        Some(i) if i < SEGMENTS => bytes("video/mp2t", ts_segment(ti, i, t.segment_len)),
        _ => StatusCode::NOT_FOUND.into_response(),
    }
}

/// The encrypted pieces' key (`/hls/{track}/key.bin`), their playlist's first sequence number.
const ENC_KEY: [u8; 16] = *b"mdm-lab-key-0769";
const ENC_SEQUENCE: u64 = 100;

/// An IV of `n`, right-aligned in 16 bytes (as a playlist's `IV=0x…`, and a piece's sequence
/// number when it has none).
fn enc_iv(n: u64) -> [u8; 16] {
    let mut iv = [0u8; 16];
    iv[8..].copy_from_slice(&n.to_be_bytes());
    iv
}

/// `data` encrypted with AES-128-CBC and PKCS#7 padding, as an HLS server sends a piece.
fn encrypt(data: &[u8], iv: &[u8; 16]) -> Vec<u8> {
    use aws_lc_rs::cipher::{
        AES_128, EncryptionContext, PaddedBlockEncryptingKey, UnboundCipherKey,
    };
    let key = PaddedBlockEncryptingKey::cbc_pkcs7(
        UnboundCipherKey::new(&AES_128, &ENC_KEY).expect("a 16-byte key"),
    )
    .expect("CBC");
    let mut out = data.to_vec();
    key.less_safe_encrypt(&mut out, EncryptionContext::Iv128((*iv).into()))
        .expect("encrypted");
    out
}

/// `N` of `{prefix}N{suffix}`.
fn numbered(file: &str, prefix: &str, suffix: &str) -> Option<u32> {
    file.strip_prefix(prefix)?
        .strip_suffix(suffix)?
        .parse()
        .ok()
}

/// `/dash/manifest.mpd`.
pub(crate) async fn dash_manifest() -> Response {
    text("application/dash+xml", mpd(false))
}

/// `/dash/protected.mpd`: the same tracks, each marked as encrypted (CENC, with Widevine's key
/// system), as a DRM-protected stream's manifest is. Its segments aren't encrypted: no client
/// should get that far.
pub(crate) async fn dash_protected() -> Response {
    text("application/dash+xml", mpd(true))
}

/// Widevine's DASH key system.
const WIDEVINE: &str = "urn:uuid:edef8ba9-79d6-4ace-a3c8-27dcd51d21ed";

fn mpd(protected: bool) -> String {
    let protection = if protected {
        format!(
            "<ContentProtection schemeIdUri=\"urn:mpeg:dash:mp4protection:2011\" value=\"cenc\"/>\
             <ContentProtection schemeIdUri=\"{WIDEVINE}\"/>"
        )
    } else {
        String::new()
    };
    let total = SEGMENTS * SEGMENT_SECS;
    let template = format!(
        "<SegmentTemplate initialization=\"$RepresentationID$/init.mp4\" \
         media=\"$RepresentationID$/seg$Number$.m4s\" startNumber=\"1\" \
         duration=\"{SEGMENT_SECS}\" timescale=\"1\"/>"
    );
    let mut video = String::new();
    let mut audio = String::new();
    for t in &TRACKS {
        match t.video {
            Some((w, h)) => {
                video += &format!(
                    "<Representation id=\"{}\" bandwidth=\"{}\" width=\"{w}\" height=\"{h}\" \
                     codecs=\"avc1.4d401f\"/>",
                    t.name, t.bandwidth
                )
            }
            None => {
                audio += &format!(
                    "<Representation id=\"{}\" bandwidth=\"{}\" codecs=\"mp4a.40.2\" \
                     audioSamplingRate=\"48000\"/>",
                    t.name, t.bandwidth
                )
            }
        }
    }
    let s = format!(
        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
         <MPD xmlns=\"urn:mpeg:dash:schema:mpd:2011\" type=\"static\" \
         profiles=\"urn:mpeg:dash:profile:isoff-live:2011\" minBufferTime=\"PT2S\" \
         mediaPresentationDuration=\"PT{total}S\">\n\
         <Period id=\"0\" start=\"PT0S\">\n\
         <AdaptationSet id=\"0\" contentType=\"video\" mimeType=\"video/mp4\" \
         segmentAlignment=\"true\">{protection}{template}{video}</AdaptationSet>\n\
         <AdaptationSet id=\"1\" contentType=\"audio\" mimeType=\"audio/mp4\" lang=\"en\" \
         segmentAlignment=\"true\">{protection}{template}{audio}</AdaptationSet>\n\
         </Period>\n</MPD>\n"
    );
    s
}

/// #171: the same HLS stream with no `.m3u8` in any address and served as `text/plain`, as some
/// CDNs and players do: `/hls2/playlist?id=ID` (the master), `/hls2/playlist?id=ID&track=T` (a
/// media playlist) and `/hls2/seg?id=ID&track=T&i=N` (`application/octet-stream`).
pub(crate) async fn hls2_playlist(Query(q): Query<HashMap<String, String>>) -> Response {
    let id = utf8_percent_encode(q.get("id").map_or("lab", String::as_str), NON_ALPHANUMERIC)
        .to_string();
    let Some(name) = q.get("track") else {
        let mut s = format!(
            "#EXTM3U\n#EXT-X-VERSION:3\n\
             #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"aud\",NAME=\"English\",LANGUAGE=\"en\",\
             DEFAULT=YES,AUTOSELECT=YES,URI=\"playlist?id={id}&track=audio\"\n"
        );
        for t in &TRACKS {
            if let Some((w, h)) = t.video {
                s += &format!(
                    "#EXT-X-STREAM-INF:BANDWIDTH={},RESOLUTION={w}x{h},\
                     CODECS=\"avc1.4d401f,mp4a.40.2\",AUDIO=\"aud\"\nplaylist?id={id}&track={}\n",
                    t.bandwidth, t.name
                );
            }
        }
        return text(PLAIN, s);
    };
    if track(name).is_none() {
        return StatusCode::NOT_FOUND.into_response();
    }
    let mut s = format!(
        "#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:{SEGMENT_SECS}\n\
         #EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-PLAYLIST-TYPE:VOD\n"
    );
    for i in 0..SEGMENTS {
        s += &format!("#EXTINF:{SEGMENT_SECS}.000,\nseg?id={id}&track={name}&i={i}\n");
    }
    s += "#EXT-X-ENDLIST\n";
    text(PLAIN, s)
}

const PLAIN: &str = "text/plain; charset=utf-8";

/// `/hls2/seg?track=T&i=N`: segment N of track T, as `application/octet-stream`.
pub(crate) async fn hls2_segment(Query(q): Query<HashMap<String, String>>) -> Response {
    let found = q.get("track").and_then(|n| track(n));
    let i = q.get("i").and_then(|i| i.parse::<u32>().ok());
    match (found, i) {
        (Some((ti, t)), Some(i)) if i < SEGMENTS => {
            bytes("application/octet-stream", ts_segment(ti, i, t.segment_len))
        }
        _ => StatusCode::NOT_FOUND.into_response(),
    }
}

/// `/dash2/manifest?id=ID`: the DASH manifest with no `.mpd` in its address, served as
/// `application/octet-stream`; its segments are `/dash2/{rep}/init.mp4` and `seg{n}.m4s`.
pub(crate) async fn dash2_manifest() -> Response {
    text("application/octet-stream", mpd(false))
}

/// `/hls2/config?id=ID`: a player's settings as `text/plain`, not a manifest (the pages' decoy:
/// the extension may read it and must never list it).
pub(crate) async fn hls2_config() -> Response {
    text(PLAIN, "{\"autoplay\":false,\"volume\":0.8}\n".to_owned())
}

/// `/dash/{rep}/init.mp4` and `/dash/{rep}/seg{n}.m4s` (n from 1).
pub(crate) async fn dash_file(Path((name, file)): Path<(String, String)>) -> Response {
    let Some((ti, t)) = track(&name) else {
        return StatusCode::NOT_FOUND.into_response();
    };
    let ty = if t.video.is_some() {
        "video/mp4"
    } else {
        "audio/mp4"
    };
    if file == "init.mp4" {
        let mut buf = segment(ti, 999, 1024);
        let ftyp = b"\0\0\0\x18ftypiso6\0\0\0\0iso6dash";
        buf[..ftyp.len()].copy_from_slice(ftyp);
        return bytes(ty, buf);
    }
    match numbered(&file, "seg", ".m4s") {
        Some(n) if (1..=SEGMENTS).contains(&n) => bytes(ty, segment(ti, n, t.segment_len)),
        _ => StatusCode::NOT_FOUND.into_response(),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// CRC-32/MPEG-2's check value.
    #[test]
    fn sections_carry_mpeg_crcs() {
        assert_eq!(crc32_mpeg(b"123456789"), 0x0376_e6e7);
    }

    #[test]
    fn segments_are_whole_packets_with_their_stream_first() {
        for (ti, t) in TRACKS.iter().enumerate() {
            for i in 0..SEGMENTS {
                let s = ts_segment(ti, i, t.segment_len);
                assert_eq!(s.len(), t.segment_len);
                assert!(s.chunks(TS_PACKET).all(|p| p[0] == 0x47));
                let pid = |p: &[u8]| (u16::from(p[1] & 0x1f) << 8) | u16::from(p[2]);
                let pids: Vec<u16> = s.chunks(TS_PACKET).map(pid).collect();
                assert_eq!(&pids[..2], [0, PMT_PID]);
                let audio = pids.iter().filter(|p| **p == AUDIO_PID).count();
                assert!(pids[2..2 + audio].iter().all(|p| *p == AUDIO_PID));
                assert!(pids[2 + audio..].iter().all(|p| *p == NULL_PID));
                // The PES starts in the first sound packet, after its adaptation field.
                let first = &s[2 * TS_PACKET..3 * TS_PACKET];
                assert_eq!(first[1] & 0x40, 0x40);
                let payload = &first[5 + first[4] as usize..];
                assert_eq!(&payload[..4], [0, 0, 1, 0xc0]);
                // ADTS frames back to back.
                assert_eq!(&payload[14..16], [0xff, 0xf1]);
                assert_eq!(&payload[14 + 11..14 + 13], [0xff, 0xf1]);
            }
        }
    }
}