Skip to content

misterdownloadmanager: The ultimate open-source, clean-room download manager written from scratch in pure Rust for Windows, Mac, and Linux

38.6 KB
Raw Blame
//! A video's streams downloaded by MDM's engine (#769). yt-dlp looks the page up and chooses the
//! formats (its `-f`); MDM's engine downloads each stream as it downloads any file, over the
//! connections Options → Connection gives, under the speed limits, with pause and resume and
//! retries, into the very file yt-dlp would have written (`<stem>.f<id>.<ext>` for a stream it
//! merges, `<stem>.<ext>` for a choice of one). yt-dlp then does what it does after its own
//! download: it finds the streams there ("has already been downloaded"), merges them, saves and
//! embeds the subtitles and the metadata, converts the sound. Its post-processing stays the one
//! source of truth for the finished file; only the bytes come down through the engine.
//!
//! A stream in pieces (HLS, DASH) comes down piece by piece, several at once, and is joined into
//! that file as yt-dlp's own downloaders join it ([`crate::pieces`]).
//!
//! What the engine can't download stays yt-dlp's to download ([`Helper`]): a live stream still
//! being recorded, a protocol other than HTTP (RTMP, RTSP, MMS, FTP…), protected streams.
//!
//! A stream's address is signed and expires (YouTube's after six hours). The engine asks for
//! today's one when it dies, or when a download continues after a restart, through
//! [`StreamLink`], an [`Unlocker`] that looks the page up again and takes the same format's
//! address.

use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;

use futures::future::BoxFuture;
use mdm_engine::download::{ConnView, SegmentView, State as DlState};
use mdm_engine::{Destination, Download, Request, ServiceFault, Unlocked, Unlocker};
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use serde_json::Value;
use tokio_util::sync::CancellationToken;
use url::Url;

/// Why yt-dlp downloads a choice itself rather than the engine.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Helper {
    /// A live stream still being recorded: it has no end to download to.
    Live,
    /// Pieces of a stream the engine doesn't list (Adobe's HDS, Microsoft's Smooth Streaming, a
    /// DASH stream made as it plays), yt-dlp's name for its protocol.
    Pieces(String),
    /// A protocol the engine doesn't speak (RTMP, RTSP, MMS, FTP…), yt-dlp's name for it.
    Protocol(String),
    /// Protected (DRM): yt-dlp refuses it, as before.
    Protected,
    /// yt-dlp gave no address (or no file name) to download it from.
    NoAddress,
}

impl Helper {
    /// For the download's log (English, as yt-dlp's own lines).
    pub fn english(&self) -> String {
        match self {
            Helper::Live => "a live stream still being recorded".into(),
            Helper::Pieces(p) => format!("a stream in pieces ({p})"),
            Helper::Protocol(p) => format!("its protocol ({p})"),
            Helper::Protected => "a protected stream".into(),
            Helper::NoAddress => "yt-dlp gave no address for it".into(),
        }
    }
}

/// One stream of a choice as the engine downloads it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Stream {
    /// yt-dlp's format id (`137`, `hls-720p`).
    pub format_id: String,
    /// The file yt-dlp looks for: where the engine saves it.
    pub path: PathBuf,
    /// Its size: exact (yt-dlp's `filesize`) or yt-dlp's estimate (`exact` false).
    pub size: Option<u64>,
    pub exact: bool,
    pub url: Url,
    /// yt-dlp's headers for it (`http_headers`) and the cookies it would send (`Cookie`).
    pub headers: Vec<(String, String)>,
    pub kind: Kind,
}

/// How a stream comes down.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Kind {
    /// One file (`http`, `https`), in requests of at most `chunk` bytes (yt-dlp's
    /// `http_chunk_size`, `Options::request_cap`).
    File { chunk: Option<u64> },
    /// An HLS media playlist at the stream's address (`m3u8_native`, `m3u8`), read as yt-dlp's
    /// hlsnative reads it, with yt-dlp's extra queries for its pieces and keys.
    Hls {
        extra: Option<String>,
        extra_key: Option<String>,
    },
    /// DASH's pieces (`http_dash_segments`), as yt-dlp lists them.
    Dash(Vec<crate::pieces::Piece>),
}

/// A choice as the engine downloads it: its streams, one after another, and the file yt-dlp makes
/// of them.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Plan {
    pub output: PathBuf,
    pub streams: Vec<Stream>,
}

impl Plan {
    /// Every stream's size, as far as known: the progress's total from the start.
    pub fn total(&self) -> Option<u64> {
        self.streams.iter().map(|s| s.size).sum()
    }
}

fn text<'a>(v: &'a Value, key: &str) -> Option<&'a str> {
    v.get(key).and_then(Value::as_str).filter(|s| !s.is_empty())
}

fn number(v: &Value, key: &str) -> Option<u64> {
    let n = v.get(key)?;
    n.as_u64()
        .or_else(|| n.as_f64().filter(|f| *f >= 0.0).map(|f| f as u64))
        .filter(|n| *n > 0)
}

/// Python's `os.path.splitext` on a file name: (`a.b`, `.mp4`) of `a.b.mp4`; a name made of a
/// leading dot alone (`.mp4`) has no extension.
fn splitext(name: &str) -> (&str, &str) {
    match name.rfind('.') {
        Some(i) if name[..i].chars().any(|c| c != '.') => (&name[..i], &name[i..]),
        _ => (name, ""),
    }
}

/// Where yt-dlp saves format `id` (extension `ext`) of a merged choice whose file is `output`
/// (its extension `merged`): `prepend_extension(correct_ext(output, ext), f<id>, ext)`, as its
/// `process_info` names each of `requested_formats` (`<stem>.f137.mp4`, `<stem>.f251.webm`).
pub fn stream_path(output: &Path, merged: &str, id: &str, ext: &str) -> PathBuf {
    let name = output
        .file_name()
        .map(|n| n.to_string_lossy().into_owned())
        .unwrap_or_default();
    // correct_ext: the output's extension, when it is the merge's or the stream's, gives way.
    let (stem, real) = splitext(&name);
    let real = real.trim_start_matches('.');
    let base = if real == merged || real == ext {
        stem
    } else {
        &name
    };
    let corrected = format!("{base}.{ext}");
    // prepend_extension: `f<id>` before the extension it now has.
    let (stem, real) = splitext(&corrected);
    let named = if real.trim_start_matches('.') == ext {
        format!("{stem}.f{id}{real}")
    } else {
        format!("{corrected}.f{id}")
    };
    output.with_file_name(named)
}

/// The `Cookie` header of yt-dlp's `cookies` for a format (`a=1; Domain=.x.com; Path=/; b=2;
/// Secure`): the pairs, without their attributes.
pub fn cookie_header(cookies: &str) -> Option<String> {
    const ATTRIBUTES: [&str; 9] = [
        "domain", "path", "expires", "max-age", "secure", "httponly", "version", "samesite",
        "comment",
    ];
    let pairs: Vec<&str> = cookies
        .split(';')
        .map(str::trim)
        .filter(|p| !p.is_empty())
        .filter(|p| {
            let name = p.split_once('=').map_or(*p, |(n, _)| n).trim();
            p.contains('=') && !ATTRIBUTES.contains(&name.to_ascii_lowercase().as_str())
        })
        .collect();
    (!pairs.is_empty()).then(|| pairs.join("; "))
}

/// yt-dlp's headers for format `f` (`http_headers`), with its cookies as a `Cookie` header.
fn headers_of(f: &Value) -> Vec<(String, String)> {
    let mut out: Vec<(String, String)> = f
        .get("http_headers")
        .and_then(Value::as_object)
        .map(|h| {
            h.iter()
                .filter_map(|(k, v)| Some((k.clone(), v.as_str()?.to_owned())))
                .filter(|(k, _)| !k.eq_ignore_ascii_case("cookie"))
                .collect()
        })
        .unwrap_or_default();
    if let Some(c) = text(f, "cookies").and_then(cookie_header) {
        out.push(("Cookie".into(), c));
    }
    out
}

/// yt-dlp's protocol for format `f`: its `protocol`, else the address's (`determine_protocol`).
fn protocol_of(f: &Value) -> String {
    if let Some(p) = text(f, "protocol") {
        return p.to_owned();
    }
    let url = text(f, "url").unwrap_or("");
    let lower = url.to_ascii_lowercase();
    if lower.starts_with("rtmp") {
        "rtmp".into()
    } else if lower.contains(".m3u8") {
        "m3u8".into()
    } else if lower.contains(".f4m") {
        "f4m".into()
    } else {
        url.split_once(':')
            .map_or_else(String::new, |(s, _)| s.to_ascii_lowercase())
    }
}

/// Format `f` as the engine downloads it into `path`, or why yt-dlp downloads it.
fn stream(f: &Value, path: PathBuf) -> Result<Stream, Helper> {
    if f.get("has_drm").and_then(Value::as_bool) == Some(true) {
        return Err(Helper::Protected);
    }
    let protocol = protocol_of(f);
    let fragments = f
        .get("fragments")
        .is_some_and(|v| !v.is_null() && v.as_array().is_none_or(|a| !a.is_empty()));
    let kind = match protocol.as_str() {
        "http" | "https" if !fragments => Kind::File {
            chunk: f
                .get("downloader_options")
                .and_then(|o| number(o, "http_chunk_size")),
        },
        // A key of the site's own (`hls_aes`) stays yt-dlp's.
        "m3u8" | "m3u8_native" if f.get("hls_aes").is_none_or(Value::is_null) => Kind::Hls {
            extra: text(f, "extra_param_to_segment_url").map(str::to_owned),
            extra_key: text(f, "extra_param_to_key_url").map(str::to_owned),
        },
        "http_dash_segments" => {
            Kind::Dash(crate::pieces::dash(f).ok_or_else(|| Helper::Pieces(protocol.clone()))?)
        }
        "m3u8"
        | "m3u8_native"
        | "http_dash_segments_generator"
        | "f4m"
        | "ism"
        | "https"
        | "http" => return Err(Helper::Pieces(protocol)),
        _ => return Err(Helper::Protocol(protocol)),
    };
    // DASH's address is its manifest's (or none): its first piece's stands for it.
    let url = text(f, "url")
        .and_then(|u| Url::parse(u).ok())
        .or_else(|| match &kind {
            Kind::Dash(p) => p.first().map(|p| p.url.clone()),
            _ => None,
        })
        .filter(|u| matches!(u.scheme(), "http" | "https"))
        .ok_or(Helper::NoAddress)?;
    let exact = number(f, "filesize");
    Ok(Stream {
        format_id: text(f, "format_id").unwrap_or("0").to_owned(),
        path,
        size: exact.or_else(|| number(f, "filesize_approx")),
        exact: exact.is_some(),
        url,
        headers: headers_of(f),
        kind,
    })
}

/// Whether yt-dlp's page is a live stream still being recorded.
pub fn is_live(info: &Value) -> bool {
    info.get("is_live").and_then(Value::as_bool) == Some(true)
        || text(info, "live_status").is_some_and(|s| s == "is_live" || s == "is_upcoming")
}

/// The engine's plan for yt-dlp's choice (`-O "%()j"` after its format selection: the formats in
/// `requested_formats`, or the one format's own fields, and `filename`), or why yt-dlp downloads
/// it. One stream that can't be the engine's leaves the whole choice to yt-dlp.
pub fn plan(selected: &Value) -> Result<Plan, Helper> {
    if is_live(selected) {
        return Err(Helper::Live);
    }
    let output = text(selected, "filename")
        .map(PathBuf::from)
        .ok_or(Helper::NoAddress)?;
    let merged = text(selected, "ext").unwrap_or("");
    let streams = match selected.get("requested_formats").and_then(Value::as_array) {
        Some(formats) if !formats.is_empty() => formats
            .iter()
            .map(|f| {
                let id = text(f, "format_id").unwrap_or("0");
                let ext = text(f, "ext").unwrap_or(merged);
                stream(f, stream_path(&output, merged, id, ext))
            })
            .collect::<Result<Vec<_>, _>>()?,
        _ => vec![stream(selected, output.clone())?],
    };
    Ok(Plan { output, streams })
}

/// The chosen format `id` in yt-dlp's choice `selected` (a look-up chosen again with `-f <id>`).
pub fn format_of<'a>(selected: &'a Value, id: &str) -> Option<&'a Value> {
    let formats = selected
        .get("requested_formats")
        .and_then(Value::as_array)
        .map(|a| a.iter().collect::<Vec<_>>())
        .unwrap_or_else(|| vec![selected]);
    formats
        .into_iter()
        .find(|f| text(f, "format_id") == Some(id))
}

/// A format as yt-dlp gives it (a look-up's, again): its stream (no file of its own).
pub fn stream_of(format: &Value) -> Option<Stream> {
    stream(format, PathBuf::new()).ok()
}

fn header_map(headers: &[(String, String)]) -> HeaderMap {
    let mut map = HeaderMap::new();
    for (k, v) in headers {
        if let (Ok(k), Ok(v)) = (
            HeaderName::from_bytes(k.as_bytes()),
            HeaderValue::from_str(v),
        ) {
            map.insert(k, v);
        }
    }
    map
}

/// Looks the page up again for a format as yt-dlp gives it today (blocking: yt-dlp runs).
pub type Relink = Arc<dyn Fn(&str) -> Result<Value, String> + Send + Sync>;

/// A stream's address, from yt-dlp (#769): the engine's [`Unlocker`] for a stream. The first ask
/// takes the address this run's look-up gave; every later one (the address died, the download
/// continues after a stop) looks the page up again for the same format's.
pub struct StreamLink {
    format_id: String,
    size: Option<u64>,
    first: Mutex<Option<Unlocked>>,
    relink: Relink,
}

impl std::fmt::Debug for StreamLink {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "StreamLink({})", self.format_id)
    }
}

fn unlocked(url: Url, headers: &[(String, String)], size: Option<u64>) -> Unlocked {
    Unlocked {
        url,
        file_name: None,
        size,
        max_connections: None,
        headers: header_map(headers),
    }
}

impl StreamLink {
    pub fn new(s: &Stream, relink: Relink) -> StreamLink {
        let size = s.exact.then_some(s.size).flatten();
        StreamLink {
            format_id: s.format_id.clone(),
            size,
            first: Mutex::new(Some(unlocked(s.url.clone(), &s.headers, size))),
            relink,
        }
    }
}

impl Unlocker for StreamLink {
    fn name(&self) -> &str {
        "yt-dlp"
    }

    fn unlock<'a>(
        &'a self,
        _url: &'a Url,
        note: &'a (dyn Fn(String, Option<f32>) + Send + Sync),
        _cancel: &'a CancellationToken,
    ) -> BoxFuture<'a, Result<Option<Unlocked>, ServiceFault>> {
        Box::pin(async move {
            if let Some(u) = self.first.lock().unwrap().take() {
                return Ok(Some(u));
            }
            note(
                format!(
                    "Asking yt-dlp for a new address for format {}",
                    self.format_id
                ),
                None,
            );
            let (relink, id) = (self.relink.clone(), self.format_id.clone());
            let got = tokio::task::spawn_blocking(move || relink(&id))
                .await
                .map_err(|_| ServiceFault::Other("yt-dlp stopped".into()))?;
            match got.map(|f| stream_of(&f)) {
                Ok(Some(s)) => Ok(Some(unlocked(s.url, &s.headers, self.size))),
                Ok(None) => Err(ServiceFault::Other(format!(
                    "yt-dlp gave no address for format {}",
                    self.format_id
                ))),
                Err(e) => Err(ServiceFault::Other(e)),
            }
        })
    }

    fn noted(&self, u: &Unlocked) -> String {
        format!(
            "yt-dlp's address for format {}: from {}",
            self.format_id,
            u.url.host_str().unwrap_or("?")
        )
    }
}

/// MDM's engine for a video's streams: the app's runtime, and the options a file from the page
/// would get (connections, proxy, network adapter, DNS, the limit for all downloads, the hosts'
/// learned limits, this download's own speed limit).
#[derive(Clone)]
pub struct Engine {
    pub handle: tokio::runtime::Handle,
    pub opts: mdm_engine::Options,
}

/// The engine's download under way and its speed limit, shared with [`crate::video::VideoJob`]:
/// a new limit applies to it at once, and Stop cancels it.
pub struct Current {
    dl: Mutex<Option<Download>>,
    /// A stream in pieces under way: Stop cancels it.
    pieces: Mutex<Option<CancellationToken>>,
    limit: Mutex<Option<u64>>,
    /// The pieces' own speed limit (the download's), shared by their connections.
    limiter: Arc<mdm_engine::limiter::Limiter>,
    /// An engine download is under way or ending: its resume state isn't saved yet.
    busy: AtomicBool,
}

impl Current {
    pub fn new(limit: Option<u64>) -> Current {
        Current {
            dl: Mutex::new(None),
            pieces: Mutex::new(None),
            limit: Mutex::new(limit),
            limiter: Arc::new(mdm_engine::limiter::Limiter::new(limit)),
            busy: AtomicBool::new(false),
        }
    }

    pub fn set_limit(&self, rate: Option<u64>) {
        *self.limit.lock().unwrap() = rate;
        self.limiter.set(rate);
        if let Some(dl) = self.dl.lock().unwrap().as_ref() {
            dl.set_speed_limit(rate);
        }
    }

    pub fn limit(&self) -> Option<u64> {
        *self.limit.lock().unwrap()
    }

    pub fn cancel(&self) {
        if let Some(dl) = self.dl.lock().unwrap().as_ref() {
            dl.cancel();
        }
        if let Some(t) = self.pieces.lock().unwrap().as_ref() {
            t.cancel();
        }
    }

    /// An engine download is still ending (Stop was asked, its state isn't saved yet).
    pub fn busy(&self) -> bool {
        self.busy.load(Ordering::Acquire)
    }

    /// Pieces under way (a gallery's files, #769), stopped by [`Current::cancel`].
    pub fn set_pieces(&self, token: Option<CancellationToken>) {
        self.busy.store(token.is_some(), Ordering::Release);
        *self.pieces.lock().unwrap() = token;
    }

    /// The download's own speed limit, shared by its pieces' connections.
    pub fn limiter(&self) -> Arc<mdm_engine::limiter::Limiter> {
        self.limiter.clone()
    }
}

/// What the engine's part showed last: the streams' segments one after another, the
/// connections of the stream under way, its server and its address.
#[derive(Clone, Debug, Default)]
pub struct View {
    pub segments: Vec<SegmentView>,
    pub connections: Vec<ConnView>,
    pub server: Option<mdm_engine::Server>,
    /// The address the stream under way comes from, as yt-dlp gave it (#795): the windows'
    /// "Address", the page being the web page it was obtained from.
    pub address: Option<Url>,
}

/// How the engine's part of a run ended.
#[derive(Debug)]
pub enum Fetched {
    /// Every stream is in place.
    Done,
    Stopped,
    Failed(String),
    /// A stream turned out to be yt-dlp's to download (a playlist without an end yet: live).
    Helper(Helper),
}

/// What [`fetch`] reports as it goes: bytes in, the total, the speed (bytes/s), the view, and the
/// engine's own log lines.
pub trait Report {
    fn progress(&self, downloaded: u64, total: Option<u64>, speed: f64, view: View);
    fn note(&self, text: String);
    fn stopped(&self) -> bool;
}

/// Each stream's size for the total: the engine's own once it knows it, else the plan's.
fn sizes(plan: &Plan, known: &[Option<u64>]) -> Vec<Option<u64>> {
    plan.streams
        .iter()
        .zip(known)
        .map(|(s, k)| k.or(s.size))
        .collect()
}

/// The streams' segments one after another: those before `at` whole, `at`'s from the engine
/// (`now`), those after it waiting.
fn segments(sizes: &[Option<u64>], at: usize, now: &[SegmentView]) -> Vec<SegmentView> {
    let mut out = Vec::new();
    let mut offset = 0u64;
    for (i, size) in sizes.iter().enumerate() {
        let len = size.unwrap_or(0);
        if i < at {
            out.push(SegmentView {
                start: offset,
                cursor: offset + len,
                end: offset + len,
            });
        } else if i == at {
            out.extend(now.iter().map(|s| SegmentView {
                start: offset + s.start,
                cursor: offset + s.cursor,
                end: offset + s.end,
            }));
        } else if len > 0 {
            out.push(SegmentView {
                start: offset,
                cursor: offset,
                end: offset + len,
            });
        }
        offset += len;
    }
    out
}

/// Download `plan`'s streams with the engine, one after another, each over every connection it
/// may open, into the files yt-dlp looks for. A stream already whole is kept; one with a resume
/// state continues (its address asked for again through `relink`). `page` stands for every
/// stream's address in the list and the resume state (its own address expires).
pub fn fetch(
    plan: &Plan,
    engine: &Engine,
    page: &Url,
    relink: &Relink,
    current: &Current,
    report: &dyn Report,
) -> Fetched {
    let mut known: Vec<Option<u64>> = vec![None; plan.streams.len()];
    for (i, s) in plan.streams.iter().enumerate() {
        if report.stopped() {
            return Fetched::Stopped;
        }
        let state = mdm_engine::state::state_path(&s.path);
        // Whole already (an earlier run's, its merge not done): yt-dlp takes it as it is.
        if !state.exists() && s.path.is_file() {
            known[i] = std::fs::metadata(&s.path).ok().map(|m| m.len());
            continue;
        }
        if let Some(dir) = s.path.parent() {
            let _ = std::fs::create_dir_all(dir);
        }
        let chunk = match &s.kind {
            Kind::File { chunk } => *chunk,
            Kind::Hls { .. } | Kind::Dash(_) => {
                match fetch_pieces(plan, i, &mut known, engine, relink, current, report) {
                    Fetched::Done => continue,
                    other => return other,
                }
            }
        };
        let mut opts = engine.opts.clone();
        opts.unlocker = Some(Arc::new(StreamLink::new(s, relink.clone())));
        opts.request_cap = chunk;
        opts.speed_limit = current.limit();
        // yt-dlp's headers are the stream's: no other User-Agent, address, type or form.
        if let Some((_, ua)) = s
            .headers
            .iter()
            .find(|(k, _)| k.eq_ignore_ascii_case("user-agent"))
        {
            opts.speak_as(ua.clone());
        }
        opts.agent_fallback = None;
        opts.browser_final = None;
        opts.expected_type = None;
        opts.expected_size = None;
        opts.post = None;
        opts.checksum = None;
        opts.on_file_created = None;
        let dl = {
            let _rt = engine.handle.enter();
            if state.exists() {
                match Download::resume(state.clone(), HeaderMap::new(), opts.clone()) {
                    Ok(dl) => dl,
                    Err(e) => {
                        // A state that can't be read (another release's, damaged): the stream
                        // starts again rather than the video failing.
                        report.note(format!(
                            "Format {}: its saved progress can't be continued ({e}); starting it again",
                            s.format_id
                        ));
                        let _ = std::fs::remove_file(&state);
                        let _ = std::fs::remove_file(mdm_engine::naming::part_path(&s.path));
                        fresh(page, s, opts)
                    }
                }
            } else {
                fresh(page, s, opts)
            }
        };
        current.busy.store(true, Ordering::Release);
        *current.dl.lock().unwrap() = Some(dl);
        let ended = follow(plan, i, &mut known, current, report);
        let dl = current.dl.lock().unwrap().take();
        let out = dl.map(|dl| engine.handle.block_on(dl.wait()));
        current.busy.store(false, Ordering::Release);
        match (ended, out) {
            (DlState::Done, Some(Ok(o))) if o.path == s.path => known[i] = Some(o.size),
            (DlState::Done, Some(Ok(o))) => {
                return Fetched::Failed(format!(
                    "the stream was saved as {} instead of {}",
                    o.path.display(),
                    s.path.display()
                ));
            }
            (DlState::Cancelled, _) | (_, Some(Err(mdm_engine::Error::Cancelled))) => {
                return Fetched::Stopped;
            }
            (_, Some(Err(e))) => return Fetched::Failed(e.to_string()),
            (_, _) => return Fetched::Failed("the engine stopped".into()),
        }
    }
    Fetched::Done
}

/// Why a stream's pieces couldn't be listed.
enum Unlisted {
    /// It is yt-dlp's to download.
    Helper(Helper),
    Failed(String),
}

/// The pieces of stream `s` (an HLS playlist fetched and read, DASH's as yt-dlp listed them).
async fn list_pieces(
    s: &Stream,
    opts: &mdm_engine::Options,
) -> Result<Vec<crate::pieces::Piece>, Unlisted> {
    use crate::pieces::Unlisted as U;
    let (extra, extra_key) = match &s.kind {
        Kind::Dash(p) => return Ok(p.clone()),
        Kind::Hls { extra, extra_key } => (extra.as_deref(), extra_key.as_deref()),
        Kind::File { .. } => return Err(Unlisted::Failed("not a stream in pieces".into())),
    };
    let failed = |e: String| Unlisted::Failed(format!("the playlist: {e}"));
    let client = mdm_engine::http_client(opts, &s.url).map_err(|e| failed(e.to_string()))?;
    let resp = client
        .get(s.url.clone())
        .headers(header_map(&s.headers))
        .send()
        .await
        .map_err(|e| failed(e.to_string()))?;
    if !resp.status().is_success() {
        return Err(failed(format!("the server answered {}", resp.status())));
    }
    // Its pieces' addresses are taken from where it ended up, as hlsnative takes them.
    let base = resp.url().clone();
    let text = resp.text().await.map_err(|e| failed(e.to_string()))?;
    crate::pieces::hls(&text, &base, extra, extra_key).map_err(|e| match e {
        U::Live => Unlisted::Helper(Helper::Live),
        U::Protected(_) => Unlisted::Helper(Helper::Protected),
        U::NoPieces => Unlisted::Failed("the playlist lists no pieces".into()),
    })
}

/// Stream `i` of `plan`, in pieces: listed, downloaded several at once into its pieces' folder
/// (a stop keeps the pieces whole), joined into its file. A piece refused has the stream asked of
/// yt-dlp again (`relink`) for today's list.
fn fetch_pieces(
    plan: &Plan,
    i: usize,
    known: &mut [Option<u64>],
    engine: &Engine,
    relink: &Relink,
    current: &Current,
    report: &dyn Report,
) -> Fetched {
    let s = &plan.streams[i];
    let list = match engine.handle.block_on(list_pieces(s, &engine.opts)) {
        Ok(l) => l,
        Err(Unlisted::Helper(why)) => return Fetched::Helper(why),
        Err(Unlisted::Failed(e)) => return Fetched::Failed(e),
    };
    let n = list.len();
    let relist: crate::pieces::Relist = {
        let (relink, id, opts) = (relink.clone(), s.format_id.clone(), engine.opts.clone());
        let handle = engine.handle.clone();
        Arc::new(move || {
            let f = relink(&id)?;
            let s =
                stream_of(&f).ok_or_else(|| format!("yt-dlp gave no address for format {id}"))?;
            handle
                .block_on(list_pieces(&s, &opts))
                .map_err(|e| match e {
                    Unlisted::Failed(e) => e,
                    Unlisted::Helper(h) => h.english(),
                })
        })
    };
    let token = CancellationToken::new();
    *current.pieces.lock().unwrap() = Some(token.clone());
    current.busy.store(true, Ordering::Release);
    let shared = Arc::new(crate::pieces::Shared::default());
    let task = engine.handle.spawn(crate::pieces::download(
        list,
        crate::pieces::Target::Join(s.path.clone()),
        header_map(&s.headers),
        engine.opts.clone(),
        current.limiter.clone(),
        token.clone(),
        shared.clone(),
        relist,
    ));
    let mut cancelled = false;
    while !task.is_finished() {
        for note in shared.take_notes() {
            report.note(format!("Format {}: {note}", s.format_id));
        }
        known[i] = shared.total(n, s.size);
        let sizes = sizes(plan, known);
        let before: u64 = sizes[..i].iter().map(|s| s.unwrap_or(0)).sum();
        let total = sizes.iter().copied().sum::<Option<u64>>();
        let connections = shared.connections();
        let speed: u64 = connections.iter().map(|c| c.speed).sum();
        let here = shared.segments(known[i].unwrap_or(0));
        report.progress(
            before + shared.downloaded(),
            total,
            speed as f64,
            View {
                segments: segments(&sizes, i, &here),
                connections,
                server: None,
                address: Some(s.url.clone()),
            },
        );
        if report.stopped() && !cancelled {
            cancelled = true;
            token.cancel();
        }
        std::thread::sleep(Duration::from_millis(200));
    }
    let ended = engine.handle.block_on(task);
    for note in shared.take_notes() {
        report.note(format!("Format {}: {note}", s.format_id));
    }
    *current.pieces.lock().unwrap() = None;
    current.busy.store(false, Ordering::Release);
    match ended {
        Ok(crate::pieces::Ended::Done(size)) => {
            known[i] = Some(size);
            report.note(format!(
                "Format {}: {n} pieces joined into {}",
                s.format_id,
                s.path.display()
            ));
            Fetched::Done
        }
        Ok(crate::pieces::Ended::Stopped) => Fetched::Stopped,
        Ok(crate::pieces::Ended::Failed(e)) => Fetched::Failed(e),
        Err(_) => Fetched::Failed("the pieces stopped on an internal error".into()),
    }
}

fn fresh(page: &Url, s: &Stream, opts: mdm_engine::Options) -> Download {
    Download::start(
        Request {
            url: page.clone(),
            headers: HeaderMap::new(),
            dest: Destination::File(s.path.clone()),
        },
        opts,
    )
}

/// Follow stream `i`'s engine download until it ends, reporting the whole choice's progress;
/// Stop cancels it. Its state at the end.
fn follow(
    plan: &Plan,
    i: usize,
    known: &mut [Option<u64>],
    current: &Current,
    report: &dyn Report,
) -> DlState {
    let mut seen = 0u64;
    let mut cancelled = false;
    loop {
        let Some(s) = current.dl.lock().unwrap().as_ref().map(Download::snapshot) else {
            return DlState::Failed;
        };
        let from = seen;
        for e in s.events.iter().filter(|e| e.seq > from) {
            seen = e.seq;
            report.note(e.text.clone());
        }
        if s.size.is_some() {
            known[i] = s.size;
        }
        let sizes = sizes(plan, known);
        let before: u64 = sizes[..i].iter().map(|s| s.unwrap_or(0)).sum();
        let total = sizes.iter().copied().sum::<Option<u64>>();
        let speed: u64 = s.connections.iter().map(|c| c.speed).sum();
        report.progress(
            before + s.downloaded,
            total,
            speed as f64,
            View {
                segments: segments(&sizes, i, &s.segments),
                connections: s.connections.clone(),
                server: s.server.clone(),
                address: Some(plan.streams[i].url.clone()),
            },
        );
        if matches!(
            s.state,
            DlState::Done | DlState::Failed | DlState::Cancelled
        ) {
            return s.state;
        }
        if report.stopped() && !cancelled {
            cancelled = true;
            current.cancel();
        }
        std::thread::sleep(Duration::from_millis(200));
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    #[test]
    fn yt_dlp_names_each_stream_of_a_merge() {
        let out = Path::new("/v/Big Buck.mp4");
        assert_eq!(
            stream_path(out, "mp4", "399", "mp4"),
            Path::new("/v/Big Buck.f399.mp4")
        );
        assert_eq!(
            stream_path(out, "mp4", "251", "webm"),
            Path::new("/v/Big Buck.f251.webm")
        );
        // A stem with dots of its own keeps them.
        assert_eq!(
            stream_path(Path::new("/v/a.b.c.mkv"), "mkv", "hls-720p", "mp4"),
            Path::new("/v/a.b.c.fhls-720p.mp4")
        );
    }

    #[test]
    fn cookies_lose_their_attributes() {
        assert_eq!(
            cookie_header("a=1; Domain=.example.com; Path=/; Secure; Expires=1794448205; b=x=y")
                .as_deref(),
            Some("a=1; b=x=y")
        );
        assert_eq!(cookie_header("Domain=.x; Path=/"), None);
        assert_eq!(cookie_header(""), None);
    }

    fn youtube() -> Value {
        json!({
            "id": "aqz-KE-bpKQ", "ext": "mp4", "format_id": "399+258",
            "filename": "/v/Big Buck.mp4", "is_live": false,
            "requested_formats": [
                {"format_id": "399", "ext": "mp4", "protocol": "https", "filesize": 124386876,
                 "url": "https://rr2---sn-a.googlevideo.com/videoplayback?itag=399",
                 "http_headers": {"User-Agent": "UA/1", "Accept": "*/*"},
                 "downloader_options": {"http_chunk_size": 10485760}},
                {"format_id": "258", "ext": "m4a", "protocol": "https", "filesize_approx": 30767602,
                 "url": "https://rr2---sn-a.googlevideo.com/videoplayback?itag=258",
                 "http_headers": {"User-Agent": "UA/1"},
                 "cookies": "SID=s; Domain=.googlevideo.com; Path=/"}
            ]
        })
    }

    #[test]
    fn a_merged_choice_is_its_streams_one_after_another() {
        let p = plan(&youtube()).unwrap();
        assert_eq!(p.output, Path::new("/v/Big Buck.mp4"));
        assert_eq!(p.streams.len(), 2);
        let (v, a) = (&p.streams[0], &p.streams[1]);
        assert_eq!(v.format_id, "399");
        assert_eq!(v.path, Path::new("/v/Big Buck.f399.mp4"));
        assert_eq!((v.size, v.exact), (Some(124386876), true));
        assert_eq!(
            v.kind,
            Kind::File {
                chunk: Some(10485760)
            }
        );
        assert!(v.headers.contains(&("User-Agent".into(), "UA/1".into())));
        assert_eq!(a.path, Path::new("/v/Big Buck.f258.m4a"));
        assert_eq!((a.size, a.exact), (Some(30767602), false));
        assert_eq!(a.kind, Kind::File { chunk: None });
        assert!(a.headers.contains(&("Cookie".into(), "SID=s".into())));
        assert_eq!(p.total(), Some(124386876 + 30767602));
        let again = stream_of(format_of(&youtube(), "258").unwrap()).unwrap();
        assert!(again.url.as_str().ends_with("itag=258"));
        assert!(again.headers.iter().any(|(k, _)| k == "Cookie"));
        assert!(format_of(&youtube(), "137").is_none());
    }

    #[test]
    fn a_choice_of_one_format_is_one_stream_at_the_file() {
        let one = json!({
            "format_id": "18", "ext": "mp4", "protocol": "https", "filename": "/v/Talk.mp4",
            "url": "https://upload.example/talk.mp4", "filesize": 1000
        });
        let p = plan(&one).unwrap();
        assert_eq!(p.streams.len(), 1);
        assert_eq!(p.streams[0].path, Path::new("/v/Talk.mp4"));
        assert_eq!(p.streams[0].kind, Kind::File { chunk: None });
        // No protocol said: the address's.
        let mut bare = one.clone();
        bare.as_object_mut().unwrap().remove("protocol");
        assert!(plan(&bare).is_ok());
    }

    #[test]
    fn what_stays_with_yt_dlp() {
        let with = |key: &str, v: Value| {
            let mut f = youtube();
            f["requested_formats"][1][key] = v;
            plan(&f)
        };
        assert_eq!(
            with("protocol", json!("http_dash_segments")),
            Err(Helper::Pieces("http_dash_segments".into())),
            "DASH without its pieces listed"
        );
        assert_eq!(
            with("fragments", json!([{"url": "a"}])),
            Err(Helper::Pieces("https".into()))
        );
        assert_eq!(
            with("protocol", json!("f4m")),
            Err(Helper::Pieces("f4m".into()))
        );
        let mut aes = youtube();
        aes["requested_formats"][1]["protocol"] = json!("m3u8_native");
        aes["requested_formats"][1]["hls_aes"] = json!({"key": "00"});
        assert_eq!(plan(&aes), Err(Helper::Pieces("m3u8_native".into())));
        assert_eq!(
            with("protocol", json!("rtmp")),
            Err(Helper::Protocol("rtmp".into()))
        );
        assert_eq!(with("has_drm", json!(true)), Err(Helper::Protected));
        assert_eq!(with("url", json!("ftp://x/y")), Err(Helper::NoAddress));
        let mut live = youtube();
        live["is_live"] = json!(true);
        assert_eq!(plan(&live), Err(Helper::Live));
        let mut live = youtube();
        live["live_status"] = json!("is_live");
        assert_eq!(plan(&live), Err(Helper::Live));
        let mut unnamed = youtube();
        unnamed.as_object_mut().unwrap().remove("filename");
        assert_eq!(plan(&unnamed), Err(Helper::NoAddress));
    }

    /// #769: HLS and DASH streams are the engine's, in pieces.
    #[test]
    fn streams_in_pieces_are_the_engines() {
        let mut f = youtube();
        f["requested_formats"][0]["protocol"] = json!("m3u8_native");
        f["requested_formats"][0]["url"] = json!("https://cdn.example/v/index.m3u8");
        f["requested_formats"][0]["extra_param_to_segment_url"] = json!("tok=1");
        f["requested_formats"][1]["protocol"] = json!("http_dash_segments");
        f["requested_formats"][1]["fragment_base_url"] = json!("https://cdn.example/a/");
        f["requested_formats"][1]["fragments"] = json!([{"path": "init.mp4"}, {"path": "s1.m4s"}]);
        f["requested_formats"][1]
            .as_object_mut()
            .unwrap()
            .remove("url");
        let p = plan(&f).unwrap();
        assert_eq!(
            p.streams[0].kind,
            Kind::Hls {
                extra: Some("tok=1".into()),
                extra_key: None
            }
        );
        let Kind::Dash(pieces) = &p.streams[1].kind else {
            panic!("{:?}", p.streams[1].kind)
        };
        assert_eq!(pieces.len(), 2);
        assert_eq!(p.streams[1].url.as_str(), "https://cdn.example/a/init.mp4");
    }

    #[test]
    fn the_segment_map_runs_through_every_stream() {
        let sizes = [Some(100), Some(50)];
        // The video under way, half of its first half in.
        let now = [
            SegmentView {
                start: 0,
                cursor: 25,
                end: 50,
            },
            SegmentView {
                start: 50,
                cursor: 60,
                end: 100,
            },
        ];
        let map = segments(&sizes, 0, &now);
        assert_eq!(map.len(), 3);
        assert_eq!(
            map[2],
            SegmentView {
                start: 100,
                cursor: 100,
                end: 150
            }
        );
        // The sound under way: the video whole before it.
        let map = segments(&sizes, 1, &now[..1]);
        assert_eq!(
            map[0],
            SegmentView {
                start: 0,
                cursor: 100,
                end: 100
            }
        );
        assert_eq!(
            map[1],
            SegmentView {
                start: 100,
                cursor: 125,
                end: 150
            }
        );
    }
}