Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

67.4 KB
Raw Blame
//! Deterministic fault-injection tests for the share and Instant upload paths.
//!
//! The real upload code runs against `MockCap`, an in-process HTTP/1.1 mock of
//! the Cap upload API and its object store, on a paused tokio clock. Retry
//! backoff, outages and request timeouts therefore cost virtual time only.
//! The clock advances in 1 ms steps, and freezes while the mock is serving a
//! request, and for a few real milliseconds after the mock or any other task
//! last did work. Bytes still crossing the loopback socket therefore never
//! see virtual time move, and a far timer (a 5 minute request timeout, say)
//! never fires early.
//!
//! Every case checks the stored objects against the local files byte for byte
//! and holds request counts and re-uploaded bytes to budgets taken from the
//! behaviour of main when the case was written. Lower is allowed; higher fails.

use super::*;
use cap_recording::upload_verification::UploadVerification;
use sha2::{Digest as _, Sha256};
use std::collections::BTreeMap;
use std::sync::atomic::AtomicBool;
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
use tokio::net::{TcpListener, TcpStream};

pub(super) const API_KEY: &str = "verify-api-key";
pub(super) const OWNER: &str = "verify-owner";
pub(super) const SAMPLE_MP4: &[u8] =
    include_bytes!("../../../media-server/src/__tests__/fixtures/test-with-audio.mp4");

const MIB: usize = 1024 * 1024;
const PART: u64 = 5 * MIB as u64;
const STUDIO_SIZE: usize = 5 * 5 * MIB + MIB + 12_345;
const STUDIO_LAST_PART: u64 = STUDIO_SIZE as u64 - 5 * PART;
const VIDEO_SEGMENT: usize = 256 * 1024;
const AUDIO_SEGMENT: usize = 32 * 1024;
const INSTANT_PAIRS: u32 = 8;
const INSTANT_VIDEO_ID: &str = "instantverify01";
const RETRY_WINDOW: Duration = Duration::from_secs(5 * 60);

const SETTLE: Duration = Duration::from_millis(50);
const WORKED: Duration = Duration::from_micros(200);

#[derive(Default)]
struct Activity {
    in_flight: std::sync::atomic::AtomicUsize,
    last: Mutex<Option<std::time::Instant>>,
}

impl Activity {
    fn touch(&self) {
        *self.last.lock().unwrap_or_else(|error| error.into_inner()) =
            Some(std::time::Instant::now());
    }

    fn busy(&self) -> bool {
        self.in_flight.load(Ordering::Acquire) > 0
            || self
                .last
                .lock()
                .unwrap_or_else(|error| error.into_inner())
                .is_some_and(|at| at.elapsed() < SETTLE)
    }
}

struct InFlight(Arc<Activity>);

impl InFlight {
    fn new(activity: &Arc<Activity>) -> Self {
        activity.in_flight.fetch_add(1, Ordering::AcqRel);
        activity.touch();
        Self(activity.clone())
    }
}

impl Drop for InFlight {
    fn drop(&mut self) {
        self.0.in_flight.fetch_sub(1, Ordering::AcqRel);
        self.0.touch();
    }
}

thread_local! {
    static ACTIVITY: std::cell::RefCell<Option<Arc<Activity>>> = const {
        std::cell::RefCell::new(None)
    };
}

pub(super) fn virtual_time<F: Future>(future: F) -> F::Output {
    let activity = Arc::new(Activity::default());
    ACTIVITY.with(|current| *current.borrow_mut() = Some(activity.clone()));
    let output = tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .start_paused(true)
        .build()
        .unwrap()
        .block_on(async {
            let ticker = tokio::spawn(async move {
                loop {
                    if activity.busy() {
                        std::thread::sleep(Duration::from_micros(50));
                        tokio::task::yield_now().await;
                    } else {
                        let step = std::time::Instant::now();
                        tokio::time::sleep(Duration::from_millis(1)).await;
                        if step.elapsed() > WORKED {
                            activity.touch();
                        }
                    }
                }
            });
            let output = future.await;
            ticker.abort();
            output
        });
    ACTIVITY.with(|current| *current.borrow_mut() = None);
    output
}

#[derive(Clone, Copy, Debug)]
pub(super) enum Route {
    Part(u32),
    Object(&'static str),
    Api(&'static str),
}

impl Route {
    fn matches(self, method: &str, path: &str) -> bool {
        match self {
            Route::Part(number) => {
                method == "PUT"
                    && path.starts_with("/s3mp/")
                    && path.rsplit('/').next() == Some(number.to_string().as_str())
            }
            Route::Object(fragment) => {
                method == "PUT" && path.starts_with("/s3/") && path.contains(fragment)
            }
            Route::Api(fragment) => path.starts_with("/api/") && path.contains(fragment),
        }
    }
}

#[derive(Clone, Copy, Debug)]
pub(super) enum Fault {
    /// Read the whole body, then answer with this status.
    Status(u16),
    /// Read this fraction of the body, then reset the connection.
    Reset(f64),
    /// Read this fraction of the body, then take the network down: every open
    /// connection is reset and new ones are reset on accept until it ends.
    Outage(f64, Duration),
    /// Read the body, wait, then answer normally.
    Delay(Duration),
    /// Read the body and never answer.
    Hang,
    /// Read this fraction of the body, raise the test's cancel flag (or quit
    /// signal), then finish the request normally.
    Trip(f64),
}

pub(super) struct Rule {
    route: Route,
    fault: Fault,
    skip: u32,
    count: u32,
    seen: u32,
    fired: u32,
}

pub(super) fn rule(route: Route, fault: Fault) -> Rule {
    Rule {
        route,
        fault,
        skip: 0,
        count: 1,
        seen: 0,
        fired: 0,
    }
}

impl Rule {
    pub(super) fn times(mut self, count: u32) -> Self {
        self.count = count;
        self
    }
}

#[derive(Default)]
struct Multipart {
    video_id: String,
    parts: BTreeMap<u32, (String, Vec<u8>)>,
    completed: bool,
    aborted: bool,
}

#[derive(Default)]
struct MockState {
    rules: Vec<Rule>,
    requests: Vec<(String, String, u16, Duration)>,
    epoch: Option<tokio::time::Instant>,
    media_put_bytes: u64,
    useful_offset: u64,
    objects: BTreeMap<String, Vec<u8>>,
    uploads: BTreeMap<String, Multipart>,
    outage_until: Option<tokio::time::Instant>,
    reset_on_accept: u32,
    accepted: usize,
    connections: Vec<tokio::task::AbortHandle>,
    next_id: u32,
    confirmations: Vec<Value>,
    deletes: u32,
    created: Vec<String>,
    deleted: Vec<String>,
    problems: Vec<String>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) struct Stats {
    /// HTTP requests that reached the mock (API calls and object PUTs).
    pub(super) requests: usize,
    /// PUTs of media objects and multipart parts (recording metadata excluded).
    pub(super) media_puts: usize,
    /// PUTs of `segments/manifest.json`.
    pub(super) manifest_puts: usize,
    /// Media body bytes the server received, whole or cut off.
    pub(super) media_bytes: u64,
    /// Media body bytes the server received beyond what it finally kept:
    /// failed or reset bodies, rewritten objects and parts of multipart
    /// uploads that were never completed.
    pub(super) reupload_bytes: u64,
    /// Multipart uploads initiated and never completed or aborted.
    pub(super) orphans: usize,
    /// Connections the mock accepted and served.
    pub(super) connections: usize,
    /// Connections reset on accept during an outage.
    pub(super) reset_on_accept: u32,
}

#[derive(Debug, Clone, Copy)]
pub(super) struct Budget {
    pub(super) requests: usize,
    pub(super) media_puts: usize,
    pub(super) manifest_puts: usize,
    pub(super) reupload_bytes: u64,
    pub(super) orphans: usize,
    pub(super) connections: usize,
}

/// Every budget, in one table: main's measured behaviour at 83237e994 (#2522
/// merged: multipart parts go one at a time over per-upload clients). A
/// change that should move a number, like a retry fix, updates it here and
/// says why; anything that goes over fails.
///
/// Studio: a 26 MiB export in six parts (five of 5 MiB, one of 1 MiB + 12345
/// bytes), each part a presign and a PUT. Instant: 2 init segments and 8
/// audio/video pairs on URLs from one batch presign, progress manifests and
/// the completion signal. Manifests are PUT to the URL the batch presign
/// returned for them, so Instant request budgets no longer count a presign
/// per manifest.
pub(super) mod budget {
    use super::{Budget, PART, STUDIO_LAST_PART, STUDIO_SIZE, VIDEO_SEGMENT};

    const fn budget(
        requests: usize,
        media_puts: usize,
        manifest_puts: usize,
        reupload_bytes: u64,
        orphans: usize,
        connections: usize,
    ) -> Budget {
        Budget {
            requests,
            media_puts,
            manifest_puts,
            reupload_bytes,
            orphans,
            connections,
        }
    }

    // budget(requests, media_puts, manifest_puts, reupload_bytes, orphans, connections)
    const SEGMENT_3: u64 = VIDEO_SEGMENT as u64 + 21;
    /// Instant segments share one pooled client, and whether a finished
    /// request's connection is back in the pool when the next one starts is
    /// a race: under heavy load main was seen opening up to 2 more
    /// connections than its usual count. Studio connection counts are exact.
    const POOL_RACE: usize = 3;
    const SEGMENT_4: u64 = VIDEO_SEGMENT as u64 + 28;

    pub(in crate::upload) const STUDIO_SHARE_CLEAN: Budget = budget(18, 7, 0, 0, 0, 8);
    pub(in crate::upload) const STUDIO_SHARE_PART_503: Budget = budget(20, 8, 0, PART, 0, 8);
    pub(in crate::upload) const STUDIO_SHARE_RESET_MID_PART: Budget =
        budget(19, 8, 0, PART / 2, 0, 9);
    pub(in crate::upload) const STUDIO_SHARE_OUTAGE_MID_PART: Budget =
        budget(19, 8, 0, PART / 2, 0, 10);
    pub(in crate::upload) const STUDIO_SHARE_PRESIGN_500: Budget = budget(19, 7, 0, 0, 0, 8);
    pub(in crate::upload) const STUDIO_SHARE_HUNG_PART: Budget = budget(19, 8, 0, PART, 0, 10);
    pub(in crate::upload) const STUDIO_SHARE_SLOW_PART: Budget = budget(18, 7, 0, 0, 0, 8);
    pub(in crate::upload) const STUDIO_SHARE_PERMANENT_403: Budget =
        budget(17, 7, 0, 6 * PART + STUDIO_LAST_PART, 1, 5);
    pub(in crate::upload) const STUDIO_SHARE_COMPLETE_500: Budget =
        budget(16, 6, 0, STUDIO_SIZE as u64, 1, 6);
    pub(in crate::upload) const STUDIO_SHARE_CANCEL_THEN_RESUME: Budget =
        budget(26, 10, 0, 3 * PART, 1, 12);

    pub(in crate::upload) const INSTANT_LIVE_CLEAN: Budget =
        budget(29, 18, 9, 0, 0, 13 + POOL_RACE);
    pub(in crate::upload) const INSTANT_LIVE_SEGMENT_503: Budget =
        budget(32, 19, 10, SEGMENT_3, 0, 15 + POOL_RACE);
    pub(in crate::upload) const INSTANT_LIVE_SEGMENT_503X3: Budget =
        budget(35, 21, 9, 3 * SEGMENT_3, 0, 16 + POOL_RACE);
    pub(in crate::upload) const INSTANT_LIVE_SEGMENT_RESET: Budget =
        budget(35, 21, 9, 3 * (SEGMENT_4 / 2), 0, 18 + POOL_RACE);
    pub(in crate::upload) const INSTANT_LIVE_OUTAGE: Budget =
        budget(34, 19, 7, SEGMENT_3 / 2, 0, 22 + POOL_RACE);
    pub(in crate::upload) const INSTANT_LIVE_SLOW_SEGMENT: Budget =
        budget(29, 18, 9, 0, 0, 13 + POOL_RACE);
    pub(in crate::upload) const INSTANT_LIVE_OUTAGE_PAST_WINDOW: Budget =
        budget(12, 8, 3, SEGMENT_3 / 2, 0, 6 + POOL_RACE);
    pub(in crate::upload) const INSTANT_LIVE_PERMANENT_403: Budget =
        budget(18, 11, 4, 3 * SEGMENT_3, 0, 9 + POOL_RACE);
    pub(in crate::upload) const INSTANT_LIVE_CANCEL: Budget =
        budget(15, 10, 4, 0, 0, 7 + POOL_RACE);
    pub(in crate::upload) const INSTANT_LIVE_MANIFEST_403: Budget =
        budget(31, 18, 10, 0, 0, 13 + POOL_RACE);
    pub(in crate::upload) const INSTANT_LIVE_COMPLETION_500: Budget =
        budget(31, 18, 9, 0, 0, 15 + POOL_RACE);

    /// The relaunch session only: every segment and the thumbnail once.
    pub(in crate::upload) const QUEUE_RESUME_AFTER_QUIT_MID_TRANSFER: Budget =
        budget(22, 15, 2, 1_795_340, 0, 12 + POOL_RACE);
    pub(in crate::upload) const QUEUE_RESUME_AFTER_QUIT_DURING_VERIFICATION: Budget =
        budget(1, 0, 0, 0, 0, 1);
}

pub(super) fn check_budget(case: &str, mock: &MockCap, budget: Budget) {
    let stats = mock.stats();
    eprintln!("upload-verify {case}: {stats:?}");
    let over = [
        ("requests", stats.requests as u64, budget.requests as u64),
        (
            "media_puts",
            stats.media_puts as u64,
            budget.media_puts as u64,
        ),
        (
            "manifest_puts",
            stats.manifest_puts as u64,
            budget.manifest_puts as u64,
        ),
        (
            "reupload_bytes",
            stats.reupload_bytes,
            budget.reupload_bytes,
        ),
        ("orphans", stats.orphans as u64, budget.orphans as u64),
        (
            "connections",
            stats.connections as u64,
            budget.connections as u64,
        ),
    ]
    .into_iter()
    .filter(|(_, actual, limit)| actual > limit)
    .map(|(name, actual, limit)| format!("{name} {actual} > budget {limit}"))
    .collect::<Vec<_>>();
    assert!(
        over.is_empty(),
        "{case} is over budget: {over:?} ({stats:?})\nrequests (virtual time):\n{}",
        mock.timeline()
    );
}

pub(super) struct MockCap {
    pub(super) url: String,
    activity: Arc<Activity>,
    state: Arc<Mutex<MockState>>,
    cancel: Arc<AtomicBool>,
    tripped: Arc<tokio::sync::Notify>,
    accept: tokio::task::JoinHandle<()>,
}

impl Drop for MockCap {
    fn drop(&mut self) {
        self.accept.abort();
        for connection in self.lock().connections.drain(..) {
            connection.abort();
        }
    }
}

impl MockCap {
    pub(super) async fn start(rules: Vec<Rule>) -> Self {
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let url = format!("http://{}", listener.local_addr().unwrap());
        let state = Arc::new(Mutex::new(MockState {
            rules,
            epoch: Some(tokio::time::Instant::now()),
            ..Default::default()
        }));
        let cancel = Arc::new(AtomicBool::new(false));
        let tripped = Arc::new(tokio::sync::Notify::new());
        let activity: Arc<Activity> = ACTIVITY
            .with(|current| current.borrow().clone())
            .unwrap_or_default();
        let server = Server {
            activity: activity.clone(),
            url: url.clone(),
            state: state.clone(),
            cancel: cancel.clone(),
            tripped: tripped.clone(),
        };
        let accept = tokio::spawn(server.accept(listener));
        Self {
            url,
            activity,
            state,
            cancel,
            tripped,
            accept,
        }
    }

    fn lock(&self) -> std::sync::MutexGuard<'_, MockState> {
        self.state.lock().unwrap_or_else(|error| error.into_inner())
    }

    pub(super) fn cancel_flag(&self) -> Arc<AtomicBool> {
        self.cancel.clone()
    }

    pub(super) async fn tripped(&self) {
        self.tripped.notified().await;
    }

    pub(super) fn object(&self, key: &str) -> Option<Vec<u8>> {
        self.lock().objects.get(key).cloned()
    }

    pub(super) fn confirmations(&self) -> Vec<Value> {
        self.lock().confirmations.clone()
    }

    pub(super) fn created(&self) -> Vec<String> {
        self.lock().created.clone()
    }

    pub(super) fn deleted(&self) -> Vec<String> {
        self.lock().deleted.clone()
    }

    pub(super) fn deletes(&self) -> u32 {
        self.lock().deletes
    }

    pub(super) fn count(&self, method: &str, fragment: &str) -> usize {
        self.lock()
            .requests
            .iter()
            .filter(|(m, path, ..)| m == method && path.contains(fragment))
            .count()
    }

    pub(super) fn responses(&self, method: &str, fragment: &str) -> Vec<(u16, Duration)> {
        self.lock()
            .requests
            .iter()
            .filter(|(m, path, ..)| m == method && path.contains(fragment))
            .map(|(_, _, status, at)| (*status, *at))
            .collect()
    }

    pub(super) fn timeline(&self) -> String {
        self.lock()
            .requests
            .iter()
            .map(|(method, path, status, at)| {
                format!("{:>10.3}s {status:>3} {method} {path}\n", at.as_secs_f64())
            })
            .collect()
    }

    pub(super) fn assert_clean(&self) {
        let problems = self.lock().problems.clone();
        assert!(
            problems.is_empty(),
            "upload protocol problems: {problems:?}"
        );
    }

    /// Waits until the mock has been idle for the settle window, so requests
    /// a quit app already handed to the kernel are served before the next
    /// session starts counting.
    pub(super) async fn drain(&self) {
        while self.activity.busy() {
            tokio::time::sleep(Duration::from_millis(1)).await;
        }
    }

    /// Forget every request so far and drop every open connection, as a quit
    /// does. Stored objects survive, so a relaunch can be measured on its own.
    pub(super) fn new_session(&self) {
        let mut state = self.lock();
        for connection in state.connections.drain(..) {
            connection.abort();
        }
        state.requests.clear();
        state.media_put_bytes = 0;
        state.reset_on_accept = 0;
        state.accepted = 0;
        state.useful_offset = state.useful_bytes();
    }

    pub(super) fn stats(&self) -> Stats {
        let state = self.lock();
        let is_media = |path: &str| {
            path.starts_with("/s3mp/") || (path.starts_with("/s3/") && !is_metadata_object(path))
        };
        Stats {
            requests: state.requests.len(),
            media_puts: state
                .requests
                .iter()
                .filter(|(method, path, ..)| method == "PUT" && is_media(path))
                .count(),
            manifest_puts: state
                .requests
                .iter()
                .filter(|(method, path, ..)| method == "PUT" && path.ends_with("/manifest.json"))
                .count(),
            media_bytes: state.media_put_bytes,
            reupload_bytes: state
                .media_put_bytes
                .saturating_sub(state.useful_bytes().saturating_sub(state.useful_offset)),
            orphans: state
                .uploads
                .values()
                .filter(|upload| !upload.completed && !upload.aborted)
                .count(),
            connections: state.accepted,
            reset_on_accept: state.reset_on_accept,
        }
    }
}

#[derive(Clone)]
struct Server {
    activity: Arc<Activity>,
    url: String,
    state: Arc<Mutex<MockState>>,
    cancel: Arc<AtomicBool>,
    tripped: Arc<tokio::sync::Notify>,
}

struct Head {
    method: String,
    path: String,
    query: String,
    headers: HashMap<String, String>,
}

impl Server {
    fn lock(&self) -> std::sync::MutexGuard<'_, MockState> {
        self.state.lock().unwrap_or_else(|error| error.into_inner())
    }

    async fn accept(self, listener: TcpListener) {
        loop {
            let Ok((stream, _)) = listener.accept().await else {
                continue;
            };
            self.activity.touch();
            let _ = stream.set_zero_linger();
            let _ = stream.set_nodelay(true);
            let mut state = self.lock();
            if state
                .outage_until
                .is_some_and(|until| tokio::time::Instant::now() < until)
            {
                state.reset_on_accept += 1;
                continue;
            }
            state.accepted += 1;
            let connection = tokio::spawn(self.clone().connection(stream));
            state
                .connections
                .retain(|connection| !connection.is_finished());
            state.connections.push(connection.abort_handle());
        }
    }

    fn take_fault(&self, method: &str, path: &str) -> Option<Fault> {
        let mut state = self.lock();
        for rule in &mut state.rules {
            if rule.route.matches(method, path) {
                rule.seen += 1;
                if rule.seen > rule.skip && rule.fired < rule.count {
                    rule.fired += 1;
                    return Some(rule.fault);
                }
            }
        }
        None
    }

    fn record(&self, head: &Head, status: u16) {
        let mut state = self.lock();
        let at = state.epoch.map_or(Duration::ZERO, |epoch| epoch.elapsed());
        state
            .requests
            .push((head.method.clone(), head.path.clone(), status, at));
    }

    fn outage(&self, lasts: Duration) {
        let mut state = self.lock();
        state.outage_until = Some(tokio::time::Instant::now() + lasts);
        for connection in state.connections.drain(..) {
            connection.abort();
        }
    }

    async fn connection(self, mut stream: TcpStream) {
        let mut pending = Vec::new();
        while let Some(head) = read_head(&mut stream, &mut pending).await {
            let mut in_flight = Some(InFlight::new(&self.activity));
            let length = head
                .headers
                .get("content-length")
                .and_then(|value| value.parse::<usize>().ok())
                .unwrap_or(0);
            let media = head.method == "PUT"
                && (head.path.starts_with("/s3mp/")
                    || (head.path.starts_with("/s3/") && !is_metadata_object(&head.path)));
            let fault = self.take_fault(&head.method, &head.path);
            let cut = match fault {
                Some(Fault::Reset(at) | Fault::Outage(at, _) | Fault::Trip(at)) => {
                    ((length as f64) * at) as usize
                }
                _ => length,
            };
            let mut body = Vec::with_capacity(length);
            if !self
                .read_body(&mut stream, &mut pending, &mut body, cut, media)
                .await
            {
                self.record(&head, 0);
                return;
            }
            match fault {
                Some(Fault::Reset(_)) => {
                    self.record(&head, 0);
                    return;
                }
                Some(Fault::Outage(_, lasts)) => {
                    self.record(&head, 0);
                    self.outage(lasts);
                    return;
                }
                Some(Fault::Trip(_)) => {
                    self.cancel.store(true, Ordering::Release);
                    self.tripped.notify_one();
                    if !self
                        .read_body(&mut stream, &mut pending, &mut body, length, media)
                        .await
                    {
                        self.record(&head, 0);
                        return;
                    }
                }
                Some(Fault::Delay(delay)) => {
                    drop(in_flight.take());
                    tokio::time::sleep(delay).await;
                    in_flight = Some(InFlight::new(&self.activity));
                }
                Some(Fault::Hang) => {
                    self.record(&head, 0);
                    drop(in_flight.take());
                    std::future::pending::<()>().await;
                }
                Some(Fault::Status(status)) => {
                    self.record(&head, status);
                    if respond(&mut stream, status, &[], br#"{"error":"injected"}"#)
                        .await
                        .is_err()
                    {
                        return;
                    }
                    continue;
                }
                None => {}
            }
            let (status, headers, payload) = self.handle(&head, body);
            self.record(&head, status);
            if respond(&mut stream, status, &headers, &payload)
                .await
                .is_err()
            {
                return;
            }
            drop(in_flight);
        }
    }

    async fn read_body(
        &self,
        stream: &mut TcpStream,
        pending: &mut Vec<u8>,
        body: &mut Vec<u8>,
        until: usize,
        media: bool,
    ) -> bool {
        let mut buffer = vec![0u8; 64 * 1024];
        while body.len() < until {
            let wanted = until - body.len();
            let read = if pending.is_empty() {
                match stream.read(&mut buffer[..wanted.min(64 * 1024)]).await {
                    Ok(0) | Err(_) => return false,
                    Ok(read) => {
                        body.extend_from_slice(&buffer[..read]);
                        read
                    }
                }
            } else {
                let read = wanted.min(pending.len());
                body.extend(pending.drain(..read));
                read
            };
            self.activity.touch();
            if media {
                self.lock().media_put_bytes += read as u64;
            }
        }
        true
    }

    fn handle(&self, head: &Head, body: Vec<u8>) -> (u16, Vec<(String, String)>, Vec<u8>) {
        let mut guard = self.lock();
        let state = &mut *guard;
        let path = head.path.as_str();
        if head.method == "PUT" && path.starts_with("/s3mp/") {
            let mut parts = path["/s3mp/".len()..].splitn(2, '/');
            let upload_id = parts.next().unwrap_or_default().to_string();
            let number = parts
                .next()
                .and_then(|n| n.parse::<u32>().ok())
                .unwrap_or(0);
            let etag = to_hex(&cap_gpui_kernels::codec::md5(&body));
            if let Some(md5) = head.headers.get("content-md5") {
                let actual = base64::Engine::encode(
                    &base64::engine::general_purpose::STANDARD,
                    cap_gpui_kernels::codec::md5(&body),
                );
                if md5 != &actual {
                    state
                        .problems
                        .push(format!("Content-MD5 mismatch on {path}"));
                    return (400, Vec::new(), b"{}".to_vec());
                }
            }
            let Some(upload) = state.uploads.get_mut(&upload_id) else {
                state
                    .problems
                    .push(format!("part for unknown upload {path}"));
                return (404, Vec::new(), b"{}".to_vec());
            };
            if upload.completed || upload.aborted {
                state.problems.push(format!("part after completion {path}"));
                return (404, Vec::new(), b"{}".to_vec());
            }
            upload.parts.insert(number, (etag.clone(), body));
            return (
                200,
                vec![("ETag".into(), format!("\"{etag}\""))],
                Vec::new(),
            );
        }
        if head.method == "PUT" && path.starts_with("/s3/") {
            state.objects.insert(path["/s3/".len()..].to_string(), body);
            return (200, Vec::new(), Vec::new());
        }

        if head.headers.get("authorization").map(String::as_str)
            != Some(format!("Bearer {API_KEY}").as_str())
        {
            state
                .problems
                .push(format!("unauthenticated {} {path}", head.method));
            return (401, Vec::new(), b"{}".to_vec());
        }
        let request: Value = serde_json::from_slice(&body).unwrap_or(Value::Null);
        let query = head
            .query
            .split('&')
            .filter_map(|pair| pair.split_once('='))
            .map(|(key, value)| (key.to_string(), value.to_string()))
            .collect::<HashMap<_, _>>();
        let text = |value: &Value, key: &str| {
            value
                .get(key)
                .and_then(Value::as_str)
                .unwrap_or_default()
                .to_string()
        };
        let reply =
            |status: u16, value: Value| (status, Vec::new(), value.to_string().into_bytes());
        match (head.method.as_str(), path) {
            ("GET", "/api/desktop/video/new-id") => {
                state.next_id += 1;
                reply(
                    200,
                    json!({ "id": format!("verifyvideo{:04}", state.next_id) }),
                )
            }
            ("GET", "/api/desktop/video/create") => {
                let id = query.get("videoId").cloned().unwrap_or_else(|| {
                    state.next_id += 1;
                    format!("verifyvideo{:04}", state.next_id)
                });
                state.created.push(id.clone());
                reply(200, json!({ "id": id }))
            }
            ("POST", "/api/upload/multipart/initiate") => {
                state.next_id += 1;
                let upload_id = format!("upload{:04}", state.next_id);
                state.uploads.insert(
                    upload_id.clone(),
                    Multipart {
                        video_id: text(&request, "videoId"),
                        ..Default::default()
                    },
                );
                reply(200, json!({ "uploadId": upload_id }))
            }
            ("POST", "/api/upload/multipart/presign-part") => {
                let upload_id = text(&request, "uploadId");
                let number = request.get("partNumber").and_then(Value::as_u64);
                if !state.uploads.contains_key(&upload_id) || number.is_none() {
                    state.problems.push(format!("bad presign {request}"));
                    return reply(404, json!({}));
                }
                reply(
                    200,
                    json!({ "presignedUrl": format!("{}/s3mp/{upload_id}/{}", self.url, number.unwrap()) }),
                )
            }
            ("POST", "/api/upload/multipart/complete") => {
                let upload_id = text(&request, "uploadId");
                let listed = request
                    .get("parts")
                    .and_then(Value::as_array)
                    .cloned()
                    .unwrap_or_default();
                let Some(upload) = state.uploads.get(&upload_id) else {
                    state
                        .problems
                        .push(format!("complete for unknown upload {upload_id}"));
                    return reply(404, json!({}));
                };
                let mut problems = Vec::new();
                let mut object = Vec::new();
                for (position, part) in listed.iter().enumerate() {
                    let number = part.get("partNumber").and_then(Value::as_u64).unwrap_or(0);
                    if number != position as u64 + 1 {
                        problems.push(format!("part {number} listed at {position}"));
                    }
                    match upload.parts.get(&(number as u32)) {
                        Some((etag, bytes))
                            if Some(etag.as_str()) == part.get("etag").and_then(Value::as_str) =>
                        {
                            if part.get("size").and_then(Value::as_u64) != Some(bytes.len() as u64)
                            {
                                problems.push(format!("part {number} size mismatch"));
                            }
                            object.extend_from_slice(bytes);
                        }
                        Some(_) => problems.push(format!("part {number} etag mismatch")),
                        None => problems.push(format!("part {number} missing")),
                    }
                }
                if upload.parts.len() != listed.len() {
                    problems.push(format!(
                        "{} parts stored, {} listed",
                        upload.parts.len(),
                        listed.len()
                    ));
                }
                if !problems.is_empty() {
                    state.problems.extend(problems);
                    return reply(400, json!({ "error": "invalid parts" }));
                }
                state
                    .objects
                    .insert(format!("{}/result.mp4", upload.video_id), object);
                state.uploads.get_mut(&upload_id).unwrap().completed = true;
                reply(200, json!({ "success": true }))
            }
            ("POST", "/api/upload/multipart/abort") => {
                if let Some(upload) = state.uploads.get_mut(&text(&request, "uploadId")) {
                    upload.aborted = true;
                }
                reply(200, json!({ "success": true }))
            }
            ("POST", "/api/upload/signed") => {
                let url = format!(
                    "{}/s3/{}/{}",
                    self.url,
                    text(&request, "videoId"),
                    text(&request, "subpath")
                );
                reply(
                    200,
                    json!({ "presignedPutData": { "url": url, "headers": {} } }),
                )
            }
            ("POST", "/api/upload/signed/batch") => {
                let video_id = text(&request, "videoId");
                let urls = request
                    .get("subpaths")
                    .and_then(Value::as_array)
                    .into_iter()
                    .flatten()
                    .filter_map(Value::as_str)
                    .map(|subpath| {
                        (
                            subpath.to_string(),
                            Value::String(format!("{}/s3/{video_id}/{subpath}", self.url)),
                        )
                    })
                    .collect::<serde_json::Map<_, _>>();
                reply(200, json!({ "urls": urls }))
            }
            ("POST", "/api/recording/prepare") => reply(404, json!({ "error": "not found" })),
            ("POST", "/api/upload/recording-complete") => {
                state.confirmations.push(request.clone());
                let video_id = text(&request, "videoId");
                let Some(verification) = request.get("verification") else {
                    return reply(200, json!({ "success": true }));
                };
                match state.verify_segments(&video_id, verification) {
                    Ok(receipt) => reply(
                        200,
                        json!({ "success": true, "status": "verified", "verification": receipt }),
                    ),
                    Err(problem) => {
                        state.problems.push(problem);
                        reply(409, json!({ "status": "reupload-required" }))
                    }
                }
            }
            ("DELETE", "/api/desktop/video/delete") => {
                state.deletes += 1;
                if let Some(id) = query.get("videoId") {
                    state.deleted.push(id.clone());
                }
                reply(200, json!({ "success": true }))
            }
            _ => {
                state
                    .problems
                    .push(format!("unknown route {} {path}", head.method));
                reply(404, json!({}))
            }
        }
    }
}

fn is_metadata_object(path: &str) -> bool {
    path.ends_with("/manifest.json") || path.ends_with("/recording.json")
}

impl MockState {
    fn useful_bytes(&self) -> u64 {
        self.objects
            .iter()
            .filter(|(key, _)| !is_metadata_object(key))
            .map(|(_, object)| object.len() as u64)
            .sum()
    }

    fn verify_segments(&self, video_id: &str, verification: &Value) -> Result<Value, String> {
        let claimed: UploadVerification = serde_json::from_value(verification.clone())
            .map_err(|error| format!("unreadable verification: {error}"))?;
        let stored = self
            .objects
            .get(&format!("{video_id}/segments/manifest.json"))
            .ok_or("verification without a stored manifest")?;
        if UploadVerification::segments(stored, claimed.required_audio) != claimed {
            return Err("verification does not match the stored manifest".into());
        }
        let manifest: SegmentUploadManifest =
            serde_json::from_slice(stored).map_err(|error| error.to_string())?;
        if !manifest.is_complete || !manifest.video_init_uploaded {
            return Err("stored manifest is not complete".into());
        }
        let mut keys = vec!["segments/video/init.mp4".to_string()];
        if manifest.audio_init_uploaded {
            keys.push("segments/audio/init.mp4".into());
        }
        keys.extend(
            manifest
                .video_segments
                .iter()
                .map(|s| format!("segments/video/segment_{:03}.m4s", s.index)),
        );
        keys.extend(
            manifest
                .audio_segments
                .iter()
                .map(|s| format!("segments/audio/segment_{:03}.m4s", s.index)),
        );
        let mut size = 0;
        for key in keys {
            size += self
                .objects
                .get(&format!("{video_id}/{key}"))
                .ok_or_else(|| format!("manifest lists missing object {key}"))?
                .len() as u64;
        }
        Ok(json!({
            "version": 1,
            "videoId": video_id,
            "artifact": claimed.artifact,
            "fileSize": size,
            "duration": manifest.video_segments.iter().map(|s| s.duration).sum::<f64>(),
            "hasAudio": !manifest.audio_segments.is_empty(),
            "fullDecode": true,
            "requiredAudioVerified": claimed.required_audio && !manifest.audio_segments.is_empty(),
        }))
    }
}

async fn read_head(stream: &mut TcpStream, pending: &mut Vec<u8>) -> Option<Head> {
    let mut buffer = [0u8; 16 * 1024];
    let end = loop {
        if let Some(end) = pending.windows(4).position(|window| window == b"\r\n\r\n") {
            break end;
        }
        match stream.read(&mut buffer).await {
            Ok(0) | Err(_) => return None,
            Ok(read) => pending.extend_from_slice(&buffer[..read]),
        }
    };
    let text = String::from_utf8_lossy(&pending[..end]).into_owned();
    pending.drain(..end + 4);
    let mut lines = text.split("\r\n");
    let mut request = lines.next()?.split(' ');
    let method = request.next()?.to_string();
    let target = request.next()?;
    let (path, query) = target.split_once('?').unwrap_or((target, ""));
    let headers = lines
        .filter_map(|line| line.split_once(':'))
        .map(|(name, value)| (name.trim().to_ascii_lowercase(), value.trim().to_string()))
        .collect();
    Some(Head {
        method,
        path: path.to_string(),
        query: query.to_string(),
        headers,
    })
}

async fn respond(
    stream: &mut TcpStream,
    status: u16,
    headers: &[(String, String)],
    body: &[u8],
) -> std::io::Result<()> {
    let mut response = format!(
        "HTTP/1.1 {status} Mock\r\nContent-Type: application/json\r\nContent-Length: {}\r\n",
        body.len()
    );
    for (name, value) in headers {
        response.push_str(&format!("{name}: {value}\r\n"));
    }
    response.push_str("\r\n");
    let mut bytes = response.into_bytes();
    bytes.extend_from_slice(body);
    stream.write_all(&bytes).await
}

fn to_hex(bytes: &[u8]) -> String {
    bytes.iter().map(|byte| format!("{byte:02x}")).collect()
}

pub(super) fn noise(seed: u64, len: usize) -> Vec<u8> {
    let mut state = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1;
    let mut bytes = Vec::with_capacity(len + 8);
    while bytes.len() < len {
        state ^= state << 13;
        state ^= state >> 7;
        state ^= state << 17;
        bytes.extend_from_slice(&state.to_le_bytes());
    }
    bytes.truncate(len);
    bytes
}

pub(super) struct Scratch(pub(super) PathBuf);

impl Scratch {
    pub(super) fn new(tag: &str) -> Self {
        let root = std::env::temp_dir().join(format!(
            "cap-upload-verify-{tag}-{}",
            crate::store::new_uuid_v4()
        ));
        std::fs::create_dir_all(&root).unwrap();
        Self(root)
    }

    /// Signs the test thread in to `server` through the thread-local store.
    pub(super) fn sign_in(&self, server: &str) {
        let path = self.0.join("store");
        std::fs::write(
            &path,
            serde_json::to_vec(&json!({
                "general_settings": {
                    "serverUrl": server,
                    "enableNotifications": false,
                    "hasCompletedOnboarding": true,
                },
                "auth": {
                    "secret": { "api_key": API_KEY },
                    "user_id": OWNER,
                    "plan": { "upgraded": true, "manual": true },
                    "organizations": [],
                },
            }))
            .unwrap(),
        )
        .unwrap();
        crate::store::set_test_tauri_store_path(Some(path));
    }
}

impl Drop for Scratch {
    fn drop(&mut self) {
        crate::store::set_test_tauri_store_path(None);
        let _ = std::fs::remove_dir_all(&self.0);
    }
}

pub(super) fn write_thumbnail(path: &Path) {
    std::fs::create_dir_all(path.parent().unwrap()).unwrap();
    image::RgbImage::from_fn(64, 36, |x, y| {
        image::Rgb([(x * 4) as u8, (y * 7) as u8, 90])
    })
    .save(path)
    .unwrap();
}

fn studio_project(root: &Path) -> PathBuf {
    let project = root.join("share.cap");
    std::fs::create_dir_all(project.join("output")).unwrap();
    std::fs::write(
        project.join("recording-meta.json"),
        r#"{"pretty_name":"Upload verify","sharing":null,"segments":[{"display":{"path":"content/segments/segment-0/display.mp4","fps":30}}]}"#,
    )
    .unwrap();
    let mut file = SAMPLE_MP4.to_vec();
    let pad = STUDIO_SIZE - file.len();
    file.extend_from_slice(&(pad as u32).to_be_bytes());
    file.extend_from_slice(b"free");
    file.extend(noise(0x5743_4150, pad - 8));
    std::fs::write(project.join("output/result.mp4"), file).unwrap();
    write_thumbnail(&project.join("screenshots/display.jpg"));
    project
}

/// A fragmented-MP4-shaped segment: a `moof` box then an `mdat` box, sized
/// and filled deterministically.
fn fragment(seed: u64, size: usize) -> Vec<u8> {
    let mut bytes = Vec::with_capacity(size);
    bytes.extend_from_slice(&32u32.to_be_bytes());
    bytes.extend_from_slice(b"moof");
    bytes.extend(noise(seed, 24));
    bytes.extend_from_slice(&((size - 32) as u32).to_be_bytes());
    bytes.extend_from_slice(b"mdat");
    bytes.extend(noise(seed ^ 0xA5A5, size - 40));
    bytes
}

/// Writes `content/display` and `content/audio` the way the Instant recorder
/// leaves them and returns their segment events, init first.
pub(super) fn instant_tracks(
    project: &Path,
    pairs: u32,
) -> (Vec<SegmentCompletedEvent>, Vec<SegmentCompletedEvent>) {
    let track = |name: &str, media: SegmentMediaType, size: usize, version: u32, kind: &str| {
        let dir = project.join("content").join(name);
        std::fs::create_dir_all(&dir).unwrap();
        let init = dir.join("init.mp4");
        std::fs::write(&init, SAMPLE_MP4).unwrap();
        let mut events = vec![SegmentCompletedEvent {
            path: init,
            index: 0,
            duration: 0.0,
            file_size: SAMPLE_MP4.len() as u64,
            is_init: true,
            media_type: media,
        }];
        let mut listed = Vec::new();
        for index in 1..=pairs {
            let bytes = fragment(
                u64::from(index) * 2 + (media == SegmentMediaType::Audio) as u64,
                size + index as usize * 7,
            );
            let path = dir.join(format!("segment_{index:03}.m4s"));
            std::fs::write(&path, &bytes).unwrap();
            listed.push(json!({
                "path": format!("segment_{index:03}.m4s"),
                "index": index,
                "duration": 2.0,
                "is_complete": true,
                "file_size": bytes.len(),
            }));
            events.push(SegmentCompletedEvent {
                path,
                index,
                duration: 2.0,
                file_size: bytes.len() as u64,
                is_init: false,
                media_type: media,
            });
        }
        std::fs::write(
            dir.join("manifest.json"),
            serde_json::to_vec(&json!({
                "version": version,
                "type": kind,
                "init_segment": "init.mp4",
                "segments": listed,
                "is_complete": true,
            }))
            .unwrap(),
        )
        .unwrap();
        events
    };
    (
        track(
            "display",
            SegmentMediaType::Video,
            VIDEO_SEGMENT,
            5,
            "m4s_segments",
        ),
        track(
            "audio",
            SegmentMediaType::Audio,
            AUDIO_SEGMENT,
            2,
            "m4s_audio_segments",
        ),
    )
}

pub(super) fn expected_manifest(
    video: &[SegmentCompletedEvent],
    audio: &[SegmentCompletedEvent],
) -> SegmentUploadManifest {
    let mut manifest = SegmentUploadManifest::default();
    for event in video.iter().chain(audio) {
        manifest.record(event);
    }
    manifest.is_complete = true;
    manifest
}

/// Every local segment is stored byte for byte and the final manifest lists
/// exactly the local recording.
pub(super) fn assert_segments_stored(
    mock: &MockCap,
    video_id: &str,
    video: &[SegmentCompletedEvent],
    audio: &[SegmentCompletedEvent],
) {
    for event in video.iter().chain(audio) {
        let key = format!("{video_id}/{}", segment_subpath(event));
        let stored = mock.object(&key);
        assert!(
            stored.as_deref() == Some(std::fs::read(&event.path).unwrap().as_slice()),
            "{key} is missing or differs from the local file"
        );
    }
    let manifest: SegmentUploadManifest = serde_json::from_slice(
        &mock
            .object(&format!("{video_id}/segments/manifest.json"))
            .expect("no manifest uploaded"),
    )
    .unwrap();
    assert_eq!(manifest, expected_manifest(video, audio));
}

struct StudioRun {
    result: Result<UploadResult, String>,
    progress: Vec<f64>,
}

async fn share(project: &Path, cancel: Arc<AtomicBool>) -> StudioRun {
    let progress = Arc::new(Mutex::new(Vec::new()));
    let sink = progress.clone();
    let result = upload_exported_video(
        project.to_path_buf(),
        None,
        move |value| sink.lock().unwrap().push(value),
        cancel,
    )
    .await;
    let progress = progress.lock().unwrap().clone();
    assert!(
        progress.first() == Some(&0.0)
            && progress.windows(2).all(|pair| pair[1] >= pair[0])
            && progress.iter().all(|value| (0.0..=1.0).contains(value)),
        "progress is not monotonic within 0..=1: {progress:?}"
    );
    // 100% means shared: only the final report after completion may be 1.0.
    assert!(
        progress
            .split_last()
            .is_none_or(|(_, earlier)| earlier.iter().all(|value| *value < 1.0))
            && (result.is_ok() || progress.iter().all(|value| *value < 1.0)),
        "progress reached 100% before the upload completed: {progress:?}"
    );
    StudioRun { result, progress }
}

fn assert_shared(mock: &MockCap, project: &Path, run: &StudioRun) {
    let meta = RecordingMeta::load_for_project(project).unwrap();
    let sharing = meta.sharing.as_ref().expect("no sharing state persisted");
    assert_eq!(
        run.result,
        Ok(UploadResult::Success(format!(
            "{}/s/{}",
            mock.url, sharing.id
        )))
    );
    assert_eq!(run.progress.last(), Some(&1.0));
    assert!(matches!(meta.upload, Some(UploadMeta::Complete)));
    let local = std::fs::read(project.join("output/result.mp4")).unwrap();
    let stored = mock
        .object(&format!("{}/result.mp4", sharing.id))
        .expect("no object stored");
    assert!(
        stored == local,
        "stored object ({} bytes, sha256 {}) differs from the local file ({} bytes, sha256 {})",
        stored.len(),
        to_hex(&Sha256::digest(&stored)),
        local.len(),
        to_hex(&Sha256::digest(&local)),
    );
    assert!(
        mock.object(&format!("{}/screenshot/screen-capture.jpg", sharing.id))
            .is_some_and(|thumbnail| thumbnail.starts_with(&[0xFF, 0xD8])),
        "no JPEG thumbnail stored"
    );
    mock.assert_clean();
}

fn assert_share_failed(project: &Path, run: &StudioRun, error: &str) {
    let failure = run
        .result
        .as_ref()
        .expect_err("share unexpectedly succeeded");
    assert!(failure.contains(error), "unexpected error: {failure}");
    let meta = RecordingMeta::load_for_project(project).unwrap();
    assert!(
        matches!(meta.upload, Some(UploadMeta::SinglePartUpload { .. })),
        "the reserved video id must stay persisted for a retry: {:?}",
        meta.upload
    );
    assert!(meta.sharing.is_none());
}

fn studio_case(
    case: &str,
    rules: Vec<Rule>,
    budget: Budget,
    check: impl FnOnce(&MockCap, &Path, StudioRun),
) {
    ffmpeg::init().unwrap();
    let scratch = Scratch::new(case);
    virtual_time(async {
        let mock = MockCap::start(rules).await;
        scratch.sign_in(&mock.url);
        let project = studio_project(&scratch.0);
        let started = tokio::time::Instant::now();
        let run = share(&project, Arc::new(AtomicBool::new(false))).await;
        eprintln!("upload-verify {case}: virtual {:?}", started.elapsed());
        check(&mock, &project, run);
        check_budget(case, &mock, budget);
    });
}

#[test]
fn studio_share_clean() {
    studio_case(
        "studio_share_clean",
        Vec::new(),
        budget::STUDIO_SHARE_CLEAN,
        |mock, project, run| {
            assert_shared(mock, project, &run);
            assert_eq!(mock.count("PUT", "/s3mp/"), 6);
        },
    );
}

#[test]
fn studio_share_part_503_retries_the_part_at_the_end() {
    studio_case(
        "studio_share_part_503",
        vec![rule(Route::Part(3), Fault::Status(503))],
        budget::STUDIO_SHARE_PART_503,
        |mock, project, run| assert_shared(mock, project, &run),
    );
}

#[test]
fn studio_share_reset_mid_part_retries_in_place() {
    studio_case(
        "studio_share_reset_mid_part",
        vec![rule(Route::Part(4), Fault::Reset(0.5))],
        budget::STUDIO_SHARE_RESET_MID_PART,
        |mock, project, run| assert_shared(mock, project, &run),
    );
}

/// The network drops while part 2 is half sent and stays down for 5 s. Main
/// has only that part in flight, so it loses half a part and retries it after
/// the outage (2 s, then 4 s backoff).
#[test]
fn studio_share_outage_mid_part() {
    studio_case(
        "studio_share_outage_mid_part",
        vec![rule(
            Route::Part(2),
            Fault::Outage(0.5, Duration::from_secs(5)),
        )],
        budget::STUDIO_SHARE_OUTAGE_MID_PART,
        |mock, project, run| {
            assert_shared(mock, project, &run);
            assert!(mock.stats().reset_on_accept >= 1);
        },
    );
}

#[test]
fn studio_share_presign_500_retries_the_part_at_the_end() {
    studio_case(
        "studio_share_presign_500",
        vec![Rule {
            skip: 3,
            ..rule(Route::Api("multipart/presign-part"), Fault::Status(500))
        }],
        budget::STUDIO_SHARE_PRESIGN_500,
        |mock, project, run| assert_shared(mock, project, &run),
    );
}

/// A part whose response never arrives hits the 5 minute request timeout
/// (virtual) and is retried in place.
#[test]
fn studio_share_hung_part_times_out_and_retries() {
    studio_case(
        "studio_share_hung_part",
        vec![rule(Route::Part(5), Fault::Hang)],
        budget::STUDIO_SHARE_HUNG_PART,
        |mock, project, run| assert_shared(mock, project, &run),
    );
}

#[test]
fn studio_share_slow_part_costs_nothing_extra() {
    studio_case(
        "studio_share_slow_part",
        vec![rule(Route::Part(3), Fault::Delay(Duration::from_secs(40)))],
        budget::STUDIO_SHARE_SLOW_PART,
        |mock, project, run| assert_shared(mock, project, &run),
    );
}

#[test]
fn studio_share_permanent_403_fails_with_the_part_error() {
    studio_case(
        "studio_share_permanent_403",
        vec![rule(Route::Part(2), Fault::Status(403)).times(u32::MAX)],
        budget::STUDIO_SHARE_PERMANENT_403,
        |mock, project, run| {
            assert_share_failed(project, &run, "uploader/part/2/status/403");
            assert_eq!(mock.count("POST", "multipart/complete"), 0);
            mock.assert_clean();
        },
    );
}

#[test]
fn studio_share_complete_500_fails_and_keeps_the_video_id() {
    studio_case(
        "studio_share_complete_500",
        vec![rule(Route::Api("multipart/complete"), Fault::Status(500))],
        budget::STUDIO_SHARE_COMPLETE_500,
        |mock, project, run| {
            assert_share_failed(project, &run, "upload_multipart_complete/500");
            mock.assert_clean();
        },
    );
}

/// Cancel (or quit) while part 3 is half sent, then share again. The second
/// attempt reuses the persisted video id and starts a new multipart upload.
#[test]
fn studio_share_cancel_then_resume_reuses_the_persisted_video() {
    ffmpeg::init().unwrap();
    let case = "studio_share_cancel_then_resume";
    let scratch = Scratch::new(case);
    virtual_time(async {
        let mock = MockCap::start(vec![rule(Route::Part(3), Fault::Trip(0.5))]).await;
        scratch.sign_in(&mock.url);
        let project = studio_project(&scratch.0);
        let first = share(&project, mock.cancel_flag()).await;
        assert_share_failed(&project, &first, "Export cancelled");
        let reserved = match RecordingMeta::load_for_project(&project).unwrap().upload {
            Some(UploadMeta::SinglePartUpload { video_id, .. }) => video_id,
            other => panic!("unexpected upload state {other:?}"),
        };
        let second = share(&project, Arc::new(AtomicBool::new(false))).await;
        assert_shared(&mock, &project, &second);
        assert_eq!(
            RecordingMeta::load_for_project(&project)
                .unwrap()
                .sharing
                .unwrap()
                .id,
            reserved
        );
        assert_eq!(mock.count("GET", "video/new-id"), 1);
        check_budget(case, &mock, budget::STUDIO_SHARE_CANCEL_THEN_RESUME);
    });
}

/// Feeds the recorder's segment events on a fixed virtual schedule: both init
/// segments at 0 s, then audio segment k at 2k s and video segment k at
/// 2k + 0.6 s, so no two segments are in flight at the same moment unless a
/// retry puts them there.
async fn feed(
    video: Vec<SegmentCompletedEvent>,
    audio: Vec<SegmentCompletedEvent>,
    events: flume::Sender<SegmentCompletedEvent>,
) {
    let start = tokio::time::Instant::now();
    let _ = events.send(video[0].clone());
    let _ = events.send(audio[0].clone());
    for k in 1..video.len() {
        let at = start + Duration::from_secs(2 * k as u64);
        tokio::time::sleep_until(at).await;
        let _ = events.send(audio[k].clone());
        tokio::time::sleep_until(at + Duration::from_millis(600)).await;
        let _ = events.send(video[k].clone());
    }
}

struct InstantRun {
    result: Result<(), String>,
    elapsed: Duration,
    video: Vec<SegmentCompletedEvent>,
    audio: Vec<SegmentCompletedEvent>,
}

fn instant_case(
    case: &str,
    rules: Vec<Rule>,
    budget: Budget,
    check: impl FnOnce(&MockCap, InstantRun),
) {
    let scratch = Scratch::new(case);
    virtual_time(async {
        let mock = MockCap::start(rules).await;
        scratch.sign_in(&mock.url);
        let (video, audio) = instant_tracks(&scratch.0, INSTANT_PAIRS);
        let (sender, events) = flume::unbounded();
        let feeder = tokio::spawn(feed(video.clone(), audio.clone(), sender));
        let started = tokio::time::Instant::now();
        let result = upload_segments(
            &LiveSegmentTransport::default(),
            INSTANT_VIDEO_ID,
            events,
            mock.cancel_flag(),
            None,
        )
        .await;
        let elapsed = started.elapsed();
        eprintln!("upload-verify {case}: virtual {elapsed:?}");
        feeder.abort();
        check(
            &mock,
            InstantRun {
                result,
                elapsed,
                video,
                audio,
            },
        );
        check_budget(case, &mock, budget);
    });
}

fn assert_instant_complete(mock: &MockCap, run: &InstantRun) {
    assert_eq!(run.result, Ok(()));
    assert_segments_stored(mock, INSTANT_VIDEO_ID, &run.video, &run.audio);
    let confirmations = mock.confirmations();
    assert_eq!(confirmations.len(), 1, "{confirmations:?}");
    assert_eq!(confirmations[0], json!({ "videoId": INSTANT_VIDEO_ID }));
    mock.assert_clean();
}

#[test]
fn instant_live_clean() {
    instant_case(
        "instant_live_clean",
        Vec::new(),
        budget::INSTANT_LIVE_CLEAN,
        |mock, run| {
            assert_instant_complete(mock, &run);
        },
    );
}

fn manifest_presigns(mock: &MockCap) -> usize {
    mock.lock()
        .requests
        .iter()
        .filter(|(method, path, ..)| method == "POST" && path == "/api/upload/signed")
        .count()
}

#[test]
fn instant_live_manifests_reuse_the_batch_presigned_url() {
    instant_case(
        "instant_live_manifest_reuse",
        Vec::new(),
        budget::INSTANT_LIVE_CLEAN,
        |mock, run| {
            assert_instant_complete(mock, &run);
            assert!(mock.count("PUT", "segments/manifest.json") > 1);
            assert_eq!(manifest_presigns(mock), 0, "{}", mock.timeline());
        },
    );
}

#[test]
fn instant_live_manifest_signs_a_new_url_when_the_reused_one_fails() {
    instant_case(
        "instant_live_manifest_reuse_rejected",
        vec![rule(
            Route::Object("segments/manifest.json"),
            Fault::Status(403),
        )],
        budget::INSTANT_LIVE_MANIFEST_403,
        |mock, run| {
            assert_instant_complete(mock, &run);
            assert_eq!(manifest_presigns(mock), 1, "{}", mock.timeline());
        },
    );
}

/// A reused URL that times out is not retried at once with a fresh one: the
/// attempt fails and the manifest retry loop backs off first, so a hung
/// network costs one request timeout per attempt, as before.
#[test]
fn instant_live_manifest_timeout_on_the_reused_url_backs_off_before_resigning() {
    instant_case(
        "instant_live_manifest_reuse_timeout",
        vec![rule(Route::Object("segments/manifest.json"), Fault::Hang)],
        budget::INSTANT_LIVE_MANIFEST_403,
        |mock, run| {
            assert_instant_complete(mock, &run);
            let hung = mock
                .responses("PUT", "segments/manifest.json")
                .into_iter()
                .find(|(status, _)| *status == 0)
                .map(|(_, at)| at)
                .expect("no hung manifest PUT");
            let resigned = mock
                .responses("POST", "/api/upload/signed")
                .into_iter()
                .filter(|(_, at)| *at > hung)
                .map(|(_, at)| at)
                .min()
                .unwrap_or_else(|| panic!("manifest was never re-signed:\n{}", mock.timeline()));
            assert!(
                resigned >= hung + Duration::from_secs(5 * 60) + Duration::from_millis(200),
                "re-signed without backing off:\n{}",
                mock.timeline()
            );
        },
    );
}

#[test]
fn instant_live_manifest_url_is_resigned_before_it_can_expire() {
    let scratch = Scratch::new("instant_live_manifest_url_expiry");
    virtual_time(async {
        let mock = MockCap::start(Vec::new()).await;
        scratch.sign_in(&mock.url);
        let (video, audio) = instant_tracks(&scratch.0, INSTANT_PAIRS);
        let (sender, events) = flume::unbounded();
        let feeder = tokio::spawn({
            let (video, audio) = (video.clone(), audio.clone());
            async move {
                let start = tokio::time::Instant::now();
                let _ = sender.send(video[0].clone());
                let _ = sender.send(audio[0].clone());
                for k in 1..video.len() {
                    tokio::time::sleep_until(start + Duration::from_secs(100 * k as u64)).await;
                    let _ = sender.send(audio[k].clone());
                    let _ = sender.send(video[k].clone());
                }
            }
        });
        let result = upload_segments(
            &LiveSegmentTransport::default(),
            INSTANT_VIDEO_ID,
            events,
            mock.cancel_flag(),
            None,
        )
        .await;
        feeder.await.unwrap();
        assert_eq!(result, Ok(()));
        assert_segments_stored(&mock, INSTANT_VIDEO_ID, &video, &audio);
        assert_eq!(manifest_presigns(&mock), 1, "{}", mock.timeline());
        mock.assert_clean();
    });
}

#[test]
fn only_object_store_urls_are_reused_for_manifests() {
    let target = |url: &str| SignedUploadTarget {
        url: url.into(),
        headers: HashMap::new(),
    };
    let now = tokio::time::Instant::now();
    assert!(
        ReusableUploadTarget::new(
            target("https://bucket.s3.us-east-1.amazonaws.com/o/v/segments/manifest.json?X-Amz-Signature=1"),
            now,
        )
        .is_some()
    );
    assert!(
        ReusableUploadTarget::new(
            target(
                "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&upload_id=1"
            ),
            now,
        )
        .is_none()
    );
}

#[test]
fn instant_live_segment_503_retries_in_call() {
    instant_case(
        "instant_live_segment_503",
        vec![rule(
            Route::Object("segments/video/segment_003"),
            Fault::Status(503),
        )],
        budget::INSTANT_LIVE_SEGMENT_503,
        |mock, run| assert_instant_complete(mock, &run),
    );
}

/// Three 503s exhaust the in-call attempts; the segment is deferred and
/// retried in place instead of failing the live upload.
#[test]
fn instant_live_segment_503_three_times_is_deferred_not_fatal() {
    instant_case(
        "instant_live_segment_503x3",
        vec![
            rule(
                Route::Object("segments/video/segment_003"),
                Fault::Status(503),
            )
            .times(3),
        ],
        budget::INSTANT_LIVE_SEGMENT_503X3,
        |mock, run| assert_instant_complete(mock, &run),
    );
}

#[test]
fn instant_live_segment_reset_mid_body() {
    instant_case(
        "instant_live_segment_reset",
        vec![
            rule(
                Route::Object("segments/video/segment_004"),
                Fault::Reset(0.5),
            )
            .times(3),
        ],
        budget::INSTANT_LIVE_SEGMENT_RESET,
        |mock, run| assert_instant_complete(mock, &run),
    );
}

#[test]
fn instant_live_deferred_retries_back_off_after_recording_ends() {
    let scratch = Scratch::new("instant_live_deferred_retries_back_off");
    virtual_time(async {
        let mock = MockCap::start(vec![
            rule(
                Route::Object("segments/video/segment_008"),
                Fault::Status(503),
            )
            .times(12),
        ])
        .await;
        scratch.sign_in(&mock.url);
        let (video, audio) = instant_tracks(&scratch.0, INSTANT_PAIRS);
        let (sender, events) = flume::unbounded();
        let feeder = tokio::spawn(feed(video.clone(), audio.clone(), sender));
        let result = upload_segments(
            &LiveSegmentTransport::default(),
            INSTANT_VIDEO_ID,
            events,
            mock.cancel_flag(),
            None,
        )
        .await;
        feeder.await.unwrap();
        assert_eq!(result, Ok(()));
        assert_segments_stored(&mock, INSTANT_VIDEO_ID, &video, &audio);
        assert_eq!(mock.confirmations().len(), 1);
        let attempts = mock
            .lock()
            .requests
            .iter()
            .filter(|(method, path, ..)| {
                method == "PUT" && path.ends_with("segments/video/segment_008.m4s")
            })
            .map(|(_, _, status, at)| (*status, *at))
            .collect::<Vec<_>>();
        assert_eq!(attempts.len(), 13, "{}", mock.timeline());
        assert!(attempts[..12].iter().all(|(status, _)| *status == 503));
        assert_eq!(attempts[12].0, 200);
        for (round, seconds) in [2, 4, 8, 16].into_iter().enumerate() {
            let last_failure = attempts[round * 3 + 2].1;
            let next_attempt = attempts[round * 3 + 3].1;
            assert!(
                next_attempt.saturating_sub(last_failure) >= Duration::from_secs(seconds),
                "deferred retry round {round} did not wait {seconds}s:\n{}",
                mock.timeline()
            );
        }
        mock.assert_clean();
    });
}

/// The network drops while video segment 3 is half sent and stays down for
/// 5.7 s, past the in-call attempts. The live upload survives it.
#[test]
fn instant_live_outage_mid_segment() {
    instant_case(
        "instant_live_outage",
        vec![rule(
            Route::Object("segments/video/segment_003"),
            Fault::Outage(0.5, Duration::from_millis(5_700)),
        )],
        budget::INSTANT_LIVE_OUTAGE,
        |mock, run| {
            assert_instant_complete(mock, &run);
            assert!(mock.stats().reset_on_accept >= 1);
        },
    );
}

#[test]
fn instant_live_slow_segment() {
    instant_case(
        "instant_live_slow_segment",
        vec![rule(
            Route::Object("segments/video/segment_002"),
            Fault::Delay(Duration::from_millis(9_300)),
        )],
        budget::INSTANT_LIVE_SLOW_SEGMENT,
        |mock, run| assert_instant_complete(mock, &run),
    );
}

/// An outage that outlasts the 5 minute retry window fails the live upload
/// with the first failure once the window has passed, leaving the rest to the
/// queue.
#[test]
fn instant_live_outage_past_the_retry_window_fails() {
    instant_case(
        "instant_live_outage_past_window",
        vec![rule(
            Route::Object("segments/video/segment_003"),
            Fault::Outage(0.5, Duration::from_secs(6 * 60)),
        )],
        budget::INSTANT_LIVE_OUTAGE_PAST_WINDOW,
        |mock, run| {
            let error = run.result.expect_err("upload outlived its retry window");
            assert!(
                error.contains("segments/video/segment_003.m4s"),
                "unexpected error: {error}"
            );
            assert!(
                (RETRY_WINDOW..RETRY_WINDOW + Duration::from_secs(30)).contains(&run.elapsed),
                "gave up after {:?}, not once the retry window had passed",
                run.elapsed
            );
            assert!(mock.confirmations().is_empty());
            mock.assert_clean();
        },
    );
}

#[test]
fn instant_live_permanent_403_is_fatal() {
    instant_case(
        "instant_live_permanent_403",
        vec![
            rule(
                Route::Object("segments/video/segment_003"),
                Fault::Status(403),
            )
            .times(u32::MAX),
        ],
        budget::INSTANT_LIVE_PERMANENT_403,
        |mock, run| {
            let error = run.result.expect_err("a 403 must not be retried forever");
            assert!(
                error.contains("segment_003.m4s") && error.contains("403"),
                "unexpected error: {error}"
            );
            assert!(mock.confirmations().is_empty());
            mock.assert_clean();
        },
    );
}

#[test]
fn instant_live_cancel_stops_without_completing() {
    instant_case(
        "instant_live_cancel",
        vec![rule(
            Route::Object("segments/video/segment_004"),
            Fault::Trip(0.5),
        )],
        budget::INSTANT_LIVE_CANCEL,
        |mock, run| {
            assert_eq!(
                run.result,
                Err("Instant recording upload cancelled".to_string())
            );
            assert!(mock.confirmations().is_empty());
            assert_eq!(mock.count("PUT", "segments/video/segment_005"), 0);
            mock.assert_clean();
        },
    );
}

#[test]
fn instant_live_completion_signal_retries() {
    instant_case(
        "instant_live_completion_500",
        vec![rule(Route::Api("recording-complete"), Fault::Status(500)).times(2)],
        budget::INSTANT_LIVE_COMPLETION_500,
        |mock, run| {
            assert_eq!(run.result, Ok(()));
            assert_segments_stored(mock, INSTANT_VIDEO_ID, &run.video, &run.audio);
            assert_eq!(mock.count("POST", "recording-complete"), 3);
            mock.assert_clean();
        },
    );
}