misterdownloadmanager: The ultimate open-source, clean-room download manager written from scratch in pure Rust for Windows, Mac, and Linux
Rust bittorrent-clientdebriddownload-acceleratordownload-managerdownloadmanagereguifile-downloadergallery-dlidmidm-alternativeinternet-download-managerm3u8-downloadermagnetplaylist-downloaderreal-debridrusttorrent-clientvideo-downloaderyoutube-downloaderyt-dlp
//! FTP and FTPS: `ftp://`, `ftpes://` (explicit TLS: `AUTH TLS` on port 21) and `ftps://`
//! (implicit TLS, port 990), downloaded like HTTP (`download.rs`): the first connection asks
//! `SIZE` and `MDTM` and starts `RETR` at byte 0 while further connections fetch parts with
//! `REST` + `RETR`. Each connection is a control session with its data connection, passive by
//! default (`EPSV` when the server offers it, else `PASV`), active (`PORT`/`EPRT`) when
//! `Options::ftp_passive` is off.
//!
//! A part ends where the next one begins, usually before the end of the file: its data
//! connection is dropped and the session goes back to the download's pool, where the next
//! command first reads the server's answer to the dropped transfer. Reusing sessions saves a
//! connect, a TLS handshake and a login per part.
//!
//! The login comes from the address (`ftp://user:pass@host/…`), else from an
//! `Authorization: Basic` header (the app's "Use authorization" and Sites Logins), else
//! anonymous. The path is relative to the login folder (RFC 1738; `%2F` makes it absolute).
//! TLS uses the system's certificate store, like HTTPS. FTP doesn't go through the HTTP proxy.
use std::net::{IpAddr, SocketAddr};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant, SystemTime};
use base64::Engine;
use bytes::{Bytes, BytesMut};
use percent_encoding::percent_decode_str;
use reqwest::header::{AUTHORIZATION, HeaderMap};
use rustls::pki_types::CertificateDer;
use suppaftp::tokio::{
AsyncRustlsConnector, AsyncRustlsFtpStream, AsyncRustlsStream, TransferStream,
};
use suppaftp::types::FileType;
use suppaftp::{FtpError, Mode, Status};
use tokio::io::AsyncReadExt;
use url::{Host, Url};
use crate::download::{Error, NetKind, Options};
type Session = AsyncRustlsFtpStream;
/// An idle pooled session older than this is closed rather than reused: servers drop idle
/// sessions (often after 60 s), and a dead one costs a failed try.
const IDLE: Duration = Duration::from_secs(30);
/// Bytes read from a data connection at a time.
const READ: usize = 64 * 1024;
/// Whether MDM downloads `url` over FTP.
pub fn is_ftp(url: &Url) -> bool {
matches!(url.scheme(), "ftp" | "ftps" | "ftpes")
}
/// `url` without its password, as it may be written to disk (the resume state): whoever
/// resumes gives the login again.
pub fn without_password(url: &Url) -> Url {
let mut u = url.clone();
let _ = u.set_password(None);
u
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Security {
Plain,
/// `AUTH TLS` after connecting.
Explicit,
/// TLS from the first byte.
Implicit,
}
/// What the server tells about the file before sending it.
#[derive(Clone, Debug)]
pub(crate) struct Facts {
pub size: Option<u64>,
pub modified: Option<SystemTime>,
/// `FEAT` lists `REST STREAM`: parts can be fetched. Unlisted, the first part tells.
pub rest: bool,
}
/// One server and file: how to reach and log in, and the sessions not in use.
pub(crate) struct Site {
host: String,
port: u16,
/// Sent as it is to `SIZE`, `MDTM` and `RETR`.
path: String,
user: String,
pass: String,
security: Security,
passive: bool,
connect_timeout: Duration,
read_timeout: Duration,
/// `Options::hosts`: fixed addresses for names.
hosts: Vec<(String, Vec<IpAddr>)>,
/// `Options::dns` and the certificates it may trust besides the system's.
dns: crate::dns::DnsChoice,
extra_roots: Vec<Vec<u8>>,
/// The network adapter its connections go through (#35).
route: crate::net::Route,
tls: Option<tokio_rustls::TlsConnector>,
/// The server offers `EPSV` (learned from `FEAT`): no address in the answer to get wrong
/// behind NAT.
epsv: AtomicBool,
pool: Mutex<Vec<(Session, Instant)>>,
max_pool: usize,
}
impl Site {
pub fn new(url: &Url, headers: &HeaderMap, opts: &Options) -> Result<Site, Error> {
let security = match url.scheme() {
"ftps" => Security::Implicit,
"ftpes" => Security::Explicit,
_ => Security::Plain,
};
let host = match url.host() {
Some(Host::Domain(d)) => d.to_owned(),
Some(Host::Ipv4(a)) => a.to_string(),
Some(Host::Ipv6(a)) => a.to_string(),
None => return Err(net(NetKind::Connect, "the address has no server")),
};
let decode = |s: &str| percent_decode_str(s).decode_utf8_lossy().into_owned();
let (user, pass) = if url.password().is_some() || header_login(headers).is_none() {
(decode(url.username()), decode(url.password().unwrap_or("")))
} else {
header_login(headers).unwrap_or_default()
};
let (user, pass) = match user.as_str() {
"" | "anonymous" | "ftp" if pass.is_empty() => ("anonymous".into(), "mdm@".into()),
_ => (user, pass),
};
// The leading slash only separates the path from the server; `;type=i` is a transfer
// type (always binary here).
let raw = url.path().strip_prefix('/').unwrap_or(url.path());
let raw = raw.rsplit_once(";type=").map_or(raw, |(p, _)| p);
let tls = match security {
Security::Plain => None,
_ => Some(connector(&opts.extra_roots)?),
};
Ok(Site {
host,
port: url.port().unwrap_or(if security == Security::Implicit {
990
} else {
21
}),
path: decode(raw),
user,
pass,
security,
passive: opts.ftp_passive,
connect_timeout: opts.connect_timeout,
read_timeout: opts.read_timeout,
hosts: opts.hosts.clone(),
dns: opts.dns.clone(),
extra_roots: opts.extra_roots.clone(),
route: opts.route.clone(),
tls,
epsv: AtomicBool::new(false),
pool: Mutex::new(Vec::new()),
max_pool: opts.connections() as usize,
})
}
/// A new session: connected, secured, logged in, binary.
async fn connect(&self) -> Result<Session, Error> {
let mut last = net(NetKind::Connect, "the server's name has no address");
let mut addrs = self.addresses().await?;
// Only those the chosen adapter can reach (#35); else say why none.
if let Some(first) = addrs.first() {
let refused = self.route.check(first.ip()).err();
addrs.retain(|a| self.route.check(a.ip()).is_ok());
if let (Some(e), true) = (refused, addrs.is_empty()) {
return Err(e.into());
}
}
if let crate::net::Route::Down { label } = &self.route {
return Err(crate::net::NetError::Down {
label: label.clone(),
}
.into());
}
for addr in addrs {
match tokio::time::timeout(self.connect_timeout, self.open(addr)).await {
Ok(Ok(s)) => return Ok(s),
// Another address may answer; a refusal from the server itself won't change.
Ok(Err(e @ Error::Net { .. })) => last = e,
Ok(Err(e)) => return Err(e),
Err(_) => last = net(NetKind::Timeout, ""),
}
}
Err(last)
}
async fn addresses(&self) -> Result<Vec<SocketAddr>, Error> {
if let Ok(ip) = self.host.parse::<IpAddr>() {
return Ok(vec![SocketAddr::new(ip, self.port)]);
}
if let Some((_, ips)) = self
.hosts
.iter()
.find(|(h, _)| h.eq_ignore_ascii_case(&self.host))
{
return Ok(ips
.iter()
.map(|ip| SocketAddr::new(*ip, self.port))
.collect());
}
let found = tokio::time::timeout(
self.connect_timeout,
crate::dns::lookup(
&self.dns,
&self.host,
self.port,
&self.extra_roots,
&self.route,
),
)
.await
.map_err(|_| net(NetKind::Timeout, ""))?
.map_err(|e| net(NetKind::Connect, &e.to_string()))?;
Ok(found
.into_iter()
.map(|ip| SocketAddr::new(ip, self.port))
.collect())
}
async fn open(&self, addr: SocketAddr) -> Result<Session, Error> {
let connect = |e| failure(e, NetKind::Connect);
let tls =
|| AsyncRustlsConnector::from(self.tls.clone().expect("TLS site has a connector"));
let mut s = match (&self.security, &self.route) {
(Security::Implicit, crate::net::Route::Any) => {
Session::connect_secure_implicit(addr, tls(), &self.host)
.await
.map_err(connect)?
}
// suppaftp opens implicit TLS's connection itself (no way to give it a bound one):
// refused rather than sent around the adapter.
(Security::Implicit, route) => {
return Err(crate::net::NetError::NotSupported {
label: route.label().unwrap_or_default().to_owned(),
what: "ftps://".into(),
}
.into());
}
(_, crate::net::Route::Any) => Session::connect(addr).await.map_err(connect)?,
(_, route) => {
let stream = crate::net::tcp_connect(route, addr)
.await
.map_err(|e| connect(FtpError::ConnectionError(e)))?;
Session::connect_with_stream(stream)
.await
.map_err(connect)?
}
};
match self.security {
Security::Explicit => s = s.into_secure(tls(), &self.host).await.map_err(connect)?,
// Implicit TLS protects the data connections too; say so, as clients do (some
// servers otherwise expect them in the clear).
Security::Implicit => {
s.custom_command("PBSZ 0", &[Status::CommandOk])
.await
.map_err(connect)?;
s.custom_command("PROT P", &[Status::CommandOk])
.await
.map_err(connect)?;
}
Security::Plain => {}
}
s.login(&self.user, &self.pass).await.map_err(connect)?;
s.transfer_type(FileType::Binary).await.map_err(connect)?;
Ok(self.data_mode(s))
}
fn data_mode(&self, mut s: Session) -> Session {
if !self.passive {
return s.active_mode(self.read_timeout);
}
if self.epsv.load(Ordering::Relaxed) {
s.set_mode(Mode::ExtendedPassive);
} else {
s.set_mode(Mode::Passive);
// A server behind NAT often answers PASV with its private address: connect to
// the address the session already reached instead.
s.set_passive_nat_workaround(true);
}
// Data connections through the chosen adapter too (#35). (Active mode listens on the
// control connection's own address, which already is the adapter's.)
if !self.route.is_any() {
let route = Arc::new(self.route.clone());
s = s.passive_stream_builder(move |addr| {
let route = route.clone();
Box::pin(async move {
crate::net::tcp_connect(&route, addr)
.await
.map_err(FtpError::ConnectionError)
})
});
}
s
}
/// What the server says about the file, on `s`.
async fn facts(&self, s: &mut Session) -> Result<Facts, Error> {
let lost = |e| failure(e, NetKind::Lost);
// Not every server has FEAT; without it, nothing is assumed.
let features = s.feat().await.unwrap_or_default();
let has = |name: &str, value: Option<&str>| {
features.iter().any(|(k, v)| {
k.eq_ignore_ascii_case(name)
&& value.is_none_or(|want| {
v.as_deref().is_some_and(|v| {
v.split_whitespace().any(|w| w.eq_ignore_ascii_case(want))
})
})
})
};
if has("EPSV", None) && self.passive {
self.epsv.store(true, Ordering::Relaxed);
s.set_mode(Mode::ExtendedPassive);
}
let rest = has("REST", Some("STREAM"));
let size = match s.size(&self.path).await {
Ok(n) => Some(n as u64),
// A missing file is said now, not after a RETR.
Err(FtpError::UnexpectedResponse(r)) if r.status.code() == 550 => {
return Err(reply(&r));
}
Err(FtpError::UnexpectedResponse(_)) => None,
Err(e) => return Err(lost(e)),
};
let modified = match s.mdtm(&self.path).await {
Ok(t) => u64::try_from(t.and_utc().timestamp())
.ok()
.map(|secs| SystemTime::UNIX_EPOCH + Duration::from_secs(secs)),
Err(FtpError::UnexpectedResponse(_)) => None,
Err(e) => return Err(lost(e)),
};
Ok(Facts {
size,
modified,
rest,
})
}
/// The first connection: the facts, and the file from byte 0 on the same session.
pub async fn start(self: &Arc<Self>) -> Result<(Facts, Transfer), Error> {
let mut s = self.connect().await?;
let facts = within(self.read_timeout, self.facts(&mut s)).await?;
let stream = within(self.read_timeout, retr(&mut s, &self.path, 0)).await?;
Ok((
facts.clone(),
Transfer {
site: self.clone(),
session: Some(s),
stream: Some(stream),
expected: facts.size,
},
))
}
/// The facts on a session of their own (kept for the transfers).
pub async fn check(&self) -> Result<Facts, Error> {
let mut s = self.connect().await?;
let facts = within(self.read_timeout, self.facts(&mut s)).await?;
self.put_back(s);
Ok(facts)
}
/// The file from byte `from` (of `size`, if known), on a pooled session if one is idle.
pub async fn open_at(
self: &Arc<Self>,
from: u64,
size: Option<u64>,
) -> Result<Transfer, Error> {
let transfer = |s, stream| Transfer {
site: self.clone(),
session: Some(s),
stream: Some(stream),
expected: size.map(|n| n.saturating_sub(from)),
};
if let Some(mut s) = self.take() {
match within(self.read_timeout, retr(&mut s, &self.path, from)).await {
Ok(stream) => return Ok(transfer(s, stream)),
// The server's own refusal: a new session would hear the same.
Err(e @ Error::FtpReply { code: 500.., .. }) => return Err(e),
// A session the server closed meanwhile: a fresh one.
Err(_) => {}
}
}
let mut s = self.connect().await?;
let stream = within(self.read_timeout, retr(&mut s, &self.path, from)).await?;
Ok(transfer(s, stream))
}
fn take(&self) -> Option<Session> {
let mut pool = self.pool.lock().unwrap();
pool.retain(|(_, at)| at.elapsed() < IDLE);
pool.pop().map(|(s, _)| s)
}
fn put_back(&self, s: Session) {
let mut pool = self.pool.lock().unwrap();
if pool.len() < self.max_pool {
pool.push((s, Instant::now()));
}
}
}
/// `REST from` (when not 0) and `RETR path`.
async fn retr(
s: &mut Session,
path: &str,
from: u64,
) -> Result<TransferStream<AsyncRustlsStream>, Error> {
let lost = |e| failure(e, NetKind::Lost);
s.resume_transfer(from as usize).await.map_err(lost)?;
s.retr_as_stream(path).await.map_err(lost)
}
/// A file (or the rest of it from an offset) coming in.
pub(crate) struct Transfer {
site: Arc<Site>,
session: Option<Session>,
stream: Option<TransferStream<AsyncRustlsStream>>,
expected: Option<u64>,
}
impl Transfer {
/// The bytes still to come, if the size is known.
pub fn expected(&self) -> Option<u64> {
self.expected
}
/// The next bytes; None at the end of the file.
pub async fn chunk(&mut self) -> Result<Option<Bytes>, Error> {
let Some(stream) = self.stream.as_mut() else {
return Ok(None);
};
let mut buf = BytesMut::with_capacity(READ);
match tokio::time::timeout(self.site.read_timeout, stream.read_buf(&mut buf)).await {
Err(_) => Err(net(NetKind::Timeout, "")),
Ok(Err(e)) => Err(net(NetKind::Lost, &e.to_string())),
Ok(Ok(0)) => Ok(None),
Ok(Ok(_)) => Ok(Some(buf.freeze())),
}
}
/// Done with this transfer: at its end (`at_end`, the server's "complete" is read), or
/// before (the data connection is dropped). The session goes back to the pool.
pub async fn done(mut self, at_end: bool) {
let Some(stream) = self.stream.take() else {
return;
};
let ok = if at_end {
matches!(
tokio::time::timeout(self.site.read_timeout, stream.finish()).await,
Ok(Ok(()))
)
} else {
drop(stream);
true
};
if ok && let Some(s) = self.session.take() {
self.site.put_back(s);
}
}
}
/// The client configuration for FTPS: the system's verifier (as reqwest uses for HTTPS), with
/// `extra` certificates also trusted.
fn connector(extra: &[Vec<u8>]) -> Result<tokio_rustls::TlsConnector, Error> {
let config = client_config(extra).map_err(|e| net(NetKind::Tls, &e.to_string()))?;
Ok(tokio_rustls::TlsConnector::from(Arc::new(config)))
}
/// MDM's own TLS client configuration (FTPS, DNS over TLS and HTTPS): the system's verifier, as
/// reqwest uses for HTTPS, with `extra` certificates also trusted (the tests' own CA). On a
/// system without certificates, one that refuses every server, saying why (#264): the DNS
/// setting that holds it may still use plain DNS, and FTP stays plain FTP.
pub(crate) fn client_config(extra: &[Vec<u8>]) -> Result<rustls::ClientConfig, rustls::Error> {
use rustls_platform_verifier::Verifier;
if extra.is_empty() && !crate::tls::system_has_roots() {
return Ok(crate::tls::no_roots_config());
}
let provider = crate::tls::provider();
let verifier = if extra.is_empty() {
Verifier::new(provider.clone())
} else {
Verifier::new_with_extra_roots(
extra.iter().map(|c| CertificateDer::from(c.clone())),
provider.clone(),
)
}?;
Ok(rustls::ClientConfig::builder_with_provider(provider)
.with_safe_default_protocol_versions()?
.dangerous()
.with_custom_certificate_verifier(Arc::new(verifier))
.with_no_client_auth())
}
/// The login an `Authorization: Basic` header carries.
fn header_login(headers: &HeaderMap) -> Option<(String, String)> {
let v = headers.get(AUTHORIZATION)?.to_str().ok()?;
let token = v
.strip_prefix("Basic ")
.or_else(|| v.strip_prefix("basic "))?;
let text = base64::engine::general_purpose::STANDARD
.decode(token.trim())
.ok()?;
let text = String::from_utf8(text).ok()?;
let (user, pass) = text.split_once(':')?;
Some((user.to_owned(), pass.to_owned()))
}
async fn within<T>(limit: Duration, f: impl Future<Output = Result<T, Error>>) -> Result<T, Error> {
tokio::time::timeout(limit, f)
.await
.unwrap_or_else(|_| Err(net(NetKind::Timeout, "")))
}
fn net(kind: NetKind, cause: &str) -> Error {
if cause.contains(crate::tls::NO_ROOTS) {
return Error::NoCertificates;
}
Error::Net {
kind,
cause: crate::download::strip_os_error_code(cause),
}
}
/// The server's answer, without its code.
fn reply(r: &suppaftp::types::Response) -> Error {
let text = String::from_utf8_lossy(&r.body);
let text = text.trim();
let code = r.status.code();
let text = text
.strip_prefix(&code.to_string())
.map_or(text, |t| t.trim_start_matches(['-', ' ']));
Error::FtpReply {
code,
text: text.lines().next().unwrap_or("").trim().to_owned(),
}
}
/// An FTP failure as an engine error; `kind` says what was being done when the network failed.
fn failure(e: FtpError, kind: NetKind) -> Error {
match e {
FtpError::UnexpectedResponse(r) => reply(&r),
FtpError::ConnectionError(io) if io.kind() == std::io::ErrorKind::TimedOut => {
net(NetKind::Timeout, "")
}
FtpError::ConnectionError(io) => net(kind, &io.to_string()),
FtpError::SecureError(s) => net(NetKind::Tls, &s),
other => net(kind, &other.to_string()),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn site(url: &str, headers: &HeaderMap) -> Site {
Site::new(&url.parse().unwrap(), headers, &Options::default()).unwrap()
}
#[test]
fn addresses_name_the_server_file_and_login() {
let none = HeaderMap::new();
let s = site("ftp://ftp.example.com/pub/a%20b.iso;type=i", &none);
assert_eq!((s.host.as_str(), s.port), ("ftp.example.com", 21));
assert_eq!(s.path, "pub/a b.iso");
assert_eq!((s.user.as_str(), s.pass.as_str()), ("anonymous", "mdm@"));
let s = site("ftps://me:p%40ss@[::1]/%2Fabs/x", &none);
assert_eq!(
(s.host.as_str(), s.port, s.security),
("::1", 990, Security::Implicit)
);
assert_eq!(
(s.user.as_str(), s.pass.as_str(), s.path.as_str()),
("me", "p@ss", "/abs/x")
);
let s = site("ftpes://host:2121/x", &none);
assert_eq!((s.port, s.security), (2121, Security::Explicit));
}
#[test]
fn a_basic_header_logs_in_when_the_address_has_no_password() {
let mut h = HeaderMap::new();
let token = base64::engine::general_purpose::STANDARD.encode("bob:secret");
h.insert(AUTHORIZATION, format!("Basic {token}").parse().unwrap());
let s = site("ftp://bob@host/x", &h);
assert_eq!((s.user.as_str(), s.pass.as_str()), ("bob", "secret"));
// The address's own password wins.
let s = site("ftp://ann:pw@host/x", &h);
assert_eq!((s.user.as_str(), s.pass.as_str()), ("ann", "pw"));
let u: Url = "ftp://ann:pw@host/x".parse().unwrap();
assert_eq!(without_password(&u).as_str(), "ftp://ann@host/x");
}
}