π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Add domain icon fetching crate
Neeraj Gupta54a1e0fa2870
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex fcbbf527c1..2fa6c9e2c2 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -680,9 +680,9 @@ dependencies = [[[package]]name = "aws-lc-rs"-version = "1.17.0"+version = "1.18.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00"+checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e"dependencies = ["aws-lc-sys","zeroize",@@ -690,14 +690,15 @@ dependencies = [[[package]]name = "aws-lc-sys"-version = "0.41.0"+version = "0.45.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4"+checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27"dependencies = ["cc","cmake","dunce","fs_extra",+ "pkg-config",][[package]]@@ -2274,6 +2275,12 @@ version = "1.0.20"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"+[[package]]+name = "ego-tree"+version = "0.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b04dc5a38e4f151a79d9f2451ae6037fb6eaf5cba34771f44781f80e508498e3"+[[package]]name = "either"version = "1.15.0"@@ -2672,6 +2679,22 @@ dependencies = ["log",]+[[package]]+name = "ente-icons"+version = "0.0.0"+dependencies = [+ "encoding_rs",+ "ente-core",+ "futures-util",+ "image",+ "mockito",+ "psl",+ "scraper",+ "thiserror 2.0.21",+ "tokio",+ "url",+]+[[package]]name = "ente-image"version = "0.0.0"@@ -7473,6 +7496,21 @@ dependencies = ["syn 2.0.117",]+[[package]]+name = "psl"+version = "2.1.239"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5736f71353f867ae262bd123b86ddb161f50a2db0300ab762c8f4a0e659635d3"+dependencies = [+ "psl-types",+]++[[package]]+name = "psl-types"+version = "2.0.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"+[[package]]name = "pxfm"version = "0.1.29"@@ -8201,9 +8239,9 @@ dependencies = [[[package]]name = "rustls"-version = "0.23.40"+version = "0.23.45"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"+checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"dependencies = ["aws-lc-rs","once_cell",@@ -8265,9 +8303,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"[[package]]name = "rustls-webpki"-version = "0.103.13"+version = "0.103.15"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"+checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"dependencies = ["aws-lc-rs","ring",@@ -8386,6 +8424,20 @@ version = "1.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"+[[package]]+name = "scraper"+version = "0.27.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bdd0be4d296f048bfb06dd01bbc80ef789ddd2e55583e8d2e6b804942abfabc2"+dependencies = [+ "cssparser",+ "ego-tree",+ "html5ever",+ "precomputed-hash",+ "selectors",+ "tendril",+]+[[package]]name = "scratch"version = "1.0.9"diff --git a/rust/Cargo.toml b/rust/Cargo.tomlindex 3c2b636ee3..3067ae73d3 100644--- a/rust/Cargo.toml+++ b/rust/Cargo.toml@@ -31,6 +31,7 @@ members = ["crates/ensu","crates/ensu-crypto","crates/exif",+ "crates/icons","crates/image","crates/legacy","crates/location",@@ -55,6 +56,7 @@ chrono = { version = "0.4.45", default-features = false }clap = "4.6.7"dialoguer = { version = "0.12.0", default-features = false }dirs = "7.0.0"+encoding_rs = "0.8.35"ente-accounts = { path = "crates/accounts" }ente-assets = { path = "crates/assets" }ente-cast = { path = "crates/cast" }@@ -66,6 +68,7 @@ ente-ensu-crypto = { path = "crates/ensu-crypto" }ente-exif = { path = "crates/exif" }ente-frb-lib = { path = "bindings/frb/lib" }ente-frb-log = { path = "bindings/frb/log" }+ente-icons = { path = "crates/icons" }ente-image = { path = "crates/image" }ente-legacy = { path = "crates/legacy" }ente-location = { path = "crates/location" }@@ -96,10 +99,12 @@ md-5 = "0.11.0"mockito = "1.7.2"ort = { version = "=2.0.0-rc.13", default-features = false }ort-sys = { version = "=2.0.0-rc.13", default-features = false }+psl = "2.1.239"quick-xml = "0.42.0"regex = "1.13.1"reqwest = { version = "0.12.28", default-features = false }rusqlite = { version = "0.40.2", default-features = false, features = ["bundled"] }+scraper = { version = "0.27.0", default-features = false }serde = { version = "1.0.229", features = ["derive"] }serde-wasm-bindgen = "0.6.5"serde_json = "1.0.151"diff --git a/rust/crates/core/src/http.rs b/rust/crates/core/src/http.rsindex b18f68aa58..6112fd8c5e 100644--- a/rust/crates/core/src/http.rs+++ b/rust/crates/core/src/http.rs@@ -361,6 +361,10 @@ impl Response {pub struct SuccessResponse(reqwest::Response);impl SuccessResponse {+ pub fn url(&self) -> &reqwest::Url {+ self.0.url()+ }+pub fn headers(&self) -> &reqwest::header::HeaderMap {self.0.headers()}diff --git a/rust/crates/icons/Cargo.toml b/rust/crates/icons/Cargo.tomlnew file mode 100644index 0000000000..34bf79ed30--- /dev/null+++ b/rust/crates/icons/Cargo.toml@@ -0,0 +1,21 @@+[package]+name = "ente-icons"+edition.workspace = true++[target.'cfg(not(target_arch = "wasm32"))'.dependencies]+encoding_rs.workspace = true+ente-core.workspace = true+futures-util.workspace = true+image.workspace = true+psl.workspace = true+scraper.workspace = true+thiserror.workspace = true+tokio = { workspace = true, features = ["rt", "time"] }+url.workspace = true++[dev-dependencies]+mockito.workspace = true+tokio = { workspace = true, features = ["macros"] }++[lints]+workspace = truediff --git a/rust/crates/icons/README.md b/rust/crates/icons/README.mdnew file mode 100644index 0000000000..16e602cb24--- /dev/null+++ b/rust/crates/icons/README.md@@ -0,0 +1,5 @@+Fetch a website icon from one HTTP(S) URL using `ente-core::http`. Returns a static PNG, or `None` if no supported icon is found. Requires native Rust with Tokio.++```rust,no_run+let icon = ente_icons::Fetcher::new()?.fetch("https://example.com").await?;+```diff --git a/rust/crates/icons/examples/fetch.rs b/rust/crates/icons/examples/fetch.rsnew file mode 100644index 0000000000..5fee251ed8--- /dev/null+++ b/rust/crates/icons/examples/fetch.rs@@ -0,0 +1,13 @@+#[tokio::main(flavor = "current_thread")]+async fn main() -> Result<(), Box<dyn std::error::Error>> {+ let mut args = std::env::args().skip(1);+ let url = args+ .next()+ .filter(|_| args.next().is_none())+ .ok_or("usage: cargo run -p ente-icons --example fetch -- https://example.com")?;+ match ente_icons::Fetcher::new()?.fetch(&url).await? {+ Some(icon) => println!("{} ({} bytes, image/png)", icon.url, icon.data.len()),+ None => println!("No icon found"),+ }+ Ok(())+}diff --git a/rust/crates/icons/src/discovery.rs b/rust/crates/icons/src/discovery.rsnew file mode 100644index 0000000000..7e85fb2ab0--- /dev/null+++ b/rust/crates/icons/src/discovery.rs@@ -0,0 +1,202 @@+use std::collections::HashSet;++use scraper::{ElementRef, Html};+use url::Url;++const MAX_CANDIDATES: usize = 8;+const PREFERRED_SIZE: u32 = 128;++pub(crate) fn icon_urls(html: &str, page: &Url) -> Vec<Url> {+ let document = Html::parse_document(html);+ let Some(head) = document+ .root_element()+ .descendants()+ .filter_map(ElementRef::wrap)+ .find(|element| element.value().name() == "head")+ else {+ return Vec::new();+ };+ let base = head+ .descendants()+ .filter_map(ElementRef::wrap)+ .find(|element| element.value().name() == "base" && element.attr("href").is_some())+ .and_then(|element| page.join(element.attr("href")?.trim()).ok())+ .unwrap_or_else(|| page.clone());++ let mut candidates = Vec::new();+ for element in head.descendants().filter_map(ElementRef::wrap) {+ if element.value().name() != "link" {+ continue;+ }+ let Some(rel) = element.attr("rel") else {+ continue;+ };+ let icon = rel+ .split_ascii_whitespace()+ .any(|token| token.eq_ignore_ascii_case("icon"));+ let touch = rel.split_ascii_whitespace().any(|token| {+ token.eq_ignore_ascii_case("apple-touch-icon")+ || token.eq_ignore_ascii_case("apple-touch-icon-precomposed")+ });+ if !icon && !touch {+ continue;+ }+ let svg = element+ .attr("type")+ .is_some_and(|kind| kind.trim().eq_ignore_ascii_case("image/svg+xml"));+ let Some(href) = element+ .attr("href")+ .map(str::trim)+ .filter(|href| !href.is_empty())+ else {+ continue;+ };+ let Ok(mut url) = base.join(href) else {+ continue;+ };+ if !matches!(url.scheme(), "http" | "https") {+ continue;+ }+ url.set_fragment(None);+ let rank = (svg, size_rank(element.attr("sizes").unwrap_or("")), !icon);+ candidates.push((rank, url));+ }+ candidates.sort_by_key(|(rank, _)| *rank);+ let mut seen = HashSet::new();+ candidates+ .into_iter()+ .map(|(_, url)| url)+ .filter(|url| seen.insert(url.clone()))+ .take(MAX_CANDIDATES)+ .collect()+}++fn size_rank(sizes: &str) -> (u8, u32) {+ sizes+ .split_ascii_whitespace()+ .filter_map(|size| {+ let (width, height) = size.split_once(['x', 'X'])?;+ let width: u32 = width.parse().ok()?;+ let height: u32 = height.parse().ok()?;+ (width > 0 && width == height).then_some((+ u8::from(width < PREFERRED_SIZE),+ width.abs_diff(PREFERRED_SIZE),+ ))+ })+ .min()+ .unwrap_or((2, 0))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn resolves_base_relative_and_cdn_links_and_orders_sizes() -> Result<(), url::ParseError> {+ let html = r#"<!doctype html><head>+ <link rel=ICON sizes=16x16 href=/small.png>+ <link rel="shortcut icon" sizes=128x128 href="best.png?a=1&b=2">+ <base href="/assets/">+ <link rel=icon sizes=256x256 href=large.png>+ <link rel=apple-touch-icon href="//cdn.example.com/touch.png">+ <link rel=icon sizes=128x128 href="best.png?a=1&b=2#duplicate">+ <link rel=stylesheet href=style.css>+ <link rel=icon href="http://cdn.example.com/plain.png">+ <link rel=icon href="data:image/png;base64,abc">+ </head><body><link rel=icon href=body.png>"#;+ let urls = icon_urls(html, &Url::parse("https://www.example.com/new/page")?);+ let urls: Vec<_> = urls.iter().map(Url::as_str).collect();+ assert_eq!(+ urls,+ [+ "https://www.example.com/assets/best.png?a=1&b=2",+ "https://www.example.com/assets/large.png",+ "https://www.example.com/small.png",+ "http://cdn.example.com/plain.png",+ "https://cdn.example.com/touch.png",+ ]+ );+ Ok(())+ }++ #[test]+ fn excludes_script_text_and_limits_candidate_requests() -> Result<(), url::ParseError> {+ let links: String = (0..20)+ .map(|i| format!("<link rel=icon href=/{i}.ico>"))+ .collect();+ let html = format!("<head><script>'<link rel=icon href=/fake.ico>'</script>{links}</head>");+ let urls = icon_urls(&html, &Url::parse("https://example.com")?);+ assert_eq!(urls.len(), MAX_CANDIDATES);+ assert_eq!(urls[0].path(), "/0.ico");+ Ok(())+ }++ #[test]+ fn declared_svgs_do_not_crowd_out_a_raster_candidate() -> Result<(), url::ParseError> {+ let mut html = "<head>".to_owned();+ for i in 0..MAX_CANDIDATES {+ html.push_str(&format!(+ "<link rel=icon type='image/svg+xml' sizes=any href=/{i}.svg>"+ ));+ }+ html.push_str("<link rel=icon sizes=128x128 href=/icon.png></head>");+ let urls = icon_urls(&html, &Url::parse("https://example.com/")?);+ assert_eq!(urls[0], Url::parse("https://example.com/icon.png")?);+ assert_eq!(urls.len(), MAX_CANDIDATES);+ Ok(())+ }++ #[test]+ fn prefers_sharp_rasters_including_touch_icons() -> Result<(), url::ParseError> {+ let html = "<head>+ <link rel=icon sizes=any href=/scalable.ico>+ <link rel=icon sizes=32x32 href=/small.png>+ <link rel=apple-touch-icon sizes=180x180 href=/touch.png>+ <link rel=apple-touch-icon sizes=128x128 href=/touch-128.png>+ <link rel=icon sizes='16x16 128x128' href=/icon.png>+ </head>";+ let urls = icon_urls(html, &Url::parse("https://example.com/")?);+ let paths: Vec<_> = urls.iter().map(Url::path).collect();+ assert_eq!(+ paths,+ [+ "/icon.png",+ "/touch-128.png",+ "/touch.png",+ "/small.png",+ "/scalable.ico"+ ]+ );+ Ok(())+ }++ #[test]+ fn accepts_http_and_https_links_with_custom_ports() -> Result<(), url::ParseError> {+ let page = Url::parse("https://example.com/")?;+ let html = "<head>+ <link rel=icon href=https://cdn.example.com:443/icon.png>+ <link rel=icon href=http://cdn.example.com:80/icon.png>+ <link rel=icon href=https://cdn.example.com:8443/icon.png>+ <link rel=icon href=http://cdn.example.com:8080/icon.png>+ </head>";+ let urls = icon_urls(html, &page);+ let urls: Vec<_> = urls.iter().map(Url::as_str).collect();+ assert_eq!(+ urls,+ [+ "https://cdn.example.com/icon.png",+ "http://cdn.example.com/icon.png",+ "https://cdn.example.com:8443/icon.png",+ "http://cdn.example.com:8080/icon.png"+ ]+ );+ assert_eq!(+ icon_urls(+ "<head><base href=https://cdn.example.com:8443/><link rel=icon href=icon.png></head>",+ &page,+ ),+ [Url::parse("https://cdn.example.com:8443/icon.png")?]+ );+ Ok(())+ }+}diff --git a/rust/crates/icons/src/fetch.rs b/rust/crates/icons/src/fetch.rsnew file mode 100644index 0000000000..88c848051c--- /dev/null+++ b/rust/crates/icons/src/fetch.rs@@ -0,0 +1,174 @@+use std::collections::HashSet;+use std::time::Duration;++use encoding_rs::Encoding;+use ente_core::http::{self, Http, SuccessResponse};+use futures_util::StreamExt;+use url::Url;++use crate::{discovery, format};++const HTML_LIMIT: usize = 256 * 1024;+const LOOKUP_TIMEOUT: Duration = Duration::from_secs(15);++#[derive(Debug, thiserror::Error)]+pub enum Error {+ #[error("expected an HTTP or HTTPS URL")]+ InvalidUrl,+ #[error("website request failed: {0}")]+ Http(#[from] http::Error),+ #[error("icon lookup timed out")]+ Timeout,+ #[error("icon exceeds the 1 MiB download limit")]+ TooLarge,+ #[error("image validation task failed: {0}")]+ Worker(#[from] tokio::task::JoinError),+}++#[derive(Debug)]+pub struct Icon {+ pub url: Url,+ pub data: Vec<u8>,+}++pub struct Fetcher {+ http: Http,+}++impl Fetcher {+ pub fn new() -> Result<Self, Error> {+ Ok(Self { http: Http::new()? })+ }++ pub async fn fetch(&self, url: &str) -> Result<Option<Icon>, Error> {+ let page = Url::parse(url)+ .ok()+ .filter(|url| matches!(url.scheme(), "http" | "https"))+ .ok_or(Error::InvalidUrl)?;+ tokio::time::timeout(LOOKUP_TIMEOUT, self.find(page))+ .await+ .map_err(|_| Error::Timeout)?+ }++ async fn find(&self, page: Url) -> Result<Option<Icon>, Error> {+ let parent = parent_url(&page);+ let mut error = None;+ for page in std::iter::once(page).chain(parent) {+ match self.site(page).await {+ Ok(Some(icon)) => return Ok(Some(icon)),+ Ok(None) => {}+ Err(failure) => error = Some(failure),+ }+ }+ error.map_or(Ok(None), Err)+ }++ async fn site(&self, page: Url) -> Result<Option<Icon>, Error> {+ let mut error = None;+ let mut candidates = Vec::new();+ let mut fallback_page = page.clone();+ match self.get(&page).await {+ Ok(Some(response)) => {+ fallback_page = response.url().clone();+ let encoding = response+ .headers()+ .get("content-type")+ .and_then(|header| header.to_str().ok())+ .and_then(|header| {+ header.split(';').find_map(|parameter| {+ let (name, value) = parameter.trim().split_once('=')?;+ name.eq_ignore_ascii_case("charset")+ .then(|| {+ Encoding::for_label(value.trim_matches(['\'', '"']).as_bytes())+ })+ .flatten()+ })+ });+ match body(response, HTML_LIMIT, true).await {+ Ok(bytes) => {+ let text = encoding.unwrap_or(encoding_rs::UTF_8).decode(&bytes).0;+ candidates = discovery::icon_urls(&text, &fallback_page);+ }+ Err(failure) => error = Some(failure),+ }+ }+ Ok(None) => {}+ Err(failure) => error = Some(failure),+ }+ if let Ok(fallback) = fallback_page.join("/favicon.ico") {+ candidates.push(fallback);+ }+ if fallback_page.origin() != page.origin()+ && let Ok(fallback) = page.join("/favicon.ico")+ {+ candidates.push(fallback);+ }+ let mut seen = HashSet::new();+ for url in candidates {+ if !seen.insert(url.clone()) {+ continue;+ }+ match self.icon(&url).await {+ Ok(Some(icon)) => return Ok(Some(icon)),+ Ok(None) => {}+ Err(failure) => error = Some(failure),+ }+ }+ error.map_or(Ok(None), Err)+ }++ async fn get(&self, url: &Url) -> Result<Option<SuccessResponse>, Error> {+ let response = self.http.get(url.as_str()).send().await?;+ if matches!(response.status(), 404 | 410) {+ return Ok(None);+ }+ Ok(Some(response.error_for_status()?))+ }++ async fn icon(&self, url: &Url) -> Result<Option<Icon>, Error> {+ let Some(response) = self.get(url).await? else {+ return Ok(None);+ };+ let url = response.url().clone();+ let data = body(response, format::MAX_BYTES, false).await?;+ Ok(+ tokio::task::spawn_blocking(move || format::png(&data).map(|data| Icon { url, data }))+ .await?,+ )+ }+}++async fn body(response: SuccessResponse, limit: usize, prefix: bool) -> Result<Vec<u8>, Error> {+ let mut bytes = Vec::new();+ let mut chunks = std::pin::pin!(response.bytes_stream());+ while let Some(chunk) = chunks.next().await {+ let chunk = chunk?;+ let remaining = limit - bytes.len();+ if !prefix && chunk.len() > remaining {+ return Err(Error::TooLarge);+ }+ bytes.extend_from_slice(&chunk[..chunk.len().min(remaining)]);+ if prefix && bytes.len() == limit {+ break;+ }+ }+ Ok(bytes)+}++fn parent_url(page: &Url) -> Option<Url> {+ let host = page.domain()?;+ let domain = psl::domain(host.as_bytes())?;+ if !domain.suffix().is_known() || domain.as_bytes() == host.as_bytes() {+ return None;+ }+ let mut parent = page.join("/").ok()?;+ parent+ .set_host(Some(std::str::from_utf8(domain.as_bytes()).ok()?))+ .ok()?;+ parent.set_username("").ok()?;+ parent.set_password(None).ok()?;+ Some(parent)+}++#[cfg(test)]+mod tests;diff --git a/rust/crates/icons/src/fetch/tests.rs b/rust/crates/icons/src/fetch/tests.rsnew file mode 100644index 0000000000..85567b16e5--- /dev/null+++ b/rust/crates/icons/src/fetch/tests.rs@@ -0,0 +1,110 @@+use super::*;++type TestResult = Result<(), Box<dyn std::error::Error>>;++#[tokio::test]+async fn shared_http_follows_redirects_and_resolves_declared_icons() -> TestResult {+ let mut server = mockito::Server::new_async().await;+ let page = server+ .mock("GET", "/")+ .with_status(302)+ .with_header("location", "/home")+ .create_async()+ .await;+ let html = server+ .mock("GET", "/home")+ .with_body("<head><base href=/assets/><link rel=icon href=logo.png></head>")+ .create_async()+ .await;+ let redirect = server+ .mock("GET", "/assets/logo.png")+ .with_status(302)+ .with_header("location", "/logo.png")+ .create_async()+ .await;+ let mut png = std::io::Cursor::new(Vec::new());+ image::DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(+ 2,+ 2,+ image::Rgba([10, 20, 30, 255]),+ ))+ .write_to(&mut png, image::ImageFormat::Png)?;+ let image = server+ .mock("GET", "/logo.png")+ .with_body(png.into_inner())+ .create_async()+ .await;+ let icon = Fetcher::new()?+ .fetch(&format!("{}/", server.url()))+ .await?+ .ok_or("missing icon")?;+ assert_eq!(icon.url.as_str(), format!("{}/logo.png", server.url()));+ assert_eq!(image::load_from_memory(&icon.data)?.width(), 2);+ page.assert_async().await;+ html.assert_async().await;+ redirect.assert_async().await;+ image.assert_async().await;+ Ok(())+}++#[test]+fn parent_fallback_respects_registration_boundaries() -> TestResult {+ for (input, expected) in [+ (+ "https://login.example.co.uk/signin?q=1",+ Some("https://example.co.uk/"),+ ),+ (+ "http://a.user.github.io:8080/login",+ Some("http://user.github.io:8080/"),+ ),+ (+ "https://user:[email protected]/",+ Some("https://example.com/"),+ ),+ ("https://example.com/", None),+ ("https://gov.uk/", None),+ ("http://localhost/", None),+ ("http://127.0.0.1/", None),+ ] {+ let parent = parent_url(&Url::parse(input)?);+ assert_eq!(parent.as_ref().map(Url::as_str), expected, "{input}");+ }+ Ok(())+}++#[tokio::test]+async fn rejects_missing_urls_and_unsupported_schemes() -> TestResult {+ let fetcher = Fetcher::new()?;+ for input in [+ "",+ "example.com",+ "file:///tmp/icon.png",+ "data:image/png;base64,abc",+ ] {+ assert!(+ matches!(fetcher.fetch(input).await, Err(Error::InvalidUrl)),+ "{input}"+ );+ }+ Ok(())+}++#[tokio::test]+async fn bounds_streamed_bodies_without_trusting_content_length() -> TestResult {+ let mut server = mockito::Server::new_async().await;+ let response = server+ .mock("GET", "/body")+ .with_chunked_body(|writer| writer.write_all(&[b'a'; 20]))+ .expect(2)+ .create_async()+ .await;+ let http = Http::new()?;+ let url = format!("{}/body", server.url());+ let prefix = http.get(&url).send().await?.error_for_status()?;+ assert_eq!(body(prefix, 10, true).await?, vec![b'a'; 10]);+ let full = http.get(&url).send().await?.error_for_status()?;+ assert!(matches!(body(full, 10, false).await, Err(Error::TooLarge)));+ response.assert_async().await;+ Ok(())+}diff --git a/rust/crates/icons/src/format.rs b/rust/crates/icons/src/format.rsnew file mode 100644index 0000000000..19d8546ad8--- /dev/null+++ b/rust/crates/icons/src/format.rs@@ -0,0 +1,291 @@+use std::io::Cursor;++use image::{DynamicImage, ImageDecoder, ImageFormat, ImageReader, Limits};++pub(crate) const MAX_BYTES: usize = 1024 * 1024;++pub(crate) fn png(data: &[u8]) -> Option<Vec<u8>> {+ let format = image::guess_format(data).ok()?;+ if !matches!(+ format,+ ImageFormat::Png+ | ImageFormat::Jpeg+ | ImageFormat::Gif+ | ImageFormat::WebP+ | ImageFormat::Ico+ ) {+ return None;+ }+ if format == ImageFormat::Ico {+ validate_ico(data)?;+ }+ let mut limits = Limits::default();+ limits.max_image_width = Some(1024);+ limits.max_image_height = Some(1024);+ limits.max_alloc = Some(16 * 1024 * 1024);+ let mut reader = ImageReader::with_format(Cursor::new(data), format);+ reader.limits(limits.clone());+ let mut decoder = reader.into_decoder().ok()?;+ limits.reserve(decoder.total_bytes()).ok()?;+ decoder.set_limits(limits).ok()?;+ let orientation = decoder.orientation().ok()?;+ let mut image = DynamicImage::from_decoder(decoder).ok()?;+ image.apply_orientation(orientation);+ let mut output = Cursor::new(Vec::new());+ image.write_to(&mut output, ImageFormat::Png).ok()?;+ let output = output.into_inner();+ (output.len() <= MAX_BYTES).then_some(output)+}++fn validate_ico(data: &[u8]) -> Option<()> {+ let count = data.get(4..6)?;+ let count = usize::from(u16::from_le_bytes([count[0], count[1]]));+ let directory = data.get(6..6 + count * 16)?;+ let mut budget = MAX_BYTES;+ for entry in directory.chunks_exact(16) {+ let length = u32::from_le_bytes([entry[8], entry[9], entry[10], entry[11]]) as usize;+ if length < 8 {+ return None;+ }+ let offset = u32::from_le_bytes([entry[12], entry[13], entry[14], entry[15]]) as usize;+ let payload = data.get(offset..offset.checked_add(length)?)?;+ if payload.starts_with(b"\x89PNG\r\n\x1a\n") {+ validate_png(payload, &mut budget)?;+ }+ }+ Some(())+}++fn validate_png(data: &[u8], budget: &mut usize) -> Option<()> {+ let mut data = data.get(8..)?;+ loop {+ let header = data.get(..8)?;+ *budget = budget.checked_sub(12)?;+ let length = u32::from_be_bytes([header[0], header[1], header[2], header[3]]) as usize;+ data = data.get(length.checked_add(12)?..)?;+ if &header[4..8] == b"iCCP" {+ return None;+ }+ if &header[4..8] == b"IEND" {+ return (length == 0).then_some(());+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use image::ImageEncoder;++ fn ico(entries: &[(u8, Vec<u8>)]) -> Result<Vec<u8>, Box<dyn std::error::Error>> {+ let mut output = vec![0, 0, 1, 0];+ output.extend_from_slice(&u16::try_from(entries.len())?.to_le_bytes());+ let mut offset = 6 + entries.len() * 16;+ for (size, bytes) in entries {+ output.extend_from_slice(&[*size, *size, 0, 0, 1, 0, 32, 0]);+ output.extend_from_slice(&u32::try_from(bytes.len())?.to_le_bytes());+ output.extend_from_slice(&u32::try_from(offset)?.to_le_bytes());+ offset += bytes.len();+ }+ for (_, bytes) in entries {+ output.extend_from_slice(bytes);+ }+ Ok(output)+ }++ fn png() -> Result<Vec<u8>, image::ImageError> {+ let image = DynamicImage::new_rgba8(2, 2);+ let mut bytes = Cursor::new(Vec::new());+ image.write_to(&mut bytes, ImageFormat::Png)?;+ Ok(bytes.into_inner())+ }++ #[test]+ fn rejects_truncated_images_html_and_svg() -> Result<(), image::ImageError> {+ let data = png()?;+ assert!(super::png(&data).is_some());+ for data in [+ &data[..20],+ b"<html>Not found</html>".as_slice(),+ b"<svg xmlns='http://www.w3.org/2000/svg'><path d='M0 0'/></svg>",+ b"",+ ] {+ assert!(super::png(data).is_none());+ }+ Ok(())+ }++ #[test]+ fn normalizes_supported_rasters_without_resizing() -> Result<(), Box<dyn std::error::Error>> {+ let pixels = image::RgbaImage::from_fn(128, 128, |x, y| {+ if (x + y) % 2 == 0 {+ image::Rgba([20, 80, 200, 255])+ } else {+ image::Rgba([0, 0, 0, 0])+ }+ });+ let image = DynamicImage::ImageRgba8(pixels);+ for encoding in [+ ImageFormat::Png,+ ImageFormat::Jpeg,+ ImageFormat::Gif,+ ImageFormat::WebP,+ ImageFormat::Ico,+ ] {+ let mut bytes = Cursor::new(Vec::new());+ if encoding == ImageFormat::Jpeg {+ DynamicImage::ImageRgb8(image.to_rgb8()).write_to(&mut bytes, encoding)?;+ } else {+ image.write_to(&mut bytes, encoding)?;+ }+ let output = super::png(bytes.get_ref()).ok_or("normalization failed")?;+ assert_eq!(image::guess_format(&output)?, ImageFormat::Png);+ let decoded = image::load_from_memory(&output)?;+ assert_eq!((decoded.width(), decoded.height()), (128, 128));+ if encoding != ImageFormat::Jpeg {+ assert_eq!(decoded.to_rgba8(), image.to_rgba8());+ }+ }+ Ok(())+ }++ #[test]+ fn emits_only_the_first_animation_frame() -> Result<(), Box<dyn std::error::Error>> {+ let first = image::RgbaImage::from_pixel(8, 8, image::Rgba([255, 0, 0, 255]));+ let second = image::RgbaImage::from_pixel(8, 8, image::Rgba([0, 255, 0, 255]));+ let mut input = Vec::new();+ image::codecs::gif::GifEncoder::new(&mut input)+ .encode_frames([image::Frame::new(first.clone()), image::Frame::new(second)])?;+ let output = super::png(&input).ok_or("animated image rejected")?;+ assert_eq!(image::guess_format(&output)?, ImageFormat::Png);+ assert_eq!(image::load_from_memory(&output)?.to_rgba8(), first);+ Ok(())+ }++ #[test]+ fn selects_a_larger_ico_entry_and_preserves_transparency()+ -> Result<(), Box<dyn std::error::Error>> {+ let small = DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(+ 16,+ 16,+ image::Rgba([0, 255, 0, 255]),+ ));+ let large = DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(+ 128,+ 128,+ image::Rgba([255, 0, 0, 80]),+ ));+ let mut entries = Vec::new();+ for image in [&small, &large] {+ let mut bytes = Cursor::new(Vec::new());+ image.write_to(&mut bytes, ImageFormat::Png)?;+ entries.push((u8::try_from(image.width())?, bytes.into_inner()));+ }+ let input = ico(&entries)?;+ let output = super::png(&input).ok_or("ICO rejected")?;+ assert_eq!(+ image::load_from_memory(&output)?.to_rgba8(),+ large.to_rgba8()+ );+ Ok(())+ }++ #[test]+ fn accepts_bitmap_backed_ico() -> Result<(), Box<dyn std::error::Error>> {+ let mut bitmap = Vec::new();+ for value in [40u32, 2, 4, 0x0020_0001, 0, 16, 0, 0, 0, 0] {+ bitmap.extend_from_slice(&value.to_le_bytes());+ }+ for _ in 0..4 {+ bitmap.extend_from_slice(&[255, 0, 0, 255]);+ }+ bitmap.extend_from_slice(&[0; 8]);+ let output = super::png(&ico(&[(2, bitmap)])?).ok_or("bitmap icon rejected")?;+ let image = image::load_from_memory(&output)?.to_rgba8();+ assert_eq!(+ image,+ image::RgbaImage::from_pixel(2, 2, image::Rgba([0, 0, 255, 255]))+ );+ Ok(())+ }++ #[test]+ fn rejects_profile_tagged_ico_before_decoding() -> Result<(), Box<dyn std::error::Error>> {+ let mut bytes = Vec::new();+ let mut encoder = image::codecs::png::PngEncoder::new(&mut bytes);+ encoder.set_icc_profile(vec![0; 128])?;+ encoder.write_image(&[0, 0, 255, 255], 1, 1, image::ExtendedColorType::Rgba8)?;+ assert!(super::png(&bytes).is_some());+ let icon = ico(&[(1, bytes)])?;+ assert!(validate_ico(&icon).is_none());+ assert!(super::png(&icon).is_none());+ Ok(())+ }++ #[test]+ fn rejects_incomplete_ico_directory_and_png_chunk_framing()+ -> Result<(), Box<dyn std::error::Error>> {+ assert!(validate_ico(&[0, 0, 1, 0, 1, 0]).is_none());+ let mut icon = ico(&[(2, png()?)])?;+ for length in 0..8u32 {+ let mut short = icon.clone();+ short[14..18].copy_from_slice(&length.to_le_bytes());+ assert!(validate_ico(&short).is_none());+ }+ icon[18..22].copy_from_slice(&u32::MAX.to_le_bytes());+ assert!(validate_ico(&icon).is_none());+ let truncated = ico(&[(2, b"\x89PNG\r\n\x1a\n\xff\xff\xff\xffIDAT".to_vec())])?;+ assert!(validate_ico(&truncated).is_none());+ let short = ico(&[(2, png()?[..20].to_vec())])?;+ assert!(validate_ico(&short).is_none());+ Ok(())+ }++ #[test]+ fn rejects_oversized_dimensions() -> Result<(), image::ImageError> {+ let image = DynamicImage::new_rgb8(1025, 1);+ let mut bytes = Cursor::new(Vec::new());+ image.write_to(&mut bytes, ImageFormat::Png)?;+ assert!(super::png(bytes.get_ref()).is_none());+ Ok(())+ }++ #[test]+ fn applies_orientation_before_removing_metadata() -> Result<(), Box<dyn std::error::Error>> {+ let mut bytes = Vec::new();+ let mut encoder = image::codecs::png::PngEncoder::new(&mut bytes);+ encoder.set_exif_metadata(+ b"II*\0\x08\0\0\0\x01\0\x12\x01\x03\0\x01\0\0\0\x06\0\0\0\0\0\0\0".to_vec(),+ )?;+ encoder.write_image(+ &[255, 0, 0, 255, 0, 255, 0, 255],+ 2,+ 1,+ image::ExtendedColorType::Rgba8,+ )?;+ let output = super::png(&bytes).ok_or("oriented image rejected")?;+ let image = image::load_from_memory(&output)?.to_rgba8();+ assert_eq!(image.dimensions(), (1, 2));+ assert_eq!(image.get_pixel(0, 0), &image::Rgba([255, 0, 0, 255]));+ assert_eq!(image.get_pixel(0, 1), &image::Rgba([0, 255, 0, 255]));+ Ok(())+ }++ #[test]+ fn rejects_png_output_exceeding_the_byte_limit() -> Result<(), image::ImageError> {+ let mut state = 1u32;+ let pixels = image::RgbImage::from_fn(768, 768, |_, _| {+ state ^= state << 13;+ state ^= state >> 17;+ state ^= state << 5;+ let bytes = state.to_le_bytes();+ image::Rgb([bytes[0], bytes[1], bytes[2]])+ });+ let mut bytes = Cursor::new(Vec::new());+ image::codecs::jpeg::JpegEncoder::new_with_quality(&mut bytes, 70)+ .encode_image(&DynamicImage::ImageRgb8(pixels))?;+ assert!(bytes.get_ref().len() <= MAX_BYTES);+ assert!(super::png(bytes.get_ref()).is_none());+ Ok(())+ }+}diff --git a/rust/crates/icons/src/lib.rs b/rust/crates/icons/src/lib.rsnew file mode 100644index 0000000000..1b707b4c8b--- /dev/null+++ b/rust/crates/icons/src/lib.rs@@ -0,0 +1,7 @@+#![cfg(not(target_arch = "wasm32"))]++mod discovery;+mod fetch;+mod format;++pub use fetch::{Error, Fetcher, Icon};