Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Add domain icon fetching crate

Neeraj Gupta54a1e0fa2870
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index fcbbf527c1..2fa6c9e2c2 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -680,9 +680,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
-version = "1.17.0"
+version = "1.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00"
+checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -690,14 +690,15 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
-version = "0.41.0"
+version = "0.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4"
+checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27"
dependencies = [
"cc",
"cmake",
"dunce",
"fs_extra",
+ "pkg-config",
]
[[package]]
@@ -2274,6 +2275,12 @@ version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
+[[package]]
+name = "ego-tree"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b04dc5a38e4f151a79d9f2451ae6037fb6eaf5cba34771f44781f80e508498e3"
+
[[package]]
name = "either"
version = "1.15.0"
@@ -2672,6 +2679,22 @@ dependencies = [
"log",
]
+[[package]]
+name = "ente-icons"
+version = "0.0.0"
+dependencies = [
+ "encoding_rs",
+ "ente-core",
+ "futures-util",
+ "image",
+ "mockito",
+ "psl",
+ "scraper",
+ "thiserror 2.0.21",
+ "tokio",
+ "url",
+]
+
[[package]]
name = "ente-image"
version = "0.0.0"
@@ -7473,6 +7496,21 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "psl"
+version = "2.1.239"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5736f71353f867ae262bd123b86ddb161f50a2db0300ab762c8f4a0e659635d3"
+dependencies = [
+ "psl-types",
+]
+
+[[package]]
+name = "psl-types"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
+
[[package]]
name = "pxfm"
version = "0.1.29"
@@ -8201,9 +8239,9 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.23.40"
+version = "0.23.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
+checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
dependencies = [
"aws-lc-rs",
"once_cell",
@@ -8265,9 +8303,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
-version = "0.103.13"
+version = "0.103.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
+checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
dependencies = [
"aws-lc-rs",
"ring",
@@ -8386,6 +8424,20 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+[[package]]
+name = "scraper"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bdd0be4d296f048bfb06dd01bbc80ef789ddd2e55583e8d2e6b804942abfabc2"
+dependencies = [
+ "cssparser",
+ "ego-tree",
+ "html5ever",
+ "precomputed-hash",
+ "selectors",
+ "tendril",
+]
+
[[package]]
name = "scratch"
version = "1.0.9"
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 3c2b636ee3..3067ae73d3 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -31,6 +31,7 @@ members = [
"crates/ensu",
"crates/ensu-crypto",
"crates/exif",
+ "crates/icons",
"crates/image",
"crates/legacy",
"crates/location",
@@ -55,6 +56,7 @@ chrono = { version = "0.4.45", default-features = false }
clap = "4.6.7"
dialoguer = { version = "0.12.0", default-features = false }
dirs = "7.0.0"
+encoding_rs = "0.8.35"
ente-accounts = { path = "crates/accounts" }
ente-assets = { path = "crates/assets" }
ente-cast = { path = "crates/cast" }
@@ -66,6 +68,7 @@ ente-ensu-crypto = { path = "crates/ensu-crypto" }
ente-exif = { path = "crates/exif" }
ente-frb-lib = { path = "bindings/frb/lib" }
ente-frb-log = { path = "bindings/frb/log" }
+ente-icons = { path = "crates/icons" }
ente-image = { path = "crates/image" }
ente-legacy = { path = "crates/legacy" }
ente-location = { path = "crates/location" }
@@ -96,10 +99,12 @@ md-5 = "0.11.0"
mockito = "1.7.2"
ort = { version = "=2.0.0-rc.13", default-features = false }
ort-sys = { version = "=2.0.0-rc.13", default-features = false }
+psl = "2.1.239"
quick-xml = "0.42.0"
regex = "1.13.1"
reqwest = { version = "0.12.28", default-features = false }
rusqlite = { version = "0.40.2", default-features = false, features = ["bundled"] }
+scraper = { version = "0.27.0", default-features = false }
serde = { version = "1.0.229", features = ["derive"] }
serde-wasm-bindgen = "0.6.5"
serde_json = "1.0.151"
diff --git a/rust/crates/core/src/http.rs b/rust/crates/core/src/http.rs
index b18f68aa58..6112fd8c5e 100644
--- a/rust/crates/core/src/http.rs
+++ b/rust/crates/core/src/http.rs
@@ -361,6 +361,10 @@ impl Response {
pub struct SuccessResponse(reqwest::Response);
impl SuccessResponse {
+ pub fn url(&self) -> &reqwest::Url {
+ self.0.url()
+ }
+
pub fn headers(&self) -> &reqwest::header::HeaderMap {
self.0.headers()
}
diff --git a/rust/crates/icons/Cargo.toml b/rust/crates/icons/Cargo.toml
new file mode 100644
index 0000000000..34bf79ed30
--- /dev/null
+++ b/rust/crates/icons/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "ente-icons"
+edition.workspace = true
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+encoding_rs.workspace = true
+ente-core.workspace = true
+futures-util.workspace = true
+image.workspace = true
+psl.workspace = true
+scraper.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["rt", "time"] }
+url.workspace = true
+
+[dev-dependencies]
+mockito.workspace = true
+tokio = { workspace = true, features = ["macros"] }
+
+[lints]
+workspace = true
diff --git a/rust/crates/icons/README.md b/rust/crates/icons/README.md
new file mode 100644
index 0000000000..16e602cb24
--- /dev/null
+++ b/rust/crates/icons/README.md
@@ -0,0 +1,5 @@
+Fetch a website icon from one HTTP(S) URL using `ente-core::http`. Returns a static PNG, or `None` if no supported icon is found. Requires native Rust with Tokio.
+
+```rust,no_run
+let icon = ente_icons::Fetcher::new()?.fetch("https://example.com").await?;
+```
diff --git a/rust/crates/icons/examples/fetch.rs b/rust/crates/icons/examples/fetch.rs
new file mode 100644
index 0000000000..5fee251ed8
--- /dev/null
+++ b/rust/crates/icons/examples/fetch.rs
@@ -0,0 +1,13 @@
+#[tokio::main(flavor = "current_thread")]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ let mut args = std::env::args().skip(1);
+ let url = args
+ .next()
+ .filter(|_| args.next().is_none())
+ .ok_or("usage: cargo run -p ente-icons --example fetch -- https://example.com")?;
+ match ente_icons::Fetcher::new()?.fetch(&url).await? {
+ Some(icon) => println!("{} ({} bytes, image/png)", icon.url, icon.data.len()),
+ None => println!("No icon found"),
+ }
+ Ok(())
+}
diff --git a/rust/crates/icons/src/discovery.rs b/rust/crates/icons/src/discovery.rs
new file mode 100644
index 0000000000..7e85fb2ab0
--- /dev/null
+++ b/rust/crates/icons/src/discovery.rs
@@ -0,0 +1,202 @@
+use std::collections::HashSet;
+
+use scraper::{ElementRef, Html};
+use url::Url;
+
+const MAX_CANDIDATES: usize = 8;
+const PREFERRED_SIZE: u32 = 128;
+
+pub(crate) fn icon_urls(html: &str, page: &Url) -> Vec<Url> {
+ let document = Html::parse_document(html);
+ let Some(head) = document
+ .root_element()
+ .descendants()
+ .filter_map(ElementRef::wrap)
+ .find(|element| element.value().name() == "head")
+ else {
+ return Vec::new();
+ };
+ let base = head
+ .descendants()
+ .filter_map(ElementRef::wrap)
+ .find(|element| element.value().name() == "base" && element.attr("href").is_some())
+ .and_then(|element| page.join(element.attr("href")?.trim()).ok())
+ .unwrap_or_else(|| page.clone());
+
+ let mut candidates = Vec::new();
+ for element in head.descendants().filter_map(ElementRef::wrap) {
+ if element.value().name() != "link" {
+ continue;
+ }
+ let Some(rel) = element.attr("rel") else {
+ continue;
+ };
+ let icon = rel
+ .split_ascii_whitespace()
+ .any(|token| token.eq_ignore_ascii_case("icon"));
+ let touch = rel.split_ascii_whitespace().any(|token| {
+ token.eq_ignore_ascii_case("apple-touch-icon")
+ || token.eq_ignore_ascii_case("apple-touch-icon-precomposed")
+ });
+ if !icon && !touch {
+ continue;
+ }
+ let svg = element
+ .attr("type")
+ .is_some_and(|kind| kind.trim().eq_ignore_ascii_case("image/svg+xml"));
+ let Some(href) = element
+ .attr("href")
+ .map(str::trim)
+ .filter(|href| !href.is_empty())
+ else {
+ continue;
+ };
+ let Ok(mut url) = base.join(href) else {
+ continue;
+ };
+ if !matches!(url.scheme(), "http" | "https") {
+ continue;
+ }
+ url.set_fragment(None);
+ let rank = (svg, size_rank(element.attr("sizes").unwrap_or("")), !icon);
+ candidates.push((rank, url));
+ }
+ candidates.sort_by_key(|(rank, _)| *rank);
+ let mut seen = HashSet::new();
+ candidates
+ .into_iter()
+ .map(|(_, url)| url)
+ .filter(|url| seen.insert(url.clone()))
+ .take(MAX_CANDIDATES)
+ .collect()
+}
+
+fn size_rank(sizes: &str) -> (u8, u32) {
+ sizes
+ .split_ascii_whitespace()
+ .filter_map(|size| {
+ let (width, height) = size.split_once(['x', 'X'])?;
+ let width: u32 = width.parse().ok()?;
+ let height: u32 = height.parse().ok()?;
+ (width > 0 && width == height).then_some((
+ u8::from(width < PREFERRED_SIZE),
+ width.abs_diff(PREFERRED_SIZE),
+ ))
+ })
+ .min()
+ .unwrap_or((2, 0))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn resolves_base_relative_and_cdn_links_and_orders_sizes() -> Result<(), url::ParseError> {
+ let html = r#"<!doctype html><head>
+ <link rel=ICON sizes=16x16 href=/small.png>
+ <link rel="shortcut icon" sizes=128x128 href="best.png?a=1&amp;b=2">
+ <base href="/assets/">
+ <link rel=icon sizes=256x256 href=large.png>
+ <link rel=apple-touch-icon href="//cdn.example.com/touch.png">
+ <link rel=icon sizes=128x128 href="best.png?a=1&amp;b=2#duplicate">
+ <link rel=stylesheet href=style.css>
+ <link rel=icon href="http://cdn.example.com/plain.png">
+ <link rel=icon href="data:image/png;base64,abc">
+ </head><body><link rel=icon href=body.png>"#;
+ let urls = icon_urls(html, &Url::parse("https://www.example.com/new/page")?);
+ let urls: Vec<_> = urls.iter().map(Url::as_str).collect();
+ assert_eq!(
+ urls,
+ [
+ "https://www.example.com/assets/best.png?a=1&b=2",
+ "https://www.example.com/assets/large.png",
+ "https://www.example.com/small.png",
+ "http://cdn.example.com/plain.png",
+ "https://cdn.example.com/touch.png",
+ ]
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn excludes_script_text_and_limits_candidate_requests() -> Result<(), url::ParseError> {
+ let links: String = (0..20)
+ .map(|i| format!("<link rel=icon href=/{i}.ico>"))
+ .collect();
+ let html = format!("<head><script>'<link rel=icon href=/fake.ico>'</script>{links}</head>");
+ let urls = icon_urls(&html, &Url::parse("https://example.com")?);
+ assert_eq!(urls.len(), MAX_CANDIDATES);
+ assert_eq!(urls[0].path(), "/0.ico");
+ Ok(())
+ }
+
+ #[test]
+ fn declared_svgs_do_not_crowd_out_a_raster_candidate() -> Result<(), url::ParseError> {
+ let mut html = "<head>".to_owned();
+ for i in 0..MAX_CANDIDATES {
+ html.push_str(&format!(
+ "<link rel=icon type='image/svg+xml' sizes=any href=/{i}.svg>"
+ ));
+ }
+ html.push_str("<link rel=icon sizes=128x128 href=/icon.png></head>");
+ let urls = icon_urls(&html, &Url::parse("https://example.com/")?);
+ assert_eq!(urls[0], Url::parse("https://example.com/icon.png")?);
+ assert_eq!(urls.len(), MAX_CANDIDATES);
+ Ok(())
+ }
+
+ #[test]
+ fn prefers_sharp_rasters_including_touch_icons() -> Result<(), url::ParseError> {
+ let html = "<head>
+ <link rel=icon sizes=any href=/scalable.ico>
+ <link rel=icon sizes=32x32 href=/small.png>
+ <link rel=apple-touch-icon sizes=180x180 href=/touch.png>
+ <link rel=apple-touch-icon sizes=128x128 href=/touch-128.png>
+ <link rel=icon sizes='16x16 128x128' href=/icon.png>
+ </head>";
+ let urls = icon_urls(html, &Url::parse("https://example.com/")?);
+ let paths: Vec<_> = urls.iter().map(Url::path).collect();
+ assert_eq!(
+ paths,
+ [
+ "/icon.png",
+ "/touch-128.png",
+ "/touch.png",
+ "/small.png",
+ "/scalable.ico"
+ ]
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn accepts_http_and_https_links_with_custom_ports() -> Result<(), url::ParseError> {
+ let page = Url::parse("https://example.com/")?;
+ let html = "<head>
+ <link rel=icon href=https://cdn.example.com:443/icon.png>
+ <link rel=icon href=http://cdn.example.com:80/icon.png>
+ <link rel=icon href=https://cdn.example.com:8443/icon.png>
+ <link rel=icon href=http://cdn.example.com:8080/icon.png>
+ </head>";
+ let urls = icon_urls(html, &page);
+ let urls: Vec<_> = urls.iter().map(Url::as_str).collect();
+ assert_eq!(
+ urls,
+ [
+ "https://cdn.example.com/icon.png",
+ "http://cdn.example.com/icon.png",
+ "https://cdn.example.com:8443/icon.png",
+ "http://cdn.example.com:8080/icon.png"
+ ]
+ );
+ assert_eq!(
+ icon_urls(
+ "<head><base href=https://cdn.example.com:8443/><link rel=icon href=icon.png></head>",
+ &page,
+ ),
+ [Url::parse("https://cdn.example.com:8443/icon.png")?]
+ );
+ Ok(())
+ }
+}
diff --git a/rust/crates/icons/src/fetch.rs b/rust/crates/icons/src/fetch.rs
new file mode 100644
index 0000000000..88c848051c
--- /dev/null
+++ b/rust/crates/icons/src/fetch.rs
@@ -0,0 +1,174 @@
+use std::collections::HashSet;
+use std::time::Duration;
+
+use encoding_rs::Encoding;
+use ente_core::http::{self, Http, SuccessResponse};
+use futures_util::StreamExt;
+use url::Url;
+
+use crate::{discovery, format};
+
+const HTML_LIMIT: usize = 256 * 1024;
+const LOOKUP_TIMEOUT: Duration = Duration::from_secs(15);
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error("expected an HTTP or HTTPS URL")]
+ InvalidUrl,
+ #[error("website request failed: {0}")]
+ Http(#[from] http::Error),
+ #[error("icon lookup timed out")]
+ Timeout,
+ #[error("icon exceeds the 1 MiB download limit")]
+ TooLarge,
+ #[error("image validation task failed: {0}")]
+ Worker(#[from] tokio::task::JoinError),
+}
+
+#[derive(Debug)]
+pub struct Icon {
+ pub url: Url,
+ pub data: Vec<u8>,
+}
+
+pub struct Fetcher {
+ http: Http,
+}
+
+impl Fetcher {
+ pub fn new() -> Result<Self, Error> {
+ Ok(Self { http: Http::new()? })
+ }
+
+ pub async fn fetch(&self, url: &str) -> Result<Option<Icon>, Error> {
+ let page = Url::parse(url)
+ .ok()
+ .filter(|url| matches!(url.scheme(), "http" | "https"))
+ .ok_or(Error::InvalidUrl)?;
+ tokio::time::timeout(LOOKUP_TIMEOUT, self.find(page))
+ .await
+ .map_err(|_| Error::Timeout)?
+ }
+
+ async fn find(&self, page: Url) -> Result<Option<Icon>, Error> {
+ let parent = parent_url(&page);
+ let mut error = None;
+ for page in std::iter::once(page).chain(parent) {
+ match self.site(page).await {
+ Ok(Some(icon)) => return Ok(Some(icon)),
+ Ok(None) => {}
+ Err(failure) => error = Some(failure),
+ }
+ }
+ error.map_or(Ok(None), Err)
+ }
+
+ async fn site(&self, page: Url) -> Result<Option<Icon>, Error> {
+ let mut error = None;
+ let mut candidates = Vec::new();
+ let mut fallback_page = page.clone();
+ match self.get(&page).await {
+ Ok(Some(response)) => {
+ fallback_page = response.url().clone();
+ let encoding = response
+ .headers()
+ .get("content-type")
+ .and_then(|header| header.to_str().ok())
+ .and_then(|header| {
+ header.split(';').find_map(|parameter| {
+ let (name, value) = parameter.trim().split_once('=')?;
+ name.eq_ignore_ascii_case("charset")
+ .then(|| {
+ Encoding::for_label(value.trim_matches(['\'', '"']).as_bytes())
+ })
+ .flatten()
+ })
+ });
+ match body(response, HTML_LIMIT, true).await {
+ Ok(bytes) => {
+ let text = encoding.unwrap_or(encoding_rs::UTF_8).decode(&bytes).0;
+ candidates = discovery::icon_urls(&text, &fallback_page);
+ }
+ Err(failure) => error = Some(failure),
+ }
+ }
+ Ok(None) => {}
+ Err(failure) => error = Some(failure),
+ }
+ if let Ok(fallback) = fallback_page.join("/favicon.ico") {
+ candidates.push(fallback);
+ }
+ if fallback_page.origin() != page.origin()
+ && let Ok(fallback) = page.join("/favicon.ico")
+ {
+ candidates.push(fallback);
+ }
+ let mut seen = HashSet::new();
+ for url in candidates {
+ if !seen.insert(url.clone()) {
+ continue;
+ }
+ match self.icon(&url).await {
+ Ok(Some(icon)) => return Ok(Some(icon)),
+ Ok(None) => {}
+ Err(failure) => error = Some(failure),
+ }
+ }
+ error.map_or(Ok(None), Err)
+ }
+
+ async fn get(&self, url: &Url) -> Result<Option<SuccessResponse>, Error> {
+ let response = self.http.get(url.as_str()).send().await?;
+ if matches!(response.status(), 404 | 410) {
+ return Ok(None);
+ }
+ Ok(Some(response.error_for_status()?))
+ }
+
+ async fn icon(&self, url: &Url) -> Result<Option<Icon>, Error> {
+ let Some(response) = self.get(url).await? else {
+ return Ok(None);
+ };
+ let url = response.url().clone();
+ let data = body(response, format::MAX_BYTES, false).await?;
+ Ok(
+ tokio::task::spawn_blocking(move || format::png(&data).map(|data| Icon { url, data }))
+ .await?,
+ )
+ }
+}
+
+async fn body(response: SuccessResponse, limit: usize, prefix: bool) -> Result<Vec<u8>, Error> {
+ let mut bytes = Vec::new();
+ let mut chunks = std::pin::pin!(response.bytes_stream());
+ while let Some(chunk) = chunks.next().await {
+ let chunk = chunk?;
+ let remaining = limit - bytes.len();
+ if !prefix && chunk.len() > remaining {
+ return Err(Error::TooLarge);
+ }
+ bytes.extend_from_slice(&chunk[..chunk.len().min(remaining)]);
+ if prefix && bytes.len() == limit {
+ break;
+ }
+ }
+ Ok(bytes)
+}
+
+fn parent_url(page: &Url) -> Option<Url> {
+ let host = page.domain()?;
+ let domain = psl::domain(host.as_bytes())?;
+ if !domain.suffix().is_known() || domain.as_bytes() == host.as_bytes() {
+ return None;
+ }
+ let mut parent = page.join("/").ok()?;
+ parent
+ .set_host(Some(std::str::from_utf8(domain.as_bytes()).ok()?))
+ .ok()?;
+ parent.set_username("").ok()?;
+ parent.set_password(None).ok()?;
+ Some(parent)
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/rust/crates/icons/src/fetch/tests.rs b/rust/crates/icons/src/fetch/tests.rs
new file mode 100644
index 0000000000..85567b16e5
--- /dev/null
+++ b/rust/crates/icons/src/fetch/tests.rs
@@ -0,0 +1,110 @@
+use super::*;
+
+type TestResult = Result<(), Box<dyn std::error::Error>>;
+
+#[tokio::test]
+async fn shared_http_follows_redirects_and_resolves_declared_icons() -> TestResult {
+ let mut server = mockito::Server::new_async().await;
+ let page = server
+ .mock("GET", "/")
+ .with_status(302)
+ .with_header("location", "/home")
+ .create_async()
+ .await;
+ let html = server
+ .mock("GET", "/home")
+ .with_body("<head><base href=/assets/><link rel=icon href=logo.png></head>")
+ .create_async()
+ .await;
+ let redirect = server
+ .mock("GET", "/assets/logo.png")
+ .with_status(302)
+ .with_header("location", "/logo.png")
+ .create_async()
+ .await;
+ let mut png = std::io::Cursor::new(Vec::new());
+ image::DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(
+ 2,
+ 2,
+ image::Rgba([10, 20, 30, 255]),
+ ))
+ .write_to(&mut png, image::ImageFormat::Png)?;
+ let image = server
+ .mock("GET", "/logo.png")
+ .with_body(png.into_inner())
+ .create_async()
+ .await;
+ let icon = Fetcher::new()?
+ .fetch(&format!("{}/", server.url()))
+ .await?
+ .ok_or("missing icon")?;
+ assert_eq!(icon.url.as_str(), format!("{}/logo.png", server.url()));
+ assert_eq!(image::load_from_memory(&icon.data)?.width(), 2);
+ page.assert_async().await;
+ html.assert_async().await;
+ redirect.assert_async().await;
+ image.assert_async().await;
+ Ok(())
+}
+
+#[test]
+fn parent_fallback_respects_registration_boundaries() -> TestResult {
+ for (input, expected) in [
+ (
+ "https://login.example.co.uk/signin?q=1",
+ Some("https://example.co.uk/"),
+ ),
+ (
+ "http://a.user.github.io:8080/login",
+ Some("http://user.github.io:8080/"),
+ ),
+ (
+ "https://user:[email protected]/",
+ Some("https://example.com/"),
+ ),
+ ("https://example.com/", None),
+ ("https://gov.uk/", None),
+ ("http://localhost/", None),
+ ("http://127.0.0.1/", None),
+ ] {
+ let parent = parent_url(&Url::parse(input)?);
+ assert_eq!(parent.as_ref().map(Url::as_str), expected, "{input}");
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn rejects_missing_urls_and_unsupported_schemes() -> TestResult {
+ let fetcher = Fetcher::new()?;
+ for input in [
+ "",
+ "example.com",
+ "file:///tmp/icon.png",
+ "data:image/png;base64,abc",
+ ] {
+ assert!(
+ matches!(fetcher.fetch(input).await, Err(Error::InvalidUrl)),
+ "{input}"
+ );
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn bounds_streamed_bodies_without_trusting_content_length() -> TestResult {
+ let mut server = mockito::Server::new_async().await;
+ let response = server
+ .mock("GET", "/body")
+ .with_chunked_body(|writer| writer.write_all(&[b'a'; 20]))
+ .expect(2)
+ .create_async()
+ .await;
+ let http = Http::new()?;
+ let url = format!("{}/body", server.url());
+ let prefix = http.get(&url).send().await?.error_for_status()?;
+ assert_eq!(body(prefix, 10, true).await?, vec![b'a'; 10]);
+ let full = http.get(&url).send().await?.error_for_status()?;
+ assert!(matches!(body(full, 10, false).await, Err(Error::TooLarge)));
+ response.assert_async().await;
+ Ok(())
+}
diff --git a/rust/crates/icons/src/format.rs b/rust/crates/icons/src/format.rs
new file mode 100644
index 0000000000..19d8546ad8
--- /dev/null
+++ b/rust/crates/icons/src/format.rs
@@ -0,0 +1,291 @@
+use std::io::Cursor;
+
+use image::{DynamicImage, ImageDecoder, ImageFormat, ImageReader, Limits};
+
+pub(crate) const MAX_BYTES: usize = 1024 * 1024;
+
+pub(crate) fn png(data: &[u8]) -> Option<Vec<u8>> {
+ let format = image::guess_format(data).ok()?;
+ if !matches!(
+ format,
+ ImageFormat::Png
+ | ImageFormat::Jpeg
+ | ImageFormat::Gif
+ | ImageFormat::WebP
+ | ImageFormat::Ico
+ ) {
+ return None;
+ }
+ if format == ImageFormat::Ico {
+ validate_ico(data)?;
+ }
+ let mut limits = Limits::default();
+ limits.max_image_width = Some(1024);
+ limits.max_image_height = Some(1024);
+ limits.max_alloc = Some(16 * 1024 * 1024);
+ let mut reader = ImageReader::with_format(Cursor::new(data), format);
+ reader.limits(limits.clone());
+ let mut decoder = reader.into_decoder().ok()?;
+ limits.reserve(decoder.total_bytes()).ok()?;
+ decoder.set_limits(limits).ok()?;
+ let orientation = decoder.orientation().ok()?;
+ let mut image = DynamicImage::from_decoder(decoder).ok()?;
+ image.apply_orientation(orientation);
+ let mut output = Cursor::new(Vec::new());
+ image.write_to(&mut output, ImageFormat::Png).ok()?;
+ let output = output.into_inner();
+ (output.len() <= MAX_BYTES).then_some(output)
+}
+
+fn validate_ico(data: &[u8]) -> Option<()> {
+ let count = data.get(4..6)?;
+ let count = usize::from(u16::from_le_bytes([count[0], count[1]]));
+ let directory = data.get(6..6 + count * 16)?;
+ let mut budget = MAX_BYTES;
+ for entry in directory.chunks_exact(16) {
+ let length = u32::from_le_bytes([entry[8], entry[9], entry[10], entry[11]]) as usize;
+ if length < 8 {
+ return None;
+ }
+ let offset = u32::from_le_bytes([entry[12], entry[13], entry[14], entry[15]]) as usize;
+ let payload = data.get(offset..offset.checked_add(length)?)?;
+ if payload.starts_with(b"\x89PNG\r\n\x1a\n") {
+ validate_png(payload, &mut budget)?;
+ }
+ }
+ Some(())
+}
+
+fn validate_png(data: &[u8], budget: &mut usize) -> Option<()> {
+ let mut data = data.get(8..)?;
+ loop {
+ let header = data.get(..8)?;
+ *budget = budget.checked_sub(12)?;
+ let length = u32::from_be_bytes([header[0], header[1], header[2], header[3]]) as usize;
+ data = data.get(length.checked_add(12)?..)?;
+ if &header[4..8] == b"iCCP" {
+ return None;
+ }
+ if &header[4..8] == b"IEND" {
+ return (length == 0).then_some(());
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use image::ImageEncoder;
+
+ fn ico(entries: &[(u8, Vec<u8>)]) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
+ let mut output = vec![0, 0, 1, 0];
+ output.extend_from_slice(&u16::try_from(entries.len())?.to_le_bytes());
+ let mut offset = 6 + entries.len() * 16;
+ for (size, bytes) in entries {
+ output.extend_from_slice(&[*size, *size, 0, 0, 1, 0, 32, 0]);
+ output.extend_from_slice(&u32::try_from(bytes.len())?.to_le_bytes());
+ output.extend_from_slice(&u32::try_from(offset)?.to_le_bytes());
+ offset += bytes.len();
+ }
+ for (_, bytes) in entries {
+ output.extend_from_slice(bytes);
+ }
+ Ok(output)
+ }
+
+ fn png() -> Result<Vec<u8>, image::ImageError> {
+ let image = DynamicImage::new_rgba8(2, 2);
+ let mut bytes = Cursor::new(Vec::new());
+ image.write_to(&mut bytes, ImageFormat::Png)?;
+ Ok(bytes.into_inner())
+ }
+
+ #[test]
+ fn rejects_truncated_images_html_and_svg() -> Result<(), image::ImageError> {
+ let data = png()?;
+ assert!(super::png(&data).is_some());
+ for data in [
+ &data[..20],
+ b"<html>Not found</html>".as_slice(),
+ b"<svg xmlns='http://www.w3.org/2000/svg'><path d='M0 0'/></svg>",
+ b"",
+ ] {
+ assert!(super::png(data).is_none());
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn normalizes_supported_rasters_without_resizing() -> Result<(), Box<dyn std::error::Error>> {
+ let pixels = image::RgbaImage::from_fn(128, 128, |x, y| {
+ if (x + y) % 2 == 0 {
+ image::Rgba([20, 80, 200, 255])
+ } else {
+ image::Rgba([0, 0, 0, 0])
+ }
+ });
+ let image = DynamicImage::ImageRgba8(pixels);
+ for encoding in [
+ ImageFormat::Png,
+ ImageFormat::Jpeg,
+ ImageFormat::Gif,
+ ImageFormat::WebP,
+ ImageFormat::Ico,
+ ] {
+ let mut bytes = Cursor::new(Vec::new());
+ if encoding == ImageFormat::Jpeg {
+ DynamicImage::ImageRgb8(image.to_rgb8()).write_to(&mut bytes, encoding)?;
+ } else {
+ image.write_to(&mut bytes, encoding)?;
+ }
+ let output = super::png(bytes.get_ref()).ok_or("normalization failed")?;
+ assert_eq!(image::guess_format(&output)?, ImageFormat::Png);
+ let decoded = image::load_from_memory(&output)?;
+ assert_eq!((decoded.width(), decoded.height()), (128, 128));
+ if encoding != ImageFormat::Jpeg {
+ assert_eq!(decoded.to_rgba8(), image.to_rgba8());
+ }
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn emits_only_the_first_animation_frame() -> Result<(), Box<dyn std::error::Error>> {
+ let first = image::RgbaImage::from_pixel(8, 8, image::Rgba([255, 0, 0, 255]));
+ let second = image::RgbaImage::from_pixel(8, 8, image::Rgba([0, 255, 0, 255]));
+ let mut input = Vec::new();
+ image::codecs::gif::GifEncoder::new(&mut input)
+ .encode_frames([image::Frame::new(first.clone()), image::Frame::new(second)])?;
+ let output = super::png(&input).ok_or("animated image rejected")?;
+ assert_eq!(image::guess_format(&output)?, ImageFormat::Png);
+ assert_eq!(image::load_from_memory(&output)?.to_rgba8(), first);
+ Ok(())
+ }
+
+ #[test]
+ fn selects_a_larger_ico_entry_and_preserves_transparency()
+ -> Result<(), Box<dyn std::error::Error>> {
+ let small = DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(
+ 16,
+ 16,
+ image::Rgba([0, 255, 0, 255]),
+ ));
+ let large = DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(
+ 128,
+ 128,
+ image::Rgba([255, 0, 0, 80]),
+ ));
+ let mut entries = Vec::new();
+ for image in [&small, &large] {
+ let mut bytes = Cursor::new(Vec::new());
+ image.write_to(&mut bytes, ImageFormat::Png)?;
+ entries.push((u8::try_from(image.width())?, bytes.into_inner()));
+ }
+ let input = ico(&entries)?;
+ let output = super::png(&input).ok_or("ICO rejected")?;
+ assert_eq!(
+ image::load_from_memory(&output)?.to_rgba8(),
+ large.to_rgba8()
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn accepts_bitmap_backed_ico() -> Result<(), Box<dyn std::error::Error>> {
+ let mut bitmap = Vec::new();
+ for value in [40u32, 2, 4, 0x0020_0001, 0, 16, 0, 0, 0, 0] {
+ bitmap.extend_from_slice(&value.to_le_bytes());
+ }
+ for _ in 0..4 {
+ bitmap.extend_from_slice(&[255, 0, 0, 255]);
+ }
+ bitmap.extend_from_slice(&[0; 8]);
+ let output = super::png(&ico(&[(2, bitmap)])?).ok_or("bitmap icon rejected")?;
+ let image = image::load_from_memory(&output)?.to_rgba8();
+ assert_eq!(
+ image,
+ image::RgbaImage::from_pixel(2, 2, image::Rgba([0, 0, 255, 255]))
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn rejects_profile_tagged_ico_before_decoding() -> Result<(), Box<dyn std::error::Error>> {
+ let mut bytes = Vec::new();
+ let mut encoder = image::codecs::png::PngEncoder::new(&mut bytes);
+ encoder.set_icc_profile(vec![0; 128])?;
+ encoder.write_image(&[0, 0, 255, 255], 1, 1, image::ExtendedColorType::Rgba8)?;
+ assert!(super::png(&bytes).is_some());
+ let icon = ico(&[(1, bytes)])?;
+ assert!(validate_ico(&icon).is_none());
+ assert!(super::png(&icon).is_none());
+ Ok(())
+ }
+
+ #[test]
+ fn rejects_incomplete_ico_directory_and_png_chunk_framing()
+ -> Result<(), Box<dyn std::error::Error>> {
+ assert!(validate_ico(&[0, 0, 1, 0, 1, 0]).is_none());
+ let mut icon = ico(&[(2, png()?)])?;
+ for length in 0..8u32 {
+ let mut short = icon.clone();
+ short[14..18].copy_from_slice(&length.to_le_bytes());
+ assert!(validate_ico(&short).is_none());
+ }
+ icon[18..22].copy_from_slice(&u32::MAX.to_le_bytes());
+ assert!(validate_ico(&icon).is_none());
+ let truncated = ico(&[(2, b"\x89PNG\r\n\x1a\n\xff\xff\xff\xffIDAT".to_vec())])?;
+ assert!(validate_ico(&truncated).is_none());
+ let short = ico(&[(2, png()?[..20].to_vec())])?;
+ assert!(validate_ico(&short).is_none());
+ Ok(())
+ }
+
+ #[test]
+ fn rejects_oversized_dimensions() -> Result<(), image::ImageError> {
+ let image = DynamicImage::new_rgb8(1025, 1);
+ let mut bytes = Cursor::new(Vec::new());
+ image.write_to(&mut bytes, ImageFormat::Png)?;
+ assert!(super::png(bytes.get_ref()).is_none());
+ Ok(())
+ }
+
+ #[test]
+ fn applies_orientation_before_removing_metadata() -> Result<(), Box<dyn std::error::Error>> {
+ let mut bytes = Vec::new();
+ let mut encoder = image::codecs::png::PngEncoder::new(&mut bytes);
+ encoder.set_exif_metadata(
+ b"II*\0\x08\0\0\0\x01\0\x12\x01\x03\0\x01\0\0\0\x06\0\0\0\0\0\0\0".to_vec(),
+ )?;
+ encoder.write_image(
+ &[255, 0, 0, 255, 0, 255, 0, 255],
+ 2,
+ 1,
+ image::ExtendedColorType::Rgba8,
+ )?;
+ let output = super::png(&bytes).ok_or("oriented image rejected")?;
+ let image = image::load_from_memory(&output)?.to_rgba8();
+ assert_eq!(image.dimensions(), (1, 2));
+ assert_eq!(image.get_pixel(0, 0), &image::Rgba([255, 0, 0, 255]));
+ assert_eq!(image.get_pixel(0, 1), &image::Rgba([0, 255, 0, 255]));
+ Ok(())
+ }
+
+ #[test]
+ fn rejects_png_output_exceeding_the_byte_limit() -> Result<(), image::ImageError> {
+ let mut state = 1u32;
+ let pixels = image::RgbImage::from_fn(768, 768, |_, _| {
+ state ^= state << 13;
+ state ^= state >> 17;
+ state ^= state << 5;
+ let bytes = state.to_le_bytes();
+ image::Rgb([bytes[0], bytes[1], bytes[2]])
+ });
+ let mut bytes = Cursor::new(Vec::new());
+ image::codecs::jpeg::JpegEncoder::new_with_quality(&mut bytes, 70)
+ .encode_image(&DynamicImage::ImageRgb8(pixels))?;
+ assert!(bytes.get_ref().len() <= MAX_BYTES);
+ assert!(super::png(bytes.get_ref()).is_none());
+ Ok(())
+ }
+}
diff --git a/rust/crates/icons/src/lib.rs b/rust/crates/icons/src/lib.rs
new file mode 100644
index 0000000000..1b707b4c8b
--- /dev/null
+++ b/rust/crates/icons/src/lib.rs
@@ -0,0 +1,7 @@
+#![cfg(not(target_arch = "wasm32"))]
+
+mod discovery;
+mod fetch;
+mod format;
+
+pub use fetch::{Error, Fetcher, Icon};