Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Remove domain icon fetching crate

Neeraj Gupta04bd9efff958
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 24a2fde91c..0290516f33 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2291,12 +2291,6 @@ version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
-[[package]]
-name = "ego-tree"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b04dc5a38e4f151a79d9f2451ae6037fb6eaf5cba34771f44781f80e508498e3"
-
[[package]]
name = "either"
version = "1.15.0"
@@ -2700,22 +2694,6 @@ dependencies = [
"log",
]
-[[package]]
-name = "ente-icons"
-version = "0.0.0"
-dependencies = [
- "encoding_rs",
- "ente-core",
- "futures-util",
- "image",
- "mockito",
- "psl",
- "scraper",
- "thiserror 2.0.21",
- "tokio",
- "url",
-]
-
[[package]]
name = "ente-image"
version = "0.0.0"
@@ -7514,21 +7492,6 @@ dependencies = [
"syn 2.0.117",
]
-[[package]]
-name = "psl"
-version = "2.1.239"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5736f71353f867ae262bd123b86ddb161f50a2db0300ab762c8f4a0e659635d3"
-dependencies = [
- "psl-types",
-]
-
-[[package]]
-name = "psl-types"
-version = "2.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
-
[[package]]
name = "pxfm"
version = "0.1.29"
@@ -8442,20 +8405,6 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
-[[package]]
-name = "scraper"
-version = "0.27.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdd0be4d296f048bfb06dd01bbc80ef789ddd2e55583e8d2e6b804942abfabc2"
-dependencies = [
- "cssparser",
- "ego-tree",
- "html5ever",
- "precomputed-hash",
- "selectors",
- "tendril",
-]
-
[[package]]
name = "scratch"
version = "1.0.9"
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 3067ae73d3..3c2b636ee3 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -31,7 +31,6 @@ members = [
"crates/ensu",
"crates/ensu-crypto",
"crates/exif",
- "crates/icons",
"crates/image",
"crates/legacy",
"crates/location",
@@ -56,7 +55,6 @@ chrono = { version = "0.4.45", default-features = false }
clap = "4.6.7"
dialoguer = { version = "0.12.0", default-features = false }
dirs = "7.0.0"
-encoding_rs = "0.8.35"
ente-accounts = { path = "crates/accounts" }
ente-assets = { path = "crates/assets" }
ente-cast = { path = "crates/cast" }
@@ -68,7 +66,6 @@ ente-ensu-crypto = { path = "crates/ensu-crypto" }
ente-exif = { path = "crates/exif" }
ente-frb-lib = { path = "bindings/frb/lib" }
ente-frb-log = { path = "bindings/frb/log" }
-ente-icons = { path = "crates/icons" }
ente-image = { path = "crates/image" }
ente-legacy = { path = "crates/legacy" }
ente-location = { path = "crates/location" }
@@ -99,12 +96,10 @@ md-5 = "0.11.0"
mockito = "1.7.2"
ort = { version = "=2.0.0-rc.13", default-features = false }
ort-sys = { version = "=2.0.0-rc.13", default-features = false }
-psl = "2.1.239"
quick-xml = "0.42.0"
regex = "1.13.1"
reqwest = { version = "0.12.28", default-features = false }
rusqlite = { version = "0.40.2", default-features = false, features = ["bundled"] }
-scraper = { version = "0.27.0", default-features = false }
serde = { version = "1.0.229", features = ["derive"] }
serde-wasm-bindgen = "0.6.5"
serde_json = "1.0.151"
diff --git a/rust/crates/core/src/http.rs b/rust/crates/core/src/http.rs
index 6112fd8c5e..b18f68aa58 100644
--- a/rust/crates/core/src/http.rs
+++ b/rust/crates/core/src/http.rs
@@ -361,10 +361,6 @@ impl Response {
pub struct SuccessResponse(reqwest::Response);
impl SuccessResponse {
- pub fn url(&self) -> &reqwest::Url {
- self.0.url()
- }
-
pub fn headers(&self) -> &reqwest::header::HeaderMap {
self.0.headers()
}
diff --git a/rust/crates/icons/Cargo.toml b/rust/crates/icons/Cargo.toml
deleted file mode 100644
index 34bf79ed30..0000000000
--- a/rust/crates/icons/Cargo.toml
+++ /dev/null
@@ -1,21 +0,0 @@
-[package]
-name = "ente-icons"
-edition.workspace = true
-
-[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-encoding_rs.workspace = true
-ente-core.workspace = true
-futures-util.workspace = true
-image.workspace = true
-psl.workspace = true
-scraper.workspace = true
-thiserror.workspace = true
-tokio = { workspace = true, features = ["rt", "time"] }
-url.workspace = true
-
-[dev-dependencies]
-mockito.workspace = true
-tokio = { workspace = true, features = ["macros"] }
-
-[lints]
-workspace = true
diff --git a/rust/crates/icons/README.md b/rust/crates/icons/README.md
deleted file mode 100644
index 16e602cb24..0000000000
--- a/rust/crates/icons/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-Fetch a website icon from one HTTP(S) URL using `ente-core::http`. Returns a static PNG, or `None` if no supported icon is found. Requires native Rust with Tokio.
-
-```rust,no_run
-let icon = ente_icons::Fetcher::new()?.fetch("https://example.com").await?;
-```
diff --git a/rust/crates/icons/examples/fetch.rs b/rust/crates/icons/examples/fetch.rs
deleted file mode 100644
index 5fee251ed8..0000000000
--- a/rust/crates/icons/examples/fetch.rs
+++ /dev/null
@@ -1,13 +0,0 @@
-#[tokio::main(flavor = "current_thread")]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
- let mut args = std::env::args().skip(1);
- let url = args
- .next()
- .filter(|_| args.next().is_none())
- .ok_or("usage: cargo run -p ente-icons --example fetch -- https://example.com")?;
- match ente_icons::Fetcher::new()?.fetch(&url).await? {
- Some(icon) => println!("{} ({} bytes, image/png)", icon.url, icon.data.len()),
- None => println!("No icon found"),
- }
- Ok(())
-}
diff --git a/rust/crates/icons/src/discovery.rs b/rust/crates/icons/src/discovery.rs
deleted file mode 100644
index beb92e22b1..0000000000
--- a/rust/crates/icons/src/discovery.rs
+++ /dev/null
@@ -1,206 +0,0 @@
-use std::collections::HashSet;
-
-use scraper::{ElementRef, Html};
-use url::Url;
-
-const MAX_CANDIDATES: usize = 8;
-const PREFERRED_SIZE: u32 = 128;
-
-pub(crate) fn icon_urls(html: &str, page: &Url) -> Vec<Url> {
- let document = Html::parse_document(html);
- let Some(head) = document
- .root_element()
- .descendants()
- .filter_map(ElementRef::wrap)
- .find(|element| element.value().name() == "head")
- else {
- return Vec::new();
- };
- let base = head
- .descendants()
- .filter_map(ElementRef::wrap)
- .find(|element| element.value().name() == "base" && element.attr("href").is_some())
- .and_then(|element| page.join(element.attr("href")?.trim()).ok())
- .unwrap_or_else(|| page.clone());
-
- let mut candidates = Vec::new();
- for element in head.descendants().filter_map(ElementRef::wrap) {
- if element.value().name() != "link" {
- continue;
- }
- let Some(rel) = element.attr("rel") else {
- continue;
- };
- let icon = rel
- .split_ascii_whitespace()
- .any(|token| token.eq_ignore_ascii_case("icon"));
- let touch = rel.split_ascii_whitespace().any(|token| {
- token.eq_ignore_ascii_case("apple-touch-icon")
- || token.eq_ignore_ascii_case("apple-touch-icon-precomposed")
- });
- if !icon && !touch {
- continue;
- }
- let Some(href) = element
- .attr("href")
- .map(str::trim)
- .filter(|href| !href.is_empty())
- else {
- continue;
- };
- let Ok(mut url) = base.join(href) else {
- continue;
- };
- if !matches!(url.scheme(), "http" | "https") {
- continue;
- }
- url.set_fragment(None);
- let svg = element
- .attr("type")
- .is_some_and(|kind| kind.trim().eq_ignore_ascii_case("image/svg+xml"))
- || url
- .path()
- .rsplit_once('.')
- .is_some_and(|(_, extension)| extension.eq_ignore_ascii_case("svg"));
- let rank = (svg, size_rank(element.attr("sizes").unwrap_or("")), !icon);
- candidates.push((rank, url));
- }
- candidates.sort_by_key(|(rank, _)| *rank);
- let mut seen = HashSet::new();
- candidates
- .into_iter()
- .map(|(_, url)| url)
- .filter(|url| seen.insert(url.clone()))
- .take(MAX_CANDIDATES)
- .collect()
-}
-
-fn size_rank(sizes: &str) -> (u8, u32) {
- sizes
- .split_ascii_whitespace()
- .filter_map(|size| {
- let (width, height) = size.split_once(['x', 'X'])?;
- let width: u32 = width.parse().ok()?;
- let height: u32 = height.parse().ok()?;
- (width > 0 && width == height).then_some((
- u8::from(width < PREFERRED_SIZE),
- width.abs_diff(PREFERRED_SIZE),
- ))
- })
- .min()
- .unwrap_or((2, 0))
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn resolves_base_relative_and_cdn_links_and_orders_sizes() -> Result<(), url::ParseError> {
- let html = r#"<!doctype html><head>
- <link rel=ICON sizes=16x16 href=/small.png>
- <link rel="shortcut icon" sizes=128x128 href="best.png?a=1&amp;b=2">
- <base href="/assets/">
- <link rel=icon sizes=256x256 href=large.png>
- <link rel=apple-touch-icon href="//cdn.example.com/touch.png">
- <link rel=icon sizes=128x128 href="best.png?a=1&amp;b=2#duplicate">
- <link rel=stylesheet href=style.css>
- <link rel=icon href="http://cdn.example.com/plain.png">
- <link rel=icon href="data:image/png;base64,abc">
- </head><body><link rel=icon href=body.png>"#;
- let urls = icon_urls(html, &Url::parse("https://www.example.com/new/page")?);
- let urls: Vec<_> = urls.iter().map(Url::as_str).collect();
- assert_eq!(
- urls,
- [
- "https://www.example.com/assets/best.png?a=1&b=2",
- "https://www.example.com/assets/large.png",
- "https://www.example.com/small.png",
- "http://cdn.example.com/plain.png",
- "https://cdn.example.com/touch.png",
- ]
- );
- Ok(())
- }
-
- #[test]
- fn excludes_script_text_and_limits_candidate_requests() -> Result<(), url::ParseError> {
- let links: String = (0..20)
- .map(|i| format!("<link rel=icon href=/{i}.ico>"))
- .collect();
- let html = format!("<head><script>'<link rel=icon href=/fake.ico>'</script>{links}</head>");
- let urls = icon_urls(&html, &Url::parse("https://example.com")?);
- assert_eq!(urls.len(), MAX_CANDIDATES);
- assert_eq!(urls[0].path(), "/0.ico");
- Ok(())
- }
-
- #[test]
- fn declared_svgs_do_not_crowd_out_a_raster_candidate() -> Result<(), url::ParseError> {
- let mut html = "<head>".to_owned();
- for i in 0..MAX_CANDIDATES {
- html.push_str(&format!(
- "<link rel=icon type='image/svg+xml' sizes=any href=/{i}.svg>"
- ));
- }
- html.push_str("<link rel=icon sizes=128x128 href=/icon.png></head>");
- let urls = icon_urls(&html, &Url::parse("https://example.com/")?);
- assert_eq!(urls[0], Url::parse("https://example.com/icon.png")?);
- assert_eq!(urls.len(), MAX_CANDIDATES);
- Ok(())
- }
-
- #[test]
- fn prefers_sharp_rasters_including_touch_icons() -> Result<(), url::ParseError> {
- let html = "<head>
- <link rel=icon sizes=any href=/scalable.ico>
- <link rel=icon sizes=32x32 href=/small.png>
- <link rel=apple-touch-icon sizes=180x180 href=/touch.png>
- <link rel=apple-touch-icon sizes=128x128 href=/touch-128.png>
- <link rel=icon sizes='16x16 128x128' href=/icon.png>
- </head>";
- let urls = icon_urls(html, &Url::parse("https://example.com/")?);
- let paths: Vec<_> = urls.iter().map(Url::path).collect();
- assert_eq!(
- paths,
- [
- "/icon.png",
- "/touch-128.png",
- "/touch.png",
- "/small.png",
- "/scalable.ico"
- ]
- );
- Ok(())
- }
-
- #[test]
- fn accepts_http_and_https_links_with_custom_ports() -> Result<(), url::ParseError> {
- let page = Url::parse("https://example.com/")?;
- let html = "<head>
- <link rel=icon href=https://cdn.example.com:443/icon.png>
- <link rel=icon href=http://cdn.example.com:80/icon.png>
- <link rel=icon href=https://cdn.example.com:8443/icon.png>
- <link rel=icon href=http://cdn.example.com:8080/icon.png>
- </head>";
- let urls = icon_urls(html, &page);
- let urls: Vec<_> = urls.iter().map(Url::as_str).collect();
- assert_eq!(
- urls,
- [
- "https://cdn.example.com/icon.png",
- "http://cdn.example.com/icon.png",
- "https://cdn.example.com:8443/icon.png",
- "http://cdn.example.com:8080/icon.png"
- ]
- );
- assert_eq!(
- icon_urls(
- "<head><base href=https://cdn.example.com:8443/><link rel=icon href=icon.png></head>",
- &page,
- ),
- [Url::parse("https://cdn.example.com:8443/icon.png")?]
- );
- Ok(())
- }
-}
diff --git a/rust/crates/icons/src/fetch.rs b/rust/crates/icons/src/fetch.rs
deleted file mode 100644
index 280b3b5935..0000000000
--- a/rust/crates/icons/src/fetch.rs
+++ /dev/null
@@ -1,208 +0,0 @@
-use std::collections::HashSet;
-use std::time::Duration;
-
-use encoding_rs::Encoding;
-use ente_core::http::{self, Http, SuccessResponse};
-use futures_util::StreamExt;
-use scraper::{ElementRef, Html};
-use url::Url;
-
-use crate::{discovery, format};
-
-const HTML_LIMIT: usize = 256 * 1024;
-const LOOKUP_TIMEOUT: Duration = Duration::from_secs(15);
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
- #[error("expected an HTTP or HTTPS URL")]
- InvalidUrl,
- #[error("website request failed: {0}")]
- Http(#[from] http::Error),
- #[error("icon lookup timed out")]
- Timeout,
- #[error("icon exceeds the 1 MiB download limit")]
- TooLarge,
- #[error("image validation task failed: {0}")]
- Worker(#[from] tokio::task::JoinError),
-}
-
-#[derive(Debug)]
-pub struct Icon {
- pub url: Url,
- pub data: Vec<u8>,
-}
-
-pub struct Fetcher {
- http: Http,
-}
-
-impl Fetcher {
- pub fn new() -> Result<Self, Error> {
- Ok(Self { http: Http::new()? })
- }
-
- pub async fn fetch(&self, url: &str) -> Result<Option<Icon>, Error> {
- let page = Url::parse(url)
- .ok()
- .filter(|url| matches!(url.scheme(), "http" | "https"))
- .ok_or(Error::InvalidUrl)?;
- tokio::time::timeout(LOOKUP_TIMEOUT, self.find(page))
- .await
- .map_err(|_| Error::Timeout)?
- }
-
- async fn find(&self, page: Url) -> Result<Option<Icon>, Error> {
- let parent = parent_url(&page);
- let mut error = None;
- for page in std::iter::once(page).chain(parent) {
- match self.site(page).await {
- Ok(Some(icon)) => return Ok(Some(icon)),
- Ok(None) => {}
- Err(failure) => error = Some(failure),
- }
- }
- error.map_or(Ok(None), Err)
- }
-
- async fn site(&self, page: Url) -> Result<Option<Icon>, Error> {
- let mut error = None;
- let mut candidates = Vec::new();
- let mut fallback_page = page.clone();
- match self.get(&page).await {
- Ok(Some(response)) => {
- fallback_page = response.url().clone();
- let encoding = response
- .headers()
- .get("content-type")
- .and_then(|header| header.to_str().ok())
- .and_then(content_type_encoding);
- match body(response, HTML_LIMIT, true).await {
- Ok(bytes) => {
- let encoding = Encoding::for_bom(&bytes)
- .map(|(encoding, _)| encoding)
- .or(encoding)
- .or_else(|| html_encoding(&bytes));
- let text = encoding.unwrap_or(encoding_rs::UTF_8).decode(&bytes).0;
- candidates = discovery::icon_urls(&text, &fallback_page);
- }
- Err(failure) => error = Some(failure),
- }
- }
- Ok(None) => {}
- Err(failure) => error = Some(failure),
- }
- if let Ok(fallback) = fallback_page.join("/favicon.ico") {
- candidates.push(fallback);
- }
- if fallback_page.origin() != page.origin()
- && let Ok(fallback) = page.join("/favicon.ico")
- {
- candidates.push(fallback);
- }
- let mut seen = HashSet::new();
- for url in candidates {
- if !seen.insert(url.clone()) {
- continue;
- }
- match self.icon(&url).await {
- Ok(Some(icon)) => return Ok(Some(icon)),
- Ok(None) => {}
- Err(failure) => error = Some(failure),
- }
- }
- error.map_or(Ok(None), Err)
- }
-
- async fn get(&self, url: &Url) -> Result<Option<SuccessResponse>, Error> {
- let response = self.http.get(url.as_str()).send().await?;
- if matches!(response.status(), 404 | 410) {
- return Ok(None);
- }
- Ok(Some(response.error_for_status()?))
- }
-
- async fn icon(&self, url: &Url) -> Result<Option<Icon>, Error> {
- let Some(response) = self.get(url).await? else {
- return Ok(None);
- };
- let url = response.url().clone();
- let data = body(response, format::MAX_BYTES, false).await?;
- Ok(
- tokio::task::spawn_blocking(move || format::png(&data).map(|data| Icon { url, data }))
- .await?,
- )
- }
-}
-
-async fn body(response: SuccessResponse, limit: usize, prefix: bool) -> Result<Vec<u8>, Error> {
- let mut bytes = Vec::new();
- let mut chunks = std::pin::pin!(response.bytes_stream());
- while let Some(chunk) = chunks.next().await {
- let chunk = chunk?;
- let remaining = limit - bytes.len();
- if !prefix && chunk.len() > remaining {
- return Err(Error::TooLarge);
- }
- bytes.extend_from_slice(&chunk[..chunk.len().min(remaining)]);
- if prefix && bytes.len() == limit {
- break;
- }
- }
- Ok(bytes)
-}
-
-fn content_type_encoding(content_type: &str) -> Option<&'static Encoding> {
- content_type.split(';').find_map(|parameter| {
- let (name, value) = parameter.trim().split_once('=')?;
- name.trim()
- .eq_ignore_ascii_case("charset")
- .then(|| Encoding::for_label(value.trim().trim_matches(['\'', '"']).as_bytes()))?
- })
-}
-
-fn html_encoding(bytes: &[u8]) -> Option<&'static Encoding> {
- let prefix = String::from_utf8_lossy(&bytes[..bytes.len().min(1024)]);
- let document = Html::parse_document(&prefix);
- let encoding = document
- .root_element()
- .descendants()
- .filter_map(ElementRef::wrap)
- .filter(|element| element.value().name() == "meta")
- .find_map(|element| {
- if let Some(charset) = element.attr("charset") {
- Encoding::for_label(charset.as_bytes())
- } else {
- element
- .attr("http-equiv")
- .filter(|value| value.eq_ignore_ascii_case("content-type"))?;
- content_type_encoding(element.attr("content")?)
- }
- })?;
- Some(
- if encoding == encoding_rs::UTF_16LE || encoding == encoding_rs::UTF_16BE {
- encoding_rs::UTF_8
- } else if encoding == encoding_rs::X_USER_DEFINED {
- encoding_rs::WINDOWS_1252
- } else {
- encoding
- },
- )
-}
-
-fn parent_url(page: &Url) -> Option<Url> {
- let host = page.domain()?;
- let domain = psl::domain(host.as_bytes())?;
- if !domain.suffix().is_known() || domain.as_bytes() == host.as_bytes() {
- return None;
- }
- let mut parent = page.join("/").ok()?;
- parent
- .set_host(Some(std::str::from_utf8(domain.as_bytes()).ok()?))
- .ok()?;
- parent.set_username("").ok()?;
- parent.set_password(None).ok()?;
- Some(parent)
-}
-
-#[cfg(test)]
-mod tests;
diff --git a/rust/crates/icons/src/fetch/tests.rs b/rust/crates/icons/src/fetch/tests.rs
deleted file mode 100644
index 85567b16e5..0000000000
--- a/rust/crates/icons/src/fetch/tests.rs
+++ /dev/null
@@ -1,110 +0,0 @@
-use super::*;
-
-type TestResult = Result<(), Box<dyn std::error::Error>>;
-
-#[tokio::test]
-async fn shared_http_follows_redirects_and_resolves_declared_icons() -> TestResult {
- let mut server = mockito::Server::new_async().await;
- let page = server
- .mock("GET", "/")
- .with_status(302)
- .with_header("location", "/home")
- .create_async()
- .await;
- let html = server
- .mock("GET", "/home")
- .with_body("<head><base href=/assets/><link rel=icon href=logo.png></head>")
- .create_async()
- .await;
- let redirect = server
- .mock("GET", "/assets/logo.png")
- .with_status(302)
- .with_header("location", "/logo.png")
- .create_async()
- .await;
- let mut png = std::io::Cursor::new(Vec::new());
- image::DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(
- 2,
- 2,
- image::Rgba([10, 20, 30, 255]),
- ))
- .write_to(&mut png, image::ImageFormat::Png)?;
- let image = server
- .mock("GET", "/logo.png")
- .with_body(png.into_inner())
- .create_async()
- .await;
- let icon = Fetcher::new()?
- .fetch(&format!("{}/", server.url()))
- .await?
- .ok_or("missing icon")?;
- assert_eq!(icon.url.as_str(), format!("{}/logo.png", server.url()));
- assert_eq!(image::load_from_memory(&icon.data)?.width(), 2);
- page.assert_async().await;
- html.assert_async().await;
- redirect.assert_async().await;
- image.assert_async().await;
- Ok(())
-}
-
-#[test]
-fn parent_fallback_respects_registration_boundaries() -> TestResult {
- for (input, expected) in [
- (
- "https://login.example.co.uk/signin?q=1",
- Some("https://example.co.uk/"),
- ),
- (
- "http://a.user.github.io:8080/login",
- Some("http://user.github.io:8080/"),
- ),
- (
- "https://user:[email protected]/",
- Some("https://example.com/"),
- ),
- ("https://example.com/", None),
- ("https://gov.uk/", None),
- ("http://localhost/", None),
- ("http://127.0.0.1/", None),
- ] {
- let parent = parent_url(&Url::parse(input)?);
- assert_eq!(parent.as_ref().map(Url::as_str), expected, "{input}");
- }
- Ok(())
-}
-
-#[tokio::test]
-async fn rejects_missing_urls_and_unsupported_schemes() -> TestResult {
- let fetcher = Fetcher::new()?;
- for input in [
- "",
- "example.com",
- "file:///tmp/icon.png",
- "data:image/png;base64,abc",
- ] {
- assert!(
- matches!(fetcher.fetch(input).await, Err(Error::InvalidUrl)),
- "{input}"
- );
- }
- Ok(())
-}
-
-#[tokio::test]
-async fn bounds_streamed_bodies_without_trusting_content_length() -> TestResult {
- let mut server = mockito::Server::new_async().await;
- let response = server
- .mock("GET", "/body")
- .with_chunked_body(|writer| writer.write_all(&[b'a'; 20]))
- .expect(2)
- .create_async()
- .await;
- let http = Http::new()?;
- let url = format!("{}/body", server.url());
- let prefix = http.get(&url).send().await?.error_for_status()?;
- assert_eq!(body(prefix, 10, true).await?, vec![b'a'; 10]);
- let full = http.get(&url).send().await?.error_for_status()?;
- assert!(matches!(body(full, 10, false).await, Err(Error::TooLarge)));
- response.assert_async().await;
- Ok(())
-}
diff --git a/rust/crates/icons/src/format.rs b/rust/crates/icons/src/format.rs
deleted file mode 100644
index c71da4dbd0..0000000000
--- a/rust/crates/icons/src/format.rs
+++ /dev/null
@@ -1,291 +0,0 @@
-use std::io::Cursor;
-
-use image::{DynamicImage, ImageDecoder, ImageFormat, ImageReader, Limits};
-
-pub(crate) const MAX_BYTES: usize = 1024 * 1024;
-
-pub(crate) fn png(data: &[u8]) -> Option<Vec<u8>> {
- let format = image::guess_format(data).ok()?;
- if !matches!(
- format,
- ImageFormat::Png
- | ImageFormat::Jpeg
- | ImageFormat::Gif
- | ImageFormat::WebP
- | ImageFormat::Ico
- ) {
- return None;
- }
- if format == ImageFormat::Ico {
- validate_ico(data)?;
- }
- let mut limits = Limits::default();
- limits.max_image_width = Some(1024);
- limits.max_image_height = Some(1024);
- limits.max_alloc = Some(16 * 1024 * 1024);
- let mut reader = ImageReader::with_format(Cursor::new(data), format);
- reader.limits(limits.clone());
- let mut decoder = reader.into_decoder().ok()?;
- limits.reserve(decoder.total_bytes()).ok()?;
- decoder.set_limits(limits).ok()?;
- let orientation = decoder.orientation().ok()?;
- let mut image = DynamicImage::from_decoder(decoder).ok()?;
- image.apply_orientation(orientation);
- let mut output = Cursor::new(Vec::new());
- image.write_to(&mut output, ImageFormat::Png).ok()?;
- let output = output.into_inner();
- (output.len() <= MAX_BYTES).then_some(output)
-}
-
-fn validate_ico(data: &[u8]) -> Option<()> {
- let count = data.get(4..6)?;
- let count = usize::from(u16::from_le_bytes([count[0], count[1]]));
- let directory = data.get(6..6 + count * 16)?;
- let mut budget = MAX_BYTES;
- for entry in directory.as_chunks::<16>().0 {
- let length = u32::from_le_bytes([entry[8], entry[9], entry[10], entry[11]]) as usize;
- if length < 8 {
- return None;
- }
- let offset = u32::from_le_bytes([entry[12], entry[13], entry[14], entry[15]]) as usize;
- let payload = data.get(offset..offset.checked_add(length)?)?;
- if payload.starts_with(b"\x89PNG\r\n\x1a\n") {
- validate_png(payload, &mut budget)?;
- }
- }
- Some(())
-}
-
-fn validate_png(data: &[u8], budget: &mut usize) -> Option<()> {
- let mut data = data.get(8..)?;
- loop {
- let header = data.get(..8)?;
- *budget = budget.checked_sub(12)?;
- let length = u32::from_be_bytes([header[0], header[1], header[2], header[3]]) as usize;
- data = data.get(length.checked_add(12)?..)?;
- if &header[4..8] == b"iCCP" {
- return None;
- }
- if &header[4..8] == b"IEND" {
- return (length == 0).then_some(());
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use image::ImageEncoder;
-
- fn ico(entries: &[(u8, Vec<u8>)]) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
- let mut output = vec![0, 0, 1, 0];
- output.extend_from_slice(&u16::try_from(entries.len())?.to_le_bytes());
- let mut offset = 6 + entries.len() * 16;
- for (size, bytes) in entries {
- output.extend_from_slice(&[*size, *size, 0, 0, 1, 0, 32, 0]);
- output.extend_from_slice(&u32::try_from(bytes.len())?.to_le_bytes());
- output.extend_from_slice(&u32::try_from(offset)?.to_le_bytes());
- offset += bytes.len();
- }
- for (_, bytes) in entries {
- output.extend_from_slice(bytes);
- }
- Ok(output)
- }
-
- fn png() -> Result<Vec<u8>, image::ImageError> {
- let image = DynamicImage::new_rgba8(2, 2);
- let mut bytes = Cursor::new(Vec::new());
- image.write_to(&mut bytes, ImageFormat::Png)?;
- Ok(bytes.into_inner())
- }
-
- #[test]
- fn rejects_truncated_images_html_and_svg() -> Result<(), image::ImageError> {
- let data = png()?;
- assert!(super::png(&data).is_some());
- for data in [
- &data[..20],
- b"<html>Not found</html>".as_slice(),
- b"<svg xmlns='http://www.w3.org/2000/svg'><path d='M0 0'/></svg>",
- b"",
- ] {
- assert!(super::png(data).is_none());
- }
- Ok(())
- }
-
- #[test]
- fn normalizes_supported_rasters_without_resizing() -> Result<(), Box<dyn std::error::Error>> {
- let pixels = image::RgbaImage::from_fn(128, 128, |x, y| {
- if (x + y) % 2 == 0 {
- image::Rgba([20, 80, 200, 255])
- } else {
- image::Rgba([0, 0, 0, 0])
- }
- });
- let image = DynamicImage::ImageRgba8(pixels);
- for encoding in [
- ImageFormat::Png,
- ImageFormat::Jpeg,
- ImageFormat::Gif,
- ImageFormat::WebP,
- ImageFormat::Ico,
- ] {
- let mut bytes = Cursor::new(Vec::new());
- if encoding == ImageFormat::Jpeg {
- DynamicImage::ImageRgb8(image.to_rgb8()).write_to(&mut bytes, encoding)?;
- } else {
- image.write_to(&mut bytes, encoding)?;
- }
- let output = super::png(bytes.get_ref()).ok_or("normalization failed")?;
- assert_eq!(image::guess_format(&output)?, ImageFormat::Png);
- let decoded = image::load_from_memory(&output)?;
- assert_eq!((decoded.width(), decoded.height()), (128, 128));
- if encoding != ImageFormat::Jpeg {
- assert_eq!(decoded.to_rgba8(), image.to_rgba8());
- }
- }
- Ok(())
- }
-
- #[test]
- fn emits_only_the_first_animation_frame() -> Result<(), Box<dyn std::error::Error>> {
- let first = image::RgbaImage::from_pixel(8, 8, image::Rgba([255, 0, 0, 255]));
- let second = image::RgbaImage::from_pixel(8, 8, image::Rgba([0, 255, 0, 255]));
- let mut input = Vec::new();
- image::codecs::gif::GifEncoder::new(&mut input)
- .encode_frames([image::Frame::new(first.clone()), image::Frame::new(second)])?;
- let output = super::png(&input).ok_or("animated image rejected")?;
- assert_eq!(image::guess_format(&output)?, ImageFormat::Png);
- assert_eq!(image::load_from_memory(&output)?.to_rgba8(), first);
- Ok(())
- }
-
- #[test]
- fn selects_a_larger_ico_entry_and_preserves_transparency()
- -> Result<(), Box<dyn std::error::Error>> {
- let small = DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(
- 16,
- 16,
- image::Rgba([0, 255, 0, 255]),
- ));
- let large = DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(
- 128,
- 128,
- image::Rgba([255, 0, 0, 80]),
- ));
- let mut entries = Vec::new();
- for image in [&small, &large] {
- let mut bytes = Cursor::new(Vec::new());
- image.write_to(&mut bytes, ImageFormat::Png)?;
- entries.push((u8::try_from(image.width())?, bytes.into_inner()));
- }
- let input = ico(&entries)?;
- let output = super::png(&input).ok_or("ICO rejected")?;
- assert_eq!(
- image::load_from_memory(&output)?.to_rgba8(),
- large.to_rgba8()
- );
- Ok(())
- }
-
- #[test]
- fn accepts_bitmap_backed_ico() -> Result<(), Box<dyn std::error::Error>> {
- let mut bitmap = Vec::new();
- for value in [40u32, 2, 4, 0x0020_0001, 0, 16, 0, 0, 0, 0] {
- bitmap.extend_from_slice(&value.to_le_bytes());
- }
- for _ in 0..4 {
- bitmap.extend_from_slice(&[255, 0, 0, 255]);
- }
- bitmap.extend_from_slice(&[0; 8]);
- let output = super::png(&ico(&[(2, bitmap)])?).ok_or("bitmap icon rejected")?;
- let image = image::load_from_memory(&output)?.to_rgba8();
- assert_eq!(
- image,
- image::RgbaImage::from_pixel(2, 2, image::Rgba([0, 0, 255, 255]))
- );
- Ok(())
- }
-
- #[test]
- fn rejects_profile_tagged_ico_before_decoding() -> Result<(), Box<dyn std::error::Error>> {
- let mut bytes = Vec::new();
- let mut encoder = image::codecs::png::PngEncoder::new(&mut bytes);
- encoder.set_icc_profile(vec![0; 128])?;
- encoder.write_image(&[0, 0, 255, 255], 1, 1, image::ExtendedColorType::Rgba8)?;
- assert!(super::png(&bytes).is_some());
- let icon = ico(&[(1, bytes)])?;
- assert!(validate_ico(&icon).is_none());
- assert!(super::png(&icon).is_none());
- Ok(())
- }
-
- #[test]
- fn rejects_incomplete_ico_directory_and_png_chunk_framing()
- -> Result<(), Box<dyn std::error::Error>> {
- assert!(validate_ico(&[0, 0, 1, 0, 1, 0]).is_none());
- let mut icon = ico(&[(2, png()?)])?;
- for length in 0..8u32 {
- let mut short = icon.clone();
- short[14..18].copy_from_slice(&length.to_le_bytes());
- assert!(validate_ico(&short).is_none());
- }
- icon[18..22].copy_from_slice(&u32::MAX.to_le_bytes());
- assert!(validate_ico(&icon).is_none());
- let truncated = ico(&[(2, b"\x89PNG\r\n\x1a\n\xff\xff\xff\xffIDAT".to_vec())])?;
- assert!(validate_ico(&truncated).is_none());
- let short = ico(&[(2, png()?[..20].to_vec())])?;
- assert!(validate_ico(&short).is_none());
- Ok(())
- }
-
- #[test]
- fn rejects_oversized_dimensions() -> Result<(), image::ImageError> {
- let image = DynamicImage::new_rgb8(1025, 1);
- let mut bytes = Cursor::new(Vec::new());
- image.write_to(&mut bytes, ImageFormat::Png)?;
- assert!(super::png(bytes.get_ref()).is_none());
- Ok(())
- }
-
- #[test]
- fn applies_orientation_before_removing_metadata() -> Result<(), Box<dyn std::error::Error>> {
- let mut bytes = Vec::new();
- let mut encoder = image::codecs::png::PngEncoder::new(&mut bytes);
- encoder.set_exif_metadata(
- b"II*\0\x08\0\0\0\x01\0\x12\x01\x03\0\x01\0\0\0\x06\0\0\0\0\0\0\0".to_vec(),
- )?;
- encoder.write_image(
- &[255, 0, 0, 255, 0, 255, 0, 255],
- 2,
- 1,
- image::ExtendedColorType::Rgba8,
- )?;
- let output = super::png(&bytes).ok_or("oriented image rejected")?;
- let image = image::load_from_memory(&output)?.to_rgba8();
- assert_eq!(image.dimensions(), (1, 2));
- assert_eq!(image.get_pixel(0, 0), &image::Rgba([255, 0, 0, 255]));
- assert_eq!(image.get_pixel(0, 1), &image::Rgba([0, 255, 0, 255]));
- Ok(())
- }
-
- #[test]
- fn rejects_png_output_exceeding_the_byte_limit() -> Result<(), image::ImageError> {
- let mut state = 1u32;
- let pixels = image::RgbImage::from_fn(768, 768, |_, _| {
- state ^= state << 13;
- state ^= state >> 17;
- state ^= state << 5;
- let bytes = state.to_le_bytes();
- image::Rgb([bytes[0], bytes[1], bytes[2]])
- });
- let mut bytes = Cursor::new(Vec::new());
- image::codecs::jpeg::JpegEncoder::new_with_quality(&mut bytes, 70)
- .encode_image(&DynamicImage::ImageRgb8(pixels))?;
- assert!(bytes.get_ref().len() <= MAX_BYTES);
- assert!(super::png(bytes.get_ref()).is_none());
- Ok(())
- }
-}
diff --git a/rust/crates/icons/src/lib.rs b/rust/crates/icons/src/lib.rs
deleted file mode 100644
index 1b707b4c8b..0000000000
--- a/rust/crates/icons/src/lib.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-#![cfg(not(target_arch = "wasm32"))]
-
-mod discovery;
-mod fetch;
-mod format;
-
-pub use fetch::{Error, Fetcher, Icon};