A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(unfurl): bound html, oembed and activitypub parsing (#3879)
Hampuse43d33728142
diff --git a/Cargo.lock b/Cargo.lockindex 6bbdfa6c4..93318327c 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1884,6 +1884,7 @@ dependencies = ["fluxer-svc","fluxer_common","hickory-resolver",+ "html5ever","infer","moka","regex",diff --git a/fluxer_unfurl/Cargo.toml b/fluxer_unfurl/Cargo.tomlindex 3c2b54a8b..76c7e3b68 100644--- a/fluxer_unfurl/Cargo.toml+++ b/fluxer_unfurl/Cargo.toml@@ -15,6 +15,7 @@ encoding_rs = "0.8.44"entities = "1.0.1"fluxer-svc = { path = "../fluxer_svc", default-features = false }hickory-resolver = { version = "0.26.3", default-features = false, features = ["tokio"] }+html5ever = "0.40"infer = "0.22.0"moka = { version = "0.12.16", features = ["future", "sync"] }regex = "1.13"diff --git a/fluxer_unfurl/src/activity_pub/formatter.rs b/fluxer_unfurl/src/activity_pub/formatter.rsindex 7663fbf9e..09924de3b 100644--- a/fluxer_unfurl/src/activity_pub/formatter.rs+++ b/fluxer_unfurl/src/activity_pub/formatter.rs@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+use super::limits;use super::types::{ActivityPubActor, ActivityPubCollectionCount, ActivityPubContext, ActivityPubPost,MastodonMediaAttachment, MastodonPost,@@ -235,7 +236,20 @@ async fn resolve_mastodon_media_embeds(let mut primary_media_claimed = false;let nsfw_str = MediaProxyClient::nsfw_mode_str(nsfw_mode);- for attachment in post.media_attachments.as_deref().unwrap_or_default() {+ let attachments = post+ .media_attachments+ .as_deref()+ .unwrap_or_default()+ .iter()+ .filter(|attachment| {+ matches!(+ attachment.attachment_type.as_deref(),+ Some("image" | "gifv" | "video")+ )+ })+ .take(limits::MAX_MEDIA_ATTACHMENTS);++ for attachment in attachments {match attachment.attachment_type.as_deref() {Some("image" | "gifv") => {let Some(image) = process_mastodon_media(attachment, media_proxy, nsfw_str).await@@ -290,7 +304,15 @@ async fn resolve_activity_pub_media_embeds(let mut primary_media_claimed = false;let nsfw_str = MediaProxyClient::nsfw_mode_str(nsfw_mode);- for attachment in post.attachment.as_deref().unwrap_or_default() {+ let attachments = post+ .attachment+ .as_deref()+ .unwrap_or_default()+ .iter()+ .filter(|attachment| attachment_is_image(attachment) || attachment_is_video(attachment))+ .take(limits::MAX_MEDIA_ATTACHMENTS);++ for attachment in attachments {if attachment_is_image(attachment) {let Some(image) = process_activity_pub_media(attachment, media_proxy, nsfw_str).awaitelse {diff --git a/fluxer_unfurl/src/activity_pub/limits.rs b/fluxer_unfurl/src/activity_pub/limits.rsindex a3c800f2e..69cf34043 100644--- a/fluxer_unfurl/src/activity_pub/limits.rs+++ b/fluxer_unfurl/src/activity_pub/limits.rs@@ -5,3 +5,5 @@ pub const INSTANCE_INFO_MAX_BYTES: usize = 256 * 1024;pub const OBJECT_MAX_BYTES: usize = 1024 * 1024;pub const MASTODON_STATUS_MAX_BYTES: usize = 1024 * 1024;++pub const MAX_MEDIA_ATTACHMENTS: usize = 10;diff --git a/fluxer_unfurl/src/html_parser.rs b/fluxer_unfurl/src/html_parser.rsindex 38bd79455..82500375e 100644--- a/fluxer_unfurl/src/html_parser.rs+++ b/fluxer_unfurl/src/html_parser.rs@@ -1,6 +1,18 @@// SPDX-License-Identifier: AGPL-3.0-or-later-use scraper::{Html, Selector};+use std::borrow::Cow;+use std::cell::Cell;+use std::rc::Rc;++use html5ever::driver::{self, ParseOpts, Parser};+use html5ever::tendril::{StrTendril, TendrilSink};+use html5ever::tree_builder::{ElementFlags, NodeOrText, QuirksMode, TreeSink};+use html5ever::{Attribute, QualName, local_name, ns};+use scraper::{Html, HtmlTreeSink, Selector};++const MAX_NESTING_DEPTH: usize = 512;++const PARSE_CHUNK_BYTES: usize = 4096;const TEXTUAL_KEYS: [&str; 5] = ["og:title",@@ -14,6 +26,171 @@ const CLASSIFYING_CARDS: [&str; 3] = ["summary_large_image", "photo", "player"];const INERT_ELEMENTS: [&str; 4] = ["script", "style", "noscript", "template"];+type Handle = <HtmlTreeSink as TreeSink>::Handle;++struct DepthLimitedSink {+ inner: HtmlTreeSink,+ too_deep: Rc<Cell<bool>>,+}++impl DepthLimitedSink {+ fn note_depth_below(&self, node: &Handle) {+ if self.too_deep.get() {+ return;+ }+ let html = self.inner.0.borrow();+ let depth = html+ .tree+ .get(*node)+ .map_or(0, |node| node.ancestors().take(MAX_NESTING_DEPTH).count());+ if depth >= MAX_NESTING_DEPTH {+ self.too_deep.set(true);+ }+ }+}++impl TreeSink for DepthLimitedSink {+ type Output = Html;+ type Handle = Handle;+ type ElemName<'a> = <HtmlTreeSink as TreeSink>::ElemName<'a>;++ fn finish(self) -> Html {+ self.inner.finish()+ }++ fn parse_error(&self, msg: Cow<'static, str>) {+ self.inner.parse_error(msg);+ }++ fn get_document(&self) -> Handle {+ self.inner.get_document()+ }++ fn elem_name<'a>(&'a self, target: &'a Handle) -> Self::ElemName<'a> {+ self.inner.elem_name(target)+ }++ fn create_element(&self, name: QualName, attrs: Vec<Attribute>, flags: ElementFlags) -> Handle {+ self.inner.create_element(name, attrs, flags)+ }++ fn create_comment(&self, text: StrTendril) -> Handle {+ self.inner.create_comment(text)+ }++ fn create_pi(&self, target: StrTendril, data: StrTendril) -> Handle {+ self.inner.create_pi(target, data)+ }++ fn append(&self, parent: &Handle, child: NodeOrText<Handle>) {+ if matches!(child, NodeOrText::AppendNode(_)) {+ self.note_depth_below(parent);+ }+ self.inner.append(parent, child);+ }++ fn append_based_on_parent_node(+ &self,+ element: &Handle,+ prev_element: &Handle,+ child: NodeOrText<Handle>,+ ) {+ if matches!(child, NodeOrText::AppendNode(_)) {+ self.note_depth_below(element);+ self.note_depth_below(prev_element);+ }+ self.inner+ .append_based_on_parent_node(element, prev_element, child);+ }++ fn append_doctype_to_document(+ &self,+ name: StrTendril,+ public_id: StrTendril,+ system_id: StrTendril,+ ) {+ self.inner+ .append_doctype_to_document(name, public_id, system_id);+ }++ fn mark_script_already_started(&self, node: &Handle) {+ self.inner.mark_script_already_started(node);+ }++ fn pop(&self, node: &Handle) {+ self.inner.pop(node);+ }++ fn get_template_contents(&self, target: &Handle) -> Handle {+ self.inner.get_template_contents(target)+ }++ fn same_node(&self, x: &Handle, y: &Handle) -> bool {+ self.inner.same_node(x, y)+ }++ fn set_quirks_mode(&self, mode: QuirksMode) {+ self.inner.set_quirks_mode(mode);+ }++ fn append_before_sibling(&self, sibling: &Handle, new_node: NodeOrText<Handle>) {+ if matches!(new_node, NodeOrText::AppendNode(_)) {+ self.note_depth_below(sibling);+ }+ self.inner.append_before_sibling(sibling, new_node);+ }++ fn add_attrs_if_missing(&self, target: &Handle, attrs: Vec<Attribute>) {+ self.inner.add_attrs_if_missing(target, attrs);+ }++ fn remove_from_parent(&self, target: &Handle) {+ self.inner.remove_from_parent(target);+ }++ fn reparent_children(&self, node: &Handle, new_parent: &Handle) {+ self.note_depth_below(new_parent);+ self.inner.reparent_children(node, new_parent);+ }+}++fn parse_with_depth_limit(+ input: &str,+ document: Html,+ start: impl FnOnce(DepthLimitedSink) -> Parser<DepthLimitedSink>,+) -> Html {+ let too_deep = Rc::new(Cell::new(false));+ let mut parser = start(DepthLimitedSink {+ inner: HtmlTreeSink::new(document),+ too_deep: Rc::clone(&too_deep),+ });+ let mut rest = input;+ while !rest.is_empty() && !too_deep.get() {+ let (chunk, tail) = rest.split_at(rest.floor_char_boundary(PARSE_CHUNK_BYTES));+ parser.process(StrTendril::from_slice(chunk));+ rest = tail;+ }+ parser.finish()+}++pub(crate) fn parse_document(html: &str) -> Html {+ parse_with_depth_limit(html, Html::new_document(), |sink| {+ driver::parse_document(sink, ParseOpts::default())+ })+}++pub(crate) fn parse_fragment(fragment: &str) -> Html {+ parse_with_depth_limit(fragment, Html::new_fragment(), |sink| {+ driver::parse_fragment(+ sink,+ ParseOpts::default(),+ QualName::new(None, ns!(html), local_name!("body")),+ Vec::new(),+ false,+ )+ })+}+#[derive(Debug, Default, Clone)]pub struct OgMetadata {pub title: Option<String>,@@ -82,7 +259,7 @@ impl MetaTags {}pub fn parse_opengraph(html: &str) -> OgMetadata {- let doc = Html::parse_document(html);+ let doc = parse_document(html);let meta = MetaTags::parse(&doc);let description = meta@@ -144,7 +321,7 @@ fn document_title(doc: &Html) -> Option<String> {}pub fn parse_twitter_card(html: &str) -> TwitterCardMetadata {- let doc = Html::parse_document(html);+ let doc = parse_document(html);let meta = MetaTags::parse(&doc);TwitterCardMetadata {@@ -157,7 +334,7 @@ pub fn parse_twitter_card(html: &str) -> TwitterCardMetadata {}pub fn find_rsd_url(html: &str) -> Option<String> {- let doc = Html::parse_document(html);+ let doc = parse_document(html);let sel = Selector::parse("link").ok()?;for el in doc.select(&sel) {@@ -185,7 +362,7 @@ pub fn find_rsd_url(html: &str) -> Option<String> {}pub fn find_activity_pub_link(html: &str) -> Option<String> {- let doc = Html::parse_document(html);+ let doc = parse_document(html);let sel = Selector::parse("link").ok()?;for el in doc.select(&sel) {@@ -217,7 +394,7 @@ pub fn find_activity_pub_link(html: &str) -> Option<String> {}pub fn has_fediverse_profile_username(html: &str) -> bool {- let doc = Html::parse_document(html);+ let doc = parse_document(html);let meta = MetaTags::parse(&doc);meta.all("og:profile:username").chain(meta.all("profile:username"))@@ -241,7 +418,7 @@ fn is_acct_shaped(value: &str) -> bool {}pub fn find_canonical_url(html: &str, base_url: &url::Url) -> Option<String> {- let doc = Html::parse_document(html);+ let doc = parse_document(html);let sel = Selector::parse("link[rel=\"canonical\"]").ok()?;let el = doc.select(&sel).next()?;let href = el.value().attr("href")?;@@ -252,7 +429,7 @@ pub fn find_canonical_url(html: &str, base_url: &url::Url) -> Option<String> {}pub fn find_apple_touch_icon(html: &str, base_url: &url::Url) -> Option<String> {- let doc = Html::parse_document(html);+ let doc = parse_document(html);for selector in [r#"link[rel="apple-touch-icon"][sizes="180x180"]"#,r#"link[rel="apple-touch-icon"]"#,@@ -315,3 +492,16 @@ fn normalize_image_reference_key(value: &str) -> Option<String> {fn is_http_url(url: &url::Url) -> bool {matches!(url.scheme(), "http" | "https")}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn deeply_nested_page_keeps_head_metadata() {+ let mut html =+ String::from(r#"<html><head><meta property="og:title" content="Kept"></head><body>"#);+ html.push_str(&"<div>".repeat(200_000));+ assert_eq!(parse_opengraph(&html).title.as_deref(), Some("Kept"));+ }+}diff --git a/fluxer_unfurl/src/http_fetch.rs b/fluxer_unfurl/src/http_fetch.rsindex 54db9bb15..788fadff3 100644--- a/fluxer_unfurl/src/http_fetch.rs+++ b/fluxer_unfurl/src/http_fetch.rs@@ -124,7 +124,8 @@ whereclient.get(current_url.clone()).headers(hop_headers).send(),).await- .map_err(|_| anyhow::anyhow!("request timed out"))??;+ .map_err(|_| anyhow::anyhow!("request timed out"))?+ .map_err(reqwest::Error::without_url)?;let status = resp.status();let headers = resp.headers().clone();@@ -189,7 +190,7 @@ pub async fn read_body(resp: reqwest::Response, max_bytes: usize) -> anyhow::Reslet mut buf = Vec::with_capacity(max_bytes.min(64 * 1024));let mut stream = resp;- while let Some(chunk) = stream.chunk().await? {+ while let Some(chunk) = stream.chunk().await.map_err(reqwest::Error::without_url)? {let next_len = buf.len().checked_add(chunk.len())diff --git a/fluxer_unfurl/src/network_policy.rs b/fluxer_unfurl/src/network_policy.rsindex eae6ead03..e54526a83 100644--- a/fluxer_unfurl/src/network_policy.rs+++ b/fluxer_unfurl/src/network_policy.rs@@ -3,10 +3,13 @@use fluxer_common::public_ip::{is_public_ip, is_valid_public_hostname};use reqwest::dns::{Addrs, Name, Resolve, Resolving};use std::net::{IpAddr, SocketAddr};+use std::time::Duration;use tokio::net::lookup_host;+use tokio::time::timeout;use url::Url;const MAX_URL_LEN: usize = 8192;+const DNS_LOOKUP_TIMEOUT: Duration = Duration::from_secs(10);#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]pub enum Error {@@ -42,8 +45,9 @@ pub async fn validate_url(url: &Url) -> Result<(), Error> {}let port = url.port_or_known_default().ok_or(Error::BlockedUrl)?;let mut seen = false;- let addrs = lookup_host((host.as_str(), port))+ let addrs = timeout(DNS_LOOKUP_TIMEOUT, lookup_host((host.as_str(), port))).await+ .map_err(|_| Error::DnsLookupFailed)?.map_err(|_| Error::DnsLookupFailed)?;for addr in addrs {seen = true;diff --git a/fluxer_unfurl/src/oembed.rs b/fluxer_unfurl/src/oembed.rsindex 1acfa932e..2290f3e9b 100644--- a/fluxer_unfurl/src/oembed.rs+++ b/fluxer_unfurl/src/oembed.rs@@ -1,10 +1,13 @@// SPDX-License-Identifier: AGPL-3.0-or-later+use crate::html_parser;use crate::http_fetch;-use scraper::{Html, Selector};+use scraper::Selector;use serde::Deserialize;use std::time::Duration;+const MAX_OEMBED_ENDPOINTS: usize = 3;+#[derive(Debug, Clone)]pub struct OEmbedEndpoint {pub url: String,@@ -36,13 +39,13 @@ pub struct OEmbedResponse {}pub fn discover_oembed_url(html: &str) -> Vec<OEmbedEndpoint> {- let doc = Html::parse_document(html);+ let doc = html_parser::parse_document(html);let sel = match Selector::parse("link") {Ok(s) => s,Err(_) => return Vec::new(),};- let mut endpoints = Vec::new();+ let mut endpoints: Vec<OEmbedEndpoint> = Vec::new();for el in doc.select(&sel) {let href = match el.value().attr("href") {@@ -54,7 +57,7 @@ pub fn discover_oembed_url(html: &str) -> Vec<OEmbedEndpoint> {None => continue,};- if !link_type.contains("+oembed") {+ if !link_type.contains("+oembed") || endpoints.iter().any(|ep| ep.url == href) {continue;}@@ -72,6 +75,9 @@ pub fn discover_oembed_url(html: &str) -> Vec<OEmbedEndpoint> {url: href.to_owned(),format,});+ if endpoints.len() >= MAX_OEMBED_ENDPOINTS {+ break;+ }}endpointsdiff --git a/fluxer_unfurl/src/resolvers/default_helpers.rs b/fluxer_unfurl/src/resolvers/default_helpers.rsindex f6f3543c4..9c61e283a 100644--- a/fluxer_unfurl/src/resolvers/default_helpers.rs+++ b/fluxer_unfurl/src/resolvers/default_helpers.rs@@ -213,7 +213,7 @@ pub fn parse_text(value: &str, max_len: usize) -> String {}pub fn decode_html_text(input: &str) -> String {- scraper::Html::parse_fragment(input)+ crate::html_parser::parse_fragment(input).root_element().text().collect()@@ -243,7 +243,10 @@ pub fn format_timestamp(unix_seconds: i64) -> Option<String> {}pub fn sanitize_absolute_url(value: &str) -> Option<String> {- Url::parse(value.trim()).ok().map(|url| url.to_string())+ Url::parse(value.trim())+ .ok()+ .filter(|url| matches!(url.scheme(), "http" | "https"))+ .map(|url| url.to_string())}#[cfg(test)]diff --git a/fluxer_unfurl/src/resolvers/default_resolver.rs b/fluxer_unfurl/src/resolvers/default_resolver.rsindex 21b87aa1b..74f78247c 100644--- a/fluxer_unfurl/src/resolvers/default_resolver.rs+++ b/fluxer_unfurl/src/resolvers/default_resolver.rs@@ -478,6 +478,7 @@ fn optional_oembed_string(value: Option<&str>) -> Option<&str> {fn optional_absolute_url(value: Option<&str>) -> Option<String> {Url::parse(optional_oembed_string(value)?).ok()+ .filter(|url| matches!(url.scheme(), "http" | "https")).map(|url| url.to_string())}diff --git a/fluxer_unfurl/src/resolvers/tenor.rs b/fluxer_unfurl/src/resolvers/tenor.rsindex 2af30ca63..7e9a24bed 100644--- a/fluxer_unfurl/src/resolvers/tenor.rs+++ b/fluxer_unfurl/src/resolvers/tenor.rs@@ -6,7 +6,7 @@ use crate::html_parser;use crate::http_fetch;use crate::media_proxy::MediaProxyClient;use crate::types::{EmbedMedia, EmbedProvider, MessageEmbed};-use scraper::{Html, Selector};+use scraper::Selector;use std::future::Future;use std::pin::Pin;use std::time::Duration;@@ -144,7 +144,7 @@ struct TenorJsonLdUrls {}fn extract_json_ld_urls(html: &str) -> Option<TenorJsonLdUrls> {- let doc = Html::parse_document(html);+ let doc = html_parser::parse_document(html);let selector = Selector::parse(r#"script.dynamic[type="application/ld+json"]"#).ok()?;for script in doc.select(&selector) {diff --git a/fluxer_unfurl/src/resolvers/xkcd.rs b/fluxer_unfurl/src/resolvers/xkcd.rsindex e51e8e681..b88a905b5 100644--- a/fluxer_unfurl/src/resolvers/xkcd.rs+++ b/fluxer_unfurl/src/resolvers/xkcd.rs@@ -51,7 +51,7 @@ impl Resolver for XkcdResolver {let title = og.title.as_deref().map(|t| parse_text(t, 70));let image_alt = {- let doc = Html::parse_document(&html);+ let doc = html_parser::parse_document(&html);extract_comic_alt(&doc)};let footer_text = image_alt.clone();diff --git a/fluxer_unfurl/src/resolvers/youtube.rs b/fluxer_unfurl/src/resolvers/youtube.rsindex b75d8b434..64aadfc8d 100644--- a/fluxer_unfurl/src/resolvers/youtube.rs+++ b/fluxer_unfurl/src/resolvers/youtube.rs@@ -4,6 +4,7 @@ use super::{ResolveContext, Resolver, ResolverResult};use crate::http_fetch;use crate::text_limits;use crate::types::{EmbedAuthor, EmbedMedia, EmbedProvider, MessageEmbed};+use reqwest::header::{HeaderMap, HeaderValue};use serde::Deserialize;use std::future::Future;use std::pin::Pin;@@ -12,6 +13,7 @@ use url::Url;const YOUTUBE_COLOR: u32 = 0xFF0000;const YOUTUBE_API_BASE: &str = "https://www.googleapis.com/youtube/v3/videos";+const YOUTUBE_API_KEY_HEADER: &str = "x-goog-api-key";const YOUTUBE_API_MAX_BYTES: usize = 256 * 1024;const YOUTUBE_API_TIMEOUT: Duration = Duration::from_secs(10);const YOUTUBE_DESCRIPTION_MAX: usize = 350;@@ -85,10 +87,20 @@ async fn fetch_api_video(video_id: &str,api_key: &str,) -> Option<YouTubeVideo> {- let api_url = build_youtube_api_url(api_base, video_id, api_key).ok()?;- let api_response = match http_fetch::fetch_url(+ let api_url = build_youtube_api_url(api_base, video_id).ok()?;+ let api_origin = api_url.origin();+ let api_response = match http_fetch::fetch_url_with_headers(&ctx.http_client,api_url.as_ref(),+ |hop| {+ let mut headers = HeaderMap::new();+ if hop.origin() == api_origin {+ let mut value = HeaderValue::from_str(api_key)?;+ value.set_sensitive(true);+ headers.insert(YOUTUBE_API_KEY_HEADER, value);+ }+ Ok(headers)+ },YOUTUBE_API_MAX_BYTES,YOUTUBE_API_TIMEOUT,)@@ -305,10 +317,9 @@ struct YouTubeThumbnail {height: Option<u32>,}-fn build_youtube_api_url(base: &str, video_id: &str, api_key: &str) -> anyhow::Result<Url> {+fn build_youtube_api_url(base: &str, video_id: &str) -> anyhow::Result<Url> {let mut url = Url::parse(base)?;url.query_pairs_mut()- .append_pair("key", api_key).append_pair("id", video_id).append_pair("part", "snippet,player,status");Ok(url)