Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(unfurl): bound html, oembed and activitypub parsing (#3879)

Hampuse43d33728142
diff --git a/Cargo.lock b/Cargo.lock
index 6bbdfa6c4..93318327c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1884,6 +1884,7 @@ dependencies = [
"fluxer-svc",
"fluxer_common",
"hickory-resolver",
+ "html5ever",
"infer",
"moka",
"regex",
diff --git a/fluxer_unfurl/Cargo.toml b/fluxer_unfurl/Cargo.toml
index 3c2b54a8b..76c7e3b68 100644
--- a/fluxer_unfurl/Cargo.toml
+++ b/fluxer_unfurl/Cargo.toml
@@ -15,6 +15,7 @@ encoding_rs = "0.8.44"
entities = "1.0.1"
fluxer-svc = { path = "../fluxer_svc", default-features = false }
hickory-resolver = { version = "0.26.3", default-features = false, features = ["tokio"] }
+html5ever = "0.40"
infer = "0.22.0"
moka = { version = "0.12.16", features = ["future", "sync"] }
regex = "1.13"
diff --git a/fluxer_unfurl/src/activity_pub/formatter.rs b/fluxer_unfurl/src/activity_pub/formatter.rs
index 7663fbf9e..09924de3b 100644
--- a/fluxer_unfurl/src/activity_pub/formatter.rs
+++ b/fluxer_unfurl/src/activity_pub/formatter.rs
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
+use super::limits;
use super::types::{
ActivityPubActor, ActivityPubCollectionCount, ActivityPubContext, ActivityPubPost,
MastodonMediaAttachment, MastodonPost,
@@ -235,7 +236,20 @@ async fn resolve_mastodon_media_embeds(
let mut primary_media_claimed = false;
let nsfw_str = MediaProxyClient::nsfw_mode_str(nsfw_mode);
- for attachment in post.media_attachments.as_deref().unwrap_or_default() {
+ let attachments = post
+ .media_attachments
+ .as_deref()
+ .unwrap_or_default()
+ .iter()
+ .filter(|attachment| {
+ matches!(
+ attachment.attachment_type.as_deref(),
+ Some("image" | "gifv" | "video")
+ )
+ })
+ .take(limits::MAX_MEDIA_ATTACHMENTS);
+
+ for attachment in attachments {
match attachment.attachment_type.as_deref() {
Some("image" | "gifv") => {
let Some(image) = process_mastodon_media(attachment, media_proxy, nsfw_str).await
@@ -290,7 +304,15 @@ async fn resolve_activity_pub_media_embeds(
let mut primary_media_claimed = false;
let nsfw_str = MediaProxyClient::nsfw_mode_str(nsfw_mode);
- for attachment in post.attachment.as_deref().unwrap_or_default() {
+ let attachments = post
+ .attachment
+ .as_deref()
+ .unwrap_or_default()
+ .iter()
+ .filter(|attachment| attachment_is_image(attachment) || attachment_is_video(attachment))
+ .take(limits::MAX_MEDIA_ATTACHMENTS);
+
+ for attachment in attachments {
if attachment_is_image(attachment) {
let Some(image) = process_activity_pub_media(attachment, media_proxy, nsfw_str).await
else {
diff --git a/fluxer_unfurl/src/activity_pub/limits.rs b/fluxer_unfurl/src/activity_pub/limits.rs
index a3c800f2e..69cf34043 100644
--- a/fluxer_unfurl/src/activity_pub/limits.rs
+++ b/fluxer_unfurl/src/activity_pub/limits.rs
@@ -5,3 +5,5 @@ pub const INSTANCE_INFO_MAX_BYTES: usize = 256 * 1024;
pub const OBJECT_MAX_BYTES: usize = 1024 * 1024;
pub const MASTODON_STATUS_MAX_BYTES: usize = 1024 * 1024;
+
+pub const MAX_MEDIA_ATTACHMENTS: usize = 10;
diff --git a/fluxer_unfurl/src/html_parser.rs b/fluxer_unfurl/src/html_parser.rs
index 38bd79455..82500375e 100644
--- a/fluxer_unfurl/src/html_parser.rs
+++ b/fluxer_unfurl/src/html_parser.rs
@@ -1,6 +1,18 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-use scraper::{Html, Selector};
+use std::borrow::Cow;
+use std::cell::Cell;
+use std::rc::Rc;
+
+use html5ever::driver::{self, ParseOpts, Parser};
+use html5ever::tendril::{StrTendril, TendrilSink};
+use html5ever::tree_builder::{ElementFlags, NodeOrText, QuirksMode, TreeSink};
+use html5ever::{Attribute, QualName, local_name, ns};
+use scraper::{Html, HtmlTreeSink, Selector};
+
+const MAX_NESTING_DEPTH: usize = 512;
+
+const PARSE_CHUNK_BYTES: usize = 4096;
const TEXTUAL_KEYS: [&str; 5] = [
"og:title",
@@ -14,6 +26,171 @@ const CLASSIFYING_CARDS: [&str; 3] = ["summary_large_image", "photo", "player"];
const INERT_ELEMENTS: [&str; 4] = ["script", "style", "noscript", "template"];
+type Handle = <HtmlTreeSink as TreeSink>::Handle;
+
+struct DepthLimitedSink {
+ inner: HtmlTreeSink,
+ too_deep: Rc<Cell<bool>>,
+}
+
+impl DepthLimitedSink {
+ fn note_depth_below(&self, node: &Handle) {
+ if self.too_deep.get() {
+ return;
+ }
+ let html = self.inner.0.borrow();
+ let depth = html
+ .tree
+ .get(*node)
+ .map_or(0, |node| node.ancestors().take(MAX_NESTING_DEPTH).count());
+ if depth >= MAX_NESTING_DEPTH {
+ self.too_deep.set(true);
+ }
+ }
+}
+
+impl TreeSink for DepthLimitedSink {
+ type Output = Html;
+ type Handle = Handle;
+ type ElemName<'a> = <HtmlTreeSink as TreeSink>::ElemName<'a>;
+
+ fn finish(self) -> Html {
+ self.inner.finish()
+ }
+
+ fn parse_error(&self, msg: Cow<'static, str>) {
+ self.inner.parse_error(msg);
+ }
+
+ fn get_document(&self) -> Handle {
+ self.inner.get_document()
+ }
+
+ fn elem_name<'a>(&'a self, target: &'a Handle) -> Self::ElemName<'a> {
+ self.inner.elem_name(target)
+ }
+
+ fn create_element(&self, name: QualName, attrs: Vec<Attribute>, flags: ElementFlags) -> Handle {
+ self.inner.create_element(name, attrs, flags)
+ }
+
+ fn create_comment(&self, text: StrTendril) -> Handle {
+ self.inner.create_comment(text)
+ }
+
+ fn create_pi(&self, target: StrTendril, data: StrTendril) -> Handle {
+ self.inner.create_pi(target, data)
+ }
+
+ fn append(&self, parent: &Handle, child: NodeOrText<Handle>) {
+ if matches!(child, NodeOrText::AppendNode(_)) {
+ self.note_depth_below(parent);
+ }
+ self.inner.append(parent, child);
+ }
+
+ fn append_based_on_parent_node(
+ &self,
+ element: &Handle,
+ prev_element: &Handle,
+ child: NodeOrText<Handle>,
+ ) {
+ if matches!(child, NodeOrText::AppendNode(_)) {
+ self.note_depth_below(element);
+ self.note_depth_below(prev_element);
+ }
+ self.inner
+ .append_based_on_parent_node(element, prev_element, child);
+ }
+
+ fn append_doctype_to_document(
+ &self,
+ name: StrTendril,
+ public_id: StrTendril,
+ system_id: StrTendril,
+ ) {
+ self.inner
+ .append_doctype_to_document(name, public_id, system_id);
+ }
+
+ fn mark_script_already_started(&self, node: &Handle) {
+ self.inner.mark_script_already_started(node);
+ }
+
+ fn pop(&self, node: &Handle) {
+ self.inner.pop(node);
+ }
+
+ fn get_template_contents(&self, target: &Handle) -> Handle {
+ self.inner.get_template_contents(target)
+ }
+
+ fn same_node(&self, x: &Handle, y: &Handle) -> bool {
+ self.inner.same_node(x, y)
+ }
+
+ fn set_quirks_mode(&self, mode: QuirksMode) {
+ self.inner.set_quirks_mode(mode);
+ }
+
+ fn append_before_sibling(&self, sibling: &Handle, new_node: NodeOrText<Handle>) {
+ if matches!(new_node, NodeOrText::AppendNode(_)) {
+ self.note_depth_below(sibling);
+ }
+ self.inner.append_before_sibling(sibling, new_node);
+ }
+
+ fn add_attrs_if_missing(&self, target: &Handle, attrs: Vec<Attribute>) {
+ self.inner.add_attrs_if_missing(target, attrs);
+ }
+
+ fn remove_from_parent(&self, target: &Handle) {
+ self.inner.remove_from_parent(target);
+ }
+
+ fn reparent_children(&self, node: &Handle, new_parent: &Handle) {
+ self.note_depth_below(new_parent);
+ self.inner.reparent_children(node, new_parent);
+ }
+}
+
+fn parse_with_depth_limit(
+ input: &str,
+ document: Html,
+ start: impl FnOnce(DepthLimitedSink) -> Parser<DepthLimitedSink>,
+) -> Html {
+ let too_deep = Rc::new(Cell::new(false));
+ let mut parser = start(DepthLimitedSink {
+ inner: HtmlTreeSink::new(document),
+ too_deep: Rc::clone(&too_deep),
+ });
+ let mut rest = input;
+ while !rest.is_empty() && !too_deep.get() {
+ let (chunk, tail) = rest.split_at(rest.floor_char_boundary(PARSE_CHUNK_BYTES));
+ parser.process(StrTendril::from_slice(chunk));
+ rest = tail;
+ }
+ parser.finish()
+}
+
+pub(crate) fn parse_document(html: &str) -> Html {
+ parse_with_depth_limit(html, Html::new_document(), |sink| {
+ driver::parse_document(sink, ParseOpts::default())
+ })
+}
+
+pub(crate) fn parse_fragment(fragment: &str) -> Html {
+ parse_with_depth_limit(fragment, Html::new_fragment(), |sink| {
+ driver::parse_fragment(
+ sink,
+ ParseOpts::default(),
+ QualName::new(None, ns!(html), local_name!("body")),
+ Vec::new(),
+ false,
+ )
+ })
+}
+
#[derive(Debug, Default, Clone)]
pub struct OgMetadata {
pub title: Option<String>,
@@ -82,7 +259,7 @@ impl MetaTags {
}
pub fn parse_opengraph(html: &str) -> OgMetadata {
- let doc = Html::parse_document(html);
+ let doc = parse_document(html);
let meta = MetaTags::parse(&doc);
let description = meta
@@ -144,7 +321,7 @@ fn document_title(doc: &Html) -> Option<String> {
}
pub fn parse_twitter_card(html: &str) -> TwitterCardMetadata {
- let doc = Html::parse_document(html);
+ let doc = parse_document(html);
let meta = MetaTags::parse(&doc);
TwitterCardMetadata {
@@ -157,7 +334,7 @@ pub fn parse_twitter_card(html: &str) -> TwitterCardMetadata {
}
pub fn find_rsd_url(html: &str) -> Option<String> {
- let doc = Html::parse_document(html);
+ let doc = parse_document(html);
let sel = Selector::parse("link").ok()?;
for el in doc.select(&sel) {
@@ -185,7 +362,7 @@ pub fn find_rsd_url(html: &str) -> Option<String> {
}
pub fn find_activity_pub_link(html: &str) -> Option<String> {
- let doc = Html::parse_document(html);
+ let doc = parse_document(html);
let sel = Selector::parse("link").ok()?;
for el in doc.select(&sel) {
@@ -217,7 +394,7 @@ pub fn find_activity_pub_link(html: &str) -> Option<String> {
}
pub fn has_fediverse_profile_username(html: &str) -> bool {
- let doc = Html::parse_document(html);
+ let doc = parse_document(html);
let meta = MetaTags::parse(&doc);
meta.all("og:profile:username")
.chain(meta.all("profile:username"))
@@ -241,7 +418,7 @@ fn is_acct_shaped(value: &str) -> bool {
}
pub fn find_canonical_url(html: &str, base_url: &url::Url) -> Option<String> {
- let doc = Html::parse_document(html);
+ let doc = parse_document(html);
let sel = Selector::parse("link[rel=\"canonical\"]").ok()?;
let el = doc.select(&sel).next()?;
let href = el.value().attr("href")?;
@@ -252,7 +429,7 @@ pub fn find_canonical_url(html: &str, base_url: &url::Url) -> Option<String> {
}
pub fn find_apple_touch_icon(html: &str, base_url: &url::Url) -> Option<String> {
- let doc = Html::parse_document(html);
+ let doc = parse_document(html);
for selector in [
r#"link[rel="apple-touch-icon"][sizes="180x180"]"#,
r#"link[rel="apple-touch-icon"]"#,
@@ -315,3 +492,16 @@ fn normalize_image_reference_key(value: &str) -> Option<String> {
fn is_http_url(url: &url::Url) -> bool {
matches!(url.scheme(), "http" | "https")
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn deeply_nested_page_keeps_head_metadata() {
+ let mut html =
+ String::from(r#"<html><head><meta property="og:title" content="Kept"></head><body>"#);
+ html.push_str(&"<div>".repeat(200_000));
+ assert_eq!(parse_opengraph(&html).title.as_deref(), Some("Kept"));
+ }
+}
diff --git a/fluxer_unfurl/src/http_fetch.rs b/fluxer_unfurl/src/http_fetch.rs
index 54db9bb15..788fadff3 100644
--- a/fluxer_unfurl/src/http_fetch.rs
+++ b/fluxer_unfurl/src/http_fetch.rs
@@ -124,7 +124,8 @@ where
client.get(current_url.clone()).headers(hop_headers).send(),
)
.await
- .map_err(|_| anyhow::anyhow!("request timed out"))??;
+ .map_err(|_| anyhow::anyhow!("request timed out"))?
+ .map_err(reqwest::Error::without_url)?;
let status = resp.status();
let headers = resp.headers().clone();
@@ -189,7 +190,7 @@ pub async fn read_body(resp: reqwest::Response, max_bytes: usize) -> anyhow::Res
let mut buf = Vec::with_capacity(max_bytes.min(64 * 1024));
let mut stream = resp;
- while let Some(chunk) = stream.chunk().await? {
+ while let Some(chunk) = stream.chunk().await.map_err(reqwest::Error::without_url)? {
let next_len = buf
.len()
.checked_add(chunk.len())
diff --git a/fluxer_unfurl/src/network_policy.rs b/fluxer_unfurl/src/network_policy.rs
index eae6ead03..e54526a83 100644
--- a/fluxer_unfurl/src/network_policy.rs
+++ b/fluxer_unfurl/src/network_policy.rs
@@ -3,10 +3,13 @@
use fluxer_common::public_ip::{is_public_ip, is_valid_public_hostname};
use reqwest::dns::{Addrs, Name, Resolve, Resolving};
use std::net::{IpAddr, SocketAddr};
+use std::time::Duration;
use tokio::net::lookup_host;
+use tokio::time::timeout;
use url::Url;
const MAX_URL_LEN: usize = 8192;
+const DNS_LOOKUP_TIMEOUT: Duration = Duration::from_secs(10);
#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]
pub enum Error {
@@ -42,8 +45,9 @@ pub async fn validate_url(url: &Url) -> Result<(), Error> {
}
let port = url.port_or_known_default().ok_or(Error::BlockedUrl)?;
let mut seen = false;
- let addrs = lookup_host((host.as_str(), port))
+ let addrs = timeout(DNS_LOOKUP_TIMEOUT, lookup_host((host.as_str(), port)))
.await
+ .map_err(|_| Error::DnsLookupFailed)?
.map_err(|_| Error::DnsLookupFailed)?;
for addr in addrs {
seen = true;
diff --git a/fluxer_unfurl/src/oembed.rs b/fluxer_unfurl/src/oembed.rs
index 1acfa932e..2290f3e9b 100644
--- a/fluxer_unfurl/src/oembed.rs
+++ b/fluxer_unfurl/src/oembed.rs
@@ -1,10 +1,13 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
+use crate::html_parser;
use crate::http_fetch;
-use scraper::{Html, Selector};
+use scraper::Selector;
use serde::Deserialize;
use std::time::Duration;
+const MAX_OEMBED_ENDPOINTS: usize = 3;
+
#[derive(Debug, Clone)]
pub struct OEmbedEndpoint {
pub url: String,
@@ -36,13 +39,13 @@ pub struct OEmbedResponse {
}
pub fn discover_oembed_url(html: &str) -> Vec<OEmbedEndpoint> {
- let doc = Html::parse_document(html);
+ let doc = html_parser::parse_document(html);
let sel = match Selector::parse("link") {
Ok(s) => s,
Err(_) => return Vec::new(),
};
- let mut endpoints = Vec::new();
+ let mut endpoints: Vec<OEmbedEndpoint> = Vec::new();
for el in doc.select(&sel) {
let href = match el.value().attr("href") {
@@ -54,7 +57,7 @@ pub fn discover_oembed_url(html: &str) -> Vec<OEmbedEndpoint> {
None => continue,
};
- if !link_type.contains("+oembed") {
+ if !link_type.contains("+oembed") || endpoints.iter().any(|ep| ep.url == href) {
continue;
}
@@ -72,6 +75,9 @@ pub fn discover_oembed_url(html: &str) -> Vec<OEmbedEndpoint> {
url: href.to_owned(),
format,
});
+ if endpoints.len() >= MAX_OEMBED_ENDPOINTS {
+ break;
+ }
}
endpoints
diff --git a/fluxer_unfurl/src/resolvers/default_helpers.rs b/fluxer_unfurl/src/resolvers/default_helpers.rs
index f6f3543c4..9c61e283a 100644
--- a/fluxer_unfurl/src/resolvers/default_helpers.rs
+++ b/fluxer_unfurl/src/resolvers/default_helpers.rs
@@ -213,7 +213,7 @@ pub fn parse_text(value: &str, max_len: usize) -> String {
}
pub fn decode_html_text(input: &str) -> String {
- scraper::Html::parse_fragment(input)
+ crate::html_parser::parse_fragment(input)
.root_element()
.text()
.collect()
@@ -243,7 +243,10 @@ pub fn format_timestamp(unix_seconds: i64) -> Option<String> {
}
pub fn sanitize_absolute_url(value: &str) -> Option<String> {
- Url::parse(value.trim()).ok().map(|url| url.to_string())
+ Url::parse(value.trim())
+ .ok()
+ .filter(|url| matches!(url.scheme(), "http" | "https"))
+ .map(|url| url.to_string())
}
#[cfg(test)]
diff --git a/fluxer_unfurl/src/resolvers/default_resolver.rs b/fluxer_unfurl/src/resolvers/default_resolver.rs
index 21b87aa1b..74f78247c 100644
--- a/fluxer_unfurl/src/resolvers/default_resolver.rs
+++ b/fluxer_unfurl/src/resolvers/default_resolver.rs
@@ -478,6 +478,7 @@ fn optional_oembed_string(value: Option<&str>) -> Option<&str> {
fn optional_absolute_url(value: Option<&str>) -> Option<String> {
Url::parse(optional_oembed_string(value)?)
.ok()
+ .filter(|url| matches!(url.scheme(), "http" | "https"))
.map(|url| url.to_string())
}
diff --git a/fluxer_unfurl/src/resolvers/tenor.rs b/fluxer_unfurl/src/resolvers/tenor.rs
index 2af30ca63..7e9a24bed 100644
--- a/fluxer_unfurl/src/resolvers/tenor.rs
+++ b/fluxer_unfurl/src/resolvers/tenor.rs
@@ -6,7 +6,7 @@ use crate::html_parser;
use crate::http_fetch;
use crate::media_proxy::MediaProxyClient;
use crate::types::{EmbedMedia, EmbedProvider, MessageEmbed};
-use scraper::{Html, Selector};
+use scraper::Selector;
use std::future::Future;
use std::pin::Pin;
use std::time::Duration;
@@ -144,7 +144,7 @@ struct TenorJsonLdUrls {
}
fn extract_json_ld_urls(html: &str) -> Option<TenorJsonLdUrls> {
- let doc = Html::parse_document(html);
+ let doc = html_parser::parse_document(html);
let selector = Selector::parse(r#"script.dynamic[type="application/ld+json"]"#).ok()?;
for script in doc.select(&selector) {
diff --git a/fluxer_unfurl/src/resolvers/xkcd.rs b/fluxer_unfurl/src/resolvers/xkcd.rs
index e51e8e681..b88a905b5 100644
--- a/fluxer_unfurl/src/resolvers/xkcd.rs
+++ b/fluxer_unfurl/src/resolvers/xkcd.rs
@@ -51,7 +51,7 @@ impl Resolver for XkcdResolver {
let title = og.title.as_deref().map(|t| parse_text(t, 70));
let image_alt = {
- let doc = Html::parse_document(&html);
+ let doc = html_parser::parse_document(&html);
extract_comic_alt(&doc)
};
let footer_text = image_alt.clone();
diff --git a/fluxer_unfurl/src/resolvers/youtube.rs b/fluxer_unfurl/src/resolvers/youtube.rs
index b75d8b434..64aadfc8d 100644
--- a/fluxer_unfurl/src/resolvers/youtube.rs
+++ b/fluxer_unfurl/src/resolvers/youtube.rs
@@ -4,6 +4,7 @@ use super::{ResolveContext, Resolver, ResolverResult};
use crate::http_fetch;
use crate::text_limits;
use crate::types::{EmbedAuthor, EmbedMedia, EmbedProvider, MessageEmbed};
+use reqwest::header::{HeaderMap, HeaderValue};
use serde::Deserialize;
use std::future::Future;
use std::pin::Pin;
@@ -12,6 +13,7 @@ use url::Url;
const YOUTUBE_COLOR: u32 = 0xFF0000;
const YOUTUBE_API_BASE: &str = "https://www.googleapis.com/youtube/v3/videos";
+const YOUTUBE_API_KEY_HEADER: &str = "x-goog-api-key";
const YOUTUBE_API_MAX_BYTES: usize = 256 * 1024;
const YOUTUBE_API_TIMEOUT: Duration = Duration::from_secs(10);
const YOUTUBE_DESCRIPTION_MAX: usize = 350;
@@ -85,10 +87,20 @@ async fn fetch_api_video(
video_id: &str,
api_key: &str,
) -> Option<YouTubeVideo> {
- let api_url = build_youtube_api_url(api_base, video_id, api_key).ok()?;
- let api_response = match http_fetch::fetch_url(
+ let api_url = build_youtube_api_url(api_base, video_id).ok()?;
+ let api_origin = api_url.origin();
+ let api_response = match http_fetch::fetch_url_with_headers(
&ctx.http_client,
api_url.as_ref(),
+ |hop| {
+ let mut headers = HeaderMap::new();
+ if hop.origin() == api_origin {
+ let mut value = HeaderValue::from_str(api_key)?;
+ value.set_sensitive(true);
+ headers.insert(YOUTUBE_API_KEY_HEADER, value);
+ }
+ Ok(headers)
+ },
YOUTUBE_API_MAX_BYTES,
YOUTUBE_API_TIMEOUT,
)
@@ -305,10 +317,9 @@ struct YouTubeThumbnail {
height: Option<u32>,
}
-fn build_youtube_api_url(base: &str, video_id: &str, api_key: &str) -> anyhow::Result<Url> {
+fn build_youtube_api_url(base: &str, video_id: &str) -> anyhow::Result<Url> {
let mut url = Url::parse(base)?;
url.query_pairs_mut()
- .append_pair("key", api_key)
.append_pair("id", video_id)
.append_pair("part", "snippet,player,status");
Ok(url)