Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(native): drop captcha widgets and embedded player documents under onlyMainContent (#5470)

Gaurav Chadhab93fc0d962bd
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rs
index 71bae49f6..72ff8d0a1 100644
--- a/apps/api/native/src/html.rs
+++ b/apps/api/native/src/html.rs
@@ -332,7 +332,7 @@ pub async fn extract_metadata(html: Option<String>) -> napi::Result<HashMap<Stri
})?
}
-const EXCLUDE_NON_MAIN_TAGS: [&str; 42] = [
+const EXCLUDE_NON_MAIN_TAGS: [&str; 48] = [
"header",
"footer",
"nav",
@@ -374,7 +374,20 @@ const EXCLUDE_NON_MAIN_TAGS: [&str; 42] = [
"#widget",
".cookie",
"#cookie",
+ "[aria-label=\"Cookie consent\" i]",
".fc-decoration",
+ ".grecaptcha-badge",
+ "[data-original-tag=\"iframe\"][src*=\"google.com/recaptcha/\"]",
+ "[data-original-tag=\"iframe\"][src*=\"recaptcha.net/recaptcha/\"]",
+ "[data-original-tag=\"iframe\"][src*=\"hcaptcha.com\"]",
+ "[data-original-tag=\"iframe\"][src*=\"challenges.cloudflare.com\"]",
+];
+
+// Inlined documents of embedded players hold only player chrome, never page content.
+const EMPTY_NON_MAIN_FRAMES: [&str; 3] = [
+ "[data-original-tag=\"iframe\"][src*=\"player.vimeo.com/\"]",
+ "[data-original-tag=\"iframe\"][src*=\"youtube.com/embed/\"]",
+ "[data-original-tag=\"iframe\"][src*=\"youtube-nocookie.com/embed/\"]",
];
const FORCE_INCLUDE_MAIN_TAGS: [&str; 13] = [
@@ -523,6 +536,18 @@ fn _transform_html_inner(
}
}
}
+
+ for x in EMPTY_NON_MAIN_FRAMES.iter() {
+ let frames: Vec<_> = document
+ .select(x)
+ .map_err(|_| "Failed to select frames")?
+ .collect();
+ for frame in frames {
+ while let Some(child) = frame.as_node().first_child() {
+ child.detach();
+ }
+ }
+ }
}
// Resolve images, including lazy-loaded ones. Many sites ship a tiny
diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
index dead67ff6..1f78d7eb5 100644
--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
@@ -48,6 +48,19 @@ const excludeNonMainTags = [
"#widget",
".cookie",
"#cookie",
+ '[aria-label="Cookie consent" i]',
+ ".grecaptcha-badge",
+ '[data-original-tag="iframe"][src*="google.com/recaptcha/"]',
+ '[data-original-tag="iframe"][src*="recaptcha.net/recaptcha/"]',
+ '[data-original-tag="iframe"][src*="hcaptcha.com"]',
+ '[data-original-tag="iframe"][src*="challenges.cloudflare.com"]',
+];
+
+// Inlined documents of embedded players hold only player chrome, never page content.
+const emptyNonMainFrames = [
+ '[data-original-tag="iframe"][src*="player.vimeo.com/"]',
+ '[data-original-tag="iframe"][src*="youtube.com/embed/"]',
+ '[data-original-tag="iframe"][src*="youtube-nocookie.com/embed/"]',
];
const forceIncludeMainTags = [
@@ -171,6 +184,8 @@ export const htmlTransform = async (
elementsToRemove.remove();
});
+
+ soup(emptyNonMainFrames.join(",")).empty();
}
// always return biggest image