Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(native): drop captcha widgets and embedded player documents under onlyMainContent (#5470)
Gaurav Chadhab93fc0d962bd
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rsindex 71bae49f6..72ff8d0a1 100644--- a/apps/api/native/src/html.rs+++ b/apps/api/native/src/html.rs@@ -332,7 +332,7 @@ pub async fn extract_metadata(html: Option<String>) -> napi::Result<HashMap<Stri})?}-const EXCLUDE_NON_MAIN_TAGS: [&str; 42] = [+const EXCLUDE_NON_MAIN_TAGS: [&str; 48] = ["header","footer","nav",@@ -374,7 +374,20 @@ const EXCLUDE_NON_MAIN_TAGS: [&str; 42] = ["#widget",".cookie","#cookie",+ "[aria-label=\"Cookie consent\" i]",".fc-decoration",+ ".grecaptcha-badge",+ "[data-original-tag=\"iframe\"][src*=\"google.com/recaptcha/\"]",+ "[data-original-tag=\"iframe\"][src*=\"recaptcha.net/recaptcha/\"]",+ "[data-original-tag=\"iframe\"][src*=\"hcaptcha.com\"]",+ "[data-original-tag=\"iframe\"][src*=\"challenges.cloudflare.com\"]",+];++// Inlined documents of embedded players hold only player chrome, never page content.+const EMPTY_NON_MAIN_FRAMES: [&str; 3] = [+ "[data-original-tag=\"iframe\"][src*=\"player.vimeo.com/\"]",+ "[data-original-tag=\"iframe\"][src*=\"youtube.com/embed/\"]",+ "[data-original-tag=\"iframe\"][src*=\"youtube-nocookie.com/embed/\"]",];const FORCE_INCLUDE_MAIN_TAGS: [&str; 13] = [@@ -523,6 +536,18 @@ fn _transform_html_inner(}}}++ for x in EMPTY_NON_MAIN_FRAMES.iter() {+ let frames: Vec<_> = document+ .select(x)+ .map_err(|_| "Failed to select frames")?+ .collect();+ for frame in frames {+ while let Some(child) = frame.as_node().first_child() {+ child.detach();+ }+ }+ }}// Resolve images, including lazy-loaded ones. Many sites ship a tinydiff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.tsindex dead67ff6..1f78d7eb5 100644--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts@@ -48,6 +48,19 @@ const excludeNonMainTags = ["#widget",".cookie","#cookie",+ '[aria-label="Cookie consent" i]',+ ".grecaptcha-badge",+ '[data-original-tag="iframe"][src*="google.com/recaptcha/"]',+ '[data-original-tag="iframe"][src*="recaptcha.net/recaptcha/"]',+ '[data-original-tag="iframe"][src*="hcaptcha.com"]',+ '[data-original-tag="iframe"][src*="challenges.cloudflare.com"]',+];++// Inlined documents of embedded players hold only player chrome, never page content.+const emptyNonMainFrames = [+ '[data-original-tag="iframe"][src*="player.vimeo.com/"]',+ '[data-original-tag="iframe"][src*="youtube.com/embed/"]',+ '[data-original-tag="iframe"][src*="youtube-nocookie.com/embed/"]',];const forceIncludeMainTags = [@@ -171,6 +184,8 @@ export const htmlTransform = async (elementsToRemove.remove();});++ soup(emptyNonMainFrames.join(",")).empty();}// always return biggest image