Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
refactor: srcset parser
Gaurav Chadha614c5748408a
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rsindex dd72351c0..120fbadfd 100644--- a/apps/api/native/src/html.rs+++ b/apps/api/native/src/html.rs@@ -585,37 +585,42 @@ fn _transform_html_inner(}}- // Make srcset URLs absolute+ // Helper function to convert srcset URLs to absolute+ let make_srcset_absolute = |srcset: &str| -> String {+ srcset+ .split(',')+ .map(|entry| {+ let entry = entry.trim();+ let parts: Vec<&str> = entry.split_whitespace().collect();+ if parts.is_empty() {+ return entry.to_string();+ }+ let img_url = parts[0];+ let descriptor = if parts.len() > 1 {+ parts[1..].join(" ")+ } else {+ String::new()+ };+ let absolute_url = url.join(img_url).map(|u| u.to_string()).unwrap_or_else(|_| img_url.to_string());+ if descriptor.is_empty() {+ absolute_url+ } else {+ format!("{} {}", absolute_url, descriptor)+ }+ })+ .collect::<Vec<String>>()+ .join(", ")+ };++ // Make srcset URLs absolute for img elementslet srcset_images_for_abs: Vec<_> = document.select("img[srcset]").map_err(|_| "Failed to select srcset images for absolute URLs")?.collect();for img in srcset_images_for_abs {- let srcset_opt = img.attributes.borrow().get("srcset").map(|x| x.to_string());- if let Some(srcset) = srcset_opt {- let absolute_srcset: Vec<String> = srcset- .split(',')- .map(|entry| {- let entry = entry.trim();- let parts: Vec<&str> = entry.split_whitespace().collect();- if parts.is_empty() {- return entry.to_string();- }- let img_url = parts[0];- let descriptor = if parts.len() > 1 {- parts[1..].join(" ")- } else {- String::new()- };- let absolute_url = url.join(img_url).map(|u| u.to_string()).unwrap_or_else(|_| img_url.to_string());- if descriptor.is_empty() {- absolute_url- } else {- format!("{} {}", absolute_url, descriptor)- }- })- .collect();- img.attributes.borrow_mut().insert("srcset", absolute_srcset.join(", "));+ if let Some(srcset) = img.attributes.borrow().get("srcset").map(|x| x.to_string()) {+ let absolute_srcset = make_srcset_absolute(&srcset);+ img.attributes.borrow_mut().insert("srcset", absolute_srcset);}}@@ -625,31 +630,9 @@ fn _transform_html_inner(.map_err(|_| "Failed to select source srcset elements")?.collect();for source in source_srcsets {- let srcset_opt = source.attributes.borrow().get("srcset").map(|x| x.to_string());- if let Some(srcset) = srcset_opt {- let absolute_srcset: Vec<String> = srcset- .split(',')- .map(|entry| {- let entry = entry.trim();- let parts: Vec<&str> = entry.split_whitespace().collect();- if parts.is_empty() {- return entry.to_string();- }- let img_url = parts[0];- let descriptor = if parts.len() > 1 {- parts[1..].join(" ")- } else {- String::new()- };- let absolute_url = url.join(img_url).map(|u| u.to_string()).unwrap_or_else(|_| img_url.to_string());- if descriptor.is_empty() {- absolute_url- } else {- format!("{} {}", absolute_url, descriptor)- }- })- .collect();- source.attributes.borrow_mut().insert("srcset", absolute_srcset.join(", "));+ if let Some(srcset) = source.attributes.borrow().get("srcset").map(|x| x.to_string()) {+ let absolute_srcset = make_srcset_absolute(&srcset);+ source.attributes.borrow_mut().insert("srcset", absolute_srcset);}}diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.tsindex edf4b4fcc..43bf0a205 100644--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts@@ -209,26 +209,30 @@ export const htmlTransform = async (} catch (_) {}});- // Make srcset URLs absolute+ // Helper function to convert srcset URLs to absolute+ const makeSrcsetAbsolute = (srcset: string): string => {+ return srcset+ .split(",")+ .map(entry => {+ const parts = entry.trim().split(/\s+/);+ if (parts.length === 0) return entry;+ const imgUrl = parts[0];+ const descriptor = parts.slice(1).join(" ");+ try {+ const absoluteUrl = new URL(imgUrl, url).href;+ return descriptor ? `${absoluteUrl} ${descriptor}` : absoluteUrl;+ } catch (_) {+ return entry.trim();+ }+ })+ .join(", ");+ };++ // Make srcset URLs absolute for img elementssoup("img[srcset]").each((_, el) => {try {const srcset = el.attribs.srcset;- const absoluteSrcset = srcset- .split(",")- .map(entry => {- const parts = entry.trim().split(/\s+/);- if (parts.length === 0) return entry;- const imgUrl = parts[0];- const descriptor = parts.slice(1).join(" ");- try {- const absoluteUrl = new URL(imgUrl, url).href;- return descriptor ? `${absoluteUrl} ${descriptor}` : absoluteUrl;- } catch (_) {- return entry.trim();- }- })- .join(", ");- el.attribs.srcset = absoluteSrcset;+ el.attribs.srcset = makeSrcsetAbsolute(srcset);} catch (_) {}});@@ -236,22 +240,7 @@ export const htmlTransform = async (soup("source[srcset]").each((_, el) => {try {const srcset = el.attribs.srcset;- const absoluteSrcset = srcset- .split(",")- .map(entry => {- const parts = entry.trim().split(/\s+/);- if (parts.length === 0) return entry;- const imgUrl = parts[0];- const descriptor = parts.slice(1).join(" ");- try {- const absoluteUrl = new URL(imgUrl, url).href;- return descriptor ? `${absoluteUrl} ${descriptor}` : absoluteUrl;- } catch (_) {- return entry.trim();- }- })- .join(", ");- el.attribs.srcset = absoluteSrcset;+ el.attribs.srcset = makeSrcsetAbsolute(srcset);} catch (_) {}});