Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: make srcset URLs absolute in HTML transformation
Devin AI2fd4bdc169cd
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rsindex e1afab37f..dd72351c0 100644--- a/apps/api/native/src/html.rs+++ b/apps/api/native/src/html.rs@@ -585,6 +585,74 @@ fn _transform_html_inner(}}+ // Make srcset URLs absolute+ let srcset_images_for_abs: Vec<_> = document+ .select("img[srcset]")+ .map_err(|_| "Failed to select srcset images for absolute URLs")?+ .collect();+ for img in srcset_images_for_abs {+ let srcset_opt = img.attributes.borrow().get("srcset").map(|x| x.to_string());+ if let Some(srcset) = srcset_opt {+ let absolute_srcset: Vec<String> = srcset+ .split(',')+ .map(|entry| {+ let entry = entry.trim();+ let parts: Vec<&str> = entry.split_whitespace().collect();+ if parts.is_empty() {+ return entry.to_string();+ }+ let img_url = parts[0];+ let descriptor = if parts.len() > 1 {+ parts[1..].join(" ")+ } else {+ String::new()+ };+ let absolute_url = url.join(img_url).map(|u| u.to_string()).unwrap_or_else(|_| img_url.to_string());+ if descriptor.is_empty() {+ absolute_url+ } else {+ format!("{} {}", absolute_url, descriptor)+ }+ })+ .collect();+ img.attributes.borrow_mut().insert("srcset", absolute_srcset.join(", "));+ }+ }++ // Make source srcset URLs absolute (for picture elements)+ let source_srcsets: Vec<_> = document+ .select("source[srcset]")+ .map_err(|_| "Failed to select source srcset elements")?+ .collect();+ for source in source_srcsets {+ let srcset_opt = source.attributes.borrow().get("srcset").map(|x| x.to_string());+ if let Some(srcset) = srcset_opt {+ let absolute_srcset: Vec<String> = srcset+ .split(',')+ .map(|entry| {+ let entry = entry.trim();+ let parts: Vec<&str> = entry.split_whitespace().collect();+ if parts.is_empty() {+ return entry.to_string();+ }+ let img_url = parts[0];+ let descriptor = if parts.len() > 1 {+ parts[1..].join(" ")+ } else {+ String::new()+ };+ let absolute_url = url.join(img_url).map(|u| u.to_string()).unwrap_or_else(|_| img_url.to_string());+ if descriptor.is_empty() {+ absolute_url+ } else {+ format!("{} {}", absolute_url, descriptor)+ }+ })+ .collect();+ source.attributes.borrow_mut().insert("srcset", absolute_srcset.join(", "));+ }+ }+let href_anchors: Vec<_> = document.select("a[href]").map_err(|_| "Failed to select href anchors")?diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.tsindex dead67ff6..edf4b4fcc 100644--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts@@ -209,6 +209,52 @@ export const htmlTransform = async (} catch (_) {}});+ // Make srcset URLs absolute+ soup("img[srcset]").each((_, el) => {+ try {+ const srcset = el.attribs.srcset;+ const absoluteSrcset = srcset+ .split(",")+ .map(entry => {+ const parts = entry.trim().split(/\s+/);+ if (parts.length === 0) return entry;+ const imgUrl = parts[0];+ const descriptor = parts.slice(1).join(" ");+ try {+ const absoluteUrl = new URL(imgUrl, url).href;+ return descriptor ? `${absoluteUrl} ${descriptor}` : absoluteUrl;+ } catch (_) {+ return entry.trim();+ }+ })+ .join(", ");+ el.attribs.srcset = absoluteSrcset;+ } catch (_) {}+ });++ // Make source srcset URLs absolute (for picture elements)+ soup("source[srcset]").each((_, el) => {+ try {+ const srcset = el.attribs.srcset;+ const absoluteSrcset = srcset+ .split(",")+ .map(entry => {+ const parts = entry.trim().split(/\s+/);+ if (parts.length === 0) return entry;+ const imgUrl = parts[0];+ const descriptor = parts.slice(1).join(" ");+ try {+ const absoluteUrl = new URL(imgUrl, url).href;+ return descriptor ? `${absoluteUrl} ${descriptor}` : absoluteUrl;+ } catch (_) {+ return entry.trim();+ }+ })+ .join(", ");+ el.attribs.srcset = absoluteSrcset;+ } catch (_) {}+ });+const cleanedHtml = soup.html();return cleanedHtml;};