Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

refactor: srcset parser

Gaurav Chadha614c5748408a
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rs
index dd72351c0..120fbadfd 100644
--- a/apps/api/native/src/html.rs
+++ b/apps/api/native/src/html.rs
@@ -585,37 +585,42 @@ fn _transform_html_inner(
}
}
- // Make srcset URLs absolute
+ // Helper function to convert srcset URLs to absolute
+ let make_srcset_absolute = |srcset: &str| -> String {
+ srcset
+ .split(',')
+ .map(|entry| {
+ let entry = entry.trim();
+ let parts: Vec<&str> = entry.split_whitespace().collect();
+ if parts.is_empty() {
+ return entry.to_string();
+ }
+ let img_url = parts[0];
+ let descriptor = if parts.len() > 1 {
+ parts[1..].join(" ")
+ } else {
+ String::new()
+ };
+ let absolute_url = url.join(img_url).map(|u| u.to_string()).unwrap_or_else(|_| img_url.to_string());
+ if descriptor.is_empty() {
+ absolute_url
+ } else {
+ format!("{} {}", absolute_url, descriptor)
+ }
+ })
+ .collect::<Vec<String>>()
+ .join(", ")
+ };
+
+ // Make srcset URLs absolute for img elements
let srcset_images_for_abs: Vec<_> = document
.select("img[srcset]")
.map_err(|_| "Failed to select srcset images for absolute URLs")?
.collect();
for img in srcset_images_for_abs {
- let srcset_opt = img.attributes.borrow().get("srcset").map(|x| x.to_string());
- if let Some(srcset) = srcset_opt {
- let absolute_srcset: Vec<String> = srcset
- .split(',')
- .map(|entry| {
- let entry = entry.trim();
- let parts: Vec<&str> = entry.split_whitespace().collect();
- if parts.is_empty() {
- return entry.to_string();
- }
- let img_url = parts[0];
- let descriptor = if parts.len() > 1 {
- parts[1..].join(" ")
- } else {
- String::new()
- };
- let absolute_url = url.join(img_url).map(|u| u.to_string()).unwrap_or_else(|_| img_url.to_string());
- if descriptor.is_empty() {
- absolute_url
- } else {
- format!("{} {}", absolute_url, descriptor)
- }
- })
- .collect();
- img.attributes.borrow_mut().insert("srcset", absolute_srcset.join(", "));
+ if let Some(srcset) = img.attributes.borrow().get("srcset").map(|x| x.to_string()) {
+ let absolute_srcset = make_srcset_absolute(&srcset);
+ img.attributes.borrow_mut().insert("srcset", absolute_srcset);
}
}
@@ -625,31 +630,9 @@ fn _transform_html_inner(
.map_err(|_| "Failed to select source srcset elements")?
.collect();
for source in source_srcsets {
- let srcset_opt = source.attributes.borrow().get("srcset").map(|x| x.to_string());
- if let Some(srcset) = srcset_opt {
- let absolute_srcset: Vec<String> = srcset
- .split(',')
- .map(|entry| {
- let entry = entry.trim();
- let parts: Vec<&str> = entry.split_whitespace().collect();
- if parts.is_empty() {
- return entry.to_string();
- }
- let img_url = parts[0];
- let descriptor = if parts.len() > 1 {
- parts[1..].join(" ")
- } else {
- String::new()
- };
- let absolute_url = url.join(img_url).map(|u| u.to_string()).unwrap_or_else(|_| img_url.to_string());
- if descriptor.is_empty() {
- absolute_url
- } else {
- format!("{} {}", absolute_url, descriptor)
- }
- })
- .collect();
- source.attributes.borrow_mut().insert("srcset", absolute_srcset.join(", "));
+ if let Some(srcset) = source.attributes.borrow().get("srcset").map(|x| x.to_string()) {
+ let absolute_srcset = make_srcset_absolute(&srcset);
+ source.attributes.borrow_mut().insert("srcset", absolute_srcset);
}
}
diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
index edf4b4fcc..43bf0a205 100644
--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
@@ -209,26 +209,30 @@ export const htmlTransform = async (
} catch (_) {}
});
- // Make srcset URLs absolute
+ // Helper function to convert srcset URLs to absolute
+ const makeSrcsetAbsolute = (srcset: string): string => {
+ return srcset
+ .split(",")
+ .map(entry => {
+ const parts = entry.trim().split(/\s+/);
+ if (parts.length === 0) return entry;
+ const imgUrl = parts[0];
+ const descriptor = parts.slice(1).join(" ");
+ try {
+ const absoluteUrl = new URL(imgUrl, url).href;
+ return descriptor ? `${absoluteUrl} ${descriptor}` : absoluteUrl;
+ } catch (_) {
+ return entry.trim();
+ }
+ })
+ .join(", ");
+ };
+
+ // Make srcset URLs absolute for img elements
soup("img[srcset]").each((_, el) => {
try {
const srcset = el.attribs.srcset;
- const absoluteSrcset = srcset
- .split(",")
- .map(entry => {
- const parts = entry.trim().split(/\s+/);
- if (parts.length === 0) return entry;
- const imgUrl = parts[0];
- const descriptor = parts.slice(1).join(" ");
- try {
- const absoluteUrl = new URL(imgUrl, url).href;
- return descriptor ? `${absoluteUrl} ${descriptor}` : absoluteUrl;
- } catch (_) {
- return entry.trim();
- }
- })
- .join(", ");
- el.attribs.srcset = absoluteSrcset;
+ el.attribs.srcset = makeSrcsetAbsolute(srcset);
} catch (_) {}
});
@@ -236,22 +240,7 @@ export const htmlTransform = async (
soup("source[srcset]").each((_, el) => {
try {
const srcset = el.attribs.srcset;
- const absoluteSrcset = srcset
- .split(",")
- .map(entry => {
- const parts = entry.trim().split(/\s+/);
- if (parts.length === 0) return entry;
- const imgUrl = parts[0];
- const descriptor = parts.slice(1).join(" ");
- try {
- const absoluteUrl = new URL(imgUrl, url).href;
- return descriptor ? `${absoluteUrl} ${descriptor}` : absoluteUrl;
- } catch (_) {
- return entry.trim();
- }
- })
- .join(", ");
- el.attribs.srcset = absoluteSrcset;
+ el.attribs.srcset = makeSrcsetAbsolute(srcset);
} catch (_) {}
});