Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Revert "Merge pull request #2515 from Chadha93/chadha93/ENG-3980"

Gergő Móricz47ce9347f971
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rs
index 120fbadfd..e1afab37f 100644
--- a/apps/api/native/src/html.rs
+++ b/apps/api/native/src/html.rs
@@ -585,57 +585,6 @@ fn _transform_html_inner(
}
}
- // Helper function to convert srcset URLs to absolute
- let make_srcset_absolute = |srcset: &str| -> String {
- srcset
- .split(',')
- .map(|entry| {
- let entry = entry.trim();
- let parts: Vec<&str> = entry.split_whitespace().collect();
- if parts.is_empty() {
- return entry.to_string();
- }
- let img_url = parts[0];
- let descriptor = if parts.len() > 1 {
- parts[1..].join(" ")
- } else {
- String::new()
- };
- let absolute_url = url.join(img_url).map(|u| u.to_string()).unwrap_or_else(|_| img_url.to_string());
- if descriptor.is_empty() {
- absolute_url
- } else {
- format!("{} {}", absolute_url, descriptor)
- }
- })
- .collect::<Vec<String>>()
- .join(", ")
- };
-
- // Make srcset URLs absolute for img elements
- let srcset_images_for_abs: Vec<_> = document
- .select("img[srcset]")
- .map_err(|_| "Failed to select srcset images for absolute URLs")?
- .collect();
- for img in srcset_images_for_abs {
- if let Some(srcset) = img.attributes.borrow().get("srcset").map(|x| x.to_string()) {
- let absolute_srcset = make_srcset_absolute(&srcset);
- img.attributes.borrow_mut().insert("srcset", absolute_srcset);
- }
- }
-
- // Make source srcset URLs absolute (for picture elements)
- let source_srcsets: Vec<_> = document
- .select("source[srcset]")
- .map_err(|_| "Failed to select source srcset elements")?
- .collect();
- for source in source_srcsets {
- if let Some(srcset) = source.attributes.borrow().get("srcset").map(|x| x.to_string()) {
- let absolute_srcset = make_srcset_absolute(&srcset);
- source.attributes.borrow_mut().insert("srcset", absolute_srcset);
- }
- }
-
let href_anchors: Vec<_> = document
.select("a[href]")
.map_err(|_| "Failed to select href anchors")?
diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
index 43bf0a205..dead67ff6 100644
--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
@@ -209,41 +209,6 @@ export const htmlTransform = async (
} catch (_) {}
});
- // Helper function to convert srcset URLs to absolute
- const makeSrcsetAbsolute = (srcset: string): string => {
- return srcset
- .split(",")
- .map(entry => {
- const parts = entry.trim().split(/\s+/);
- if (parts.length === 0) return entry;
- const imgUrl = parts[0];
- const descriptor = parts.slice(1).join(" ");
- try {
- const absoluteUrl = new URL(imgUrl, url).href;
- return descriptor ? `${absoluteUrl} ${descriptor}` : absoluteUrl;
- } catch (_) {
- return entry.trim();
- }
- })
- .join(", ");
- };
-
- // Make srcset URLs absolute for img elements
- soup("img[srcset]").each((_, el) => {
- try {
- const srcset = el.attribs.srcset;
- el.attribs.srcset = makeSrcsetAbsolute(srcset);
- } catch (_) {}
- });
-
- // Make source srcset URLs absolute (for picture elements)
- soup("source[srcset]").each((_, el) => {
- try {
- const srcset = el.attribs.srcset;
- el.attribs.srcset = makeSrcsetAbsolute(srcset);
- } catch (_) {}
- });
-
const cleanedHtml = soup.html();
return cleanedHtml;
};