Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "Merge pull request #2515 from Chadha93/chadha93/ENG-3980"
Gergő Móricz47ce9347f971
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rsindex 120fbadfd..e1afab37f 100644--- a/apps/api/native/src/html.rs+++ b/apps/api/native/src/html.rs@@ -585,57 +585,6 @@ fn _transform_html_inner(}}- // Helper function to convert srcset URLs to absolute- let make_srcset_absolute = |srcset: &str| -> String {- srcset- .split(',')- .map(|entry| {- let entry = entry.trim();- let parts: Vec<&str> = entry.split_whitespace().collect();- if parts.is_empty() {- return entry.to_string();- }- let img_url = parts[0];- let descriptor = if parts.len() > 1 {- parts[1..].join(" ")- } else {- String::new()- };- let absolute_url = url.join(img_url).map(|u| u.to_string()).unwrap_or_else(|_| img_url.to_string());- if descriptor.is_empty() {- absolute_url- } else {- format!("{} {}", absolute_url, descriptor)- }- })- .collect::<Vec<String>>()- .join(", ")- };-- // Make srcset URLs absolute for img elements- let srcset_images_for_abs: Vec<_> = document- .select("img[srcset]")- .map_err(|_| "Failed to select srcset images for absolute URLs")?- .collect();- for img in srcset_images_for_abs {- if let Some(srcset) = img.attributes.borrow().get("srcset").map(|x| x.to_string()) {- let absolute_srcset = make_srcset_absolute(&srcset);- img.attributes.borrow_mut().insert("srcset", absolute_srcset);- }- }-- // Make source srcset URLs absolute (for picture elements)- let source_srcsets: Vec<_> = document- .select("source[srcset]")- .map_err(|_| "Failed to select source srcset elements")?- .collect();- for source in source_srcsets {- if let Some(srcset) = source.attributes.borrow().get("srcset").map(|x| x.to_string()) {- let absolute_srcset = make_srcset_absolute(&srcset);- source.attributes.borrow_mut().insert("srcset", absolute_srcset);- }- }-let href_anchors: Vec<_> = document.select("a[href]").map_err(|_| "Failed to select href anchors")?diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.tsindex 43bf0a205..dead67ff6 100644--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts@@ -209,41 +209,6 @@ export const htmlTransform = async (} catch (_) {}});- // Helper function to convert srcset URLs to absolute- const makeSrcsetAbsolute = (srcset: string): string => {- return srcset- .split(",")- .map(entry => {- const parts = entry.trim().split(/\s+/);- if (parts.length === 0) return entry;- const imgUrl = parts[0];- const descriptor = parts.slice(1).join(" ");- try {- const absoluteUrl = new URL(imgUrl, url).href;- return descriptor ? `${absoluteUrl} ${descriptor}` : absoluteUrl;- } catch (_) {- return entry.trim();- }- })- .join(", ");- };-- // Make srcset URLs absolute for img elements- soup("img[srcset]").each((_, el) => {- try {- const srcset = el.attribs.srcset;- el.attribs.srcset = makeSrcsetAbsolute(srcset);- } catch (_) {}- });-- // Make source srcset URLs absolute (for picture elements)- soup("source[srcset]").each((_, el) => {- try {- const srcset = el.attribs.srcset;- el.attribs.srcset = makeSrcsetAbsolute(srcset);- } catch (_) {}- });-const cleanedHtml = soup.html();return cleanedHtml;};