Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): resolve lazy-loaded images from data-srcset/data-src (#3923)
firecrawl-spring[bot]41186201ad2e
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rsindex d0cd70a57..15c4faed4 100644--- a/apps/api/native/src/html.rs+++ b/apps/api/native/src/html.rs@@ -524,70 +524,96 @@ fn _transform_html_inner(}}- let srcset_images: Vec<_> = document- .select("img[srcset]")- .map_err(|_| "Failed to select srcset images")?+ // Resolve images, including lazy-loaded ones. Many sites ship a tiny+ // placeholder in `src` and keep the real image in `data-srcset` / `data-src`,+ // swapping it in via JS only once the image scrolls into view. When the real+ // `srcset` isn't present yet, fall back to those lazy-load attributes so we+ // emit the actual image instead of the placeholder. Images that already have+ // a `srcset` are handled exactly as before.+ let imgs: Vec<_> = document+ .select("img")+ .map_err(|_| "Failed to select images")?.collect();- for img in srcset_images {- let mut sizes: Vec<ImageSource> = img- .attributes- .borrow()- .get("srcset")- .ok_or("Failed to get srcset")?- .split(',')- .filter_map(|x| {- let tok: Vec<&str> = x.trim().split(' ').collect();- let last_token = tok[tok.len() - 1];- let (last_token, last_token_used) = if tok.len() > 1- && !last_token.is_empty()- && (last_token.ends_with('x') || last_token.ends_with('w'))- {- (last_token, true)- } else {- ("1x", false)- };-- if let Some((last_index, _)) = last_token.char_indices().last() {- if let Ok(parsed_size) = last_token[..last_index].parse() {- Some(ImageSource {- url: if last_token_used {- tok[0..tok.len() - 1].join(" ")- } else {- tok.join(" ")- },- size: parsed_size,- is_x: last_token.ends_with('x'),- })+ for img in imgs {+ let srcset = {+ let attrs = img.attributes.borrow();+ attrs+ .get("srcset")+ .filter(|x| !x.trim().is_empty())+ .or_else(|| attrs.get("data-srcset").filter(|x| !x.trim().is_empty()))+ .map(|x| x.to_string())+ };++ if let Some(srcset) = srcset {+ let mut sizes: Vec<ImageSource> = srcset+ .split(',')+ .filter_map(|x| {+ let tok: Vec<&str> = x.trim().split(' ').collect();+ let last_token = tok[tok.len() - 1];+ let (last_token, last_token_used) = if tok.len() > 1+ && !last_token.is_empty()+ && (last_token.ends_with('x') || last_token.ends_with('w'))+ {+ (last_token, true)+ } else {+ ("1x", false)+ };++ if let Some((last_index, _)) = last_token.char_indices().last() {+ if let Ok(parsed_size) = last_token[..last_index].parse() {+ Some(ImageSource {+ url: if last_token_used {+ tok[0..tok.len() - 1].join(" ")+ } else {+ tok.join(" ")+ },+ size: parsed_size,+ is_x: last_token.ends_with('x'),+ })+ } else {+ None+ }} else {None}- } else {- None- }- })- .collect();+ })+ .collect();- if sizes.iter().all(|x| x.is_x) {- if let Some(src) = img.attributes.borrow().get("src").map(|x| x.to_string()) {- sizes.push(ImageSource {- url: src,- size: 1.0,- is_x: true,- });+ if sizes.iter().all(|x| x.is_x) {+ if let Some(src) = img.attributes.borrow().get("src").map(|x| x.to_string()) {+ sizes.push(ImageSource {+ url: src,+ size: 1.0,+ is_x: true,+ });+ }}- }- sizes.sort_by(|a, b| {- b.size- .partial_cmp(&a.size)- .unwrap_or(std::cmp::Ordering::Equal)- });+ sizes.sort_by(|a, b| {+ b.size+ .partial_cmp(&a.size)+ .unwrap_or(std::cmp::Ordering::Equal)+ });- if let Some(biggest) = sizes.first() {- img- .attributes- .borrow_mut()- .insert("src", biggest.url.clone());+ if let Some(biggest) = sizes.first() {+ img+ .attributes+ .borrow_mut()+ .insert("src", biggest.url.clone());+ }+ } else {+ // No usable `srcset`/`data-srcset`: fall back to `data-src` for the+ // common lazy-load case where only a single image URL is deferred.+ let data_src = {+ let attrs = img.attributes.borrow();+ attrs+ .get("data-src")+ .filter(|x| !x.trim().is_empty())+ .map(|x| x.to_string())+ };+ if let Some(data_src) = data_src {+ img.attributes.borrow_mut().insert("src", data_src);+ }}}diff --git a/apps/api/src/lib/__tests__/html-transformer.test.ts b/apps/api/src/lib/__tests__/html-transformer.test.tsindex cee16f015..371730035 100644--- a/apps/api/src/lib/__tests__/html-transformer.test.ts+++ b/apps/api/src/lib/__tests__/html-transformer.test.ts@@ -455,6 +455,77 @@ describe("HTML Transformer", () => {expect(result).toContain("π π π");});+ it("should pick the largest image from a lazy-loaded data-srcset", async () => {+ // Lazy loaders keep a tiny placeholder in `src` and the real responsive+ // image in `data-srcset` until the image scrolls into view.+ const options: TransformHtmlOptions = {+ html: `+ <div>+ <img+ src="https://cdn.example.com/photo.png?width=5"+ data-src="https://cdn.example.com/photo.png"+ data-srcset="https://cdn.example.com/photo.png?width=320 320w, https://cdn.example.com/photo.png?width=1296 1296w">+ </div>+ `,+ url: "https://example.com",+ includeTags: [],+ excludeTags: [],+ onlyMainContent: false,+ };++ const result = await transformHtml(options);+ expect(result).toContain(+ 'src="https://cdn.example.com/photo.png?width=1296"',+ );+ expect(result).not.toContain("width=5");+ });++ it("should fall back to data-src when there is no srcset", async () => {+ const options: TransformHtmlOptions = {+ html: `+ <div>+ <img+ src="https://cdn.example.com/photo.png?width=5"+ data-src="https://cdn.example.com/photo.png">+ </div>+ `,+ url: "https://example.com",+ includeTags: [],+ excludeTags: [],+ onlyMainContent: false,+ };++ const result = await transformHtml(options);+ expect(result).toContain('src="https://cdn.example.com/photo.png"');+ expect(result).not.toContain("width=5");+ });++ it("should prefer a real srcset over data-srcset when both are present", async () => {+ // Once the lazy loader fires, `srcset` holds the real image; we must not+ // override it with the (now stale) data-* attributes.+ const options: TransformHtmlOptions = {+ html: `+ <div>+ <img+ src="https://cdn.example.com/real.png?width=800"+ srcset="https://cdn.example.com/real.png?width=400 400w, https://cdn.example.com/real.png?width=800 800w"+ data-srcset="https://cdn.example.com/placeholder.png?width=5 5w">+ </div>+ `,+ url: "https://example.com",+ includeTags: [],+ excludeTags: [],+ onlyMainContent: false,+ };++ const result = await transformHtml(options);+ expect(result).toContain(+ 'src="https://cdn.example.com/real.png?width=800"',+ );+ // The src must not be overwritten by the (stale) data-srcset placeholder.+ expect(result).not.toContain('src="https://cdn.example.com/placeholder');+ });+it("should make all URLs absolute", async () => {const options: TransformHtmlOptions = {html: `