Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(api): resolve lazy-loaded images from data-srcset/data-src (#3923)

firecrawl-spring[bot]41186201ad2e
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rs
index d0cd70a57..15c4faed4 100644
--- a/apps/api/native/src/html.rs
+++ b/apps/api/native/src/html.rs
@@ -524,70 +524,96 @@ fn _transform_html_inner(
}
}
- let srcset_images: Vec<_> = document
- .select("img[srcset]")
- .map_err(|_| "Failed to select srcset images")?
+ // Resolve images, including lazy-loaded ones. Many sites ship a tiny
+ // placeholder in `src` and keep the real image in `data-srcset` / `data-src`,
+ // swapping it in via JS only once the image scrolls into view. When the real
+ // `srcset` isn't present yet, fall back to those lazy-load attributes so we
+ // emit the actual image instead of the placeholder. Images that already have
+ // a `srcset` are handled exactly as before.
+ let imgs: Vec<_> = document
+ .select("img")
+ .map_err(|_| "Failed to select images")?
.collect();
- for img in srcset_images {
- let mut sizes: Vec<ImageSource> = img
- .attributes
- .borrow()
- .get("srcset")
- .ok_or("Failed to get srcset")?
- .split(',')
- .filter_map(|x| {
- let tok: Vec<&str> = x.trim().split(' ').collect();
- let last_token = tok[tok.len() - 1];
- let (last_token, last_token_used) = if tok.len() > 1
- && !last_token.is_empty()
- && (last_token.ends_with('x') || last_token.ends_with('w'))
- {
- (last_token, true)
- } else {
- ("1x", false)
- };
-
- if let Some((last_index, _)) = last_token.char_indices().last() {
- if let Ok(parsed_size) = last_token[..last_index].parse() {
- Some(ImageSource {
- url: if last_token_used {
- tok[0..tok.len() - 1].join(" ")
- } else {
- tok.join(" ")
- },
- size: parsed_size,
- is_x: last_token.ends_with('x'),
- })
+ for img in imgs {
+ let srcset = {
+ let attrs = img.attributes.borrow();
+ attrs
+ .get("srcset")
+ .filter(|x| !x.trim().is_empty())
+ .or_else(|| attrs.get("data-srcset").filter(|x| !x.trim().is_empty()))
+ .map(|x| x.to_string())
+ };
+
+ if let Some(srcset) = srcset {
+ let mut sizes: Vec<ImageSource> = srcset
+ .split(',')
+ .filter_map(|x| {
+ let tok: Vec<&str> = x.trim().split(' ').collect();
+ let last_token = tok[tok.len() - 1];
+ let (last_token, last_token_used) = if tok.len() > 1
+ && !last_token.is_empty()
+ && (last_token.ends_with('x') || last_token.ends_with('w'))
+ {
+ (last_token, true)
+ } else {
+ ("1x", false)
+ };
+
+ if let Some((last_index, _)) = last_token.char_indices().last() {
+ if let Ok(parsed_size) = last_token[..last_index].parse() {
+ Some(ImageSource {
+ url: if last_token_used {
+ tok[0..tok.len() - 1].join(" ")
+ } else {
+ tok.join(" ")
+ },
+ size: parsed_size,
+ is_x: last_token.ends_with('x'),
+ })
+ } else {
+ None
+ }
} else {
None
}
- } else {
- None
- }
- })
- .collect();
+ })
+ .collect();
- if sizes.iter().all(|x| x.is_x) {
- if let Some(src) = img.attributes.borrow().get("src").map(|x| x.to_string()) {
- sizes.push(ImageSource {
- url: src,
- size: 1.0,
- is_x: true,
- });
+ if sizes.iter().all(|x| x.is_x) {
+ if let Some(src) = img.attributes.borrow().get("src").map(|x| x.to_string()) {
+ sizes.push(ImageSource {
+ url: src,
+ size: 1.0,
+ is_x: true,
+ });
+ }
}
- }
- sizes.sort_by(|a, b| {
- b.size
- .partial_cmp(&a.size)
- .unwrap_or(std::cmp::Ordering::Equal)
- });
+ sizes.sort_by(|a, b| {
+ b.size
+ .partial_cmp(&a.size)
+ .unwrap_or(std::cmp::Ordering::Equal)
+ });
- if let Some(biggest) = sizes.first() {
- img
- .attributes
- .borrow_mut()
- .insert("src", biggest.url.clone());
+ if let Some(biggest) = sizes.first() {
+ img
+ .attributes
+ .borrow_mut()
+ .insert("src", biggest.url.clone());
+ }
+ } else {
+ // No usable `srcset`/`data-srcset`: fall back to `data-src` for the
+ // common lazy-load case where only a single image URL is deferred.
+ let data_src = {
+ let attrs = img.attributes.borrow();
+ attrs
+ .get("data-src")
+ .filter(|x| !x.trim().is_empty())
+ .map(|x| x.to_string())
+ };
+ if let Some(data_src) = data_src {
+ img.attributes.borrow_mut().insert("src", data_src);
+ }
}
}
diff --git a/apps/api/src/lib/__tests__/html-transformer.test.ts b/apps/api/src/lib/__tests__/html-transformer.test.ts
index cee16f015..371730035 100644
--- a/apps/api/src/lib/__tests__/html-transformer.test.ts
+++ b/apps/api/src/lib/__tests__/html-transformer.test.ts
@@ -455,6 +455,77 @@ describe("HTML Transformer", () => {
expect(result).toContain("πŸŽ‰ πŸ‘ πŸš€");
});
+ it("should pick the largest image from a lazy-loaded data-srcset", async () => {
+ // Lazy loaders keep a tiny placeholder in `src` and the real responsive
+ // image in `data-srcset` until the image scrolls into view.
+ const options: TransformHtmlOptions = {
+ html: `
+ <div>
+ <img
+ src="https://cdn.example.com/photo.png?width=5"
+ data-src="https://cdn.example.com/photo.png"
+ data-srcset="https://cdn.example.com/photo.png?width=320 320w, https://cdn.example.com/photo.png?width=1296 1296w">
+ </div>
+ `,
+ url: "https://example.com",
+ includeTags: [],
+ excludeTags: [],
+ onlyMainContent: false,
+ };
+
+ const result = await transformHtml(options);
+ expect(result).toContain(
+ 'src="https://cdn.example.com/photo.png?width=1296"',
+ );
+ expect(result).not.toContain("width=5");
+ });
+
+ it("should fall back to data-src when there is no srcset", async () => {
+ const options: TransformHtmlOptions = {
+ html: `
+ <div>
+ <img
+ src="https://cdn.example.com/photo.png?width=5"
+ data-src="https://cdn.example.com/photo.png">
+ </div>
+ `,
+ url: "https://example.com",
+ includeTags: [],
+ excludeTags: [],
+ onlyMainContent: false,
+ };
+
+ const result = await transformHtml(options);
+ expect(result).toContain('src="https://cdn.example.com/photo.png"');
+ expect(result).not.toContain("width=5");
+ });
+
+ it("should prefer a real srcset over data-srcset when both are present", async () => {
+ // Once the lazy loader fires, `srcset` holds the real image; we must not
+ // override it with the (now stale) data-* attributes.
+ const options: TransformHtmlOptions = {
+ html: `
+ <div>
+ <img
+ src="https://cdn.example.com/real.png?width=800"
+ srcset="https://cdn.example.com/real.png?width=400 400w, https://cdn.example.com/real.png?width=800 800w"
+ data-srcset="https://cdn.example.com/placeholder.png?width=5 5w">
+ </div>
+ `,
+ url: "https://example.com",
+ includeTags: [],
+ excludeTags: [],
+ onlyMainContent: false,
+ };
+
+ const result = await transformHtml(options);
+ expect(result).toContain(
+ 'src="https://cdn.example.com/real.png?width=800"',
+ );
+ // The src must not be overwritten by the (stale) data-srcset placeholder.
+ expect(result).not.toContain('src="https://cdn.example.com/placeholder');
+ });
+
it("should make all URLs absolute", async () => {
const options: TransformHtmlOptions = {
html: `