Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

added iframe links to extractLinksFromHTML

rafaelsideguide367af9512fa4
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 72a49fd87..1ae166b49 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -333,6 +333,16 @@ export class WebCrawler {
}
});
+ // Extract links from iframes with inline src
+ $("iframe").each((_, element) => {
+ const src = $(element).attr("src");
+ if (src && src.startsWith("data:text/html")) {
+ const iframeHtml = decodeURIComponent(src.split(",")[1]);
+ const iframeLinks = this.extractLinksFromHTML(iframeHtml, url);
+ links = links.concat(iframeLinks);
+ }
+ });
+
return links;
}