Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
added iframe links to extractLinksFromHTML
rafaelsideguide367af9512fa4
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 72a49fd87..1ae166b49 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -333,6 +333,16 @@ export class WebCrawler {}});+ // Extract links from iframes with inline src+ $("iframe").each((_, element) => {+ const src = $(element).attr("src");+ if (src && src.startsWith("data:text/html")) {+ const iframeHtml = decodeURIComponent(src.split(",")[1]);+ const iframeLinks = this.extractLinksFromHTML(iframeHtml, url);+ links = links.concat(iframeLinks);+ }+ });+return links;}