Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(sitemap): propagate crawlid
Móricz Gergőaa2c36906030
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex f883e4c58..ceaba6ef0 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -555,6 +555,7 @@ export class WebCrawler {sitemapCount = await getLinksFromSitemap({ sitemapUrl, urlsHandler, mode: "fire-engine" },this.logger,+ this.jobId,);} catch (error) {this.logger.debug(`Failed to fetch sitemap from ${sitemapUrl}`, {@@ -595,6 +596,7 @@ export class WebCrawler {mode: "fire-engine",},this.logger,+ this.jobId,);} catch (error) {this.logger.debug(@@ -618,6 +620,7 @@ export class WebCrawler {sitemapCount += await getLinksFromSitemap({ sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine" },this.logger,+ this.jobId,);} catch (error) {this.logger.debug(`Failed to fetch sitemap from ${baseUrlSitemap}`, {@@ -631,6 +634,7 @@ export class WebCrawler {sitemapCount += await getLinksFromSitemap({ sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine" },this.logger,+ this.jobId,);}}diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex bdd5cd659..aba0e0baa 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -18,13 +18,14 @@ export async function getLinksFromSitemap(mode?: "axios" | "fire-engine";},logger: Logger,+ crawlId: string,): Promise<number> {try {let content: string = "";try {if (mode === "fire-engine" && useFireEngine) {const fetchResponse = await scrapeURL(- "sitemap",+ "sitemap;" + crawlId,sitemapUrl,scrapeOptions.parse({ formats: ["rawHtml"] }),{ forceEngine: "fetch" },@@ -79,7 +80,7 @@ export async function getLinksFromSitemap(}} else {const fetchResponse = await scrapeURL(- "sitemap",+ "sitemap;" + crawlId,sitemapUrl,scrapeOptions.parse({ formats: ["rawHtml"] }),{ forceEngine: "fetch" },@@ -125,7 +126,7 @@ export async function getLinksFromSitemap(.map((sitemap) => sitemap.loc[0].trim());const sitemapPromises: Promise<number>[] = sitemapUrls.map((sitemapUrl) =>- getLinksFromSitemap({ sitemapUrl, urlsHandler, mode }, logger),+ getLinksFromSitemap({ sitemapUrl, urlsHandler, mode }, logger, crawlId),);const results = await Promise.all(sitemapPromises);@@ -147,6 +148,7 @@ export async function getLinksFromSitemap(getLinksFromSitemap({ sitemapUrl: sitemapUrl, urlsHandler, mode },logger,+ crawlId,),);count += (await Promise.all(sitemapPromises)).reduce(