Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(sitemap): propagate crawlid

Móricz Gergőaa2c36906030
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index f883e4c58..ceaba6ef0 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -555,6 +555,7 @@ export class WebCrawler {
sitemapCount = await getLinksFromSitemap(
{ sitemapUrl, urlsHandler, mode: "fire-engine" },
this.logger,
+ this.jobId,
);
} catch (error) {
this.logger.debug(`Failed to fetch sitemap from ${sitemapUrl}`, {
@@ -595,6 +596,7 @@ export class WebCrawler {
mode: "fire-engine",
},
this.logger,
+ this.jobId,
);
} catch (error) {
this.logger.debug(
@@ -618,6 +620,7 @@ export class WebCrawler {
sitemapCount += await getLinksFromSitemap(
{ sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine" },
this.logger,
+ this.jobId,
);
} catch (error) {
this.logger.debug(`Failed to fetch sitemap from ${baseUrlSitemap}`, {
@@ -631,6 +634,7 @@ export class WebCrawler {
sitemapCount += await getLinksFromSitemap(
{ sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine" },
this.logger,
+ this.jobId,
);
}
}
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index bdd5cd659..aba0e0baa 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -18,13 +18,14 @@ export async function getLinksFromSitemap(
mode?: "axios" | "fire-engine";
},
logger: Logger,
+ crawlId: string,
): Promise<number> {
try {
let content: string = "";
try {
if (mode === "fire-engine" && useFireEngine) {
const fetchResponse = await scrapeURL(
- "sitemap",
+ "sitemap;" + crawlId,
sitemapUrl,
scrapeOptions.parse({ formats: ["rawHtml"] }),
{ forceEngine: "fetch" },
@@ -79,7 +80,7 @@ export async function getLinksFromSitemap(
}
} else {
const fetchResponse = await scrapeURL(
- "sitemap",
+ "sitemap;" + crawlId,
sitemapUrl,
scrapeOptions.parse({ formats: ["rawHtml"] }),
{ forceEngine: "fetch" },
@@ -125,7 +126,7 @@ export async function getLinksFromSitemap(
.map((sitemap) => sitemap.loc[0].trim());
const sitemapPromises: Promise<number>[] = sitemapUrls.map((sitemapUrl) =>
- getLinksFromSitemap({ sitemapUrl, urlsHandler, mode }, logger),
+ getLinksFromSitemap({ sitemapUrl, urlsHandler, mode }, logger, crawlId),
);
const results = await Promise.all(sitemapPromises);
@@ -147,6 +148,7 @@ export async function getLinksFromSitemap(
getLinksFromSitemap(
{ sitemapUrl: sitemapUrl, urlsHandler, mode },
logger,
+ crawlId,
),
);
count += (await Promise.all(sitemapPromises)).reduce(