Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: Parallel sitemap discovery (#3571)

Baptiste Jamin9b310c77d23a
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
index c31e96937..eaecde2d7 100644
--- a/docs/public-api/crawlee-basic.api.md
+++ b/docs/public-api/crawlee-basic.api.md
@@ -830,6 +830,7 @@ export class SitemapRequestLoader implements IRequestLoader {
// @public (undocumented)
export interface SitemapRequestLoaderOptions extends UrlConstraints {
+ concurrency?: number;
enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
httpClient?: BaseHttpClient;
maxBufferSize?: number;
diff --git a/packages/basic-crawler/src/internals/sitemap-request-loader.ts b/packages/basic-crawler/src/internals/sitemap-request-loader.ts
index 19ba8c945..31d29c120 100644
--- a/packages/basic-crawler/src/internals/sitemap-request-loader.ts
+++ b/packages/basic-crawler/src/internals/sitemap-request-loader.ts
@@ -35,6 +35,7 @@ const sitemapRequestLoaderOptionsSchema = z.strictObject({
exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
persistenceOptions: z.looseObject({}).optional(),
httpClient: schemas.httpClient.optional(),
+ concurrency: z.number().int().gte(1).default(1),
});
/** @internal */
@@ -118,18 +119,34 @@ export interface SitemapRequestLoaderOptions extends UrlConstraints {
* Custom HTTP client to be used for sitemap loading.
*/
httpClient?: BaseHttpClient;
+ /**
+ * Maximum number of sitemaps to load in parallel.
+ * This includes both the initial sitemaps and any nested sitemaps discovered during parsing.
+ *
+ * @default 1
+ */
+ concurrency?: number;
}
interface SitemapParsingProgress {
- inProgressSitemapUrl: string | null;
- inProgressEntries: Set<string>;
+ inProgressSitemapUrls: Set<string>;
+ inProgressEntriesPerSitemap: Map<string, Set<string>>;
pendingSitemapUrls: Set<string>;
processedSitemapUrls: Set<string>;
}
interface SitemapRequestLoaderState {
urlQueue: string[];
- sitemapParsingProgress: Record<keyof SitemapParsingProgress, any>;
+ sitemapParsingProgress: {
+ pendingSitemapUrls: string[];
+ inProgressSitemapUrls?: string[];
+ inProgressEntriesPerSitemap?: Record<string, string[]>;
+ processedSitemapUrls?: string[];
+ /** Legacy single-sitemap format */
+ inProgressSitemapUrl?: string | null;
+ /** Legacy single-sitemap format */
+ inProgressEntries?: string[];
+ };
abortLoading: boolean;
closed: boolean;
requestData: [string, Request][];
@@ -158,16 +175,13 @@ export class SitemapRequestLoader implements IRequestLoader {
* Object for keeping track of the sitemap parsing progress.
*/
#sitemapParsingProgress: SitemapParsingProgress = {
+ inProgressSitemapUrls: new Set<string>(),
/**
- * URL of the sitemap that is currently being parsed. `null` if no sitemap is being parsed.
- */
- inProgressSitemapUrl: null,
- /**
- * Buffer for URLs from the currently parsed sitemap. Used for tracking partially loaded sitemaps across migrations.
+ * URLs already emitted from each in-progress sitemap. Used for tracking partially loaded sitemaps across migrations.
*/
- inProgressEntries: new Set<string>(),
+ inProgressEntriesPerSitemap: new Map<string, Set<string>>(),
/**
- * Set of sitemap URLs that have not been parsed yet. If the set is empty and `inProgressSitemapUrl` is `null`, the sitemap loading is finished.
+ * Set of sitemap URLs that have not been parsed yet. If this and `inProgressSitemapUrls` are empty, the sitemap loading is finished.
*/
pendingSitemapUrls: new Set<string>(),
/**
@@ -176,6 +190,11 @@ export class SitemapRequestLoader implements IRequestLoader {
processedSitemapUrls: new Set<string>(),
};
+ /**
+ * Maximum number of sitemaps to load in parallel.
+ */
+ #concurrency: number;
+
/**
* Object stream of URLs parsed from the sitemaps.
* Using `highWaterMark`, this can manage the speed of the sitemap loading.
@@ -237,6 +256,7 @@ export class SitemapRequestLoader implements IRequestLoader {
maxBufferSize,
sitemapUrls,
enqueueStrategy,
+ concurrency,
} = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
this.#log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });
@@ -254,6 +274,7 @@ export class SitemapRequestLoader implements IRequestLoader {
this.#proxyUrl = proxyUrl;
this.#enqueueStrategy = enqueueStrategy;
+ this.#concurrency = concurrency;
this.#urlQueueStream = this.#createNewStream(maxBufferSize);
@@ -344,11 +365,69 @@ export class SitemapRequestLoader implements IRequestLoader {
*/
isSitemapFullyLoaded(): boolean {
return (
- this.#sitemapParsingProgress.inProgressSitemapUrl === null &&
+ this.#sitemapParsingProgress.inProgressSitemapUrls.size === 0 &&
this.#sitemapParsingProgress.pendingSitemapUrls.size === 0
);
}
+ #isUrlAlreadyEmitted(url: string): boolean {
+ for (const entries of this.#sitemapParsingProgress.inProgressEntriesPerSitemap.values()) {
+ if (entries.has(url)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Processes a single sitemap URL and emits its URLs to the queue.
+ * Nested sitemaps are added to the pending queue for later processing.
+ */
+ async #loadSingleSitemap(
+ sitemapUrl: string,
+ parseSitemapOptions?: SitemapRequestLoaderOptions['parseSitemapOptions'],
+ ): Promise<void> {
+ const progress = this.#sitemapParsingProgress;
+ progress.inProgressSitemapUrls.add(sitemapUrl);
+
+ if (!progress.inProgressEntriesPerSitemap.has(sitemapUrl)) {
+ progress.inProgressEntriesPerSitemap.set(sitemapUrl, new Set<string>());
+ }
+ const entries = progress.inProgressEntriesPerSitemap.get(sitemapUrl)!;
+
+ try {
+ for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.#proxyUrl, {
+ ...parseSitemapOptions,
+ maxDepth: 0,
+ emitNestedSitemaps: true,
+ enqueueStrategy: this.#enqueueStrategy,
+ })) {
+ // Leave the sitemap in progress so `isSitemapFullyLoaded()` keeps reporting the abort.
+ if (this.#abortLoading) return;
+
+ if (!item.originSitemapUrl) {
+ // This is a nested sitemap
+ if (!progress.processedSitemapUrls.has(item.loc) && !progress.inProgressSitemapUrls.has(item.loc)) {
+ progress.pendingSitemapUrls.add(item.loc);
+ }
+ continue;
+ }
+
+ if (!entries.has(item.loc) && !this.#isUrlAlreadyEmitted(item.loc)) {
+ await this.#pushNextUrl(item.loc);
+ entries.add(item.loc);
+ }
+ }
+ } catch (e: any) {
+ this.#log.error(`Error loading sitemap contents from ${sitemapUrl}:`, e);
+ }
+
+ progress.inProgressSitemapUrls.delete(sitemapUrl);
+ progress.pendingSitemapUrls.delete(sitemapUrl);
+ progress.inProgressEntriesPerSitemap.delete(sitemapUrl);
+ progress.processedSitemapUrls.add(sitemapUrl);
+ }
+
/**
* Start processing the sitemaps and loading the URLs.
*
@@ -359,41 +438,37 @@ export class SitemapRequestLoader implements IRequestLoader {
}: {
parseSitemapOptions?: SitemapRequestLoaderOptions['parseSitemapOptions'];
}): Promise<void> {
- while (!this.isSitemapFullyLoaded() && !this.#abortLoading) {
- const sitemapUrl =
- this.#sitemapParsingProgress.inProgressSitemapUrl ??
- this.#sitemapParsingProgress.pendingSitemapUrls.values().next().value!;
-
- try {
- for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.#proxyUrl, {
- ...parseSitemapOptions,
- maxDepth: 0,
- emitNestedSitemaps: true,
- enqueueStrategy: this.#enqueueStrategy,
- })) {
- if (!item.originSitemapUrl) {
- // This is a nested sitemap
- if (!this.#sitemapParsingProgress.processedSitemapUrls.has(item.loc)) {
- this.#sitemapParsingProgress.pendingSitemapUrls.add(item.loc);
- }
- continue;
- }
-
- if (!this.#sitemapParsingProgress.inProgressEntries.has(item.loc)) {
- await this.#pushNextUrl(item.loc);
- this.#sitemapParsingProgress.inProgressEntries.add(item.loc);
- }
- }
- } catch (e: any) {
- this.#log.error('Error loading sitemap contents:', e);
+ const activeLoaders: Promise<void>[] = [];
+
+ const startNewLoaders = () => {
+ while (
+ !this.#abortLoading &&
+ activeLoaders.length < this.#concurrency &&
+ this.#sitemapParsingProgress.pendingSitemapUrls.size > 0
+ ) {
+ const sitemapUrl = this.#sitemapParsingProgress.pendingSitemapUrls.values().next().value!;
+ this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
+
+ const loaderPromise = this.#loadSingleSitemap(sitemapUrl, parseSitemapOptions).finally(() => {
+ const index = activeLoaders.indexOf(loaderPromise);
+ if (index !== -1) void activeLoaders.splice(index, 1);
+ });
+ activeLoaders.push(loaderPromise);
}
+ };
+
+ startNewLoaders();
- this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
- this.#sitemapParsingProgress.processedSitemapUrls.add(sitemapUrl);
- this.#sitemapParsingProgress.inProgressEntries.clear();
- this.#sitemapParsingProgress.inProgressSitemapUrl = null;
+ while (
+ (activeLoaders.length > 0 || this.#sitemapParsingProgress.pendingSitemapUrls.size > 0) &&
+ !this.#abortLoading
+ ) {
+ if (activeLoaders.length > 0) await Promise.race(activeLoaders);
+ startNewLoaders();
}
+ if (activeLoaders.length > 0) await Promise.all(activeLoaders);
+
this.#urlQueueStream.end();
}
@@ -524,11 +599,16 @@ export class SitemapRequestLoader implements IRequestLoader {
// been transferred to the new stream above).
previousStream.emit('readdata');
+ const inProgressEntriesPerSitemap: Record<string, string[]> = {};
+ for (const [sitemapUrl, entries] of this.#sitemapParsingProgress.inProgressEntriesPerSitemap) {
+ inProgressEntriesPerSitemap[sitemapUrl] = Array.from(entries);
+ }
+
await this.#store.setValue(this.#persistStateKey, {
sitemapParsingProgress: {
pendingSitemapUrls: Array.from(this.#sitemapParsingProgress.pendingSitemapUrls),
- inProgressSitemapUrl: this.#sitemapParsingProgress.inProgressSitemapUrl,
- inProgressEntries: Array.from(this.#sitemapParsingProgress.inProgressEntries),
+ inProgressSitemapUrls: Array.from(this.#sitemapParsingProgress.inProgressSitemapUrls),
+ inProgressEntriesPerSitemap,
processedSitemapUrls: Array.from(this.#sitemapParsingProgress.processedSitemapUrls),
},
// Re-queue in-progress requests to the front so they are retried if the state is restored.
@@ -553,11 +633,27 @@ export class SitemapRequestLoader implements IRequestLoader {
return;
}
+ const progress = state.sitemapParsingProgress;
+ const inProgressSitemapUrls = new Set<string>(progress.inProgressSitemapUrls ?? []);
+ if (progress.inProgressSitemapUrl && !progress.inProgressSitemapUrls) {
+ inProgressSitemapUrls.add(progress.inProgressSitemapUrl);
+ }
+
+ const inProgressEntriesPerSitemap = new Map<string, Set<string>>();
+ if (progress.inProgressEntriesPerSitemap) {
+ for (const [sitemapUrl, entries] of Object.entries(progress.inProgressEntriesPerSitemap)) {
+ inProgressEntriesPerSitemap.set(sitemapUrl, new Set(entries));
+ }
+ } else if (progress.inProgressEntries && progress.inProgressSitemapUrl) {
+ inProgressEntriesPerSitemap.set(progress.inProgressSitemapUrl, new Set(progress.inProgressEntries));
+ }
+
+ // Interrupted sitemaps are re-parsed; `inProgressEntriesPerSitemap` skips the URLs they already emitted.
this.#sitemapParsingProgress = {
- pendingSitemapUrls: new Set(state.sitemapParsingProgress.pendingSitemapUrls),
- inProgressSitemapUrl: state.sitemapParsingProgress.inProgressSitemapUrl,
- inProgressEntries: new Set(state.sitemapParsingProgress.inProgressEntries),
- processedSitemapUrls: new Set(state.sitemapParsingProgress.processedSitemapUrls ?? []),
+ pendingSitemapUrls: new Set([...progress.pendingSitemapUrls, ...inProgressSitemapUrls]),
+ inProgressSitemapUrls: new Set<string>(),
+ inProgressEntriesPerSitemap,
+ processedSitemapUrls: new Set(progress.processedSitemapUrls ?? []),
};
this.#requestData = new Map(state.requestData ?? []);
diff --git a/test/basic-crawler/sitemap-request-loader.test.ts b/test/basic-crawler/sitemap-request-loader.test.ts
index d899ca6b9..0deb58dbb 100644
--- a/test/basic-crawler/sitemap-request-loader.test.ts
+++ b/test/basic-crawler/sitemap-request-loader.test.ts
@@ -289,6 +289,49 @@ beforeAll(async () => {
].join('\n'),
);
});
+
+ // Sitemaps for concurrency testing
+ const concurrencyLoadTimes: Record<string, number> = {};
+
+ for (const n of [1, 2, 3]) {
+ app.get(`/sitemap-slow-${n}.xml`, async (req, res) => {
+ concurrencyLoadTimes[`slow-${n}-start`] = Date.now();
+ res.setHeader('content-type', 'text/xml');
+ await sleep(100);
+ res.write(
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ `<url><loc>http://not-exists.com/slow-${n}-page-1</loc></url>`,
+ `<url><loc>http://not-exists.com/slow-${n}-page-2</loc></url>`,
+ '</urlset>',
+ ].join('\n'),
+ );
+ concurrencyLoadTimes[`slow-${n}-end`] = Date.now();
+ res.end();
+ });
+ }
+
+ app.get('/sitemap-index-slow.xml', async (req, res) => {
+ res.setHeader('content-type', 'text/xml');
+ res.end(
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ ...[1, 2, 3].map((n) => `<sitemap><loc>${url}/sitemap-slow-${n}.xml</loc></sitemap>`),
+ '</sitemapindex>',
+ ].join('\n'),
+ );
+ });
+
+ app.get('/concurrency-load-times', (req, res) => {
+ res.json(concurrencyLoadTimes);
+ });
+
+ app.delete('/concurrency-load-times', (req, res) => {
+ Object.keys(concurrencyLoadTimes).forEach((key) => delete concurrencyLoadTimes[key]);
+ res.status(204).end();
+ });
});
afterAll(async () => {
@@ -768,4 +811,147 @@ describe('SitemapRequestLoader', () => {
]),
);
}, 10_000);
+
+ describe('concurrency', () => {
+ const slowIndexOptions = { enqueueStrategy: 'all' as const };
+
+ beforeEach(async () => {
+ await fetch(`${url}/concurrency-load-times`, { method: 'DELETE' });
+ });
+
+ const getStartTimes = async () => {
+ const loadTimes = await fetch(`${url}/concurrency-load-times`).then(async (r) => r.json());
+ return [loadTimes['slow-1-start'], loadTimes['slow-2-start'], loadTimes['slow-3-start']].sort(
+ (a, b) => a - b,
+ );
+ };
+
+ test('loads multiple sitemaps in parallel', async () => {
+ const list = await SitemapRequestLoader.open({
+ ...slowIndexOptions,
+ sitemapUrls: [`${url}/sitemap-index-slow.xml`],
+ concurrency: 3,
+ });
+
+ for await (const request of list) {
+ await list.markRequestAsHandled(request);
+ }
+
+ expect(await list.getHandledCount()).toBe(6);
+
+ // Sequential loading would put the starts ~100ms apart
+ const startTimes = await getStartTimes();
+ expect(startTimes[2] - startTimes[0]).toBeLessThan(80);
+ });
+
+ test('concurrency=1 loads sitemaps sequentially', async () => {
+ const list = await SitemapRequestLoader.open({
+ ...slowIndexOptions,
+ sitemapUrls: [`${url}/sitemap-index-slow.xml`],
+ concurrency: 1,
+ });
+
+ for await (const request of list) {
+ await list.markRequestAsHandled(request);
+ }
+
+ expect(await list.getHandledCount()).toBe(6);
+
+ const loadTimes = await fetch(`${url}/concurrency-load-times`).then(async (r) => r.json());
+ expect(loadTimes['slow-2-start']).toBeGreaterThanOrEqual(loadTimes['slow-1-end'] - 20);
+ });
+
+ test('respects the concurrency limit', async () => {
+ const list = await SitemapRequestLoader.open({
+ ...slowIndexOptions,
+ sitemapUrls: [`${url}/sitemap-index-slow.xml`],
+ concurrency: 2,
+ });
+
+ for await (const request of list) {
+ await list.markRequestAsHandled(request);
+ }
+
+ expect(await list.getHandledCount()).toBe(6);
+
+ const startTimes = await getStartTimes();
+ expect(startTimes[1] - startTimes[0]).toBeLessThan(50);
+ expect(startTimes[2] - startTimes[0]).toBeGreaterThan(80);
+ });
+
+ test('collects all URLs without duplicates', async () => {
+ const list = await SitemapRequestLoader.open({
+ ...slowIndexOptions,
+ sitemapUrls: [`${url}/sitemap-index-slow.xml`],
+ concurrency: 3,
+ });
+
+ const urls: string[] = [];
+ for await (const request of list) {
+ urls.push(request.url);
+ await list.markRequestAsHandled(request);
+ }
+
+ expect(urls.sort()).toEqual(
+ [1, 2, 3].flatMap((n) => [
+ `http://not-exists.com/slow-${n}-page-1`,
+ `http://not-exists.com/slow-${n}-page-2`,
+ ]),
+ );
+ });
+
+ test('abort signal stops concurrent loading', async () => {
+ const controller = new AbortController();
+
+ const list = await SitemapRequestLoader.open({
+ ...slowIndexOptions,
+ sitemapUrls: [`${url}/sitemap-index-slow.xml`],
+ concurrency: 3,
+ signal: controller.signal,
+ });
+
+ await sleep(50);
+ controller.abort();
+
+ for await (const request of list) {
+ await list.markRequestAsHandled(request);
+ }
+
+ // Let the in-flight sitemap fetches (100ms) settle, so the check isn't racing them.
+ await sleep(200);
+ expect(list.isSitemapFullyLoaded()).toBe(false);
+ await expect(list.checkReadiness()).resolves.toEqual({ status: 'finished' });
+ });
+
+ test('state persistence works with concurrency', async () => {
+ const options = {
+ ...slowIndexOptions,
+ sitemapUrls: [`${url}/sitemap-index-slow.xml`],
+ persistStateKey: 'concurrency-persist',
+ concurrency: 2,
+ };
+
+ {
+ const list = await SitemapRequestLoader.open(options);
+
+ while ((await list.checkReadiness()).status !== 'ready') {
+ await sleep(20);
+ }
+
+ const request = await list.fetchNextRequest();
+ await list.markRequestAsHandled(request!);
+
+ await list.persistState();
+ }
+
+ const newList = await SitemapRequestLoader.open(options);
+ const urls = new Set<string>();
+ for await (const request of newList) {
+ urls.add(request.url);
+ await newList.markRequestAsHandled(request);
+ }
+
+ expect(urls.size).toBeGreaterThanOrEqual(5);
+ });
+ });
});