Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: using GET for candidate checks - sitemaps (#3464)

nikitachapovskii-devf82eb04673b6
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.ts
index cc74ca776..d74eda264 100644
--- a/packages/utils/src/internals/sitemap.ts
+++ b/packages/utils/src/internals/sitemap.ts
@@ -546,19 +546,22 @@ export async function* discoverValidSitemaps(
} else {
const firstUrl = new URL(domainUrls[0]);
const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
- for (const pathname of possibleSitemapPathnames) {
+ const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
firstUrl.pathname = pathname;
- const candidateSitemapUrl = firstUrl.toString();
+ return firstUrl.toString();
+ });
+ const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));
- try {
- if (await urlExists(candidateSitemapUrl)) {
- if (addSitemapUrl(candidateSitemapUrl)) {
- yield candidateSitemapUrl;
- }
+ for (const [index, result] of candidateResults.entries()) {
+ const candidateSitemapUrl = candidateSitemapUrls[index];
+
+ if (result.status === 'fulfilled') {
+ if (result.value && addSitemapUrl(candidateSitemapUrl)) {
+ yield candidateSitemapUrl;
}
- } catch (err) {
+ } else {
log.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
- error: err,
+ error: result.reason,
});
}
}
diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.ts
index 52ad9fdeb..2fbc54928 100644
--- a/packages/utils/test/sitemap.test.ts
+++ b/packages/utils/test/sitemap.test.ts
@@ -581,12 +581,14 @@ describe('discoverValidSitemaps', () => {
urls.push(url);
}
- expect(urls).toEqual([
- 'http://domain-a.com/sitemap.xml',
- 'http://domain-b.com/sitemap.xml',
- 'http://domain-a.com/sitemap.txt',
- 'http://domain-b.com/sitemap.txt',
- ]);
+ expect(urls.slice().sort()).toEqual(
+ [
+ 'http://domain-a.com/sitemap.xml',
+ 'http://domain-b.com/sitemap.xml',
+ 'http://domain-a.com/sitemap.txt',
+ 'http://domain-b.com/sitemap.txt',
+ ].sort(),
+ );
});
it('aborts when timeoutMillis elapses', async () => {