Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor(discoverValidSitemaps): check `/sitemap_index.xml` for sitemaps (#3370)

Sviatozar Petrenko19b02548c1e4
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.ts
index 014dd20d4..1cf58e1a8 100644
--- a/packages/utils/src/internals/sitemap.ts
+++ b/packages/utils/src/internals/sitemap.ts
@@ -504,17 +504,13 @@ export async function* discoverValidSitemaps(
}
} else {
const firstUrl = new URL(domainUrls[0]);
- firstUrl.pathname = '/sitemap.xml';
- if (await urlExists(firstUrl.toString())) {
- if (addSitemapUrl(firstUrl.toString())) {
- yield firstUrl.toString();
- }
- }
-
- firstUrl.pathname = '/sitemap.txt';
- if (await urlExists(firstUrl.toString())) {
- if (addSitemapUrl(firstUrl.toString())) {
- yield firstUrl.toString();
+ const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
+ for (const pathname of possibleSitemapPathnames) {
+ firstUrl.pathname = pathname;
+ if (await urlExists(firstUrl.toString())) {
+ if (addSitemapUrl(firstUrl.toString())) {
+ yield firstUrl.toString();
+ }
}
}
}
diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.ts
index cd1c4dbbd..8bb7e65e2 100644
--- a/packages/utils/test/sitemap.test.ts
+++ b/packages/utils/test/sitemap.test.ts
@@ -469,6 +469,8 @@ describe('discoverValidSitemaps', () => {
.head('/sitemap.xml')
.reply(404, '')
.head('/sitemap.txt')
+ .reply(404, '')
+ .head('/sitemap_index.xml')
.reply(404, '');
const urls = [];
@@ -486,6 +488,8 @@ describe('discoverValidSitemaps', () => {
.head('/sitemap.xml')
.reply(200, '')
.head('/sitemap.txt')
+ .reply(404, '')
+ .head('/sitemap_index.xml')
.reply(404, '');
const urls = [];
@@ -503,7 +507,9 @@ describe('discoverValidSitemaps', () => {
.head('/sitemap.xml')
.reply(404, '')
.head('/sitemap.txt')
- .reply(200, '');
+ .reply(200, '')
+ .head('/sitemap_index.xml')
+ .reply(404, '');
const urls = [];
for await (const url of discoverValidSitemaps(['http://sitemap-discovery.com'])) {
@@ -513,6 +519,25 @@ describe('discoverValidSitemaps', () => {
expect(urls).toEqual(['http://sitemap-discovery.com/sitemap.txt']);
});
+ it('extracts sitemap from well-known paths if robots.txt is missing (sitemap_index.xml)', async () => {
+ nock('http://sitemap-discovery.com')
+ .get('/robots.txt')
+ .reply(404)
+ .head('/sitemap.xml')
+ .reply(404, '')
+ .head('/sitemap.txt')
+ .reply(404, '')
+ .head('/sitemap_index.xml')
+ .reply(200, '');
+
+ const urls = [];
+ for await (const url of discoverValidSitemaps(['http://sitemap-discovery.com'])) {
+ urls.push(url);
+ }
+
+ expect(urls).toEqual(['http://sitemap-discovery.com/sitemap_index.xml']);
+ });
+
it('extracts sitemap from input url', async () => {
nock('http://sitemap-discovery.com').get('/robots.txt').reply(404);
@@ -534,7 +559,9 @@ describe('discoverValidSitemaps', () => {
.reply(200, '')
.head('/sitemap.txt')
.delay(50)
- .reply(200, '');
+ .reply(200, '')
+ .head('/sitemap_index.xml')
+ .reply(404);
nock('http://domain-b.com')
.get('/robots.txt')
@@ -545,7 +572,9 @@ describe('discoverValidSitemaps', () => {
.reply(200, '')
.head('/sitemap.txt')
.delay(60)
- .reply(200, '');
+ .reply(200, '')
+ .head('/sitemap_index.xml')
+ .reply(404);
const urls = [];
for await (const url of discoverValidSitemaps(['http://domain-a.com', 'http://domain-b.com'])) {