Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: malformed sitemap url when sitemap index child contains querystring (#2430)

Austine4cd41c49999
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.ts
index bf3c88e3d..b8f94f164 100644
--- a/packages/utils/src/internals/sitemap.ts
+++ b/packages/utils/src/internals/sitemap.ts
@@ -149,8 +149,8 @@ export class Sitemap {
parsingState.sitemapUrls = Array.isArray(urls) ? urls : [urls];
while (parsingState.sitemapUrls.length > 0) {
- let sitemapUrl = parsingState.sitemapUrls.pop()!;
- parsingState.visitedSitemapUrls.push(sitemapUrl);
+ const sitemapUrl = new URL(parsingState.sitemapUrls.pop()!);
+ parsingState.visitedSitemapUrls.push(sitemapUrl.toString());
parsingState.resetContext();
try {
@@ -163,19 +163,19 @@ export class Sitemap {
if (sitemapStream.response!.statusCode === 200) {
await new Promise((resolve, reject) => {
let stream: Duplex = sitemapStream;
- if (sitemapUrl.endsWith('.gz')) {
+ if (sitemapUrl.pathname.endsWith('.gz')) {
stream = stream.pipe(createGunzip()).on('error', reject);
- sitemapUrl = sitemapUrl.substring(0, sitemapUrl.length - 3);
+ sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
}
const parser = (() => {
const contentType = sitemapStream.response!.headers['content-type'];
- if (['text/xml', 'application/xml'].includes(contentType ?? '') || sitemapUrl.endsWith('.xml')) {
+ if (['text/xml', 'application/xml'].includes(contentType ?? '') || sitemapUrl.pathname.endsWith('.xml')) {
return Sitemap.createXmlParser(parsingState, () => resolve(undefined), reject);
}
- if (contentType === 'text/plain' || sitemapUrl.endsWith('.txt')) {
+ if (contentType === 'text/plain' || sitemapUrl.pathname.endsWith('.txt')) {
return new SitemapTxtParser(parsingState, () => resolve(undefined));
}
diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.ts
index b3791c52c..27901079a 100644
--- a/packages/utils/test/sitemap.test.ts
+++ b/packages/utils/test/sitemap.test.ts
@@ -7,7 +7,7 @@ describe('Sitemap', () => {
beforeEach(() => {
nock.disableNetConnect();
nock('http://not-exists.com').persist()
- .get('/sitemap_child.xml')
+ .get((url) => url === '/sitemap_child.xml' || url === '/sitemap_child_2.xml')
.reply(200, [
'<?xml version="1.0" encoding="UTF-8"?>',
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
@@ -59,6 +59,10 @@ describe('Sitemap', () => {
'<loc>http://not-exists.com/sitemap_child.xml</loc>',
'<lastmod>2004-12-23</lastmod>',
'</sitemap>',
+ '<sitemap>',
+ '<loc>http://not-exists.com/sitemap_child_2.xml?from=94937939985&amp;to=1318570721404</loc>',
+ '<lastmod>2004-12-23</lastmod>',
+ '</sitemap>',
'</sitemapindex>',
].join('\n'))
.get('/not_actual_xml.xml')