Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(utils): parse gzipped sitemaps with no inner file extension (#4330)

SwastikTripathi3ecbb9a842a7
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.ts
index 1379bb937..259ec6f66 100644
--- a/packages/utils/src/internals/sitemap.ts
+++ b/packages/utils/src/internals/sitemap.ts
@@ -261,6 +261,11 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
return new SitemapTxtParser();
}
+ // A name like `sitemap.gz` says nothing about what was compressed. Sitemaps are XML unless named otherwise.
+ if (mimeType?.essence === 'application/gzip') {
+ return SitemapXmlParser.create();
+ }
+
throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);
};
diff --git a/test/utils/sitemap.test.ts b/test/utils/sitemap.test.ts
index e479860a2..41ca668d1 100644
--- a/test/utils/sitemap.test.ts
+++ b/test/utils/sitemap.test.ts
@@ -1,3 +1,5 @@
+import { gzipSync } from 'node:zlib';
+
import { FetchHttpClient } from '@crawlee/http-client';
import nock from 'nock';
import { beforeEach, describe, expect, it } from 'vitest';
@@ -448,6 +450,25 @@ describe('Sitemap', () => {
expect(logger.warningOnce).not.toHaveBeenCalled();
});
+ it('loads a gzipped XML sitemap whose name has no inner extension', async () => {
+ nock('http://gzipped.com')
+ .get('/sitemap.gz')
+ .reply(
+ 200,
+ gzipSync(
+ '<?xml version="1.0" encoding="UTF-8"?><urlset><url><loc>http://gzipped.com/page</loc></url></urlset>',
+ ),
+ { 'content-type': 'application/gzip' },
+ );
+
+ const sitemap = await Sitemap.load('http://gzipped.com/sitemap.gz', undefined, {
+ httpClient: new FetchHttpClient(),
+ sitemapRetries: 0,
+ });
+
+ expect(sitemap.urls).toEqual(['http://gzipped.com/page']);
+ });
+
it('keeps sitemap.txt lines apart when a network chunk ends on a line break', async () => {
const urls = Array.from({ length: 300 }, (_, i) => `http://not-exists.com/page-${i}`);
const httpClient = new FetchHttpClient();