Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: correctly report gzip decompression errors (#2368)
Hendrik Loewe84a2f1733033
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.tsindex f652e70e1..3c9f89030 100644--- a/packages/utils/src/internals/sitemap.ts+++ b/packages/utils/src/internals/sitemap.ts@@ -162,7 +162,7 @@ export class Sitemap {await new Promise((resolve, reject) => {let stream: Duplex = sitemapStream;if (sitemapUrl.endsWith('.gz')) {- stream = stream.pipe(createGunzip());+ stream = stream.pipe(createGunzip()).on('error', reject);sitemapUrl = sitemapUrl.substring(0, sitemapUrl.length - 3);}diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.tsindex 3e21fcbe9..a766fec6c 100644--- a/packages/utils/test/sitemap.test.ts+++ b/packages/utils/test/sitemap.test.ts@@ -46,6 +46,11 @@ describe('Sitemap', () => {'nr/w/Eb2Ll2RVWLcvwrWMlWtWLWJcBIl6TdW/R/ZZp3soAdV/Yy2w1mOUI63tz4itCRd3Cz9882y','NfMGy9bJ8CfTZkU4fXUavAGtDs17GwMAAA==',].join('\n'), 'base64'))+ .get('/invalid_sitemap_child.xml.gz')+ .reply(200, Buffer.from([+ 'H4sIAAAAAAAAA62S306DMBTG73kK0gtvDLSFLSKWcucTzOulKR00QottGZtPbxfQEEWXqElzkvMv',+ 'NfMGy9bJ8CfTZkU4fXUavAGtDs17GwMAAA==',+ ].join('\n'), 'base64')).get('/sitemap_parent.xml').reply(200, ['<?xml version="1.0" encoding="UTF-8"?>',@@ -115,6 +120,13 @@ describe('Sitemap', () => {]));});+ it('identifies incorrect gzipped sitemaps as malformed', async () => {+ const sitemap = await Sitemap.load(+ 'http://not-exists.com/invalid_sitemap_child.xml.gz',+ );+ expect(new Set(sitemap.urls)).toEqual(new Set([]));+ });+it('follows links in sitemap indexes', async () => {const sitemap = await Sitemap.load('http://not-exists.com/sitemap_parent.xml');expect(new Set(sitemap.urls)).toEqual(new Set([