Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(utils): keep sitemap URLs written as CDATA surrounded by whitespace (#4250)
Jay Hemnani829fd5743355
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.tsindex 5587d27ab..d4c16ecd6 100644--- a/packages/utils/src/internals/sitemap.ts+++ b/packages/utils/src/internals/sitemap.ts@@ -152,19 +152,24 @@ class SitemapXmlParser extends Transform {}#onText(text: string) {+ text = text.trim();++ // Whitespace around a CDATA section arrives as separate text events.+ if (text === '') {+ return;+ }+if (this.#currentTag === 'loc') {if (this.#rootTagName === 'sitemapindex') {- this.push({ type: 'sitemapUrl', url: text.trim() } satisfies SitemapItem);+ this.push({ type: 'sitemapUrl', url: text } satisfies SitemapItem);}if (this.#rootTagName === 'urlset') {this.#url ??= {};- this.#url.loc = text.trim();+ this.#url.loc = text;}}- text = text.trim();-if (this.#currentTag === 'lastmod') {const lastmod = new Date(text);if (!Number.isNaN(lastmod.getTime())) {diff --git a/test/utils/sitemap.test.ts b/test/utils/sitemap.test.tsindex 72b5a7501..0defe2247 100644--- a/test/utils/sitemap.test.ts+++ b/test/utils/sitemap.test.ts@@ -167,6 +167,20 @@ describe('Sitemap', () => {'</urlset>',].join('\n'),)+ .get('/sitemap_cdata_pretty.xml')+ .reply(+ 200,+ [+ '<?xml version="1.0" encoding="UTF-8"?>',+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',+ '<url>',+ `<loc>+ <![CDATA[http://not-exists.com/catalog]]>+ </loc>`,+ '</url>',+ '</urlset>',+ ].join('\n'),+ ).get('/sitemap_pretty.xml').reply(200,@@ -400,6 +414,13 @@ describe('Sitemap', () => {expect(new Set(sitemap.urls)).toEqual(new Set(['http://not-exists.com/catalog']));});+ it('handles CDATA surrounded by whitespace in loc tags', async () => {+ const sitemap = await Sitemap.load('http://not-exists.com/sitemap_cdata_pretty.xml', undefined, {+ httpClient: new FetchHttpClient(),+ });+ expect(sitemap.urls).toEqual(['http://not-exists.com/catalog']);+ });+it('autodetects sitemaps', async () => {const sitemap = await Sitemap.tryCommonNames('http://not-exists.com/arbitrary_url?search=xyz', undefined, {httpClient: new FetchHttpClient(),