Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(utils): keep sitemap URLs written as CDATA surrounded by whitespace (#4250)

Jay Hemnani829fd5743355
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.ts
index 5587d27ab..d4c16ecd6 100644
--- a/packages/utils/src/internals/sitemap.ts
+++ b/packages/utils/src/internals/sitemap.ts
@@ -152,19 +152,24 @@ class SitemapXmlParser extends Transform {
}
#onText(text: string) {
+ text = text.trim();
+
+ // Whitespace around a CDATA section arrives as separate text events.
+ if (text === '') {
+ return;
+ }
+
if (this.#currentTag === 'loc') {
if (this.#rootTagName === 'sitemapindex') {
- this.push({ type: 'sitemapUrl', url: text.trim() } satisfies SitemapItem);
+ this.push({ type: 'sitemapUrl', url: text } satisfies SitemapItem);
}
if (this.#rootTagName === 'urlset') {
this.#url ??= {};
- this.#url.loc = text.trim();
+ this.#url.loc = text;
}
}
- text = text.trim();
-
if (this.#currentTag === 'lastmod') {
const lastmod = new Date(text);
if (!Number.isNaN(lastmod.getTime())) {
diff --git a/test/utils/sitemap.test.ts b/test/utils/sitemap.test.ts
index 72b5a7501..0defe2247 100644
--- a/test/utils/sitemap.test.ts
+++ b/test/utils/sitemap.test.ts
@@ -167,6 +167,20 @@ describe('Sitemap', () => {
'</urlset>',
].join('\n'),
)
+ .get('/sitemap_cdata_pretty.xml')
+ .reply(
+ 200,
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ '<url>',
+ `<loc>
+ <![CDATA[http://not-exists.com/catalog]]>
+ </loc>`,
+ '</url>',
+ '</urlset>',
+ ].join('\n'),
+ )
.get('/sitemap_pretty.xml')
.reply(
200,
@@ -400,6 +414,13 @@ describe('Sitemap', () => {
expect(new Set(sitemap.urls)).toEqual(new Set(['http://not-exists.com/catalog']));
});
+ it('handles CDATA surrounded by whitespace in loc tags', async () => {
+ const sitemap = await Sitemap.load('http://not-exists.com/sitemap_cdata_pretty.xml', undefined, {
+ httpClient: new FetchHttpClient(),
+ });
+ expect(sitemap.urls).toEqual(['http://not-exists.com/catalog']);
+ });
+
it('autodetects sitemaps', async () => {
const sitemap = await Sitemap.tryCommonNames('http://not-exists.com/arbitrary_url?search=xyz', undefined, {
httpClient: new FetchHttpClient(),