Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: `.trim()` urls from pretty-printed sitemap.xml files (#2709)
Jindřich Bär802a6fea7b21
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.tsindex 7c3cc1249..471230c29 100644--- a/packages/utils/src/internals/sitemap.ts+++ b/packages/utils/src/internals/sitemap.ts@@ -141,12 +141,12 @@ class SitemapXmlParser extends Transform {private onText(text: string) {if (this.currentTag === 'loc') {if (this.rootTagName === 'sitemapindex') {- this.push({ type: 'sitemapUrl', url: text } satisfies SitemapItem);+ this.push({ type: 'sitemapUrl', url: text.trim() } satisfies SitemapItem);}if (this.rootTagName === 'urlset') {this.url ??= {};- this.url.loc = text;+ this.url.loc = text.trim();}}diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.tsindex f1691e96e..5b6225ddd 100644--- a/packages/utils/test/sitemap.test.ts+++ b/packages/utils/test/sitemap.test.ts@@ -117,6 +117,31 @@ describe('Sitemap', () => {'</sitemapindex>',].join('\n'),)+ .get('/sitemap_parent_pretty.xml')+ .reply(+ 200,+ [+ '<?xml version="1.0" encoding="UTF-8"?>',+ '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',+ '<sitemap>',+ `<loc>+ http://not-exists.com/sitemap_child.xml+ </loc>`,+ `<lastmod>+ 2004-12-23+ </lastmod>`,+ '</sitemap>',+ '<sitemap>',+ `<loc>+ http://not-exists.com/sitemap_child_2.xml?from=94937939985&to=1318570721404+ </loc>`,+ `<lastmod>+ 2004-12-23+ </lastmod>`,+ '</sitemap>',+ '</sitemapindex>',+ ].join('\n'),+ ).get('/not_actual_xml.xml').reply(200,@@ -141,6 +166,30 @@ describe('Sitemap', () => {'</urlset>',].join('\n'),)+ .get('/sitemap_pretty.xml')+ .reply(+ 200,+ [+ '<?xml version="1.0" encoding="UTF-8"?>',+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',+ '<url>',+ `<loc>+ http://not-exists.com/catalog?item=80&desc=vacation_turkey+ </loc>`,+ `<lastmod>+ 2005-02-03+ </lastmod>`,+ `<changefreq>++ monthly+ </changefreq>`,+ `<priority>+ 0.8+ </priority>`,+ '</url>',+ '</urlset>',+ ].join('\n'),+ ).get('/sitemap.xml').reply(200,@@ -269,6 +318,42 @@ describe('Sitemap', () => {);});+ it('handles pretty-printed XML correctly', async () => {+ const sitemap = await Sitemap.load('http://not-exists.com/sitemap_pretty.xml');+ expect(new Set(sitemap.urls)).toEqual(new Set(['http://not-exists.com/catalog?item=80&desc=vacation_turkey']));+ });++ it('extracts metadata from pretty-printed XML', async () => {+ const items: SitemapUrl[] = [];++ for await (const item of parseSitemap([{ type: 'url', url: 'http://not-exists.com/sitemap_pretty.xml' }])) {+ items.push(item);+ }++ expect(items).toHaveLength(1);+ expect(items).toContainEqual(+ expect.objectContaining({+ loc: 'http://not-exists.com/catalog?item=80&desc=vacation_turkey',+ priority: 0.8,+ changefreq: 'monthly',+ lastmod: new Date('2005-02-03'),+ }),+ );+ });++ it('handles pretty-printed nested sitemaps XML correctly', async () => {+ const sitemap = await Sitemap.load('http://not-exists.com/sitemap_parent_pretty.xml');+ expect(new Set(sitemap.urls)).toEqual(+ new Set([+ 'http://not-exists.com/',+ 'http://not-exists.com/catalog?item=12&desc=vacation_hawaii',+ 'http://not-exists.com/catalog?item=73&desc=vacation_new_zealand',+ 'http://not-exists.com/catalog?item=74&desc=vacation_newfoundland',+ 'http://not-exists.com/catalog?item=83&desc=vacation_usa',+ ]),+ );+ });+it('loads sitemaps from string', async () => {const sitemap = await Sitemap.fromXmlString([