Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(utils): prevent sitemap metadata leak across url without loc (#3795)
Anas Khan895ed6264d7e
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.tsindex 4b4fbada3..e44948e4e 100644--- a/packages/utils/src/internals/sitemap.ts+++ b/packages/utils/src/internals/sitemap.ts@@ -136,8 +136,10 @@ class SitemapXmlParser extends Transform {this.currentTag = undefined;}- if (name === 'url' && this.url.loc !== undefined) {- this.push({ type: 'url', ...this.url, loc: this.url.loc } satisfies SitemapItem);+ if (name === 'url') {+ if (this.url.loc !== undefined) {+ this.push({ type: 'url', ...this.url, loc: this.url.loc } satisfies SitemapItem);+ }this.url = {};}}@@ -157,7 +159,10 @@ class SitemapXmlParser extends Transform {text = text.trim();if (this.currentTag === 'lastmod') {- this.url.lastmod = new Date(text);+ const lastmod = new Date(text);+ if (!Number.isNaN(lastmod.getTime())) {+ this.url.lastmod = lastmod;+ }}if (this.currentTag === 'priority') {diff --git a/packages/utils/test/sitemap.test.ts b/packages/utils/test/sitemap.test.tsindex 6a8545cb0..52831ff04 100644--- a/packages/utils/test/sitemap.test.ts+++ b/packages/utils/test/sitemap.test.ts@@ -423,6 +423,39 @@ describe('Sitemap', () => {);});+ it('does not leak metadata from a url without loc and drops invalid lastmod', async () => {+ const items: SitemapUrl[] = [];++ for await (const item of parseSitemap([+ {+ type: 'raw',+ content: [+ '<?xml version="1.0" encoding="UTF-8"?>',+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',+ '<url>',+ '<loc>http://not-exists.com/a</loc>',+ '<lastmod>not-a-date</lastmod>',+ '</url>',+ '<url>',+ '<lastmod>2020-01-01</lastmod>',+ '<priority>0.5</priority>',+ '</url>',+ '<url>',+ '<loc>http://not-exists.com/c</loc>',+ '</url>',+ '</urlset>',+ ].join('\n'),+ },+ ])) {+ items.push(item);+ }++ expect(items.map((item) => item.loc)).toEqual(['http://not-exists.com/a', 'http://not-exists.com/c']);+ expect(items[0].lastmod).toBeUndefined();+ expect(items[1].lastmod).toBeUndefined();+ expect(items[1].priority).toBeUndefined();+ });+it('loads sitemaps that reference other sitemaps from string', async () => {const sitemap = await Sitemap.fromXmlString([