Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(utils): parse sitemaps whose elements use a namespace prefix (#4393)

SwastikTripathi258787857d0b
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.ts
index 510c7a5e6..595aa3e6e 100644
--- a/packages/utils/src/internals/sitemap.ts
+++ b/packages/utils/src/internals/sitemap.ts
@@ -67,6 +67,14 @@ class SitemapTxtParser extends Transform {
}
}
+/** Namespaces of the sitemap protocol (https://www.sitemaps.org/protocol.html), including Google's legacy ones. */
+const SITEMAP_NAMESPACES = new Set([
+ 'http://www.sitemaps.org/schemas/sitemap/0.9',
+ 'https://www.sitemaps.org/schemas/sitemap/0.9',
+ 'http://www.google.com/schemas/sitemap/0.9',
+ 'http://www.google.com/schemas/sitemap/0.84',
+]);
+
class SitemapXmlParser extends Transform {
#decoder: StringDecoder = new StringDecoder('utf8');
#parser: sax.SAXParser;
@@ -74,6 +82,8 @@ class SitemapXmlParser extends Transform {
#rootTagName?: 'sitemapindex' | 'urlset';
#currentTag?: 'loc' | 'lastmod' | 'changefreq' | 'priority' = undefined;
#url: Partial<SitemapUrl> = {};
+ /** Prefixes bound to the sitemap namespace, e.g. `ns0` for `<ns0:urlset xmlns:ns0="...">`. */
+ #sitemapPrefixes = new Set<string>();
static async create(): Promise<SitemapXmlParser> {
const { SAXParser } = (await import('sax')).default;
@@ -108,26 +118,41 @@ class SitemapXmlParser extends Transform {
this.#parser.onerror = this.destroy.bind(this);
}
+ /**
+ * Returns the local name of a sitemap protocol element, or `undefined` for an element of another
+ * namespace (such as `image:loc`). Unprefixed elements are sitemap elements, as before.
+ */
+ #sitemapTagName(qualifiedName: string): string | undefined {
+ const colon = qualifiedName.indexOf(':');
+ if (colon === -1) return qualifiedName;
+ return this.#sitemapPrefixes.has(qualifiedName.slice(0, colon)) ? qualifiedName.slice(colon + 1) : undefined;
+ }
+
#onOpenTag(node: sax.Tag | sax.QualifiedTag) {
+ for (const [attribute, value] of Object.entries(node.attributes)) {
+ if (attribute.startsWith('xmlns:') && SITEMAP_NAMESPACES.has(String(value).trim())) {
+ this.#sitemapPrefixes.add(attribute.slice('xmlns:'.length));
+ }
+ }
+
+ const name = this.#sitemapTagName(node.name);
+
if (this.#rootTagName !== undefined) {
- if (
- node.name === 'loc' ||
- node.name === 'lastmod' ||
- node.name === 'priority' ||
- node.name === 'changefreq'
- ) {
- this.#currentTag = node.name;
+ if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
+ this.#currentTag = name;
}
}
- if (node.name === 'urlset') {
+ if (name === 'urlset') {
this.#rootTagName = 'urlset';
}
- if (node.name === 'sitemapindex') {
+ if (name === 'sitemapindex') {
this.#rootTagName = 'sitemapindex';
}
}
- #onCloseTag(name: string) {
+ #onCloseTag(qualifiedName: string) {
+ const name = this.#sitemapTagName(qualifiedName);
+
if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
this.#currentTag = undefined;
}
diff --git a/test/utils/sitemap.test.ts b/test/utils/sitemap.test.ts
index 43c401dd3..3fc4bf11e 100644
--- a/test/utils/sitemap.test.ts
+++ b/test/utils/sitemap.test.ts
@@ -701,6 +701,91 @@ describe('Sitemap', () => {
);
});
+ it('loads sitemaps whose elements use a namespace prefix', async () => {
+ // Python's xml.etree.ElementTree writes the sitemap namespace with an `ns0` prefix by default
+ const sitemap = await Sitemap.fromXmlString(
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<ns0:urlset xmlns:ns0="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ '<ns0:url>',
+ '<ns0:loc>http://not-exists.com/catalog?item=80&amp;desc=vacation_turkey</ns0:loc>',
+ '<ns0:lastmod>2004-11-23</ns0:lastmod>',
+ '<ns0:priority>0.4</ns0:priority>',
+ '</ns0:url>',
+ '<ns0:url>',
+ '<ns0:loc>http://not-exists.com/catalog?item=81&amp;desc=vacation_maledives</ns0:loc>',
+ '</ns0:url>',
+ '</ns0:urlset>',
+ ].join('\n'),
+ );
+
+ expect(new Set(sitemap.urls)).toEqual(
+ new Set([
+ 'http://not-exists.com/catalog?item=80&desc=vacation_turkey',
+ 'http://not-exists.com/catalog?item=81&desc=vacation_maledives',
+ ]),
+ );
+
+ const items: SitemapUrl[] = [];
+ for await (const item of parseSitemap([
+ {
+ type: 'raw',
+ content:
+ '<sm:urlset xmlns:sm="http://www.sitemaps.org/schemas/sitemap/0.9"><sm:url><sm:loc>http://not-exists.com/a</sm:loc><sm:lastmod>2004-11-23</sm:lastmod><sm:changefreq>daily</sm:changefreq><sm:priority>0.4</sm:priority></sm:url></sm:urlset>',
+ },
+ ])) {
+ items.push(item);
+ }
+ expect(items).toHaveLength(1);
+ expect(items[0]).toMatchObject({
+ loc: 'http://not-exists.com/a',
+ lastmod: new Date('2004-11-23'),
+ changefreq: 'daily',
+ priority: 0.4,
+ });
+ });
+
+ it('loads sitemap indexes whose elements use a namespace prefix', async () => {
+ const sitemap = await Sitemap.fromXmlString(
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<ns0:sitemapindex xmlns:ns0="http://www.sitemaps.org/schemas/sitemap/0.9">',
+ '<ns0:sitemap>',
+ '<ns0:loc>http://not-exists.com/sitemap_child.xml</ns0:loc>',
+ '</ns0:sitemap>',
+ '</ns0:sitemapindex>',
+ ].join('\n'),
+ undefined,
+ { httpClient: new FetchHttpClient() },
+ );
+
+ expect(new Set(sitemap.urls)).toEqual(
+ new Set([
+ 'http://not-exists.com/',
+ 'http://not-exists.com/catalog?item=12&desc=vacation_hawaii',
+ 'http://not-exists.com/catalog?item=73&desc=vacation_new_zealand',
+ 'http://not-exists.com/catalog?item=74&desc=vacation_newfoundland',
+ 'http://not-exists.com/catalog?item=83&desc=vacation_usa',
+ ]),
+ );
+ });
+
+ it('ignores loc elements of other namespaces', async () => {
+ const sitemap = await Sitemap.fromXmlString(
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<ns0:urlset xmlns:ns0="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">',
+ '<ns0:url>',
+ '<ns0:loc>http://not-exists.com/page</ns0:loc>',
+ '<image:image><image:loc>http://not-exists.com/image.jpg</image:loc></image:image>',
+ '</ns0:url>',
+ '</ns0:urlset>',
+ ].join('\n'),
+ );
+
+ expect(sitemap.urls).toEqual(['http://not-exists.com/page']);
+ });
+
it("loads XML sitemap even though it's gzipped according to file extension", async () => {
const sitemap = await Sitemap.load('http://not-exists.com/non_gzipped_sitemap.xml.gz', undefined, {
httpClient: new FetchHttpClient(),